From 5450038680485624004629eac32620b4f038aeca Mon Sep 17 00:00:00 2001 From: Eslam-Nawara Date: Mon, 7 Oct 2024 15:23:36 +0300 Subject: [PATCH 1/9] reintegrate wg in zos4 --- cmds/modules/netlightd/main.go | 4 +- go.mod | 7 + go.sum | 12 ++ pkg/gridtypes/zos/network_light.go | 34 ++++ pkg/netlight/network.go | 163 ++++++++++++++-- pkg/netlight/resource/resource.go | 209 +++++++++++++++++++- pkg/netlight/wireguard/wireguard.go | 232 +++++++++++++++++++++++ pkg/netlight/wireguard/wireguard_test.go | 67 +++++++ pkg/network_light.go | 6 +- pkg/primitives/network-light/network.go | 3 +- pkg/stubs/network_light_stub.go | 21 +- 11 files changed, 730 insertions(+), 28 deletions(-) create mode 100644 pkg/netlight/wireguard/wireguard.go create mode 100644 pkg/netlight/wireguard/wireguard_test.go diff --git a/cmds/modules/netlightd/main.go b/cmds/modules/netlightd/main.go index 193f27321..126536578 100644 --- a/cmds/modules/netlightd/main.go +++ b/cmds/modules/netlightd/main.go @@ -11,6 +11,7 @@ import ( "github.com/oasisprotocol/curve25519-voi/primitives/x25519" "github.com/pkg/errors" + "github.com/threefoldtech/zos/pkg/gridtypes/zos" "github.com/threefoldtech/zos/pkg/netlight" "github.com/threefoldtech/zos/pkg/netlight/nft" "github.com/threefoldtech/zos/pkg/netlight/resource" @@ -115,8 +116,7 @@ func action(cli *cli.Context) error { _, err = resource.Create("dmz", bridge, &net.IPNet{ IP: net.ParseIP("100.127.0.2"), Mask: net.CIDRMask(16, 32), - }, netlight.NDMZGwIP, nil, myceliumSeedFromIdentity(identity.PrivateKey(cli.Context))) - + }, netlight.NDMZGwIP, nil, myceliumSeedFromIdentity(identity.PrivateKey(cli.Context)), net.IPNet{}, zos.NetworkLight{}) if err != nil { return fmt.Errorf("failed to create ndmz resource: %w", err) } diff --git a/go.mod b/go.mod index 8f35f5d66..e118fe7dc 100644 --- a/go.mod +++ b/go.mod @@ -86,6 +86,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/gtank/ristretto255 v0.1.2 // indirect github.com/hanwen/go-fuse/v2 v2.3.0 // indirect @@ -93,6 +94,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/holiman/uint256 v1.2.3 // indirect + github.com/josharian/native v1.1.0 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/lestrrat-go/backoff/v2 v2.0.7 // indirect github.com/lestrrat-go/blackmagic v1.0.0 // indirect @@ -103,6 +105,9 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mdlayher/genetlink v1.3.2 // indirect + github.com/mdlayher/netlink v1.7.2 // indirect + github.com/mdlayher/socket v0.4.1 // indirect github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/locker v1.0.1 // indirect @@ -142,6 +147,8 @@ require ( golang.org/x/net v0.22.0 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/text v0.15.0 // indirect + golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect + golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 // indirect diff --git a/go.sum b/go.sum index 49c2960f7..ff34eb1a2 100644 --- a/go.sum +++ b/go.sum @@ -298,6 +298,8 @@ github.com/jgautheron/goconst v0.0.0-20170703170152-9740945f5dcb/go.mod h1:82Txj github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/joncrlsn/dque v0.0.0-20200702023911-3e80e3146ce5 h1:bo1aoO6l128nKJCBrFflOj9s+KPqMM7ErNyB5GGBNDs= github.com/joncrlsn/dque v0.0.0-20200702023911-3e80e3146ce5/go.mod h1:dNKs71rs2VJGBAmttu7fouEsRQlRjxy0p1Sx+T5wbpY= +github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= +github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -359,6 +361,12 @@ github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw= +github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o= +github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= +github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= +github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= +github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= github.com/mibk/dupl v1.0.0/go.mod h1:pCr4pNxxIbFGvtyCOi0c7LVjmV6duhKWV+ex5vh38ME= github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b h1:QrHweqAtyJ9EwCaGHBu1fghwxIPiopAHV06JlXrMHjk= @@ -718,6 +726,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo= +golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4= +golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 h1:CawjfCvYQH2OU3/TnxLx97WDSUDRABfT18pCOYwc2GE= +golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6/go.mod h1:3rxYc4HtVcSG9gVaTs2GEBdehh+sYPOwKtyUWEOTb80= gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/pkg/gridtypes/zos/network_light.go b/pkg/gridtypes/zos/network_light.go index 6f443de56..4ec9d0b52 100644 --- a/pkg/gridtypes/zos/network_light.go +++ b/pkg/gridtypes/zos/network_light.go @@ -55,6 +55,10 @@ func NetworkIDFromWorkloadID(wl gridtypes.WorkloadID) (NetID, error) { // - For each PC or a laptop (for each wireguard peer) there must be a peer in the peer list (on all nodes) // This is why this can get complicated. type NetworkLight struct { + // IP range of the network, must be an IPv4 /16 + // for example a 10.1.0.0/16 + NetworkIPRange gridtypes.IPNet `json:"ip_range"` + // IPV4 subnet for this network resource // this must be a valid subnet of the entire network ip range. // for example 10.1.1.0/24 @@ -65,6 +69,36 @@ type NetworkLight struct { // if no mycelium configuration is provided, vms can't // get mycelium IPs. Mycelium Mycelium `json:"mycelium,omitempty"` + + // The private wg key of this node (this peer) which is installing this + // network workload right now. + // This has to be filled in by the user (and not generated for example) + // because other peers need to be installed as well (with this peer public key) + // hence it's easier to configure everything one time at the user side and then + // apply everything on all nodes at once + WGPrivateKey string `json:"wireguard_private_key"` + + // WGListenPort is the wireguard listen port on this node. this has + // to be filled in by the user for same reason as private key (other nodes need to know about it) + // To find a free port you have to ask the node first by a call over RMB about which ports are possible + // to use. + WGListenPort uint16 `json:"wireguard_listen_port"` + + // Peers is a list of other peers in this network + Peers []Peer `json:"peers"` +} + +// Peer is the description of a peer of a NetResource +type Peer struct { + // IPV4 subnet of the network resource of the peer + Subnet gridtypes.IPNet `json:"subnet"` + // WGPublicKey of the peer (driven from its private key) + WGPublicKey string `json:"wireguard_public_key"` + // Allowed Ips is related to his subnet. + // todo: remove and derive from subnet + AllowedIPs []gridtypes.IPNet `json:"allowed_ips"` + // Entrypoint of the peer + Endpoint string `json:"endpoint"` } type MyceliumPeer string diff --git a/pkg/netlight/network.go b/pkg/netlight/network.go index 92cc5633e..c1215d7bb 100644 --- a/pkg/netlight/network.go +++ b/pkg/netlight/network.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "slices" + "strings" "time" "github.com/blang/semver" @@ -25,6 +26,8 @@ import ( "github.com/threefoldtech/zos/pkg/netlight/namespace" "github.com/threefoldtech/zos/pkg/netlight/options" "github.com/threefoldtech/zos/pkg/netlight/resource" + "github.com/threefoldtech/zos/pkg/netlight/wireguard" + "github.com/threefoldtech/zos/pkg/set" "github.com/threefoldtech/zos/pkg/versioned" "github.com/vishvananda/netlink" ) @@ -38,18 +41,17 @@ const ( networkDir = "networks" ) -var ( - NDMZGwIP = &net.IPNet{ - IP: net.ParseIP("100.127.0.1"), - Mask: net.CIDRMask(16, 32), - } -) +var NDMZGwIP = &net.IPNet{ + IP: net.ParseIP("100.127.0.1"), + Mask: net.CIDRMask(16, 32), +} var NetworkSchemaLatestVersion = semver.MustParse("0.1.0") type networker struct { ipamLease string networkDir string + portSet *set.UIntSet } var _ pkg.NetworkerLight = (*networker)(nil) @@ -63,13 +65,20 @@ func NewNetworker() (pkg.NetworkerLight, error) { ipamLease := filepath.Join(vd, ipamLeaseDir) runtimeDir := filepath.Join(vd, networkDir) - return &networker{ + nw := networker{ ipamLease: ipamLease, networkDir: runtimeDir, - }, nil + portSet: set.NewInt(), + } + + if err := nw.syncWGPorts(); err != nil { + return nil, err + } + return &nw, nil } -func (n *networker) Create(name string, privateNet net.IPNet, seed []byte) error { +// func (n *networker) Create(name string, privateNet net.IPNet, seed []byte, twinID uint32, wgListenPort uint16, wgPrivateKey string, ipRange net.IPNet, nr pkg.Network) error { +func (n *networker) Create(name string, net zos.NetworkLight) error { b, err := bridge.Get(NDMZBridge) if err != nil { return err @@ -79,7 +88,67 @@ func (n *networker) Create(name string, privateNet net.IPNet, seed []byte) error return err } - _, err = resource.Create(name, b, ip, NDMZGwIP, &privateNet, seed) + storedNR, err := n.networkOf(zos.NetID(name)) + if err != nil && !os.IsNotExist(err) { + return errors.Wrap(err, "failed to load previous network setup") + } + + if err == nil { + if err := n.releasePort(storedNR.WGListenPort); err != nil { + return err + } + } + + if err := n.reservePort(net.WGListenPort); err != nil { + return err + } + + netr, err := resource.Create(name, b, ip, NDMZGwIP, &net.Subnet.IPNet, net.Mycelium.Key, net.NetworkIPRange.IPNet, net) + if err != nil { + return err + } + + cleanup := func() { + log.Error().Msg("clean up network resource") + if err := resource.Delete(name); err != nil { + log.Error().Err(err).Msg("error during deletion of network resource after failed deployment") + } + if err := n.releasePort(net.WGListenPort); err != nil { + log.Error().Err(err).Msg("release wireguard port failed") + } + } + + defer func() { + if err != nil { + cleanup() + } + }() + + wgName, err := netr.WGName() + if err != nil { + return errors.Wrap(err, "failed to get wg interface name for network resource") + } + + exists, err := netr.HasWireguard() + if err != nil { + return errors.Wrap(err, "failed to check if network resource has wireguard setup") + } + + if !exists { + var wg *wireguard.Wireguard + wg, err = wireguard.New(wgName) + if err != nil { + return errors.Wrapf(err, "failed to create wg interface for network resource '%s'", name) + } + if err = netr.SetWireguard(wg); err != nil { + return errors.Wrap(err, "failed to setup wireguard interface for network resource") + } + } + + if err = netr.ConfigureWG(net.WGPrivateKey); err != nil { + return errors.Wrap(err, "failed to configure network resource") + } + return err } @@ -89,7 +158,6 @@ func (n *networker) Delete(name string) error { } return resource.Delete(name) - } func (n *networker) AttachPrivate(name, id string, vmIp net.IP) (device pkg.TapDevice, err error) { @@ -391,10 +459,14 @@ func (n *networker) Interfaces(iface string, netns string) (pkg.Interfaces, erro } func CreateNDMZBridge() (*netlink.Bridge, error) { - return createNDMZBridge(NDMZBridge, NDMZGw) + return createNDMZBridge(NDMZBridge) } -func createNDMZBridge(name string, gw string) (*netlink.Bridge, error) { +func (n *networker) WireguardPorts() ([]uint, error) { + return n.portSet.List() +} + +func createNDMZBridge(name string) (*netlink.Bridge, error) { if !bridge.Exists(name) { if _, err := bridge.New(name); err != nil { return nil, errors.Wrapf(err, "couldn't create bridge %s", name) @@ -424,3 +496,68 @@ func createNDMZBridge(name string, gw string) (*netlink.Bridge, error) { return link.(*netlink.Bridge), nil } + +func (n *networker) reservePort(port uint16) error { + log.Debug().Uint16("port", port).Msg("reserve wireguard port") + err := n.portSet.Add(uint(port)) + if err != nil { + return errors.Wrap(err, "wireguard listen port already in use, pick another one") + } + + return nil +} + +func (n *networker) releasePort(port uint16) error { + log.Debug().Uint16("port", port).Msg("release wireguard port") + n.portSet.Remove(uint(port)) + return nil +} + +func (n *networker) syncWGPorts() error { + names, err := namespace.List("n-") + if err != nil { + return err + } + + readPort := func(name string) (int, error) { + netNS, err := namespace.GetByName(name) + if err != nil { + return 0, err + } + defer netNS.Close() + + ifaceName := strings.Replace(name, "n-", "w-", 1) + + var port int + err = netNS.Do(func(_ ns.NetNS) error { + link, err := wireguard.GetByName(ifaceName) + if err != nil { + return err + } + d, err := link.Device() + if err != nil { + return err + } + + port = d.ListenPort + return nil + }) + if err != nil { + return 0, err + } + + return port, nil + } + + for _, name := range names { + port, err := readPort(name) + if err != nil { + log.Error().Err(err).Str("namespace", name).Msgf("failed to read port for network namespace") + continue + } + // skip error cause we don't care if there are some duplicate at this point + _ = n.portSet.Add(uint(port)) + } + + return nil +} diff --git a/pkg/netlight/resource/resource.go b/pkg/netlight/resource/resource.go index 038d60318..5658771dc 100644 --- a/pkg/netlight/resource/resource.go +++ b/pkg/netlight/resource/resource.go @@ -9,14 +9,18 @@ import ( "path/filepath" "github.com/containernetworking/plugins/pkg/ns" + mapset "github.com/deckarep/golang-set" "github.com/pkg/errors" + "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg" + "github.com/threefoldtech/zos/pkg/gridtypes/zos" "github.com/threefoldtech/zos/pkg/netlight/bridge" "github.com/threefoldtech/zos/pkg/netlight/ifaceutil" "github.com/threefoldtech/zos/pkg/netlight/namespace" "github.com/threefoldtech/zos/pkg/netlight/nft" "github.com/threefoldtech/zos/pkg/netlight/options" "github.com/threefoldtech/zos/pkg/netlight/tuntap" + "github.com/threefoldtech/zos/pkg/netlight/wireguard" "github.com/threefoldtech/zos/pkg/zinit" "github.com/vishvananda/netlink" ) @@ -32,6 +36,10 @@ var nftRules embed.FS type Resource struct { name string + // local network resources + resource zos.NetworkLight + // network IP range, usually a /16 + networkIPRange net.IPNet } // Create creates a network resource (please check docs) @@ -43,7 +51,7 @@ type Resource struct { // ndmzGwIP: the gw Ip for the resource. Normally this is the ip assigned to the master bridge. // privateNet: optional private network range // seed: mycelium seed -func Create(name string, master *netlink.Bridge, ndmzIP *net.IPNet, ndmzGwIP *net.IPNet, privateNet *net.IPNet, seed []byte) (*Resource, error) { +func Create(name string, master *netlink.Bridge, ndmzIP *net.IPNet, ndmzGwIP *net.IPNet, privateNet *net.IPNet, seed []byte, ipRange net.IPNet, nr zos.NetworkLight) (*Resource, error) { privateNetBr := fmt.Sprintf("r%s", name) myBr := fmt.Sprintf("m%s", name) nsName := fmt.Sprintf("n%s", name) @@ -148,7 +156,6 @@ func Create(name string, master *netlink.Bridge, ndmzIP *net.IPNet, ndmzGwIP *ne return nil }) - if err != nil { return nil, err } @@ -162,7 +169,8 @@ func Create(name string, master *netlink.Bridge, ndmzIP *net.IPNet, ndmzGwIP *ne return nil, fmt.Errorf("failed to apply nft rules for namespace '%s': %w", name, err) } rules.Close() - return &Resource{name}, setupMycelium(netNS, infMycelium, seed) + // we need to set up the resource right and get it right. + return &Resource{name, nr, ipRange}, setupMycelium(netNS, infMycelium, seed) } func Delete(name string) error { @@ -192,7 +200,6 @@ func Delete(name string) error { } return bridge.Delete(myBr) - } func setLinkAddr(name string, ip *net.IPNet) error { @@ -221,7 +228,7 @@ func Get(name string) (*Resource, error) { nsName := fmt.Sprintf("n%s", name) if namespace.Exists(nsName) { - return &Resource{name}, nil + return &Resource{name: name}, nil } return nil, fmt.Errorf("resource not found: %s", name) @@ -340,7 +347,6 @@ func (r *Resource) AttachMycelium(id string, seed []byte) (device pkg.TapDevice, } route := pkg.Route{ - Net: net.IPNet{ IP: net.ParseIP("400::"), Mask: net.CIDRMask(7, 128), @@ -422,12 +428,9 @@ func (r *Resource) AttachMyceliumZDB(id string, zdbNS ns.NetNS) (err error) { }, Gw: gw.IP, }) - }) } return nil - // - } func (r *Resource) Seed() (seed []byte, err error) { @@ -440,3 +443,191 @@ func (r *Resource) Seed() (seed []byte, err error) { seed, err = os.ReadFile(filepath.Join(MyceliumSeedDir, name)) return } + +// WGName returns the name of the wireguard interface to create for the network resource +func (r *Resource) WGName() (string, error) { + wgName := fmt.Sprintf("w-%s", r.name) + if len(wgName) > 15 { + return "", errors.Errorf("network namespace too long %s", wgName) + } + return wgName, nil +} + +// HasWireguard checks if network resource has wireguard setup up +func (r *Resource) HasWireguard() (bool, error) { + nsName, err := r.Namespace() + if err != nil { + return false, err + } + + nrNetNS, err := namespace.GetByName(nsName) + if err != nil { + return false, err + } + + defer nrNetNS.Close() + + wgName, err := r.WGName() + if err != nil { + return false, err + } + exist := false + err = nrNetNS.Do(func(_ ns.NetNS) error { + _, err = wireguard.GetByName(wgName) + + if errors.As(err, &netlink.LinkNotFoundError{}) { + return nil + } else if err != nil { + return err + } + + exist = true + return nil + }) + + return exist, err +} + +// Namespace returns the name of the network namespace to create for the network resource +func (r *Resource) Namespace() (string, error) { + name := fmt.Sprintf("n-%s", r.name) + if len(name) > 15 { + return "", errors.Errorf("network namespace too long %s", name) + } + return name, nil +} + +// SetWireguard sets wireguard of this network resource +func (r *Resource) SetWireguard(wg *wireguard.Wireguard) error { + nsName, err := r.Namespace() + if err != nil { + return err + } + + nrNetNS, err := namespace.GetByName(nsName) + if err != nil { + return err + } + defer nrNetNS.Close() + + return netlink.LinkSetNsFd(wg, int(nrNetNS.Fd())) +} + +// ConfigureWG sets the routes and IP addresses on the +// wireguard interface of the network resources +func (r *Resource) ConfigureWG(privateKey string) error { + wgPeers, err := r.wgPeers() + if err != nil { + return errors.Wrap(err, "failed to wireguard peer configuration") + } + + nsName, err := r.Namespace() + if err != nil { + return err + } + netNS, err := namespace.GetByName(nsName) + if err != nil { + return fmt.Errorf("network namespace %s does not exits", nsName) + } + + handler := func(_ ns.NetNS) error { + wgName, err := r.WGName() + if err != nil { + return err + } + + wg, err := wireguard.GetByName(wgName) + if err != nil { + return errors.Wrapf(err, "failed to get wireguard interface %s", wgName) + } + + if err = wg.Configure(privateKey, int(r.resource.WGListenPort), wgPeers); err != nil { + return errors.Wrap(err, "failed to configure wireguard interface") + } + + addrs, err := netlink.AddrList(wg, netlink.FAMILY_ALL) + if err != nil { + return err + } + curAddrs := mapset.NewSet() + for _, addr := range addrs { + curAddrs.Add(addr.IPNet.String()) + } + + newAddrs := mapset.NewSet() + newAddrs.Add(wgIP(&r.resource.Subnet.IPNet).String()) + + toRemove := curAddrs.Difference(newAddrs) + toAdd := newAddrs.Difference(curAddrs) + + log.Info().Msgf("current %s", curAddrs.String()) + log.Info().Msgf("to add %s", toAdd.String()) + log.Info().Msgf("to remove %s", toRemove.String()) + + for addr := range toAdd.Iter() { + addr, _ := addr.(string) + log.Debug().Str("ip", addr).Msg("set ip on wireguard interface") + if err := wg.SetAddr(addr); err != nil && !os.IsExist(err) { + return errors.Wrapf(err, "failed to set address %s on wireguard interface %s", addr, wg.Attrs().Name) + } + } + + for addr := range toRemove.Iter() { + addr, _ := addr.(string) + log.Debug().Str("ip", addr).Msg("unset ip on wireguard interface") + if err := wg.UnsetAddr(addr); err != nil && !os.IsNotExist(err) { + return errors.Wrapf(err, "failed to unset address %s on wireguard interface %s", addr, wg.Attrs().Name) + } + } + + route := &netlink.Route{ + LinkIndex: wg.Attrs().Index, + Dst: &r.networkIPRange, + } + if err := netlink.RouteAdd(route); err != nil && !os.IsExist(err) { + log.Error(). + Err(err). + Str("route", route.String()). + Msg("fail to set route") + return errors.Wrapf(err, "failed to add route %s", route.String()) + } + + return nil + } + + return netNS.Do(handler) +} + +func (nr *Resource) wgPeers() ([]*wireguard.Peer, error) { + wgPeers := make([]*wireguard.Peer, 0, len(nr.resource.Peers)+1) + + for _, peer := range nr.resource.Peers { + + allowedIPs := make([]string, 0, len(peer.AllowedIPs)) + for _, ip := range peer.AllowedIPs { + allowedIPs = append(allowedIPs, ip.String()) + } + + wgPeer := &wireguard.Peer{ + PublicKey: peer.WGPublicKey, + AllowedIPs: allowedIPs, + Endpoint: peer.Endpoint, + } + + log.Info().Str("peer prefix", peer.Subnet.String()).Msg("generate wireguard configuration for peer") + wgPeers = append(wgPeers, wgPeer) + } + + return wgPeers, nil +} + +func wgIP(subnet *net.IPNet) *net.IPNet { + // example: 10.3.1.0 -> 100.64.3.1 + a := subnet.IP[len(subnet.IP)-3] + b := subnet.IP[len(subnet.IP)-2] + + return &net.IPNet{ + IP: net.IPv4(0x64, 0x40, a, b), + Mask: net.CIDRMask(16, 32), + } +} diff --git a/pkg/netlight/wireguard/wireguard.go b/pkg/netlight/wireguard/wireguard.go new file mode 100644 index 000000000..c86c3d36a --- /dev/null +++ b/pkg/netlight/wireguard/wireguard.go @@ -0,0 +1,232 @@ +package wireguard + +import ( + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + + "github.com/vishvananda/netlink" + "golang.zx2c4.com/wireguard/wgctrl" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" +) + +// Wireguard is a netlink.Link of type wireguard +type Wireguard struct { + attrs *netlink.LinkAttrs +} + +// New create a new wireguard interface +func New(name string) (*Wireguard, error) { + attrs := netlink.NewLinkAttrs() + attrs.Name = name + attrs.MTU = 1420 + + wg := &Wireguard{attrs: &attrs} + if err := netlink.LinkAdd(wg); err != nil && !os.IsExist(err) { + return nil, err + } + + // always make sure we load the full attributes + return GetByName(name) +} + +// GetByName return a wireguard object by its name +func GetByName(name string) (*Wireguard, error) { + link, err := netlink.LinkByName(name) + if err != nil { + return nil, err + } + + if link.Type() != "wireguard" { + return nil, fmt.Errorf("link %s is not of type wireguard", name) + } + wg := &Wireguard{ + attrs: link.Attrs(), + } + return wg, nil +} + +// Type implements the netlink.Link interface +func (w *Wireguard) Type() string { + return "wireguard" +} + +// Attrs implements the netlink.Link interface +func (w *Wireguard) Attrs() *netlink.LinkAttrs { + return w.attrs +} + +// Device returns the detail of the configuration of the +// wireguard interface +func (w *Wireguard) Device() (*wgtypes.Device, error) { + wg, err := wgctrl.New() + if err != nil { + return nil, err + } + defer wg.Close() + + return wg.Device(w.attrs.Name) +} + +// SetAddr sets an IP address on the interface +func (w *Wireguard) SetAddr(cidr string) error { + addr, err := netlink.ParseAddr(cidr) + if err != nil { + return err + } + + if err := netlink.AddrAdd(w, addr); err != nil && !os.IsExist(err) { + return err + } + return nil +} + +// UnsetAddr removes an IP address from the interface +func (w *Wireguard) UnsetAddr(cidr string) error { + addr, err := netlink.ParseAddr(cidr) + if err != nil { + return err + } + + if err := netlink.AddrDel(w, addr); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// Peer represent a peer in a wireguard configuration +type Peer struct { + PublicKey string + Endpoint string + AllowedIPs []string +} + +// Configure configures the wiregard configuration +func (w *Wireguard) Configure(privateKey string, listentPort int, peers []*Peer) error { + if err := netlink.LinkSetDown(w); err != nil { + return err + } + + wc, err := wgctrl.New() + if err != nil { + return err + } + defer wc.Close() + + peersConfig := make([]wgtypes.PeerConfig, len(peers)) + for i, peer := range peers { + p, err := newPeer(peer.PublicKey, peer.Endpoint, peer.AllowedIPs) + if err != nil { + return err + } + peersConfig[i] = p + } + + key, err := wgtypes.ParseKey(privateKey) + if err != nil { + return err + } + + config := wgtypes.Config{ + PrivateKey: &key, + Peers: peersConfig, + ListenPort: &listentPort, + ReplacePeers: true, + } + log.Info().Msg("configure wg device") + + if err := wc.ConfigureDevice(w.attrs.Name, config); err != nil { + return errors.Wrap(err, "failed to configure wireguard interface") + } + + if err := netlink.LinkSetUp(w); err != nil && !os.IsExist(err) { + return errors.Wrapf(err, "failed to bring wireguard interface %s up", w.Attrs().Name) + } + return nil +} + +func newPeer(pubkey, endpoint string, allowedIPs []string) (wgtypes.PeerConfig, error) { + peer := wgtypes.PeerConfig{ + ReplaceAllowedIPs: true, + } + var err error + + duration := time.Second * 20 + peer.PersistentKeepaliveInterval = &duration + + peer.PublicKey, err = wgtypes.ParseKey(pubkey) + if err != nil { + return peer, err + } + + if endpoint != "" { + host, p, err := net.SplitHostPort(endpoint) + if err != nil { + return peer, err + } + + port, err := strconv.Atoi(p) + if err != nil { + return peer, err + } + + peer.Endpoint = &net.UDPAddr{ + IP: net.ParseIP(host), + Port: port, + } + } + + for _, allowedIP := range allowedIPs { + ip, ipNet, err := net.ParseCIDR(allowedIP) + if err != nil { + return peer, err + } + ipNet.IP = ip + peer.AllowedIPs = append(peer.AllowedIPs, *ipNet) + } + + return peer, nil +} + +// GenerateKey generates a new private key. If key already exists +// in that location, that key is returned instead. +func GenerateKey(dir string) (wgtypes.Key, error) { + path := filepath.Join(dir, "key.priv") + data, err := os.ReadFile(path) + if err == nil { + // key already exists + return wgtypes.ParseKey(string(data)) + } else if !os.IsNotExist(err) { + // another error than not exist + return wgtypes.Key{}, err + } + + key, err := wgtypes.GeneratePrivateKey() + if err != nil { + return wgtypes.Key{}, err + } + if err := os.MkdirAll(dir, 0700); err != nil { + return wgtypes.Key{}, err + } + + if err := os.WriteFile(path, []byte(key.String()), 0400); err != nil { + return wgtypes.Key{}, err + } + return key, nil +} + +// LoadKey tries to read a private key from disk +func LoadKey(dir string) (wgtypes.Key, error) { + path := filepath.Join(dir, "key.priv") + b, err := os.ReadFile(path) + if err != nil { + return wgtypes.Key{}, err + } + return wgtypes.ParseKey(string(b)) +} diff --git a/pkg/netlight/wireguard/wireguard_test.go b/pkg/netlight/wireguard/wireguard_test.go new file mode 100644 index 000000000..6ccfa2783 --- /dev/null +++ b/pkg/netlight/wireguard/wireguard_test.go @@ -0,0 +1,67 @@ +package wireguard + +import ( + "testing" + + "github.com/vishvananda/netlink" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewPeer(t *testing.T) { + endpoint := "37.187.124.71:51820" + publicKey := "mR5fBXohKe2MZ6v+GLwlKwrvkFxo1VvV3bPNHDBhOAI=" + allowedIps := []string{"172.21.0.0/24", "fe80::f002/64"} + peer, err := newPeer(publicKey, endpoint, allowedIps) + require.NoError(t, err) + + require.Equal(t, endpoint, peer.Endpoint.String()) + require.Equal(t, publicKey, peer.PublicKey.String()) + tmp := make([]string, len(peer.AllowedIPs)) + for i, ip := range peer.AllowedIPs { + tmp[i] = ip.String() + } + require.Equal(t, allowedIps, tmp) +} + +func TestConfigure(t *testing.T) { + wg, err := New("test") + require.NoError(t, err) + + defer func() { + _ = netlink.LinkDel(wg) + }() + + privateKey := "4DwTbGRWECH8oqcTXdoWXGOaWWC952QKbFE1fMzBNmA=" + publicKey := "kDd5mB6L4gkd3U5W287JeQu7urFzBYH51JQZUrJd8Hg=" + endpoint := "37.187.124.71:51820" + peerPublicKey := "mR5fBXohKe2MZ6v+GLwlKwrvkFxo1VvV3bPNHDBhOAI=" + allowedIps := []string{"172.21.0.0/24", "192.168.1.10/32", "fe80::f002/128"} + + err = wg.Configure(privateKey, 1600, []*Peer{ + { + PublicKey: peerPublicKey, + AllowedIPs: allowedIps, + Endpoint: endpoint, + }, + }) + require.NoError(t, err) + + device, err := wg.Device() + require.NoError(t, err) + + assert.Equal(t, privateKey, device.PrivateKey.String()) + assert.Equal(t, publicKey, device.PrivateKey.PublicKey().String()) + assert.Equal(t, publicKey, device.PrivateKey.PublicKey().String()) + + for _, peer := range device.Peers { + assert.Equal(t, endpoint, peer.Endpoint.String()) + + actual := make([]string, len(peer.AllowedIPs)) + for y, ip := range peer.AllowedIPs { + actual[y] = ip.String() + } + assert.Equal(t, allowedIps, actual) + } +} diff --git a/pkg/network_light.go b/pkg/network_light.go index ba0bfc828..992b87a2e 100644 --- a/pkg/network_light.go +++ b/pkg/network_light.go @@ -3,6 +3,8 @@ package pkg import ( "context" "net" + + "github.com/threefoldtech/zos/pkg/gridtypes/zos" ) type Route struct { @@ -17,7 +19,7 @@ type Route struct { // NetworkerLight is the interface for the network light module type NetworkerLight interface { - Create(name string, privateNet net.IPNet, seed []byte) error + Create(name string, net zos.NetworkLight) error Delete(name string) error AttachPrivate(name, id string, vmIp net.IP) (device TapDevice, err error) AttachMycelium(name, id string, seed []byte) (device TapDevice, err error) @@ -29,6 +31,8 @@ type NetworkerLight interface { Ready() error ZOSAddresses(ctx context.Context) <-chan NetlinkAddresses GetSubnet(networkID NetID) (net.IPNet, error) + + WireguardPorts() ([]uint, error) } type TapDevice struct { diff --git a/pkg/primitives/network-light/network.go b/pkg/primitives/network-light/network.go index 5a4e19671..d7a295601 100644 --- a/pkg/primitives/network-light/network.go +++ b/pkg/primitives/network-light/network.go @@ -39,7 +39,8 @@ func (p *Manager) networkProvisionImpl(ctx context.Context, wl *gridtypes.Worklo mgr := stubs.NewNetworkerLightStub(p.zbus) log.Debug().Str("network", fmt.Sprintf("%+v", network)).Msg("provision network") - err := mgr.Create(ctx, string(zos.NetworkID(twin, wl.Name)), network.Subnet.IPNet, network.Mycelium.Key) + // err := mgr.Create(ctx, string(zos.NetworkID(twin, wl.Name)), network.Subnet.IPNet, network.Mycelium.Key) + err := mgr.Create(ctx, string(zos.NetworkID(twin, wl.Name)), network) if err != nil { return errors.Wrapf(err, "failed to create network resource for network %s", wl.ID) } diff --git a/pkg/stubs/network_light_stub.go b/pkg/stubs/network_light_stub.go index 21d7afdc0..56461579b 100644 --- a/pkg/stubs/network_light_stub.go +++ b/pkg/stubs/network_light_stub.go @@ -80,8 +80,8 @@ func (s *NetworkerLightStub) AttachZDB(ctx context.Context, arg0 string) (ret0 s return } -func (s *NetworkerLightStub) Create(ctx context.Context, arg0 string, arg1 net.IPNet, arg2 []uint8) (ret0 error) { - args := []interface{}{arg0, arg1, arg2} +func (s *NetworkerLightStub) Create(ctx context.Context, arg0 string, arg1 zos.NetworkLight) (ret0 error) { + args := []interface{}{arg0, arg1} result, err := s.client.RequestContext(ctx, s.module, s.object, "Create", args...) if err != nil { panic(err) @@ -190,6 +190,23 @@ func (s *NetworkerLightStub) Ready(ctx context.Context) (ret0 error) { return } +func (s *NetworkerLightStub) WireguardPorts(ctx context.Context) (ret0 []uint, ret1 error) { + args := []interface{}{} + result, err := s.client.RequestContext(ctx, s.module, s.object, "WireguardPorts", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret1 = result.CallError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + func (s *NetworkerLightStub) ZDBIPs(ctx context.Context, arg0 string) (ret0 [][]uint8, ret1 error) { args := []interface{}{arg0} result, err := s.client.RequestContext(ctx, s.module, s.object, "ZDBIPs", args...) From 3d74a0a97a5ecbafc07fe29d0a3ae1033ee9287c Mon Sep 17 00:00:00 2001 From: Eslam-Nawara Date: Mon, 7 Oct 2024 15:23:50 +0300 Subject: [PATCH 2/9] add list wg ports endpoint --- pkg/gridtypes/zos/network_light.go | 7 +++---- pkg/netlight/resource/resource.go | 2 +- pkg/zos_api/network.go | 4 ++++ pkg/zos_api/routes.go | 3 +-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/gridtypes/zos/network_light.go b/pkg/gridtypes/zos/network_light.go index 4ec9d0b52..90dc88160 100644 --- a/pkg/gridtypes/zos/network_light.go +++ b/pkg/gridtypes/zos/network_light.go @@ -55,10 +55,6 @@ func NetworkIDFromWorkloadID(wl gridtypes.WorkloadID) (NetID, error) { // - For each PC or a laptop (for each wireguard peer) there must be a peer in the peer list (on all nodes) // This is why this can get complicated. type NetworkLight struct { - // IP range of the network, must be an IPv4 /16 - // for example a 10.1.0.0/16 - NetworkIPRange gridtypes.IPNet `json:"ip_range"` - // IPV4 subnet for this network resource // this must be a valid subnet of the entire network ip range. // for example 10.1.1.0/24 @@ -70,6 +66,9 @@ type NetworkLight struct { // get mycelium IPs. Mycelium Mycelium `json:"mycelium,omitempty"` + // IP range of the network, must be an IPv4 /16 + // for example a 10.1.0.0/16 + NetworkIPRange gridtypes.IPNet `json:"ip_range"` // The private wg key of this node (this peer) which is installing this // network workload right now. // This has to be filled in by the user (and not generated for example) diff --git a/pkg/netlight/resource/resource.go b/pkg/netlight/resource/resource.go index 5658771dc..6d70a802e 100644 --- a/pkg/netlight/resource/resource.go +++ b/pkg/netlight/resource/resource.go @@ -490,7 +490,7 @@ func (r *Resource) HasWireguard() (bool, error) { // Namespace returns the name of the network namespace to create for the network resource func (r *Resource) Namespace() (string, error) { - name := fmt.Sprintf("n-%s", r.name) + name := fmt.Sprintf("n%s", r.name) if len(name) > 15 { return "", errors.Errorf("network namespace too long %s", name) } diff --git a/pkg/zos_api/network.go b/pkg/zos_api/network.go index f763f2a5a..f70be0c14 100644 --- a/pkg/zos_api/network.go +++ b/pkg/zos_api/network.go @@ -10,6 +10,10 @@ import ( "github.com/threefoldtech/zos/pkg/gridtypes" ) +func (g *ZosAPI) networkListWGPortsHandler(ctx context.Context, payload []byte) (interface{}, error) { + return g.networkerLightStub.WireguardPorts(ctx) +} + func (g *ZosAPI) networkInterfacesHandler(ctx context.Context, payload []byte) (interface{}, error) { results := make(map[string][]net.IPNet) interfaces, err := g.networkerLightStub.Interfaces(ctx, "zos", "") diff --git a/pkg/zos_api/routes.go b/pkg/zos_api/routes.go index b1a0e4078..dfc37b094 100644 --- a/pkg/zos_api/routes.go +++ b/pkg/zos_api/routes.go @@ -5,7 +5,6 @@ import ( ) func (g *ZosAPI) SetupRoutes(router *peer.Router) { - root := router.SubRoute("zos") root.Use(g.log) system := root.SubRoute("system") @@ -26,7 +25,7 @@ func (g *ZosAPI) SetupRoutes(router *peer.Router) { storage.WithHandler("pools", g.storagePoolsHandler) network := root.SubRoute("network") - // network.WithHandler("list_wg_ports", g.networkListWGPortsHandler) + network.WithHandler("list_wg_ports", g.networkListWGPortsHandler) // network.WithHandler("public_config_get", g.networkPublicConfigGetHandler) network.WithHandler("interfaces", g.networkInterfacesHandler) network.WithHandler("has_ipv6", g.networkHasIPv6Handler) From c38cc5f524198f8381cd2e921e8acd971e8ac0a9 Mon Sep 17 00:00:00 2001 From: Eslam-Nawara Date: Wed, 9 Oct 2024 16:07:49 +0300 Subject: [PATCH 3/9] save network resource on creation --- pkg/netlight/network.go | 147 ++++++++++++++++-------- pkg/network_light.go | 5 +- pkg/primitives/network-light/network.go | 5 +- pkg/stubs/network_light_stub.go | 7 +- 4 files changed, 110 insertions(+), 54 deletions(-) diff --git a/pkg/netlight/network.go b/pkg/netlight/network.go index c1215d7bb..b0801a246 100644 --- a/pkg/netlight/network.go +++ b/pkg/netlight/network.go @@ -19,6 +19,7 @@ import ( "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg" "github.com/threefoldtech/zos/pkg/cache" + "github.com/threefoldtech/zos/pkg/gridtypes" "github.com/threefoldtech/zos/pkg/gridtypes/zos" "github.com/threefoldtech/zos/pkg/netlight/bridge" "github.com/threefoldtech/zos/pkg/netlight/ifaceutil" @@ -39,6 +40,7 @@ const ( ipamLeaseDir = "ndmz-lease" DefaultBridge = "zos" networkDir = "networks" + linkDir = "link" ) var NDMZGwIP = &net.IPNet{ @@ -52,6 +54,7 @@ type networker struct { ipamLease string networkDir string portSet *set.UIntSet + linkDir string } var _ pkg.NetworkerLight = (*networker)(nil) @@ -62,13 +65,19 @@ func NewNetworker() (pkg.NetworkerLight, error) { return nil, fmt.Errorf("failed to create networkd cache directory: %w", err) } - ipamLease := filepath.Join(vd, ipamLeaseDir) runtimeDir := filepath.Join(vd, networkDir) + linkDir := filepath.Join(runtimeDir, linkDir) + ipamLease := filepath.Join(vd, ipamLeaseDir) + + if err := os.MkdirAll(linkDir, 0755); err != nil { + return nil, errors.Wrapf(err, "failed to create directory: '%s'", linkDir) + } nw := networker{ ipamLease: ipamLease, networkDir: runtimeDir, portSet: set.NewInt(), + linkDir: linkDir, } if err := nw.syncWGPorts(); err != nil { @@ -78,7 +87,11 @@ func NewNetworker() (pkg.NetworkerLight, error) { } // func (n *networker) Create(name string, privateNet net.IPNet, seed []byte, twinID uint32, wgListenPort uint16, wgPrivateKey string, ipRange net.IPNet, nr pkg.Network) error { -func (n *networker) Create(name string, net zos.NetworkLight) error { +func (n *networker) Create(name string, wl gridtypes.WorkloadID, net zos.NetworkLight) error { + if err := n.storeNetwork(name, wl, net); err != nil { + return errors.Wrap(err, "failed to store network object") + } + b, err := bridge.Get(NDMZBridge) if err != nil { return err @@ -88,26 +101,6 @@ func (n *networker) Create(name string, net zos.NetworkLight) error { return err } - storedNR, err := n.networkOf(zos.NetID(name)) - if err != nil && !os.IsNotExist(err) { - return errors.Wrap(err, "failed to load previous network setup") - } - - if err == nil { - if err := n.releasePort(storedNR.WGListenPort); err != nil { - return err - } - } - - if err := n.reservePort(net.WGListenPort); err != nil { - return err - } - - netr, err := resource.Create(name, b, ip, NDMZGwIP, &net.Subnet.IPNet, net.Mycelium.Key, net.NetworkIPRange.IPNet, net) - if err != nil { - return err - } - cleanup := func() { log.Error().Msg("clean up network resource") if err := resource.Delete(name); err != nil { @@ -124,40 +117,33 @@ func (n *networker) Create(name string, net zos.NetworkLight) error { } }() - wgName, err := netr.WGName() + netr, err := resource.Create(name, b, ip, NDMZGwIP, &net.Subnet.IPNet, net.Mycelium.Key, net.NetworkIPRange.IPNet, net) if err != nil { - return errors.Wrap(err, "failed to get wg interface name for network resource") + return err } - exists, err := netr.HasWireguard() - if err != nil { - return errors.Wrap(err, "failed to check if network resource has wireguard setup") - } + return n.setupWireguard(name, net, netr) +} - if !exists { - var wg *wireguard.Wireguard - wg, err = wireguard.New(wgName) - if err != nil { - return errors.Wrapf(err, "failed to create wg interface for network resource '%s'", name) - } - if err = netr.SetWireguard(wg); err != nil { - return errors.Wrap(err, "failed to setup wireguard interface for network resource") - } +func (n *networker) Delete(wl gridtypes.WorkloadWithID) error { + if err := ipam.DeAllocateIPv4(wl.ID.String(), n.ipamLease); err != nil { + return err } - if err = netr.ConfigureWG(net.WGPrivateKey); err != nil { - return errors.Wrap(err, "failed to configure network resource") + netID, err := zos.NetworkIDFromWorkloadID(wl.ID) + if err != nil { + return err } - - return err -} - -func (n *networker) Delete(name string) error { - if err := ipam.DeAllocateIPv4(name, n.ipamLease); err != nil { + netNR, err := n.networkOf(netID) + if err != nil { return err } - return resource.Delete(name) + if err := n.releasePort(netNR.WGListenPort); err != nil { + log.Error().Err(err).Msg("release wireguard port failed") + } + + return resource.Delete(string(wl.ID)) } func (n *networker) AttachPrivate(name, id string, vmIp net.IP) (device pkg.TapDevice, err error) { @@ -497,6 +483,50 @@ func createNDMZBridge(name string) (*netlink.Bridge, error) { return link.(*netlink.Bridge), nil } +func (n networker) setupWireguard(name string, net zos.NetworkLight, netr *resource.Resource) error { + storedNR, err := n.networkOf(zos.NetID(name)) + if err != nil && !os.IsNotExist(err) { + return errors.Wrap(err, "failed to load previous network setup") + } + + if err == nil { + if err := n.releasePort(storedNR.WGListenPort); err != nil { + return err + } + } + + if err := n.reservePort(net.WGListenPort); err != nil { + return err + } + + wgName, err := netr.WGName() + if err != nil { + return errors.Wrap(err, "failed to get wg interface name for network resource") + } + + exists, err := netr.HasWireguard() + if err != nil { + return errors.Wrap(err, "failed to check if network resource has wireguard setup") + } + + if !exists { + var wg *wireguard.Wireguard + wg, err = wireguard.New(wgName) + if err != nil { + return errors.Wrapf(err, "failed to create wg interface for network resource '%s'", name) + } + if err = netr.SetWireguard(wg); err != nil { + return errors.Wrap(err, "failed to setup wireguard interface for network resource") + } + } + + if err = netr.ConfigureWG(net.WGPrivateKey); err != nil { + return errors.Wrap(err, "failed to configure network resource") + } + + return nil +} + func (n *networker) reservePort(port uint16) error { log.Debug().Uint16("port", port).Msg("reserve wireguard port") err := n.portSet.Add(uint(port)) @@ -561,3 +591,28 @@ func (n *networker) syncWGPorts() error { return nil } + +func (n *networker) storeNetwork(name string, wl gridtypes.WorkloadID, network zos.NetworkLight) error { + // map the network ID to the network namespace + path := filepath.Join(n.networkDir, name) + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + + writer, err := versioned.NewWriter(file, NetworkSchemaLatestVersion) + if err != nil { + return err + } + + enc := json.NewEncoder(writer) + if err := enc.Encode(&network); err != nil { + return err + } + link := filepath.Join(n.linkDir, wl.String()) + if err := os.Symlink(filepath.Join("../", name), link); err != nil && !os.IsExist(err) { + return errors.Wrap(err, "failed to create network symlink") + } + return nil +} diff --git a/pkg/network_light.go b/pkg/network_light.go index 992b87a2e..7a18342de 100644 --- a/pkg/network_light.go +++ b/pkg/network_light.go @@ -4,6 +4,7 @@ import ( "context" "net" + "github.com/threefoldtech/zos/pkg/gridtypes" "github.com/threefoldtech/zos/pkg/gridtypes/zos" ) @@ -19,8 +20,8 @@ type Route struct { // NetworkerLight is the interface for the network light module type NetworkerLight interface { - Create(name string, net zos.NetworkLight) error - Delete(name string) error + Create(name string, wl gridtypes.WorkloadID, net zos.NetworkLight) error + Delete(wl gridtypes.WorkloadWithID) error AttachPrivate(name, id string, vmIp net.IP) (device TapDevice, err error) AttachMycelium(name, id string, seed []byte) (device TapDevice, err error) Detach(id string) error diff --git a/pkg/primitives/network-light/network.go b/pkg/primitives/network-light/network.go index d7a295601..23999ac69 100644 --- a/pkg/primitives/network-light/network.go +++ b/pkg/primitives/network-light/network.go @@ -40,7 +40,7 @@ func (p *Manager) networkProvisionImpl(ctx context.Context, wl *gridtypes.Worklo log.Debug().Str("network", fmt.Sprintf("%+v", network)).Msg("provision network") // err := mgr.Create(ctx, string(zos.NetworkID(twin, wl.Name)), network.Subnet.IPNet, network.Mycelium.Key) - err := mgr.Create(ctx, string(zos.NetworkID(twin, wl.Name)), network) + err := mgr.Create(ctx, string(zos.NetworkID(twin, wl.Name)), wl.ID, network) if err != nil { return errors.Wrapf(err, "failed to create network resource for network %s", wl.ID) } @@ -57,10 +57,9 @@ func (p *Manager) Update(ctx context.Context, wl *gridtypes.WorkloadWithID) (int } func (p *Manager) Deprovision(ctx context.Context, wl *gridtypes.WorkloadWithID) error { - twin, _ := provision.GetDeploymentID(ctx) mgr := stubs.NewNetworkerLightStub(p.zbus) - if err := mgr.Delete(ctx, string(zos.NetworkID(twin, wl.Name))); err != nil { + if err := mgr.Delete(ctx, *wl); err != nil { return fmt.Errorf("failed to delete network resource: %w", err) } diff --git a/pkg/stubs/network_light_stub.go b/pkg/stubs/network_light_stub.go index 56461579b..b148a9c08 100644 --- a/pkg/stubs/network_light_stub.go +++ b/pkg/stubs/network_light_stub.go @@ -8,6 +8,7 @@ import ( "context" zbus "github.com/threefoldtech/zbus" pkg "github.com/threefoldtech/zos/pkg" + gridtypes "github.com/threefoldtech/zos/pkg/gridtypes" zos "github.com/threefoldtech/zos/pkg/gridtypes/zos" "net" ) @@ -80,8 +81,8 @@ func (s *NetworkerLightStub) AttachZDB(ctx context.Context, arg0 string) (ret0 s return } -func (s *NetworkerLightStub) Create(ctx context.Context, arg0 string, arg1 zos.NetworkLight) (ret0 error) { - args := []interface{}{arg0, arg1} +func (s *NetworkerLightStub) Create(ctx context.Context, arg0 string, arg1 gridtypes.WorkloadID, arg2 zos.NetworkLight) (ret0 error) { + args := []interface{}{arg0, arg1, arg2} result, err := s.client.RequestContext(ctx, s.module, s.object, "Create", args...) if err != nil { panic(err) @@ -95,7 +96,7 @@ func (s *NetworkerLightStub) Create(ctx context.Context, arg0 string, arg1 zos.N return } -func (s *NetworkerLightStub) Delete(ctx context.Context, arg0 string) (ret0 error) { +func (s *NetworkerLightStub) Delete(ctx context.Context, arg0 gridtypes.WorkloadWithID) (ret0 error) { args := []interface{}{arg0} result, err := s.client.RequestContext(ctx, s.module, s.object, "Delete", args...) if err != nil { From b92923a41c2b8f972a2f2f78b5a1818e52545f56 Mon Sep 17 00:00:00 2001 From: Eslam-Nawara Date: Wed, 9 Oct 2024 16:32:50 +0300 Subject: [PATCH 4/9] fix import --- pkg/network_light.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/network_light.go b/pkg/network_light.go index 5a12e493b..c0353a287 100644 --- a/pkg/network_light.go +++ b/pkg/network_light.go @@ -4,8 +4,8 @@ import ( "context" "net" - "github.com/threefoldtech/zos/pkg/gridtypes" - "github.com/threefoldtech/zos/pkg/gridtypes/zos" + "github.com/threefoldtech/zos4/pkg/gridtypes" + "github.com/threefoldtech/zos4/pkg/gridtypes/zos" ) type Route struct { From 84034405c8971831e79d0e7f8d2c984b44fbb330 Mon Sep 17 00:00:00 2001 From: Eslam-Nawara Date: Mon, 14 Oct 2024 16:54:42 +0300 Subject: [PATCH 5/9] update vm light to run in private network --- pkg/netlight/network.go | 60 +++++++++++++++++++------ pkg/network_light.go | 6 ++- pkg/primitives/network-light/network.go | 3 +- pkg/primitives/vm-light/utils.go | 48 +++++++++++++++++++- pkg/stubs/network_light_stub.go | 46 ++++++++++++++++--- 5 files changed, 138 insertions(+), 25 deletions(-) diff --git a/pkg/netlight/network.go b/pkg/netlight/network.go index b19ff9450..78c6dbee7 100644 --- a/pkg/netlight/network.go +++ b/pkg/netlight/network.go @@ -125,16 +125,12 @@ func (n *networker) Create(name string, wl gridtypes.WorkloadID, net zos.Network return n.setupWireguard(name, net, netr) } -func (n *networker) Delete(wl gridtypes.WorkloadWithID) error { +func (n *networker) Delete(name string, wl gridtypes.WorkloadWithID) error { if err := ipam.DeAllocateIPv4(wl.ID.String(), n.ipamLease); err != nil { return err } - netID, err := zos.NetworkIDFromWorkloadID(wl.ID) - if err != nil { - return err - } - netNR, err := n.networkOf(netID) + netNR, err := n.networkOf(name) if err != nil { return err } @@ -143,7 +139,12 @@ func (n *networker) Delete(wl gridtypes.WorkloadWithID) error { log.Error().Err(err).Msg("release wireguard port failed") } - return resource.Delete(string(wl.ID)) + if err := resource.Delete(string(wl.ID)); err != nil { + return err + } + + path := filepath.Join(n.networkDir, name) + return os.Remove(path) } func (n *networker) AttachPrivate(name, id string, vmIp net.IP) (device pkg.TapDevice, err error) { @@ -202,17 +203,48 @@ func (n *networker) AttachZDB(id string) (string, error) { // GetSubnet of a local network resource identified by the network ID, ipv4 and ipv6 // subnet respectively -func (n *networker) GetSubnet(networkID pkg.NetID) (net.IPNet, error) { - localNR, err := n.networkOf(networkID) +func (n *networker) GetSubnet(name string) (net.IPNet, error) { + localNR, err := n.networkOf(name) if err != nil { - return net.IPNet{}, errors.Wrapf(err, "couldn't load network with id (%s)", networkID) + return net.IPNet{}, errors.Wrapf(err, "couldn't load network (%s)", name) } return localNR.Subnet.IPNet, nil } -func (n *networker) networkOf(id zos.NetID) (nr pkg.Network, err error) { - path := filepath.Join(n.networkDir, string(id)) +// GetNet of a network identified by the network ID +func (n *networker) GetNet(name string) (net.IPNet, error) { + localNR, err := n.networkOf(name) + if err != nil { + return net.IPNet{}, errors.Wrapf(err, "couldn't load network (%s)", name) + } + + return localNR.NetworkIPRange.IPNet, nil +} + +// GetDefaultGwIP returns the IPs of the default gateways inside the network +// resource identified by the network ID on the local node, for IPv4 +func (n *networker) GetDefaultGwIP(name string) (net.IP, error) { + localNR, err := n.networkOf(name) + if err != nil { + return nil, errors.Wrapf(err, "couldn't load network (%s)", name) + } + + // only IP4 atm + ip := localNR.Subnet.IP.To4() + if ip == nil { + return nil, errors.New("nr subnet is not valid IPv4") + } + + // defaut gw is currently implied to be at `x.x.x.1` + // also a subnet in a NR is assumed to be a /24 + ip[len(ip)-1] = 1 + + return ip, nil +} + +func (n *networker) networkOf(name string) (nr pkg.Network, err error) { + path := filepath.Join(n.networkDir, name) file, err := os.OpenFile(path, os.O_RDWR, 0660) if err != nil { return nr, err @@ -484,11 +516,13 @@ func createNDMZBridge(name string) (*netlink.Bridge, error) { } func (n networker) setupWireguard(name string, net zos.NetworkLight, netr *resource.Resource) error { - storedNR, err := n.networkOf(zos.NetID(name)) + storedNR, err := n.networkOf(name) if err != nil && !os.IsNotExist(err) { return errors.Wrap(err, "failed to load previous network setup") } + log.Debug().Msg("setting up wireguard") + if err == nil { if err := n.releasePort(storedNR.WGListenPort); err != nil { return err diff --git a/pkg/network_light.go b/pkg/network_light.go index c0353a287..e797d9402 100644 --- a/pkg/network_light.go +++ b/pkg/network_light.go @@ -21,7 +21,7 @@ type Route struct { // NetworkerLight is the interface for the network light module type NetworkerLight interface { Create(name string, wl gridtypes.WorkloadID, net zos.NetworkLight) error - Delete(wl gridtypes.WorkloadWithID) error + Delete(name string, wl gridtypes.WorkloadWithID) error AttachPrivate(name, id string, vmIp net.IP) (device TapDevice, err error) AttachMycelium(name, id string, seed []byte) (device TapDevice, err error) Detach(id string) error @@ -31,8 +31,10 @@ type NetworkerLight interface { Namespace(id string) string Ready() error ZOSAddresses(ctx context.Context) <-chan NetlinkAddresses - GetSubnet(networkID NetID) (net.IPNet, error) + GetSubnet(name string) (net.IPNet, error) + GetDefaultGwIP(name string) (net.IP, error) + GetNet(name string) (net.IPNet, error) WireguardPorts() ([]uint, error) } diff --git a/pkg/primitives/network-light/network.go b/pkg/primitives/network-light/network.go index effdf0bf8..11fddea33 100644 --- a/pkg/primitives/network-light/network.go +++ b/pkg/primitives/network-light/network.go @@ -58,8 +58,9 @@ func (p *Manager) Update(ctx context.Context, wl *gridtypes.WorkloadWithID) (int func (p *Manager) Deprovision(ctx context.Context, wl *gridtypes.WorkloadWithID) error { mgr := stubs.NewNetworkerLightStub(p.zbus) + twin, _ := provision.GetDeploymentID(ctx) - if err := mgr.Delete(ctx, *wl); err != nil { + if err := mgr.Delete(ctx, string(zos.NetworkID(twin, wl.Name)), *wl); err != nil { return fmt.Errorf("failed to delete network resource: %w", err) } diff --git a/pkg/primitives/vm-light/utils.go b/pkg/primitives/vm-light/utils.go index 8b9d1622c..30d012e0c 100644 --- a/pkg/primitives/vm-light/utils.go +++ b/pkg/primitives/vm-light/utils.go @@ -17,6 +17,11 @@ import ( "github.com/threefoldtech/zos4/pkg/stubs" ) +var networkResourceNet = net.IPNet{ + IP: net.ParseIP("100.64.0.0"), + Mask: net.IPv4Mask(0xff, 0xff, 0, 0), +} + // fill up the VM (machine) object with write boot config for a full virtual machine (with a disk image) func (p *Manager) prepVirtualMachine( ctx context.Context, @@ -178,7 +183,6 @@ func (p *Manager) newMyceliumNetworkInterface(ctx context.Context, dl gridtypes. tapName := wl.ID.Unique(string(config.Network)) iface, err := network.AttachMycelium(ctx, string(netID), tapName, config.Seed) - if err != nil { return pkg.VMIface{}, errors.Wrap(err, "could not set up tap device") } @@ -200,6 +204,41 @@ func (p *Manager) newMyceliumNetworkInterface(ctx context.Context, dl gridtypes. func (p *Manager) newPrivNetworkInterface(ctx context.Context, dl gridtypes.Deployment, wl *gridtypes.WorkloadWithID, inf zos.MachineInterface) (pkg.VMIface, error) { network := stubs.NewNetworkerLightStub(p.zbus) netID := zos.NetworkID(dl.TwinID, inf.Network) + name := netID.String() + + subnet, err := network.GetSubnet(ctx, name) + if err != nil { + return pkg.VMIface{}, errors.Wrapf(err, "could not get network resource subnet") + } + + inf.IP = inf.IP.To4() + if inf.IP == nil { + return pkg.VMIface{}, fmt.Errorf("invalid IPv4 supplied to wg interface") + } + + if !subnet.Contains(inf.IP) { + return pkg.VMIface{}, fmt.Errorf("IP %s is not part of local nr subnet %s", inf.IP.String(), subnet.String()) + } + + // always the .1/24 ip is reserved + if inf.IP[3] == 1 { + return pkg.VMIface{}, fmt.Errorf("ip %s is reserved", inf.IP.String()) + } + + privNet, err := network.GetNet(ctx, name) + if err != nil { + return pkg.VMIface{}, errors.Wrapf(err, "could not get network range") + } + + addrCIDR := net.IPNet{ + IP: inf.IP, + Mask: subnet.Mask, + } + + gw4, err := network.GetDefaultGwIP(ctx, name) + if err != nil { + return pkg.VMIface{}, errors.Wrap(err, "could not get network resource default gateway") + } tapName := wl.ID.Unique(string(inf.Network)) iface, err := network.AttachPrivate(ctx, string(netID), tapName, inf.IP) @@ -212,9 +251,14 @@ func (p *Manager) newPrivNetworkInterface(ctx context.Context, dl gridtypes.Depl MAC: iface.Mac.String(), IPs: []net.IPNet{ *iface.IP, + addrCIDR, // privIP6, }, - Routes: iface.Routes, + // Routes: iface.Routes, + Routes: []pkg.Route{ + {Net: privNet, Gateway: gw4}, + {Net: networkResourceNet, Gateway: gw4}, + }, IP4DefaultGateway: net.IP(iface.Routes[0].Gateway), // IP6DefaultGateway: gw6, PublicIPv4: false, diff --git a/pkg/stubs/network_light_stub.go b/pkg/stubs/network_light_stub.go index 75063d55c..e071e7d15 100644 --- a/pkg/stubs/network_light_stub.go +++ b/pkg/stubs/network_light_stub.go @@ -6,13 +6,11 @@ package stubs import ( "context" - "net" - - gridtypes "github.com/threefoldtech/zos4/pkg/gridtypes" - zbus "github.com/threefoldtech/zbus" pkg "github.com/threefoldtech/zos4/pkg" + gridtypes "github.com/threefoldtech/zos4/pkg/gridtypes" zos "github.com/threefoldtech/zos4/pkg/gridtypes/zos" + "net" ) type NetworkerLightStub struct { @@ -98,8 +96,8 @@ func (s *NetworkerLightStub) Create(ctx context.Context, arg0 string, arg1 gridt return } -func (s *NetworkerLightStub) Delete(ctx context.Context, arg0 gridtypes.WorkloadWithID) (ret0 error) { - args := []interface{}{arg0} +func (s *NetworkerLightStub) Delete(ctx context.Context, arg0 string, arg1 gridtypes.WorkloadWithID) (ret0 error) { + args := []interface{}{arg0, arg1} result, err := s.client.RequestContext(ctx, s.module, s.object, "Delete", args...) if err != nil { panic(err) @@ -128,7 +126,41 @@ func (s *NetworkerLightStub) Detach(ctx context.Context, arg0 string) (ret0 erro return } -func (s *NetworkerLightStub) GetSubnet(ctx context.Context, arg0 zos.NetID) (ret0 net.IPNet, ret1 error) { +func (s *NetworkerLightStub) GetDefaultGwIP(ctx context.Context, arg0 string) (ret0 []uint8, ret1 error) { + args := []interface{}{arg0} + result, err := s.client.RequestContext(ctx, s.module, s.object, "GetDefaultGwIP", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret1 = result.CallError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *NetworkerLightStub) GetNet(ctx context.Context, arg0 string) (ret0 net.IPNet, ret1 error) { + args := []interface{}{arg0} + result, err := s.client.RequestContext(ctx, s.module, s.object, "GetNet", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret1 = result.CallError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *NetworkerLightStub) GetSubnet(ctx context.Context, arg0 string) (ret0 net.IPNet, ret1 error) { args := []interface{}{arg0} result, err := s.client.RequestContext(ctx, s.module, s.object, "GetSubnet", args...) if err != nil { From 47b57fb156c621d3aec4e30469a03515f2f843ce Mon Sep 17 00:00:00 2001 From: Eslam-Nawara Date: Tue, 15 Oct 2024 10:46:30 +0300 Subject: [PATCH 6/9] add how to reach local node with wg to docs --- docs/manual/network/readme.md | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/manual/network/readme.md b/docs/manual/network/readme.md index 636945e71..22b5bcd3f 100644 --- a/docs/manual/network/readme.md +++ b/docs/manual/network/readme.md @@ -13,4 +13,45 @@ In the simplest form a network workload consists of: Full network definition can be found [here](../../../pkg/gridtypes/zos/network.go) +## Private Networks + +To reach vms on local nodes using wireguard you need to: +- Deploy a networkwith valid pairs so you can be able to connect to the vm from your machine and add a container to this network. +For example: + +```go + WGPrivateKey: wgKey, + WGListenPort: 3011, + Peers: []zos.Peer{ + { + Subnet: gridtypes.MustParseIPNet("10.1.2.0/24"), + WGPublicKey: "4KTvZS2KPWYfMr+GbiUUly0ANVg8jBC7xP9Bl79Z8zM=", + + AllowedIPs: []gridtypes.IPNet{ + gridtypes.MustParseIPNet("10.1.2.0/24"), + gridtypes.MustParseIPNet("100.64.1.2/32"), + }, + }, + }, + +``` +>> **Note:** make sure to use valid two wg key pairs for the container and your local machine. +- After the deployment the network can be accessed through wg. + +```conf +[Interface] +Address = 100.64.1.2/32 +PrivateKey = + +[Peer] +PublicKey = cYvKjMRBLj3o3e4lxWOK6bbSyHWtgLNHkEBxIv7Olm4= +AllowedIPs = 10.1.1.0/24, 100.64.1.1/32 +PersistentKeepalive = 25 +Endpoint = 192.168.123.32:3011 +``` +- Bring wireguard interface up `wg-quick up ` +- Test the connection `wg` + +- Then you should be able to ping to the container `ping 10.1.1.2` + For more details on how the network work please refer to the [internal manual](../../internals/network/readme.md) From e1d9959d01c9298a7422f64cd06f822e64c401d3 Mon Sep 17 00:00:00 2001 From: Eslam Nawara <67752395+Eslam-Nawara@users.noreply.github.com> Date: Tue, 15 Oct 2024 10:59:48 +0300 Subject: [PATCH 7/9] Update readme.md --- docs/manual/network/readme.md | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/manual/network/readme.md b/docs/manual/network/readme.md index 22b5bcd3f..17e03120d 100644 --- a/docs/manual/network/readme.md +++ b/docs/manual/network/readme.md @@ -16,27 +16,28 @@ Full network definition can be found [here](../../../pkg/gridtypes/zos/network.g ## Private Networks To reach vms on local nodes using wireguard you need to: -- Deploy a networkwith valid pairs so you can be able to connect to the vm from your machine and add a container to this network. +- Deploy a network on your local node with valid pairs so you can be able to connect to the vm from your machine and add a container to this network. For example: ```go - WGPrivateKey: wgKey, - WGListenPort: 3011, - Peers: []zos.Peer{ - { - Subnet: gridtypes.MustParseIPNet("10.1.2.0/24"), - WGPublicKey: "4KTvZS2KPWYfMr+GbiUUly0ANVg8jBC7xP9Bl79Z8zM=", - - AllowedIPs: []gridtypes.IPNet{ - gridtypes.MustParseIPNet("10.1.2.0/24"), - gridtypes.MustParseIPNet("100.64.1.2/32"), - }, +WGPrivateKey: wgKey, +WGListenPort: 3011, +Peers: []zos.Peer{ + { + Subnet: gridtypes.MustParseIPNet("10.1.2.0/24"), + WGPublicKey: "4KTvZS2KPWYfMr+GbiUUly0ANVg8jBC7xP9Bl79Z8zM=", + + AllowedIPs: []gridtypes.IPNet{ + gridtypes.MustParseIPNet("10.1.2.0/24"), + gridtypes.MustParseIPNet("100.64.1.2/32"), }, }, +}, ``` ->> **Note:** make sure to use valid two wg key pairs for the container and your local machine. -- After the deployment the network can be accessed through wg. + +> **Note:** make sure to use valid two wg key pairs for the container and your local machine. +- After the deployment the network can be accessed through wg with the following config. ```conf [Interface] @@ -51,7 +52,10 @@ Endpoint = 192.168.123.32:3011 ``` - Bring wireguard interface up `wg-quick up ` - Test the connection `wg` +![image](https://github.com/user-attachments/assets/ca0d37e2-d586-4e0f-ae98-2d70188492bd) + +- Then you should be able to ping/access the container `ping 10.1.1.2` +![image](https://github.com/user-attachments/assets/d625a573-3d07-4980-afc0-4570acd7a21f) -- Then you should be able to ping to the container `ping 10.1.1.2` For more details on how the network work please refer to the [internal manual](../../internals/network/readme.md) From ad442af26b73a4e77f06bc14b5b2b8755e7dce20 Mon Sep 17 00:00:00 2001 From: Eslam-Nawara Date: Tue, 15 Oct 2024 12:47:31 +0300 Subject: [PATCH 8/9] add private net to routes --- pkg/primitives/vm-light/utils.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/primitives/vm-light/utils.go b/pkg/primitives/vm-light/utils.go index 30d012e0c..310312b94 100644 --- a/pkg/primitives/vm-light/utils.go +++ b/pkg/primitives/vm-light/utils.go @@ -246,6 +246,8 @@ func (p *Manager) newPrivNetworkInterface(ctx context.Context, dl gridtypes.Depl return pkg.VMIface{}, errors.Wrap(err, "could not set up tap device for private interface") } + iface.Routes = append(iface.Routes, pkg.Route{Net: privNet, Gateway: gw4}) + out := pkg.VMIface{ Tap: iface.Name, MAC: iface.Mac.String(), @@ -254,11 +256,7 @@ func (p *Manager) newPrivNetworkInterface(ctx context.Context, dl gridtypes.Depl addrCIDR, // privIP6, }, - // Routes: iface.Routes, - Routes: []pkg.Route{ - {Net: privNet, Gateway: gw4}, - {Net: networkResourceNet, Gateway: gw4}, - }, + Routes: iface.Routes, IP4DefaultGateway: net.IP(iface.Routes[0].Gateway), // IP6DefaultGateway: gw6, PublicIPv4: false, From 517f92170cb87aaded375f1cde1588dcfe4441e6 Mon Sep 17 00:00:00 2001 From: Eslam-Nawara Date: Tue, 15 Oct 2024 12:53:15 +0300 Subject: [PATCH 9/9] fix lint --- pkg/gridtypes/zos/network_light.go | 2 +- pkg/primitives/vm-light/utils.go | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/gridtypes/zos/network_light.go b/pkg/gridtypes/zos/network_light.go index f16c24288..ad5594a46 100644 --- a/pkg/gridtypes/zos/network_light.go +++ b/pkg/gridtypes/zos/network_light.go @@ -32,7 +32,7 @@ func NetworkID(twin uint32, network gridtypes.Name) NetID { if len(b) > 13 { b = b[:13] } - return NetID(string(b)) + return NetID(b) } func NetworkIDFromWorkloadID(wl gridtypes.WorkloadID) (NetID, error) { diff --git a/pkg/primitives/vm-light/utils.go b/pkg/primitives/vm-light/utils.go index 310312b94..166ba6d0b 100644 --- a/pkg/primitives/vm-light/utils.go +++ b/pkg/primitives/vm-light/utils.go @@ -17,11 +17,6 @@ import ( "github.com/threefoldtech/zos4/pkg/stubs" ) -var networkResourceNet = net.IPNet{ - IP: net.ParseIP("100.64.0.0"), - Mask: net.IPv4Mask(0xff, 0xff, 0, 0), -} - // fill up the VM (machine) object with write boot config for a full virtual machine (with a disk image) func (p *Manager) prepVirtualMachine( ctx context.Context,