reintegrate wg in zos4

cmds/modules/netlightd/main.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/oasisprotocol/curve25519-voi/primitives/x25519"
 	"github.com/pkg/errors"
+	"github.com/threefoldtech/zos/pkg/gridtypes/zos"
 	"github.com/threefoldtech/zos/pkg/netlight"
 	"github.com/threefoldtech/zos/pkg/netlight/nft"
 	"github.com/threefoldtech/zos/pkg/netlight/resource"
@@ -115,8 +116,7 @@ func action(cli *cli.Context) error {
 	_, err = resource.Create("dmz", bridge, &net.IPNet{
 		IP:   net.ParseIP("100.127.0.2"),
 		Mask: net.CIDRMask(16, 32),
-	}, netlight.NDMZGwIP, nil, myceliumSeedFromIdentity(identity.PrivateKey(cli.Context)))
-
+	}, netlight.NDMZGwIP, nil, myceliumSeedFromIdentity(identity.PrivateKey(cli.Context)), net.IPNet{}, zos.NetworkLight{})
 	if err != nil {
 		return fmt.Errorf("failed to create ndmz resource: %w", err)
 	}

go.mod

@@ -86,13 +86,15 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/gtank/ristretto255 v0.1.2 // indirect
 	github.com/hanwen/go-fuse/v2 v2.3.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/holiman/uint256 v1.2.3 // indirect
+	github.com/josharian/native v1.1.0 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/lestrrat-go/backoff/v2 v2.0.7 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.0 // indirect
@@ -103,6 +105,9 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mdlayher/genetlink v1.3.2 // indirect
+	github.com/mdlayher/netlink v1.7.2 // indirect
+	github.com/mdlayher/socket v0.4.1 // indirect
 	github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/moby/locker v1.0.1 // indirect
@@ -142,6 +147,8 @@ require (
 	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/text v0.15.0 // indirect
+	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect
+	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 // indirect
 	gonum.org/v1/gonum v0.15.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 // indirect

go.sum

@@ -298,6 +298,8 @@ github.com/jgautheron/goconst v0.0.0-20170703170152-9740945f5dcb/go.mod h1:82Txj
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/joncrlsn/dque v0.0.0-20200702023911-3e80e3146ce5 h1:bo1aoO6l128nKJCBrFflOj9s+KPqMM7ErNyB5GGBNDs=
 github.com/joncrlsn/dque v0.0.0-20200702023911-3e80e3146ce5/go.mod h1:dNKs71rs2VJGBAmttu7fouEsRQlRjxy0p1Sx+T5wbpY=
+github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
+github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -359,6 +361,12 @@ github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq
 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw=
+github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o=
+github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
+github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
+github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U=
+github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=
 github.com/mibk/dupl v1.0.0/go.mod h1:pCr4pNxxIbFGvtyCOi0c7LVjmV6duhKWV+ex5vh38ME=
 github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM=
 github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b h1:QrHweqAtyJ9EwCaGHBu1fghwxIPiopAHV06JlXrMHjk=
@@ -718,6 +726,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
+golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
+golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 h1:CawjfCvYQH2OU3/TnxLx97WDSUDRABfT18pCOYwc2GE=
+golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6/go.mod h1:3rxYc4HtVcSG9gVaTs2GEBdehh+sYPOwKtyUWEOTb80=
 gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ=
 gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=

pkg/gridtypes/zos/network_light.go

@@ -55,6 +55,10 @@ func NetworkIDFromWorkloadID(wl gridtypes.WorkloadID) (NetID, error) {
 // - For each PC or a laptop (for each wireguard peer) there must be a peer in the peer list (on all nodes)
 // This is why this can get complicated.
 type NetworkLight struct {
+	// IP range of the network, must be an IPv4 /16
+	// for example a 10.1.0.0/16
+	NetworkIPRange gridtypes.IPNet `json:"ip_range"`
+
 	// IPV4 subnet for this network resource
 	// this must be a valid subnet of the entire network ip range.
 	// for example 10.1.1.0/24
@@ -65,6 +69,36 @@ type NetworkLight struct {
 	// if no mycelium configuration is provided, vms can't
 	// get mycelium IPs.
 	Mycelium Mycelium `json:"mycelium,omitempty"`
+
+	// The private wg key of this node (this peer) which is installing this
+	// network workload right now.
+	// This has to be filled in by the user (and not generated for example)
+	// because other peers need to be installed as well (with this peer public key)
+	// hence it's easier to configure everything one time at the user side and then
+	// apply everything on all nodes at once
+	WGPrivateKey string `json:"wireguard_private_key"`
+
+	// WGListenPort is the wireguard listen port on this node. this has
+	// to be filled in by the user for same reason as private key (other nodes need to know about it)
+	// To find a free port you have to ask the node first by a call over RMB about which ports are possible
+	// to use.
+	WGListenPort uint16 `json:"wireguard_listen_port"`
+
+	// Peers is a list of other peers in this network
+	Peers []Peer `json:"peers"`
+}
+
+// Peer is the description of a peer of a NetResource
+type Peer struct {
+	// IPV4 subnet of the network resource of the peer
+	Subnet gridtypes.IPNet `json:"subnet"`
+	// WGPublicKey of the peer (driven from its private key)
+	WGPublicKey string `json:"wireguard_public_key"`
+	// Allowed Ips is related to his subnet.
+	// todo: remove and derive from subnet
+	AllowedIPs []gridtypes.IPNet `json:"allowed_ips"`
+	// Entrypoint of the peer
+	Endpoint string `json:"endpoint"`
 }
 
 type MyceliumPeer string

pkg/netlight/network.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"slices"
+	"strings"
 	"time"
 
 	"github.com/blang/semver"
@@ -25,6 +26,8 @@ import (
 	"github.com/threefoldtech/zos/pkg/netlight/namespace"
 	"github.com/threefoldtech/zos/pkg/netlight/options"
 	"github.com/threefoldtech/zos/pkg/netlight/resource"
+	"github.com/threefoldtech/zos/pkg/netlight/wireguard"
+	"github.com/threefoldtech/zos/pkg/set"
 	"github.com/threefoldtech/zos/pkg/versioned"
 	"github.com/vishvananda/netlink"
 )
@@ -38,18 +41,17 @@ const (
 	networkDir    = "networks"
 )
 
-var (
-	NDMZGwIP = &net.IPNet{
-		IP:   net.ParseIP("100.127.0.1"),
-		Mask: net.CIDRMask(16, 32),
-	}
-)
+var NDMZGwIP = &net.IPNet{
+	IP:   net.ParseIP("100.127.0.1"),
+	Mask: net.CIDRMask(16, 32),
+}
 
 var NetworkSchemaLatestVersion = semver.MustParse("0.1.0")
 
 type networker struct {
 	ipamLease  string
 	networkDir string
+	portSet    *set.UIntSet
 }
 
 var _ pkg.NetworkerLight = (*networker)(nil)
@@ -63,13 +65,20 @@ func NewNetworker() (pkg.NetworkerLight, error) {
 	ipamLease := filepath.Join(vd, ipamLeaseDir)
 	runtimeDir := filepath.Join(vd, networkDir)
 
-	return &networker{
+	nw := networker{
 		ipamLease:  ipamLease,
 		networkDir: runtimeDir,
-	}, nil
+		portSet:    set.NewInt(),
+	}
+
+	if err := nw.syncWGPorts(); err != nil {
+		return nil, err
+	}
+	return &nw, nil
 }
 
-func (n *networker) Create(name string, privateNet net.IPNet, seed []byte) error {
+// func (n *networker) Create(name string, privateNet net.IPNet, seed []byte, twinID uint32, wgListenPort uint16, wgPrivateKey string, ipRange net.IPNet, nr pkg.Network) error {
+func (n *networker) Create(name string, net zos.NetworkLight) error {
 	b, err := bridge.Get(NDMZBridge)
 	if err != nil {
 		return err
@@ -79,7 +88,67 @@ func (n *networker) Create(name string, privateNet net.IPNet, seed []byte) error
 		return err
 	}
 
-	_, err = resource.Create(name, b, ip, NDMZGwIP, &privateNet, seed)
+	storedNR, err := n.networkOf(zos.NetID(name))
+	if err != nil && !os.IsNotExist(err) {
+		return errors.Wrap(err, "failed to load previous network setup")
+	}
+
+	if err == nil {
+		if err := n.releasePort(storedNR.WGListenPort); err != nil {
+			return err
+		}
+	}
+
+	if err := n.reservePort(net.WGListenPort); err != nil {
+		return err
+	}
+
+	netr, err := resource.Create(name, b, ip, NDMZGwIP, &net.Subnet.IPNet, net.Mycelium.Key, net.NetworkIPRange.IPNet, net)
+	if err != nil {
+		return err
+	}
+
+	cleanup := func() {
+		log.Error().Msg("clean up network resource")
+		if err := resource.Delete(name); err != nil {
+			log.Error().Err(err).Msg("error during deletion of network resource after failed deployment")
+		}
+		if err := n.releasePort(net.WGListenPort); err != nil {
+			log.Error().Err(err).Msg("release wireguard port failed")
+		}
+	}
+
+	defer func() {
+		if err != nil {
+			cleanup()
+		}
+	}()
+
+	wgName, err := netr.WGName()
+	if err != nil {
+		return errors.Wrap(err, "failed to get wg interface name for network resource")
+	}
+
+	exists, err := netr.HasWireguard()
+	if err != nil {
+		return errors.Wrap(err, "failed to check if network resource has wireguard setup")
+	}
+
+	if !exists {
+		var wg *wireguard.Wireguard
+		wg, err = wireguard.New(wgName)
+		if err != nil {
+			return errors.Wrapf(err, "failed to create wg interface for network resource '%s'", name)
+		}
+		if err = netr.SetWireguard(wg); err != nil {
+			return errors.Wrap(err, "failed to setup wireguard interface for network resource")
+		}
+	}
+
+	if err = netr.ConfigureWG(net.WGPrivateKey); err != nil {
+		return errors.Wrap(err, "failed to configure network resource")
+	}
+
 	return err
 }
 
@@ -89,7 +158,6 @@ func (n *networker) Delete(name string) error {
 	}
 
 	return resource.Delete(name)
-
 }
 
 func (n *networker) AttachPrivate(name, id string, vmIp net.IP) (device pkg.TapDevice, err error) {
@@ -391,10 +459,14 @@ func (n *networker) Interfaces(iface string, netns string) (pkg.Interfaces, erro
 }
 
 func CreateNDMZBridge() (*netlink.Bridge, error) {
-	return createNDMZBridge(NDMZBridge, NDMZGw)
+	return createNDMZBridge(NDMZBridge)
 }
 
-func createNDMZBridge(name string, gw string) (*netlink.Bridge, error) {
+func (n *networker) WireguardPorts() ([]uint, error) {
+	return n.portSet.List()
+}
+
+func createNDMZBridge(name string) (*netlink.Bridge, error) {
 	if !bridge.Exists(name) {
 		if _, err := bridge.New(name); err != nil {
 			return nil, errors.Wrapf(err, "couldn't create bridge %s", name)
@@ -424,3 +496,68 @@ func createNDMZBridge(name string, gw string) (*netlink.Bridge, error) {
 
 	return link.(*netlink.Bridge), nil
 }
+
+func (n *networker) reservePort(port uint16) error {
+	log.Debug().Uint16("port", port).Msg("reserve wireguard port")
+	err := n.portSet.Add(uint(port))
+	if err != nil {
+		return errors.Wrap(err, "wireguard listen port already in use, pick another one")
+	}
+
+	return nil
+}
+
+func (n *networker) releasePort(port uint16) error {
+	log.Debug().Uint16("port", port).Msg("release wireguard port")
+	n.portSet.Remove(uint(port))
+	return nil
+}
+
+func (n *networker) syncWGPorts() error {
+	names, err := namespace.List("n-")
+	if err != nil {
+		return err
+	}
+
+	readPort := func(name string) (int, error) {
+		netNS, err := namespace.GetByName(name)
+		if err != nil {
+			return 0, err
+		}
+		defer netNS.Close()
+
+		ifaceName := strings.Replace(name, "n-", "w-", 1)
+
+		var port int
+		err = netNS.Do(func(_ ns.NetNS) error {
+			link, err := wireguard.GetByName(ifaceName)
+			if err != nil {
+				return err
+			}
+			d, err := link.Device()
+			if err != nil {
+				return err
+			}
+
+			port = d.ListenPort
+			return nil
+		})
+		if err != nil {
+			return 0, err
+		}
+
+		return port, nil
+	}
+
+	for _, name := range names {
+		port, err := readPort(name)
+		if err != nil {
+			log.Error().Err(err).Str("namespace", name).Msgf("failed to read port for network namespace")
+			continue
+		}
+		// skip error cause we don't care if there are some duplicate at this point
+		_ = n.portSet.Add(uint(port))
+	}
+
+	return nil
+}