Skip to content

Commit 5450038

Browse files
committed
reintegrate wg in zos4
1 parent 9213b66 commit 5450038

File tree

11 files changed

+730
-28
lines changed

11 files changed

+730
-28
lines changed

cmds/modules/netlightd/main.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/oasisprotocol/curve25519-voi/primitives/x25519"
1313
"github.com/pkg/errors"
14+
"github.com/threefoldtech/zos/pkg/gridtypes/zos"
1415
"github.com/threefoldtech/zos/pkg/netlight"
1516
"github.com/threefoldtech/zos/pkg/netlight/nft"
1617
"github.com/threefoldtech/zos/pkg/netlight/resource"
@@ -115,8 +116,7 @@ func action(cli *cli.Context) error {
115116
_, err = resource.Create("dmz", bridge, &net.IPNet{
116117
IP: net.ParseIP("100.127.0.2"),
117118
Mask: net.CIDRMask(16, 32),
118-
}, netlight.NDMZGwIP, nil, myceliumSeedFromIdentity(identity.PrivateKey(cli.Context)))
119-
119+
}, netlight.NDMZGwIP, nil, myceliumSeedFromIdentity(identity.PrivateKey(cli.Context)), net.IPNet{}, zos.NetworkLight{})
120120
if err != nil {
121121
return fmt.Errorf("failed to create ndmz resource: %w", err)
122122
}

go.mod

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,13 +86,15 @@ require (
8686
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
8787
github.com/golang/protobuf v1.5.2 // indirect
8888
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
89+
github.com/google/go-cmp v0.6.0 // indirect
8990
github.com/gorilla/websocket v1.5.1 // indirect
9091
github.com/gtank/ristretto255 v0.1.2 // indirect
9192
github.com/hanwen/go-fuse/v2 v2.3.0 // indirect
9293
github.com/hashicorp/errwrap v1.1.0 // indirect
9394
github.com/hashicorp/go-multierror v1.1.1 // indirect
9495
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
9596
github.com/holiman/uint256 v1.2.3 // indirect
97+
github.com/josharian/native v1.1.0 // indirect
9698
github.com/klauspost/compress v1.16.7 // indirect
9799
github.com/lestrrat-go/backoff/v2 v2.0.7 // indirect
98100
github.com/lestrrat-go/blackmagic v1.0.0 // indirect
@@ -103,6 +105,9 @@ require (
103105
github.com/mattn/go-colorable v0.1.13 // indirect
104106
github.com/mattn/go-isatty v0.0.20 // indirect
105107
github.com/mattn/go-runewidth v0.0.13 // indirect
108+
github.com/mdlayher/genetlink v1.3.2 // indirect
109+
github.com/mdlayher/netlink v1.7.2 // indirect
110+
github.com/mdlayher/socket v0.4.1 // indirect
106111
github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b // indirect
107112
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
108113
github.com/moby/locker v1.0.1 // indirect
@@ -142,6 +147,8 @@ require (
142147
golang.org/x/net v0.22.0 // indirect
143148
golang.org/x/sync v0.3.0 // indirect
144149
golang.org/x/text v0.15.0 // indirect
150+
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b // indirect
151+
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 // indirect
145152
gonum.org/v1/gonum v0.15.0 // indirect
146153
google.golang.org/appengine v1.6.7 // indirect
147154
google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 // indirect

go.sum

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,8 @@ github.com/jgautheron/goconst v0.0.0-20170703170152-9740945f5dcb/go.mod h1:82Txj
298298
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
299299
github.com/joncrlsn/dque v0.0.0-20200702023911-3e80e3146ce5 h1:bo1aoO6l128nKJCBrFflOj9s+KPqMM7ErNyB5GGBNDs=
300300
github.com/joncrlsn/dque v0.0.0-20200702023911-3e80e3146ce5/go.mod h1:dNKs71rs2VJGBAmttu7fouEsRQlRjxy0p1Sx+T5wbpY=
301+
github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
302+
github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
301303
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
302304
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
303305
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -359,6 +361,12 @@ github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq
359361
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
360362
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
361363
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
364+
github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw=
365+
github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o=
366+
github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
367+
github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
368+
github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U=
369+
github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=
362370
github.com/mibk/dupl v1.0.0/go.mod h1:pCr4pNxxIbFGvtyCOi0c7LVjmV6duhKWV+ex5vh38ME=
363371
github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM=
364372
github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b h1:QrHweqAtyJ9EwCaGHBu1fghwxIPiopAHV06JlXrMHjk=
@@ -718,6 +726,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
718726
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
719727
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
720728
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
729+
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
730+
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
731+
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 h1:CawjfCvYQH2OU3/TnxLx97WDSUDRABfT18pCOYwc2GE=
732+
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6/go.mod h1:3rxYc4HtVcSG9gVaTs2GEBdehh+sYPOwKtyUWEOTb80=
721733
gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ=
722734
gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo=
723735
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=

pkg/gridtypes/zos/network_light.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ func NetworkIDFromWorkloadID(wl gridtypes.WorkloadID) (NetID, error) {
5555
// - For each PC or a laptop (for each wireguard peer) there must be a peer in the peer list (on all nodes)
5656
// This is why this can get complicated.
5757
type NetworkLight struct {
58+
// IP range of the network, must be an IPv4 /16
59+
// for example a 10.1.0.0/16
60+
NetworkIPRange gridtypes.IPNet `json:"ip_range"`
61+
5862
// IPV4 subnet for this network resource
5963
// this must be a valid subnet of the entire network ip range.
6064
// for example 10.1.1.0/24
@@ -65,6 +69,36 @@ type NetworkLight struct {
6569
// if no mycelium configuration is provided, vms can't
6670
// get mycelium IPs.
6771
Mycelium Mycelium `json:"mycelium,omitempty"`
72+
73+
// The private wg key of this node (this peer) which is installing this
74+
// network workload right now.
75+
// This has to be filled in by the user (and not generated for example)
76+
// because other peers need to be installed as well (with this peer public key)
77+
// hence it's easier to configure everything one time at the user side and then
78+
// apply everything on all nodes at once
79+
WGPrivateKey string `json:"wireguard_private_key"`
80+
81+
// WGListenPort is the wireguard listen port on this node. this has
82+
// to be filled in by the user for same reason as private key (other nodes need to know about it)
83+
// To find a free port you have to ask the node first by a call over RMB about which ports are possible
84+
// to use.
85+
WGListenPort uint16 `json:"wireguard_listen_port"`
86+
87+
// Peers is a list of other peers in this network
88+
Peers []Peer `json:"peers"`
89+
}
90+
91+
// Peer is the description of a peer of a NetResource
92+
type Peer struct {
93+
// IPV4 subnet of the network resource of the peer
94+
Subnet gridtypes.IPNet `json:"subnet"`
95+
// WGPublicKey of the peer (driven from its private key)
96+
WGPublicKey string `json:"wireguard_public_key"`
97+
// Allowed Ips is related to his subnet.
98+
// todo: remove and derive from subnet
99+
AllowedIPs []gridtypes.IPNet `json:"allowed_ips"`
100+
// Entrypoint of the peer
101+
Endpoint string `json:"endpoint"`
68102
}
69103

70104
type MyceliumPeer string

pkg/netlight/network.go

Lines changed: 150 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"os"
1010
"path/filepath"
1111
"slices"
12+
"strings"
1213
"time"
1314

1415
"github.com/blang/semver"
@@ -25,6 +26,8 @@ import (
2526
"github.com/threefoldtech/zos/pkg/netlight/namespace"
2627
"github.com/threefoldtech/zos/pkg/netlight/options"
2728
"github.com/threefoldtech/zos/pkg/netlight/resource"
29+
"github.com/threefoldtech/zos/pkg/netlight/wireguard"
30+
"github.com/threefoldtech/zos/pkg/set"
2831
"github.com/threefoldtech/zos/pkg/versioned"
2932
"github.com/vishvananda/netlink"
3033
)
@@ -38,18 +41,17 @@ const (
3841
networkDir = "networks"
3942
)
4043

41-
var (
42-
NDMZGwIP = &net.IPNet{
43-
IP: net.ParseIP("100.127.0.1"),
44-
Mask: net.CIDRMask(16, 32),
45-
}
46-
)
44+
var NDMZGwIP = &net.IPNet{
45+
IP: net.ParseIP("100.127.0.1"),
46+
Mask: net.CIDRMask(16, 32),
47+
}
4748

4849
var NetworkSchemaLatestVersion = semver.MustParse("0.1.0")
4950

5051
type networker struct {
5152
ipamLease string
5253
networkDir string
54+
portSet *set.UIntSet
5355
}
5456

5557
var _ pkg.NetworkerLight = (*networker)(nil)
@@ -63,13 +65,20 @@ func NewNetworker() (pkg.NetworkerLight, error) {
6365
ipamLease := filepath.Join(vd, ipamLeaseDir)
6466
runtimeDir := filepath.Join(vd, networkDir)
6567

66-
return &networker{
68+
nw := networker{
6769
ipamLease: ipamLease,
6870
networkDir: runtimeDir,
69-
}, nil
71+
portSet: set.NewInt(),
72+
}
73+
74+
if err := nw.syncWGPorts(); err != nil {
75+
return nil, err
76+
}
77+
return &nw, nil
7078
}
7179

72-
func (n *networker) Create(name string, privateNet net.IPNet, seed []byte) error {
80+
// func (n *networker) Create(name string, privateNet net.IPNet, seed []byte, twinID uint32, wgListenPort uint16, wgPrivateKey string, ipRange net.IPNet, nr pkg.Network) error {
81+
func (n *networker) Create(name string, net zos.NetworkLight) error {
7382
b, err := bridge.Get(NDMZBridge)
7483
if err != nil {
7584
return err
@@ -79,7 +88,67 @@ func (n *networker) Create(name string, privateNet net.IPNet, seed []byte) error
7988
return err
8089
}
8190

82-
_, err = resource.Create(name, b, ip, NDMZGwIP, &privateNet, seed)
91+
storedNR, err := n.networkOf(zos.NetID(name))
92+
if err != nil && !os.IsNotExist(err) {
93+
return errors.Wrap(err, "failed to load previous network setup")
94+
}
95+
96+
if err == nil {
97+
if err := n.releasePort(storedNR.WGListenPort); err != nil {
98+
return err
99+
}
100+
}
101+
102+
if err := n.reservePort(net.WGListenPort); err != nil {
103+
return err
104+
}
105+
106+
netr, err := resource.Create(name, b, ip, NDMZGwIP, &net.Subnet.IPNet, net.Mycelium.Key, net.NetworkIPRange.IPNet, net)
107+
if err != nil {
108+
return err
109+
}
110+
111+
cleanup := func() {
112+
log.Error().Msg("clean up network resource")
113+
if err := resource.Delete(name); err != nil {
114+
log.Error().Err(err).Msg("error during deletion of network resource after failed deployment")
115+
}
116+
if err := n.releasePort(net.WGListenPort); err != nil {
117+
log.Error().Err(err).Msg("release wireguard port failed")
118+
}
119+
}
120+
121+
defer func() {
122+
if err != nil {
123+
cleanup()
124+
}
125+
}()
126+
127+
wgName, err := netr.WGName()
128+
if err != nil {
129+
return errors.Wrap(err, "failed to get wg interface name for network resource")
130+
}
131+
132+
exists, err := netr.HasWireguard()
133+
if err != nil {
134+
return errors.Wrap(err, "failed to check if network resource has wireguard setup")
135+
}
136+
137+
if !exists {
138+
var wg *wireguard.Wireguard
139+
wg, err = wireguard.New(wgName)
140+
if err != nil {
141+
return errors.Wrapf(err, "failed to create wg interface for network resource '%s'", name)
142+
}
143+
if err = netr.SetWireguard(wg); err != nil {
144+
return errors.Wrap(err, "failed to setup wireguard interface for network resource")
145+
}
146+
}
147+
148+
if err = netr.ConfigureWG(net.WGPrivateKey); err != nil {
149+
return errors.Wrap(err, "failed to configure network resource")
150+
}
151+
83152
return err
84153
}
85154

@@ -89,7 +158,6 @@ func (n *networker) Delete(name string) error {
89158
}
90159

91160
return resource.Delete(name)
92-
93161
}
94162

95163
func (n *networker) AttachPrivate(name, id string, vmIp net.IP) (device pkg.TapDevice, err error) {
@@ -391,10 +459,14 @@ func (n *networker) Interfaces(iface string, netns string) (pkg.Interfaces, erro
391459
}
392460

393461
func CreateNDMZBridge() (*netlink.Bridge, error) {
394-
return createNDMZBridge(NDMZBridge, NDMZGw)
462+
return createNDMZBridge(NDMZBridge)
395463
}
396464

397-
func createNDMZBridge(name string, gw string) (*netlink.Bridge, error) {
465+
func (n *networker) WireguardPorts() ([]uint, error) {
466+
return n.portSet.List()
467+
}
468+
469+
func createNDMZBridge(name string) (*netlink.Bridge, error) {
398470
if !bridge.Exists(name) {
399471
if _, err := bridge.New(name); err != nil {
400472
return nil, errors.Wrapf(err, "couldn't create bridge %s", name)
@@ -424,3 +496,68 @@ func createNDMZBridge(name string, gw string) (*netlink.Bridge, error) {
424496

425497
return link.(*netlink.Bridge), nil
426498
}
499+
500+
func (n *networker) reservePort(port uint16) error {
501+
log.Debug().Uint16("port", port).Msg("reserve wireguard port")
502+
err := n.portSet.Add(uint(port))
503+
if err != nil {
504+
return errors.Wrap(err, "wireguard listen port already in use, pick another one")
505+
}
506+
507+
return nil
508+
}
509+
510+
func (n *networker) releasePort(port uint16) error {
511+
log.Debug().Uint16("port", port).Msg("release wireguard port")
512+
n.portSet.Remove(uint(port))
513+
return nil
514+
}
515+
516+
func (n *networker) syncWGPorts() error {
517+
names, err := namespace.List("n-")
518+
if err != nil {
519+
return err
520+
}
521+
522+
readPort := func(name string) (int, error) {
523+
netNS, err := namespace.GetByName(name)
524+
if err != nil {
525+
return 0, err
526+
}
527+
defer netNS.Close()
528+
529+
ifaceName := strings.Replace(name, "n-", "w-", 1)
530+
531+
var port int
532+
err = netNS.Do(func(_ ns.NetNS) error {
533+
link, err := wireguard.GetByName(ifaceName)
534+
if err != nil {
535+
return err
536+
}
537+
d, err := link.Device()
538+
if err != nil {
539+
return err
540+
}
541+
542+
port = d.ListenPort
543+
return nil
544+
})
545+
if err != nil {
546+
return 0, err
547+
}
548+
549+
return port, nil
550+
}
551+
552+
for _, name := range names {
553+
port, err := readPort(name)
554+
if err != nil {
555+
log.Error().Err(err).Str("namespace", name).Msgf("failed to read port for network namespace")
556+
continue
557+
}
558+
// skip error cause we don't care if there are some duplicate at this point
559+
_ = n.portSet.Add(uint(port))
560+
}
561+
562+
return nil
563+
}

0 commit comments

Comments
 (0)