Merge pull request #40 from threefoldtech/main_setPublicConfig

set public config in zoslight

Author: ashraffouda

cmds/modules/netlightd/main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/threefoldtech/zos4/pkg/netlight/bridge"
 	"github.com/threefoldtech/zos4/pkg/netlight/ifaceutil"
 	"github.com/threefoldtech/zos4/pkg/netlight/nft"
+	"github.com/threefoldtech/zos4/pkg/netlight/public"
 	"github.com/threefoldtech/zos4/pkg/netlight/resource"
 	"github.com/urfave/cli/v2"
 
@@ -73,6 +74,8 @@ func action(cli *cli.Context) error {
 		return errors.Wrap(err, "fail to create module root")
 	}
 
+	public.SetPersistence(root)
+
 	waitMyceliumBin()
 
 	if err := bootstrap.DefaultBridgeValid(); err != nil {
@@ -135,7 +138,7 @@ func action(cli *cli.Context) error {
 		return fmt.Errorf("failed to drop traffic to lan: %w", err)
 	}
 
-	mod, err := netlight.NewNetworker()
+	mod, err := netlight.NewNetworker(identity)
 	if err != nil {
 		return fmt.Errorf("failed to create Networker: %w", err)
 	}

cmds/modules/noded/main.go

@@ -59,7 +59,6 @@ var Module cli.Command = cli.Command{
 }
 
 func registerationServer(ctx context.Context, msgBrokerCon string, env environment.Environment, info registrar.RegistrationInfo) error {
-
 	redis, err := zbus.NewRedisClient(msgBrokerCon)
 	if err != nil {
 		return errors.Wrap(err, "fail to connect to message broker server")
@@ -85,13 +84,19 @@ func action(cli *cli.Context) error {
 		printID      bool   = cli.Bool("id")
 		printNet     bool   = cli.Bool("net")
 	)
+
 	env := environment.MustGet()
 
 	redis, err := zbus.NewRedisClient(msgBrokerCon)
 	if err != nil {
 		return errors.Wrap(err, "fail to connect to message broker server")
 	}
 
+	consumer, err := events.NewConsumer(msgBrokerCon, module)
+	if err != nil {
+		return errors.Wrap(err, "failed to to create event consumer")
+	}
+
 	if printID {
 		sysCl := stubs.NewSystemMonitorStub(redis)
 		fmt.Println(sysCl.NodeID(cli.Context))
@@ -216,7 +221,7 @@ func action(cli *cli.Context) error {
 	}
 	go events.Start(ctx)
 
-	system, err := monitord.NewSystemMonitor(node, 2*time.Second)
+	system, err := monitord.NewSystemMonitor(node, 2*time.Second, redis)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to initialize system monitor")
 	}
@@ -232,6 +237,15 @@ func action(cli *cli.Context) error {
 
 	log.Info().Uint32("node", node).Uint32("twin", twin).Msg("node registered")
 
+	go func() {
+		for {
+			if err := startPublicConfigWatcher(ctx, node, redis, consumer); err != nil {
+				log.Error().Err(err).Msg("setting public config failed")
+				<-time.After(10 * time.Second)
+			}
+		}
+	}()
+
 	log.Info().Uint32("twin", twin).Msg("node has been registered")
 	idStub := stubs.NewIdentityManagerStub(redis)
 	fetchCtx, cancel := context.WithTimeout(ctx, 30*time.Second)

cmds/modules/noded/public.go

@@ -0,0 +1,78 @@
+package noded
+
+import (
+	"context"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+	substrate "github.com/threefoldtech/tfchain/clients/tfchain-client-go"
+	"github.com/threefoldtech/zbus"
+	"github.com/threefoldtech/zos4/pkg"
+	"github.com/threefoldtech/zos4/pkg/events"
+	"github.com/threefoldtech/zos4/pkg/stubs"
+)
+
+// public sets and watches changes to public config on chain and tries to apply the provided setup
+func startPublicConfigWatcher(ctx context.Context, nodeID uint32, cl zbus.Client, events *events.RedisConsumer) error {
+	ch, err := events.PublicConfig(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to subscribe to node events")
+	}
+
+	substrateGateway := stubs.NewSubstrateGatewayStub(cl)
+
+reapply:
+	for {
+		node, err := substrateGateway.GetNode(ctx, nodeID)
+		if err != nil {
+			return errors.Wrap(err, "failed to get node public config")
+		}
+
+		var cfg *substrate.PublicConfig
+		if node.PublicConfig.HasValue {
+			cfg = &node.PublicConfig.AsValue
+		}
+
+		if err := setPublicConfig(ctx, cl, cfg); err != nil {
+			return errors.Wrap(err, "failed to set public config (reapply)")
+		}
+
+		for {
+			select {
+			case <-ctx.Done():
+				return nil
+			case event := <-ch:
+				log.Info().Msgf("got a public config update: %+v", event.PublicConfig)
+				var cfg *substrate.PublicConfig
+				if event.PublicConfig.HasValue {
+					cfg = &event.PublicConfig.AsValue
+				}
+				if err := setPublicConfig(ctx, cl, cfg); err != nil {
+					return errors.Wrap(err, "failed to set public config")
+				}
+			case <-time.After(2 * time.Hour):
+				// last resort, if none of the events
+				// was received, it will be a good idea to just
+				// check every 2 hours for changes.
+				continue reapply
+			}
+		}
+	}
+}
+
+func setPublicConfig(ctx context.Context, cl zbus.Client, cfg *substrate.PublicConfig) error {
+	log.Info().Msg("setting node public config")
+	netMgr := stubs.NewNetworkerLightStub(cl)
+
+	if cfg == nil {
+		return netMgr.UnSetPublicConfig(ctx)
+	}
+
+	pub, err := pkg.PublicConfigFrom(*cfg)
+	if err != nil {
+		return errors.Wrap(err, "failed to create public config from setup")
+	}
+
+	return netMgr.SetPublicConfig(ctx, pub)
+}

pkg/gateway/gateway.go

@@ -20,7 +20,6 @@ import (
 	"github.com/threefoldtech/zos4/pkg/cache"
 	"github.com/threefoldtech/zos4/pkg/gridtypes"
 	"github.com/threefoldtech/zos4/pkg/gridtypes/zos"
-	"github.com/threefoldtech/zos4/pkg/kernel"
 	"github.com/threefoldtech/zos4/pkg/netlight/types"
 	"github.com/threefoldtech/zos4/pkg/stubs"
 	"github.com/threefoldtech/zos4/pkg/zinit"
@@ -342,8 +341,11 @@ func (g *gatewayModule) validateNameContracts() error {
 	ctx, cancel := context.WithTimeout(context.Background(), validationPeriod/2)
 	defer cancel()
 	e := stubs.NewProvisionStub(g.cl)
-	baseDomain, found := kernel.GetParams().GetOne("domain")
-	if !found {
+
+	netStub := stubs.NewNetworkerLightStub(g.cl)
+	config, err := netStub.LoadPublicConfig(context.Background())
+	baseDomain := config.Domain
+	if baseDomain == "" || err == nil {
 		// domain doesn't exist so no name workloads exist
 		return nil
 	}
@@ -425,12 +427,12 @@ func (g *gatewayModule) traefikBinary(ctx context.Context, z *zinit.Client) (str
 // ensureGateway makes sure that gateway infrastructure is in place and
 // that it is supported.
 func (g *gatewayModule) ensureGateway(ctx context.Context, forceResstart bool) (string, error) {
-	var (
-		flistd = stubs.NewFlisterStub(g.cl)
-	)
+	flistd := stubs.NewFlisterStub(g.cl)
 
-	domain, found := kernel.GetParams().GetOne("domain")
-	if !found {
+	netStub := stubs.NewNetworkerLightStub(g.cl)
+	config, err := netStub.LoadPublicConfig(context.Background())
+	domain := config.Domain
+	if domain == "" || err != nil {
 		return "", fmt.Errorf("gateway is not supported on this node, domain is not set")
 	}
 
@@ -478,9 +480,10 @@ func (g *gatewayModule) ensureGateway(ctx context.Context, forceResstart bool) (
 		return domain, nil
 	}
 
-	//other wise we start traefik
+	// other wise we start traefik
 	return domain, g.startTraefik(z)
 }
+
 func (g *gatewayModule) verifyDomainDestination(ctx context.Context, domain string) error {
 	networker := stubs.NewNetworkerLightStub(g.cl)
 
@@ -508,7 +511,6 @@ func (g *gatewayModule) verifyDomainDestination(ctx context.Context, domain stri
 }
 
 func (g *gatewayModule) startTraefik(z *zinit.Client) error {
-
 	cmd := fmt.Sprintf(
 		"%s --configfile %s",
 		g.binPath,
@@ -540,7 +542,6 @@ func (g *gatewayModule) configPath(name string) string {
 }
 
 func (g *gatewayModule) validateNameContract(name string, twinID uint32) error {
-
 	contractID, subErr := g.substrateGateway.GetContractIDByNameRegistration(context.Background(), name)
 	if subErr.IsCode(pkg.CodeNotFound) {
 		return ErrContractNotReserved

pkg/monitord/system.go

@@ -9,9 +9,10 @@ import (
 	"github.com/shirou/gopsutil/disk"
 	"github.com/shirou/gopsutil/mem"
 	"github.com/shirou/gopsutil/net"
+	"github.com/threefoldtech/zbus"
 	"github.com/threefoldtech/zos4/pkg"
 	"github.com/threefoldtech/zos4/pkg/gridtypes/zos"
-	"github.com/threefoldtech/zos4/pkg/kernel"
+	"github.com/threefoldtech/zos4/pkg/stubs"
 )
 
 var _ pkg.SystemMonitor = (*systemMonitor)(nil)
@@ -20,15 +21,16 @@ var _ pkg.SystemMonitor = (*systemMonitor)(nil)
 type systemMonitor struct {
 	duration time.Duration
 	node     uint32
+	cl       zbus.Client
 }
 
 // NewSystemMonitor creates new system of system monitor
-func NewSystemMonitor(node uint32, duration time.Duration) (pkg.SystemMonitor, error) {
+func NewSystemMonitor(node uint32, duration time.Duration, cl zbus.Client) (pkg.SystemMonitor, error) {
 	if duration == 0 {
 		duration = 2 * time.Second
 	}
 
-	return &systemMonitor{duration: duration, node: node}, nil
+	return &systemMonitor{duration: duration, node: node, cl: cl}, nil
 }
 
 func (m *systemMonitor) NodeID() uint32 {
@@ -212,8 +214,11 @@ func (n *systemMonitor) GetNodeFeatures() []pkg.NodeFeature {
 		pkg.NodeFeature(zos.ZLogsType),
 		pkg.NodeFeature("mycelium"),
 	}
-	_, found := kernel.GetParams().GetOne("domain")
-	if found {
+
+	netStub := stubs.NewNetworkerLightStub(n.cl)
+	config, err := netStub.LoadPublicConfig(context.Background())
+
+	if config.Domain != "" && err == nil {
 		feat = append(feat, "gateway")
 	}
 

pkg/netlight/network.go

@@ -24,7 +24,9 @@ import (
 	"github.com/threefoldtech/zos4/pkg/netlight/ipam"
 	"github.com/threefoldtech/zos4/pkg/netlight/namespace"
 	"github.com/threefoldtech/zos4/pkg/netlight/options"
+	"github.com/threefoldtech/zos4/pkg/netlight/public"
 	"github.com/threefoldtech/zos4/pkg/netlight/resource"
+	"github.com/threefoldtech/zos4/pkg/stubs"
 	"github.com/threefoldtech/zos4/pkg/versioned"
 	"github.com/vishvananda/netlink"
 )
@@ -38,23 +40,22 @@ const (
 	networkDir    = "networks"
 )
 
-var (
-	NDMZGwIP = &net.IPNet{
-		IP:   net.ParseIP("100.127.0.1"),
-		Mask: net.CIDRMask(16, 32),
-	}
-)
+var NDMZGwIP = &net.IPNet{
+	IP:   net.ParseIP("100.127.0.1"),
+	Mask: net.CIDRMask(16, 32),
+}
 
 var NetworkSchemaLatestVersion = semver.MustParse("0.1.0")
 
 type networker struct {
+	identity   *stubs.IdentityManagerStub
 	ipamLease  string
 	networkDir string
 }
 
 var _ pkg.NetworkerLight = (*networker)(nil)
 
-func NewNetworker() (pkg.NetworkerLight, error) {
+func NewNetworker(identity *stubs.IdentityManagerStub) (pkg.NetworkerLight, error) {
 	vd, err := cache.VolatileDir("networkd", 50*mib)
 	if err != nil && !os.IsExist(err) {
 		return nil, fmt.Errorf("failed to create networkd cache directory: %w", err)
@@ -64,6 +65,7 @@ func NewNetworker() (pkg.NetworkerLight, error) {
 	runtimeDir := filepath.Join(vd, networkDir)
 
 	return &networker{
+		identity:   identity,
 		ipamLease:  ipamLease,
 		networkDir: runtimeDir,
 	}, nil
@@ -89,7 +91,6 @@ func (n *networker) Delete(name string) error {
 	}
 
 	return resource.Delete(name)
-
 }
 
 func (n *networker) AttachPrivate(name, id string, vmIp net.IP) (device pkg.TapDevice, err error) {
@@ -390,6 +391,38 @@ func (n *networker) Interfaces(iface string, netns string) (pkg.Interfaces, erro
 	return pkg.Interfaces{Interfaces: interfaces}, nil
 }
 
+func (n *networker) UnSetPublicConfig() error {
+	return public.DeletePublicConfig()
+}
+
+// Set node public namespace config
+func (n *networker) SetPublicConfig(cfg pkg.PublicConfig) error {
+	if cfg.Equal(pkg.PublicConfig{}) {
+		return fmt.Errorf("public config cannot be unset, only modified")
+	}
+
+	current, err := public.LoadPublicConfig()
+	if err != nil && err != public.ErrNoPublicConfig {
+		return errors.Wrapf(err, "failed to load current public configuration")
+	}
+
+	if current != nil && current.Equal(cfg) {
+		// nothing to do
+		return nil
+	}
+
+	if err := public.SavePublicConfig(cfg); err != nil {
+		return errors.Wrap(err, "failed to store public config")
+	}
+
+	return nil
+}
+
+func (n *networker) LoadPublicConfig() (pkg.PublicConfig, error) {
+	cfg, err := public.LoadPublicConfig()
+	return *cfg, err
+}
+
 func CreateNDMZBridge() (*netlink.Bridge, error) {
 	return createNDMZBridge(NDMZBridge, NDMZGw)
 }

pkg/network_light.go

@@ -29,6 +29,9 @@ type NetworkerLight interface {
 	Ready() error
 	ZOSAddresses(ctx context.Context) <-chan NetlinkAddresses
 	GetSubnet(networkID NetID) (net.IPNet, error)
+	SetPublicConfig(cfg PublicConfig) error
+	UnSetPublicConfig() error
+	LoadPublicConfig() (PublicConfig, error)
 }
 
 type TapDevice struct {