support machine and network light deployments

examples/yaml/multiple_vms/Pulumi.yaml

@@ -10,16 +10,17 @@ resources:
       mnemonic:
 
   scheduler:
-    type: threefold:provider:Scheduler
+    type: threefold:Scheduler
     options:
       provider: ${provider}
     properties:
       mru: 4 # 4gb ram
       sru: 100 # 100gb storage
       farm_ids: [71,73, 74, 76, 77]
+      ygg: true
 
   network:
-    type: threefold:provider:Network
+    type: threefold:Network
     options:
       provider: ${provider}
       dependsOn:
@@ -35,7 +36,7 @@ resources:
       #   manual_nodeID: 9751c596c7c951aedad1a5f78f18b59515064adf660e0d55abead65e6fbbd627 # hex encoded 32 bytes [example]
 
   deployment:
-    type: threefold:provider:Deployment
+    type: threefold:Deployment
     options:
       provider: ${provider}
       dependsOn:
@@ -46,6 +47,7 @@ resources:
       network_name: test
       vms:
         - name: vm
+          node_id: ${scheduler.nodes[0]}
           flist: https://hub.grid.tf/tf-official-apps/base:latest.flist
           entrypoint: "/sbin/zinit init"
           network_name: test
@@ -61,6 +63,7 @@ resources:
             SSH_KEY:
 
         - name: vm2
+          node_id: ${scheduler.nodes[0]}
           flist: https://hub.grid.tf/tf-official-apps/base:latest.flist
           entrypoint: "/sbin/zinit init"
           network_name: test
@@ -76,6 +79,7 @@ resources:
             SSH_KEY:
 
         - name: vm3
+          node_id: ${scheduler.nodes[0]}
           flist: https://hub.grid.tf/tf-official-apps/base:latest.flist
           entrypoint: "/sbin/zinit init"
           network_name: test
@@ -101,4 +105,4 @@ outputs:
   planetary_ip2: ${deployment.vms_computed[1].planetary_ip}
   mycelium_ip2: ${deployment.vms_computed[1].mycelium_ip}
   planetary_ip3: ${deployment.vms_computed[2].planetary_ip}
-  mycelium_ip3: ${deployment.vms_computed[2].mycelium_ip}  
+  mycelium_ip3: ${deployment.vms_computed[2].mycelium_ip}

go.work.sum

@@ -699,16 +699,12 @@ github.com/pulumi/pulumi/pkg/v3 v3.128.0 h1:K3qtJYjHg4DkA7LxknY/MoQZ+QHdHQDh/k2n
 github.com/pulumi/pulumi/pkg/v3 v3.128.0/go.mod h1:/spoJXy/mqQ8fBLgXBEbUrAnL7pHdfXOviIo5fZROEY=
 github.com/pulumi/pulumi/pkg/v3 v3.131.0/go.mod h1:+Cy9VMptNdrhGMcYWg3ZqzWlih+hyWoGOImRSoPNs8o=
 github.com/pulumi/pulumi/pkg/v3 v3.135.0/go.mod h1:6RkK4xeCtl54+PljZFwLASs6TADOE+IMofX0wj1SocY=
-github.com/pulumi/pulumi/pkg/v3 v3.136.1 h1:zA8aJZ7qI0QgZkBKjjQaYHEcigK6pZfrbfG38imXzWo=
-github.com/pulumi/pulumi/pkg/v3 v3.136.1/go.mod h1:Iz8QIs07AbEdrO52hEIEM5C4VBDUYFH2NdM9u2xxBxY=
 github.com/pulumi/pulumi/sdk/v3 v3.30.0/go.mod h1:hGo/+AL1L4sPL9Ukd/i5bNFM3WHs3dHcA+GKEW7M3RA=
 github.com/pulumi/pulumi/sdk/v3 v3.129.0/go.mod h1:p1U24en3zt51agx+WlNboSOV8eLlPWYAkxMzVEXKbnY=
 github.com/pulumi/pulumi/sdk/v3 v3.131.0/go.mod h1:J5kQEX8v87aeUhk6NdQXnjCo1DbiOnOiL3Sf2DuDda8=
 github.com/pulumi/pulumi/sdk/v3 v3.133.0/go.mod h1:J5kQEX8v87aeUhk6NdQXnjCo1DbiOnOiL3Sf2DuDda8=
 github.com/pulumi/pulumi/sdk/v3 v3.134.0/go.mod h1:J5kQEX8v87aeUhk6NdQXnjCo1DbiOnOiL3Sf2DuDda8=
 github.com/pulumi/pulumi/sdk/v3 v3.135.0/go.mod h1:J5kQEX8v87aeUhk6NdQXnjCo1DbiOnOiL3Sf2DuDda8=
-github.com/pulumi/pulumi/sdk/v3 v3.136.1 h1:VJWTgdBrLvvzIkMbGq/epNEfT65P9gTvw14UF/I7hTI=
-github.com/pulumi/pulumi/sdk/v3 v3.136.1/go.mod h1:PvKsX88co8XuwuPdzolMvew5lZV+4JmZfkeSjj7A6dI=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc=
 github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
@@ -761,15 +757,19 @@ github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240416134707
 github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240416134707-eb36aa90df2d/go.mod h1:dtDKAPiUDxAwIkfHV7xcAFZcOm+xwNIuOI1MLFS+MeQ=
 github.com/threefoldtech/tfgrid-sdk-go/grid-client v0.15.11 h1:FmEQplYSZ6pYslmBlUNaUabScOdi6hJialeV/yq0BrY=
 github.com/threefoldtech/tfgrid-sdk-go/grid-client v0.15.11/go.mod h1:Obsv0kspgSSwF8jb91Z7CBY2FCQNq6RhiZXIQf26nWY=
-github.com/threefoldtech/tfgrid-sdk-go/grid-client v0.15.16 h1:zkCTHQqJguMkmeuin2ottra90WN09LWLbGPs9vQsvIw=
-github.com/threefoldtech/tfgrid-sdk-go/grid-client v0.15.16/go.mod h1:MwKEEeKN7qApCdabKRBTs18lrJ6JzA2Xx5Hd2/zb4o0=
+github.com/threefoldtech/tfgrid-sdk-go/grid-client v0.15.17 h1:JwjhOz4mSV0qL0/oF+fdLYtE+l0s7fFjObunzTairYA=
+github.com/threefoldtech/tfgrid-sdk-go/grid-client v0.15.17/go.mod h1:MwKEEeKN7qApCdabKRBTs18lrJ6JzA2Xx5Hd2/zb4o0=
 github.com/threefoldtech/tfgrid-sdk-go/grid-proxy v0.15.10 h1:If91FVqQgd2mZutMMUIrapsqZKDN8/hL5r5Yfr+vFK0=
 github.com/threefoldtech/tfgrid-sdk-go/grid-proxy v0.15.10/go.mod h1:yg1BDRR/TSr87mkjxEfsKc0VzMTJcbg87WW7C0GUPXQ=
 github.com/threefoldtech/tfgrid-sdk-go/grid-proxy v0.15.11 h1:qsSICIj5cvoVBR4GLWmdu2n3FfJG5isJ2ABnNNWykLU=
 github.com/threefoldtech/tfgrid-sdk-go/grid-proxy v0.15.11/go.mod h1:7cKHvKPpfdCpU4a/VCznRDDNMPBPI98cJ0uVJoUGrgg=
+github.com/threefoldtech/tfgrid-sdk-go/grid-proxy v0.15.18 h1:KW9pxM20y9Bp814GsPyNYR7F8ax+h6MgAmTyScrJz2c=
+github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.15.18 h1:+Eeq25Jh0VD34nhlYTrSUDEiLCp4hsz1B4QwqhWoje4=
 github.com/threefoldtech/zbus v1.0.1/go.mod h1:E/v/xEvG/l6z/Oj0aDkuSUXFm/1RVJkhKBwDTAIdsHo=
 github.com/threefoldtech/zos v0.5.6-0.20240611092312-66184cdeb953 h1:hzqMY2Ydb+N2TBrTJHpy5n5i5nflfre1dhoNQH/tfhI=
 github.com/threefoldtech/zos v0.5.6-0.20240611092312-66184cdeb953/go.mod h1:yH6T43SiHvQBBuLgKGElECsIhqzJ6fy/dJyaxtO/X0M=
+github.com/threefoldtech/zos4 v0.5.6-0.20241008102757-02d898c580c4 h1:JCExxpPL32G7evO/+gHwlZLfAX1+l9QN9t55tnPDCp0=
+github.com/threefoldtech/zos4 v0.5.6-0.20241008102757-02d898c580c4/go.mod h1:7KFtZaCcEFwQ1/cz/+hkYK616Ww04ISZgmMqLWHz6To=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=
 github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=

provider/cmd/pulumi-resource-threefold/schema.json

@@ -1,6 +1,7 @@
 {
   "name": "threefold",
   "displayName": "Threefold Grid",
+  "version": "0.7.7",
   "description": "The Pulumi Resource Provider for the Threefold Grid.",
   "keywords": [
     "pulumi",
@@ -1392,6 +1393,12 @@
         },
         "twin_id": {
           "type": "integer"
+        },
+        "wireguard": {
+          "type": "boolean"
+        },
+        "ygg": {
+          "type": "boolean"
         }
       },
       "type": "object",
@@ -1470,6 +1477,12 @@
         },
         "twin_id": {
           "type": "integer"
+        },
+        "wireguard": {
+          "type": "boolean"
+        },
+        "ygg": {
+          "type": "boolean"
         }
       }
     }

provider/deployment_parser.go

@@ -7,7 +7,7 @@ import (
 	"strconv"
 
 	"github.com/threefoldtech/tfgrid-sdk-go/grid-client/workloads"
-	"github.com/threefoldtech/zos/pkg/gridtypes/zos"
+	"github.com/threefoldtech/tfgrid-sdk-go/grid-client/zos"
 )
 
 // Disk represents the disk struct
@@ -147,7 +147,7 @@ func parseInputsToBackends(backends []Backend) workloads.Backends {
 	return bs
 }
 
-func parseInputToDeployment(deploymentArgs DeploymentArgs) (workloads.Deployment, error) {
+func parseInputToDeployment(deploymentArgs DeploymentArgs, light bool) (workloads.Deployment, error) {
 	var solutionProvider *uint64
 	if deploymentArgs.SolutionProvider != 0 {
 		solutionProviderUint := uint64(deploymentArgs.SolutionProvider)
@@ -160,6 +160,7 @@ func parseInputToDeployment(deploymentArgs DeploymentArgs) (workloads.Deployment
 	}
 
 	var vms []workloads.VM
+	var vmsLight []workloads.VMLight
 	for _, vm := range deploymentArgs.VmsInputs {
 		var mounts []workloads.Mount
 		for _, mount := range vm.Mounts {
@@ -195,6 +196,27 @@ func parseInputToDeployment(deploymentArgs DeploymentArgs) (workloads.Deployment
 			}
 		}
 
+		if light {
+			vmsLight = append(vmsLight, workloads.VMLight{
+				Name:           vm.Name,
+				NodeID:         uint32(nodeID),
+				Flist:          vm.Flist,
+				FlistChecksum:  vm.FlistChecksum,
+				NetworkName:    vm.NetworkName,
+				MyceliumIPSeed: myceliumIPSeed,
+				Description:    vm.Description,
+				GPUs:           vm.GPUs,
+				CPU:            uint8(vm.CPU),
+				MemoryMB:       uint64(vm.Memory),
+				RootfsSizeMB:   uint64(vm.RootfsSize),
+				Entrypoint:     vm.EntryPoint,
+				Mounts:         mounts,
+				Zlogs:          zlogs,
+				EnvVars:        vm.EnvVars,
+			})
+			continue
+		}
+
 		vms = append(vms, workloads.VM{
 			Name:           vm.Name,
 			NodeID:         uint32(nodeID),
@@ -279,6 +301,7 @@ func parseInputToDeployment(deploymentArgs DeploymentArgs) (workloads.Deployment
 		Disks:            disks,
 		Zdbs:             zdbs,
 		Vms:              vms,
+		VmsLight:         vmsLight,
 		QSFS:             qsfss,
 	}, nil
 }
@@ -354,6 +377,46 @@ func parseDeploymentToState(deployment workloads.Deployment) DeploymentState {
 		})
 	}
 
+	for _, vm := range deployment.VmsLight {
+		var zlogs []Zlog
+		for _, zlog := range vm.Zlogs {
+			zlogs = append(zlogs, Zlog{
+				Zmachine: zlog.Zmachine,
+				Output:   zlog.Output,
+			})
+		}
+
+		var mounts []Mount
+		for _, mount := range vm.Mounts {
+			mounts = append(mounts, Mount{
+				Name:       mount.Name,
+				MountPoint: mount.MountPoint,
+			})
+		}
+
+		if vm.NodeID == 0 {
+			vm.NodeID = deployment.NodeID
+		}
+
+		vms = append(vms, VMInput{
+			Name:           vm.Name,
+			NodeID:         vm.NodeID,
+			Flist:          vm.Flist,
+			FlistChecksum:  vm.FlistChecksum,
+			NetworkName:    vm.NetworkName,
+			MyceliumIPSeed: hex.EncodeToString(vm.MyceliumIPSeed),
+			Description:    vm.Description,
+			GPUs:           vm.GPUs,
+			CPU:            int(vm.CPU),
+			Memory:         int(vm.MemoryMB),
+			RootfsSize:     int(vm.RootfsSizeMB),
+			EntryPoint:     vm.Entrypoint,
+			Mounts:         mounts,
+			Zlogs:          zlogs,
+			EnvVars:        vm.EnvVars,
+		})
+	}
+
 	var qsfss []QSFSInput
 	for _, qsfs := range deployment.QSFS {
 		var groups []Group
@@ -425,6 +488,15 @@ func parseDeploymentToState(deployment workloads.Deployment) DeploymentState {
 		})
 	}
 
+	for _, vm := range deployment.VmsLight {
+		vmsComputed = append(vmsComputed, VMComputed{
+			MyceliumIPSeed: hex.EncodeToString(vm.MyceliumIPSeed),
+			MyceliumIP:     vm.MyceliumIP,
+			ConsoleURL:     vm.ConsoleURL,
+			IP:             vm.IP,
+		})
+	}
+
 	qsfsComputed := make([]QSFSComputed, 0)
 	for _, qsfs := range deployment.QSFS {
 		qsfsComputed = append(qsfsComputed, QSFSComputed{
@@ -445,7 +517,7 @@ func parseDeploymentToState(deployment workloads.Deployment) DeploymentState {
 	return state
 }
 
-func updateDeploymentFromState(deployment *workloads.Deployment, state DeploymentState) error {
+func updateDeploymentFromState(deployment *workloads.Deployment, state DeploymentState, light bool) error {
 	nodeDeploymentID := make(map[uint32]uint64)
 	for nodeID, deploymentID := range state.NodeDeploymentID {
 		node, err := strconv.ParseUint(nodeID, 10, 32)
@@ -464,12 +536,20 @@ func updateDeploymentFromState(deployment *workloads.Deployment, state Deploymen
 	}
 
 	for i, vm := range state.VmsComputed {
-		if len(deployment.Vms) > i {
+		if len(deployment.Vms) > i || len(deployment.VmsLight) > i {
 			myceliumIPSeed, err := hex.DecodeString(vm.MyceliumIPSeed)
 			if err != nil {
 				return err
 			}
 
+			if light {
+				deployment.VmsLight[i].MyceliumIPSeed = myceliumIPSeed
+				deployment.VmsLight[i].MyceliumIP = vm.MyceliumIP
+				deployment.VmsLight[i].ConsoleURL = vm.ConsoleURL
+				deployment.VmsLight[i].IP = vm.IP
+				continue
+			}
+
 			deployment.Vms[i].MyceliumIPSeed = myceliumIPSeed
 			deployment.Vms[i].ComputedIP = vm.ComputedIP
 			deployment.Vms[i].ComputedIP6 = vm.ComputedIP6

provider/deployment_parser_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/threefoldtech/zos/pkg/gridtypes/zos"
+	"github.com/threefoldtech/tfgrid-sdk-go/grid-client/zos"
 )
 
 func generateInputs() (Disk, ZDBInput, VMInput, QSFSInput, DeploymentArgs) {
@@ -102,7 +102,7 @@ func TestDeploymentParser(t *testing.T) {
 	diskInput, zdbInput, vmInput, qsfsInput, deploymentInput := generateInputs()
 
 	t.Run("parsing input success", func(t *testing.T) {
-		deployment, err := parseInputToDeployment(deploymentInput)
+		deployment, err := parseInputToDeployment(deploymentInput, false)
 		assert.NoError(t, err)
 		assert.Equal(t, deployment.NodeID, uint32(deploymentInput.NodeID.(int)))
 		assert.Equal(t, deployment.Name, deploymentInput.Name)
@@ -115,32 +115,46 @@ func TestDeploymentParser(t *testing.T) {
 		assert.Equal(t, deployment.QSFS[0].Name, qsfsInput.Name)
 	})
 
+	t.Run("parsing input-light success", func(t *testing.T) {
+		deployment, err := parseInputToDeployment(deploymentInput, true)
+		assert.NoError(t, err)
+		assert.Equal(t, deployment.NodeID, uint32(deploymentInput.NodeID.(int)))
+		assert.Equal(t, deployment.Name, deploymentInput.Name)
+		assert.Equal(t, deployment.NetworkName, deploymentInput.NetworkName)
+		assert.Equal(t, deployment.SolutionType, deploymentInput.SolutionType)
+		assert.Equal(t, *deployment.SolutionProvider, uint64(deploymentInput.SolutionProvider))
+		assert.Equal(t, deployment.Disks[0].Name, diskInput.Name)
+		assert.Equal(t, deployment.Zdbs[0].Name, zdbInput.Name)
+		assert.Equal(t, deployment.VmsLight[0].Name, vmInput.Name)
+		assert.Equal(t, deployment.QSFS[0].Name, qsfsInput.Name)
+	})
+
 	t.Run("parsing input failed: wrong node id type", func(t *testing.T) {
 		deploymentInput.NodeID = ""
-		_, err := parseInputToDeployment(deploymentInput)
+		_, err := parseInputToDeployment(deploymentInput, false)
 		assert.Error(t, err)
 		deploymentInput.NodeID = 1
 	})
 
 	t.Run("parsing and update deployment success", func(t *testing.T) {
-		deployment, err := parseInputToDeployment(deploymentInput)
+		deployment, err := parseInputToDeployment(deploymentInput, false)
 		assert.NoError(t, err)
 
 		deployment.NodeDeploymentID = map[uint32]uint64{1: 1}
 		state := parseDeploymentToState(deployment)
 		assert.Equal(t, deployment.NodeDeploymentID[1], uint64(state.NodeDeploymentID["1"]))
 
-		err = updateDeploymentFromState(&deployment, state)
+		err = updateDeploymentFromState(&deployment, state, false)
 		assert.NoError(t, err)
 	})
 
 	t.Run("parsing and update deployment failed: wrong node id type", func(t *testing.T) {
-		deployment, err := parseInputToDeployment(deploymentInput)
+		deployment, err := parseInputToDeployment(deploymentInput, false)
 		assert.NoError(t, err)
 
 		state := parseDeploymentToState(deployment)
 		state.NodeDeploymentID = map[string]int64{"": 1}
-		err = updateDeploymentFromState(&deployment, state)
+		err = updateDeploymentFromState(&deployment, state, false)
 		assert.Error(t, err)
 	})
 }