Merge pull request #2929 from threefoldtech/development_2.6

Development 2.6

Author: AhmedHanafy725

.github/workflows/build.yml

@@ -5,9 +5,9 @@ name: Full Clients Build
 
 on:
   push:
-    branches: [development, development_314]
+    branches: [development, development_2.6]
   pull_request:
-    branches: [development, development_314]
+    branches: [development, development_2.6]
 
 jobs:
   build:

.github/workflows/link_checker.yml

@@ -0,0 +1,17 @@
+name: Link Checker
+
+on:
+  schedule:
+    - cron: '0 8 * * *'
+
+jobs:
+  check-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Link Checker
+        id: checker
+        uses: docker://ghcr.io/threefoldfoundation/website-link-checker:latest
+        with:
+          args: 'https://staging.dashboard.dev.grid.tf -e 404 501 503 504 -w all'

.github/workflows/lint.yml

@@ -5,9 +5,9 @@ name: Full Clients Lint
 
 on:
   push:
-    branches: [development, development_314]
+    branches: [development, development_2.6]
   pull_request:
-    branches: [development, development_314]
+    branches: [development, development_2.6]
 
 jobs:
   lint:

.github/workflows/playground_build.yml

@@ -6,13 +6,13 @@ on:
   push:
     branches: 
       - development
-      - development_314
+      - development_2.6
     paths:
       - "packages/playground/**"
   pull_request:
     branches: 
       - development
-      - development_314
+      - development_2.6
     paths:
       - "packages/playground/**"
 

.github/workflows/stats_build.yaml

@@ -4,11 +4,11 @@ name: Stats Build
 
 on:
   push:
-    branches: [development, development_314]
+    branches: [development, development_2.6]
     paths:
       - "packages/stats/**"
   pull_request:
-    branches: [development, development_314]
+    branches: [development, development_2.6]
     paths:
       - "packages/stats/**"
 

.github/workflows/yarn_audit.yml

@@ -0,0 +1,24 @@
+# When the action is executed, it runs yarn audit command in all the paths that is mentioned in the input. The exit codes of the commands are compared and if it is greater than 7 (only high severity as of now), the action will try to fetch the open issues in the repo with the label provided in the input. The label is mandatory to prevent from creating duplicate issues. If there are no open issues with the given label in open state, the action will try to create a Github Issue with the details provided in the input.
+
+name: Full Clients Audit
+
+on:
+  push:
+    branches: [development, development_2.6]
+  pull_request:
+    branches: [development, development_2.6]
+
+jobs:
+  audit-and-open-issue:
+    runs-on: ubuntu-latest
+    steps:
+          - uses: actions/checkout@v2
+          - name: Yarn Audit
+            uses: pragatheeswarans/[email protected]
+            with:
+              token: ${{ github.token }}
+              label: 'audit'
+              title: "${{ github.workflow }}: Critical Security Vulnerability Identified"
+              description: 'High severity issues are identified in the repo.'
+              paths: |
+                .

packages/grid_client/scripts/single_vm.ts

@@ -1,23 +1,25 @@
-import { MachinesModel } from "../src";
+import { GridClient, MachinesModel } from "../src";
+import { type ZmachineData } from "../src/helpers/types";
 import { config, getClient } from "./client_loader";
 import { log } from "./utils";
 
-async function deploy(client, vms) {
+async function deploy(client: GridClient, vms: MachinesModel) {
   const res = await client.machines.deploy(vms);
   log("================= Deploying VM =================");
   log(res);
   log("================= Deploying VM =================");
 }
 
-async function getDeployment(client, vms) {
-  const res = await client.machines.getObj(vms);
+async function getDeployment(client: GridClient, name: string): Promise<ZmachineData[]> {
+  const res = await client.machines.getObj(name);
   log("================= Getting deployment information =================");
   log(res);
   log("================= Getting deployment information =================");
+  return res;
 }
 
-async function cancel(client, vms) {
-  const res = await client.machines.delete(vms);
+async function cancel(client: GridClient, name: string) {
+  const res = await client.machines.delete({ name: name });
   log("================= Canceling the deployment =================");
   log(res);
   log("================= Canceling the deployment =================");
@@ -69,7 +71,7 @@ async function main() {
   await getDeployment(grid3, name);
 
   //Uncomment the line below to cancel the deployment
-  // await cancel(grid3, { name });
+  // await cancel(grid3, name);
 
   await grid3.disconnect();
 }

packages/grid_client/src/helpers/index.ts

@@ -4,3 +4,5 @@ export * from "./events";
 export * from "./validator";
 export * from "./expose";
 export * from "./migration";
+export * from "./root_fs";
+export * from "./types";

packages/grid_client/src/helpers/root_fs.ts

@@ -0,0 +1,39 @@
+import { Decimal } from "decimal.js";
+
+const GB = 1024;
+
+interface RootFSOptions {
+  /** The machine CPU, should be in cores e.g. 5 cores*/
+  CPUCores: number;
+  /** The machine memory, should be in megabytes e.g. 1024 or 2048 MG*/
+  RAMInMegaBytes: number;
+}
+
+/**
+ * Calculate the root filesystem size (CU - Compute Units) based on provided options.
+ *
+ * This function calculates the compute units (CU) required based on the CPU cores and RAM in megabytes.
+ * If both CPU cores and RAM are provided in the `options` parameter, it calculates CU by multiplying
+ * the CPU cores with RAM and dividing by 8 * GB, then converting the result to an integer. If the
+ * calculated CU is zero, it returns 500 / GB; otherwise, it returns 2.
+ *
+ * @param {RootFSOptions} [options] - Optional configuration object.
+ * @param {number} [options.CPUCores] - The number of CPU cores.
+ * @param {number} [options.RAMInMegaBytes] - The RAM size in megabytes.
+ *
+ * @returns {number} - The calculated compute units (CU) based on the provided options.
+ */
+function calculateRootFileSystem(options?: RootFSOptions): number {
+  let cu = 0;
+
+  if (options && options.CPUCores && options.RAMInMegaBytes) {
+    cu = new Decimal(options.CPUCores)
+      .mul(options.RAMInMegaBytes)
+      .divToInt(8 * GB)
+      .toNumber();
+  }
+
+  return cu === 0 ? 500 / GB : 2;
+}
+
+export { calculateRootFileSystem, RootFSOptions };

packages/grid_client/src/helpers/types.ts

@@ -0,0 +1,100 @@
+import { PublicIPResult, ResultStates } from "../zos";
+
+interface NetworkInterface {
+  /** The network identifier */
+  network: string;
+  /** The IP address of the interface */
+  ip: string;
+}
+
+interface ComputeCapacity {
+  /** Number of CPU cores allocated */
+  cpu: number;
+  /** Amount of memory allocated in MB */
+  memory: number;
+}
+
+interface BaseMountData {
+  /** The name of the mount */
+  name: string;
+  /** The mount point in the filesystem */
+  mountPoint: string;
+}
+
+interface ExtendedMountData extends BaseMountData {
+  /** The size of the mount (optional) */
+  size?: number;
+  /** The state of the mount result (optional) */
+  state?: ResultStates;
+  /** Message providing additional information about the mount result (optional) */
+  message?: string;
+  /** Cache information (optional) */
+  cache?: any;
+  /** Prefix information (optional) */
+  prefix?: any;
+  /** Minimal shards (optional) */
+  minimal_shards?: number;
+  /** Expected shards (optional) */
+  expected_shards?: number;
+  /** QSFS ZDBs name (optional) */
+  qsfs_zdbs_name?: string;
+  /** Metrics endpoint (optional) */
+  metricsEndpoint?: string;
+}
+
+// Union type for the mount data
+type MountData = BaseMountData | ExtendedMountData;
+
+interface ZmachineData {
+  /** The version of the workload */
+  version: number;
+  /** The contract ID associated with the workload */
+  contractId: number;
+  /** The node ID where the workload is deployed */
+  nodeId: string;
+  /** The name of the workload */
+  name: string;
+  /** The creation timestamp of the workload result */
+  created: number;
+  /** The current state of the workload */
+  status: string;
+  /** Message providing additional information about the workload state */
+  message: string;
+  /** The flist (file list) used by the workload */
+  flist: string;
+  /** The public IP address obtained by the machine */
+  publicIP: PublicIPResult;
+  /** The planetary IP address of the machine */
+  planetary: string;
+  /** The Mycelium IP address of the machine, if applicable */
+  myceliumIP: string;
+  /** List of network interfaces */
+  interfaces: NetworkInterface[];
+  /** The compute capacity (CPU and memory) allocated to the machine */
+  capacity: ComputeCapacity;
+  /** List of mounts associated with the machine */
+  mounts: MountData[];
+  /** Environment variables set for the workload */
+  env: Record<string, unknown>;
+  /** The entrypoint command for the workload */
+  entrypoint: string;
+  /** Metadata associated with the workload */
+  metadata: string;
+  /** Description of the workload */
+  description: string;
+  /** Size of the root filesystem */
+  rootfs_size: number;
+  /** Indicates if corex is enabled */
+  corex: boolean;
+  /** The list of the GPUs */
+  gpu: string[] | undefined;
+}
+
+interface VM extends ZmachineData {
+  customDomain?: string;
+  deploymentName: string;
+  projectName: string;
+  wireguard: string;
+}
+
+export { ZmachineData, VM };

packages/grid_client/src/high_level/machine.ts

@@ -2,6 +2,7 @@ import { GridClientErrors, ValidationError } from "@threefold/types";
 import { Addr } from "netaddr";
 
 import { events } from "../helpers/events";
+import { calculateRootFileSystem } from "../helpers/root_fs";
 import { randomChoice, zeroPadding } from "../helpers/utils";
 import { validateHexSeed } from "../helpers/validator";
 import { DiskModel, MyceliumNetworkModel, QSFSDiskModel } from "../modules/models";
@@ -53,6 +54,12 @@ class VMHL extends HighLevelBase {
     zlogsOutput?: string,
     gpus: string[] = [],
   ): Promise<[TwinDeployment[], string]> {
+    if (!rootfs_size) {
+      rootfs_size = calculateRootFileSystem({
+        CPUCores: cpu,
+        RAMInMegaBytes: memory,
+      });
+    }
     const deployments: TwinDeployment[] = [];
     const workloads: Workload[] = [];
     let totalDisksSize = rootfs_size;

packages/grid_client/src/modules/base.ts

@@ -6,6 +6,7 @@ import { GqlNodeContract, RMB } from "../clients";
 import { TFClient } from "../clients/tf-grid/client";
 import { GridClientConfig } from "../config";
 import { formatErrorMessage } from "../helpers";
+import { type ZmachineData } from "../helpers/types";
 import { HighLevelBase } from "../high_level/base";
 import { KubernetesHL } from "../high_level/kubernetes";
 import { VMHL } from "../high_level/machine";
@@ -306,7 +307,7 @@ class BaseModule {
     return null;
   }
 
-  async _getZmachineData(deploymentName: string, deployments, workload: Workload): Promise<Record<string, unknown>> {
+  async _getZmachineData(deploymentName: string, deployments, workload: Workload): Promise<ZmachineData> {
     const data = workload.data as Zmachine;
     const resultData = workload.result.data as ZmachineResult;
     return {

packages/grid_client/tests/modules/zmachine.test.ts

@@ -0,0 +1,64 @@
+import { ComputeCapacity, Mount, Zmachine, ZmachineNetwork } from "../../src";
+
+const zmachine = new Zmachine();
+
+beforeAll(() => {
+  const computeCapacity = new ComputeCapacity();
+  computeCapacity.cpu = 1;
+  computeCapacity.memory = 5;
+
+  const network = new ZmachineNetwork();
+  network.planetary = true;
+  network.public_ip = "10.249.0.0/16";
+  network.interfaces = [
+    {
+      network: "znetwork",
+      ip: "10.20.2.2",
+    },
+  ];
+
+  const rootfs_size = 2;
+
+  const disks = new Mount();
+  disks.name = "zdisk";
+  disks.mountpoint = "/mnt/data";
+
+  zmachine.flist = "https://hub.grid.tf/tf-official-vms/ubuntu-22.04.flist";
+  zmachine.network = network;
+  zmachine.size = rootfs_size * 1024 ** 3;
+  zmachine.mounts = [disks];
+  zmachine.entrypoint = "/sbin/zinit init";
+  zmachine.compute_capacity = computeCapacity;
+  zmachine.env = {};
+  zmachine.corex = false;
+  zmachine.gpu = [];
+});
+
+describe("Zmachine Class Tests", () => {
+  it("should create a valid Zmachine instance", () => {
+    expect(zmachine).toBeInstanceOf(Zmachine);
+  });
+
+  it("should correctly compute the challenge string", () => {
+    const expectedChallenge =
+      "https://hub.grid.tf/tf-official-vms/ubuntu-22.04.flist" +
+      "10.249.0.0/16" +
+      "true" +
+      "znetwork" +
+      "10.20.2.22" +
+      "14748364815" +
+      "zdisk" +
+      "/mnt/data" +
+      "/sbin/zinit init";
+
+    expect(zmachine.challenge()).toBe(expectedChallenge);
+  });
+
+  it("should fail validation for missing required fields", () => {
+    const result = () => {
+      zmachine.flist = "";
+      zmachine.entrypoint = "";
+    };
+    expect(result).toThrow();
+  });
+});