feat: update deploy github actions and related scripts (#146)

.github/workflows/main.yml → .github/workflows/deploy/deploy.yml

@@ -1,7 +1,6 @@
+name: "Deploy"
+
 on:
-  pull_request:
-    branches:
-      - main
   push:
     branches:
       - main
@@ -18,6 +17,8 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.9
+      - name: Install Pytest Html
+        run: python3 -m pip install pytest-html pyyaml -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com
       - name: Git Diff
         id: git-diff
         uses: technote-space/get-diff-action@v6
@@ -77,6 +78,7 @@ jobs:
       - name: Test Correctness
         env:
           CHANGED_STACKS: ${{ needs.get-changed-project-stack.outputs.changed_stacks }}
+          WORKSPACE_FILE_DIR: workspaces
         run: python3 -m pytest -v hack/test_correctness.py --junitxml ./hack/report/test-correctness.xml --html ./hack/report/test-correctness.html
       - name: Upload Report
         if: always()
@@ -101,12 +103,15 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.9
+      - name: Install Pytest Html
+        run: python3 -m pip install pytest-html pyyaml -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com
       - name: Setup K3d&K3s
         uses: nolar/setup-k3d-k3s@v1
       - name: Preview
         id: preview
         env:
           CHANGED_STACKS: ${{ needs.get-changed-project-stack.outputs.changed_stacks }}
+          WORKSPACE_FILE_DIR: workspaces
         run: |
           #edit the profile in the post step does not work, source kusion env file manually
           source "$HOME/.kusion/.env"
@@ -132,12 +137,15 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.9
+      - name: Install Pytest Html
+        run: python3 -m pip install pytest-html pyyaml -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com
       - name: Setup K3d&K3s
         uses: nolar/setup-k3d-k3s@v1
       - name: Apply
         id: apply
         env:
           CHANGED_STACKS: ${{ needs.get-changed-project-stack.outputs.changed_stacks }}
+          WORKSPACE_FILE_DIR: workspaces
         run: |
           #edit the profile in the post step does not work, source kusion env file manually
           source "$HOME/.kusion/.env"

hack/apply_changed_stacks.py

@@ -62,6 +62,7 @@ def pack_result_files():
 
 
 stack_dirs = util.get_changed_stacks()
+util.create_workspaces(stack_dirs)
 success = apply_stacks(stack_dirs)
 if success:
     pack_result_files()

hack/check_structure.py

@@ -5,33 +5,24 @@
 """
 from pathlib import Path
 import pytest
-import yaml
 from lib.common import *
 from lib import util
 
 
 def check_project_meta(project_dir: Path):
-    yaml_content = read_to_yaml(str(project_dir / PROJECT_FILE))
+    yaml_content = util.read_to_yaml(str(project_dir / PROJECT_FILE))
     assert (
         yaml_content.get(NAME) is not None
     ), "file structure error: invalid project meta: project name undefined"
 
 
 def check_stack_meta(stack_dir: Path):
-    yaml_content = read_to_yaml(str(stack_dir / STACK_FILE))
+    yaml_content = util.read_to_yaml(str(stack_dir / STACK_FILE))
     assert (
         yaml_content.get(NAME) is not None
     ), "file structure error: invalid stack meta: stack name undefined"
 
 
-def read_to_yaml(file_path):
-    with open(file_path) as file:
-        # The FullLoader parameter handles the conversion from YAML
-        # scalar values to Python the dictionary format
-        # See: https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation#how-to-disable-the-warning
-        return yaml.load(file, Loader=yaml.FullLoader)
-
-
 project_dirs = util.get_changed_projects()
 stack_dirs = util.get_changed_stacks()
 

hack/get_changed_project_stack.py

@@ -13,7 +13,7 @@ def split_changed_paths_str(changed_paths_str: str) -> List[str]:
 
 def get_changed_project_paths(changed_paths: List[str]) -> List[str]:
     project_paths = []
-    check_files = [KCL_FILE_SUFFIX, KCL_MOD_FILE, KCL_MOD_LOCK_FILE, PROJECT_FILE, STACK_FILE]
+    check_files = [KCL_FILE_SUFFIX, KCL_MOD_FILE, PROJECT_FILE, STACK_FILE]
     for changed_path in changed_paths:
         if not changed_path:
             continue
@@ -33,7 +33,7 @@ def get_changed_project_paths(changed_paths: List[str]) -> List[str]:
 
 def get_changed_stack_paths(changed_paths: List[str], project_paths: List[str]) -> List[str]:
     stack_paths = []
-    check_files = [KCL_FILE_SUFFIX, KCL_MOD_FILE, KCL_MOD_LOCK_FILE, PROJECT_FILE, STACK_FILE]
+    check_files = [KCL_FILE_SUFFIX, KCL_MOD_FILE, PROJECT_FILE, STACK_FILE]
     for changed_path in changed_paths:
         if not changed_path:
             continue

hack/lib/common.py

@@ -1,6 +1,7 @@
 CHANGED_PROJECTS = "CHANGED_PROJECTS"
 CHANGED_STACKS = "CHANGED_STACKS"
 CHANGED_PATHS = "CHANGED_PATHS"
+WORKSPACE_FILE_DIR = "WORKSPACE_FILE_DIR"
 GITHUB_OUTPUT = "GITHUB_OUTPUT"
 STATUS_SUCCEEDED = "SUCCEEDED"
 STATUS_FAILED = "FAILED"
@@ -10,15 +11,18 @@
 PROJECT_FILE = "project.yaml"
 STACK_FILE = "stack.yaml"
 KCL_FILE_SUFFIX = ".k"
+YAML_FILE_SUFFIX = ".yaml"
 KCL_MOD_FILE = "kcl.mod"
-KCL_MOD_LOCK_FILE = "kcl.mod.lock"
 NAME = "name"
 KUSION_CMD = "kusion"
 COMPILE_CMD = "compile"
 BUILD_CMD = "build"
 PREVIEW_CMD = "preview"
 APPLY_CMD = "apply"
+WORKSPACE_CMD = "workspace"
+CREATE_CMD = "create"
 NO_STYLE_FLAG = "--no-style"
 YES_FLAG = "--yes"
+FILE_FLAG = "--file"
 
 IGNORE_PROJECTS = ["example/wordpress"]

hack/lib/util.py

@@ -1,5 +1,8 @@
 import os
+import subprocess
+from pathlib import Path
 from typing import List
+import yaml
 from .common import *
 
 
@@ -35,3 +38,52 @@ def should_ignore_stack(stack_path: str) -> bool:
         if stack_path.startswith(project_path):
             return True
     return False
+
+
+def create_workspaces(stack_paths: List[str]):
+    workspaces = detect_workspaces(stack_paths)
+    workspace_file_dir = get_workspace_file_dir()
+    for workspace in workspaces:
+        create_workspace(workspace, workspace_file_dir)
+
+
+def create_workspace(workspace: str, workspace_file_dir: str):
+    workspace_file = workspace_file_path(workspace_file_dir, workspace)
+    cmd = [KUSION_CMD, WORKSPACE_CMD, CREATE_CMD, workspace, FILE_FLAG, workspace_file]
+    process = subprocess.run(
+        cmd, capture_output=True, env=dict(os.environ)
+    )
+    if process.returncode != 0:
+        raise Exception(f"Create workspace {workspace} with file {workspace_file} failed",
+                        f"stdout = {process.stdout.decode().strip()}",
+                        f"stderr = {process.stderr.decode().strip()}",
+                        f"returncode = {process.returncode}")
+
+
+def detect_workspaces(stack_paths: List[str]) -> List[str]:
+    workspaces = []
+    for stack_path in stack_paths:
+        workspaces.append(get_stack_name(Path(stack_path)))
+    return list(set(filter(None, workspaces)))
+
+
+def get_stack_name(stack_dir: Path) -> str:
+    yaml_content = read_to_yaml(str(stack_dir / STACK_FILE))
+    return yaml_content.get(NAME) or ""
+
+
+def read_to_yaml(file_path):
+    with open(file_path) as file:
+        # The FullLoader parameter handles the conversion from YAML
+        # scalar values to Python the dictionary format
+        # See: https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation#how-to-disable-the-warning
+        return yaml.load(file, Loader=yaml.FullLoader)
+
+
+def get_workspace_file_dir() -> str:
+    return os.getenv(WORKSPACE_FILE_DIR) or "workspaces"
+
+
+def workspace_file_path(file_dir: str, name: str) -> str:
+    file = name + YAML_FILE_SUFFIX
+    return os.path.join(file_dir, file)

hack/preview_changed_stacks.py

@@ -62,6 +62,7 @@ def pack_result_files():
 
 
 stack_dirs = util.get_changed_stacks()
+util.create_workspaces(stack_dirs)
 success = preview_stacks(stack_dirs)
 if success:
     pack_result_files()

hack/test_correctness.py

@@ -6,6 +6,7 @@
 from lib import util
 
 stack_dirs = util.get_changed_stacks()
+util.create_workspaces(stack_dirs)
 
 
 @pytest.mark.parametrize("stack_dir", stack_dirs)