diff --git a/.github/workflows/main.yml b/.github/workflows/deploy/deploy.yml similarity index 89% rename from .github/workflows/main.yml rename to .github/workflows/deploy/deploy.yml index 80cd83e2..68f3c230 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/deploy/deploy.yml @@ -1,7 +1,6 @@ +name: "Deploy" + on: - pull_request: - branches: - - main push: branches: - main @@ -18,6 +17,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.9 + - name: Install Pytest Html + run: python3 -m pip install pytest-html pyyaml -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - name: Git Diff id: git-diff uses: technote-space/get-diff-action@v6 @@ -77,6 +78,7 @@ jobs: - name: Test Correctness env: CHANGED_STACKS: ${{ needs.get-changed-project-stack.outputs.changed_stacks }} + WORKSPACE_FILE_DIR: workspaces run: python3 -m pytest -v hack/test_correctness.py --junitxml ./hack/report/test-correctness.xml --html ./hack/report/test-correctness.html - name: Upload Report if: always() @@ -101,12 +103,15 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.9 + - name: Install Pytest Html + run: python3 -m pip install pytest-html pyyaml -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - name: Setup K3d&K3s uses: nolar/setup-k3d-k3s@v1 - name: Preview id: preview env: CHANGED_STACKS: ${{ needs.get-changed-project-stack.outputs.changed_stacks }} + WORKSPACE_FILE_DIR: workspaces run: | #edit the profile in the post step does not work, source kusion env file manually source "$HOME/.kusion/.env" @@ -132,12 +137,15 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.9 + - name: Install Pytest Html + run: python3 -m pip install pytest-html pyyaml -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com - name: Setup K3d&K3s uses: nolar/setup-k3d-k3s@v1 - name: Apply id: apply env: CHANGED_STACKS: ${{ needs.get-changed-project-stack.outputs.changed_stacks }} + WORKSPACE_FILE_DIR: workspaces run: | #edit the profile in the post step does not work, source kusion env file manually source "$HOME/.kusion/.env" diff --git a/hack/apply_changed_stacks.py b/hack/apply_changed_stacks.py index dff93723..56bf168e 100644 --- a/hack/apply_changed_stacks.py +++ b/hack/apply_changed_stacks.py @@ -62,6 +62,7 @@ def pack_result_files(): stack_dirs = util.get_changed_stacks() +util.create_workspaces(stack_dirs) success = apply_stacks(stack_dirs) if success: pack_result_files() diff --git a/hack/check_structure.py b/hack/check_structure.py index 33a15627..5e231be3 100644 --- a/hack/check_structure.py +++ b/hack/check_structure.py @@ -5,33 +5,24 @@ """ from pathlib import Path import pytest -import yaml from lib.common import * from lib import util def check_project_meta(project_dir: Path): - yaml_content = read_to_yaml(str(project_dir / PROJECT_FILE)) + yaml_content = util.read_to_yaml(str(project_dir / PROJECT_FILE)) assert ( yaml_content.get(NAME) is not None ), "file structure error: invalid project meta: project name undefined" def check_stack_meta(stack_dir: Path): - yaml_content = read_to_yaml(str(stack_dir / STACK_FILE)) + yaml_content = util.read_to_yaml(str(stack_dir / STACK_FILE)) assert ( yaml_content.get(NAME) is not None ), "file structure error: invalid stack meta: stack name undefined" -def read_to_yaml(file_path): - with open(file_path) as file: - # The FullLoader parameter handles the conversion from YAML - # scalar values to Python the dictionary format - # See: https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation#how-to-disable-the-warning - return yaml.load(file, Loader=yaml.FullLoader) - - project_dirs = util.get_changed_projects() stack_dirs = util.get_changed_stacks() diff --git a/hack/get_changed_project_stack.py b/hack/get_changed_project_stack.py index faee81b4..80767dea 100644 --- a/hack/get_changed_project_stack.py +++ b/hack/get_changed_project_stack.py @@ -13,7 +13,7 @@ def split_changed_paths_str(changed_paths_str: str) -> List[str]: def get_changed_project_paths(changed_paths: List[str]) -> List[str]: project_paths = [] - check_files = [KCL_FILE_SUFFIX, KCL_MOD_FILE, KCL_MOD_LOCK_FILE, PROJECT_FILE, STACK_FILE] + check_files = [KCL_FILE_SUFFIX, KCL_MOD_FILE, PROJECT_FILE, STACK_FILE] for changed_path in changed_paths: if not changed_path: continue @@ -33,7 +33,7 @@ def get_changed_project_paths(changed_paths: List[str]) -> List[str]: def get_changed_stack_paths(changed_paths: List[str], project_paths: List[str]) -> List[str]: stack_paths = [] - check_files = [KCL_FILE_SUFFIX, KCL_MOD_FILE, KCL_MOD_LOCK_FILE, PROJECT_FILE, STACK_FILE] + check_files = [KCL_FILE_SUFFIX, KCL_MOD_FILE, PROJECT_FILE, STACK_FILE] for changed_path in changed_paths: if not changed_path: continue diff --git a/hack/lib/common.py b/hack/lib/common.py index efcdb944..edcf0b0c 100644 --- a/hack/lib/common.py +++ b/hack/lib/common.py @@ -1,6 +1,7 @@ CHANGED_PROJECTS = "CHANGED_PROJECTS" CHANGED_STACKS = "CHANGED_STACKS" CHANGED_PATHS = "CHANGED_PATHS" +WORKSPACE_FILE_DIR = "WORKSPACE_FILE_DIR" GITHUB_OUTPUT = "GITHUB_OUTPUT" STATUS_SUCCEEDED = "SUCCEEDED" STATUS_FAILED = "FAILED" @@ -10,15 +11,18 @@ PROJECT_FILE = "project.yaml" STACK_FILE = "stack.yaml" KCL_FILE_SUFFIX = ".k" +YAML_FILE_SUFFIX = ".yaml" KCL_MOD_FILE = "kcl.mod" -KCL_MOD_LOCK_FILE = "kcl.mod.lock" NAME = "name" KUSION_CMD = "kusion" COMPILE_CMD = "compile" BUILD_CMD = "build" PREVIEW_CMD = "preview" APPLY_CMD = "apply" +WORKSPACE_CMD = "workspace" +CREATE_CMD = "create" NO_STYLE_FLAG = "--no-style" YES_FLAG = "--yes" +FILE_FLAG = "--file" IGNORE_PROJECTS = ["example/wordpress"] diff --git a/hack/lib/deprecated_utils.py b/hack/lib/deprecated_utils.py deleted file mode 100644 index 4f77e94f..00000000 --- a/hack/lib/deprecated_utils.py +++ /dev/null @@ -1,129 +0,0 @@ -import os -from .common import * -from pathlib import Path -from typing import List, Union -import time - -RETRY_MAX_NUM = os.getenv("RETRY_MAX_NUM", 5) # 最大重试次数（默认五次） -RETRY_INTERVAL = os.getenv("RETRY_INTERVAL", 3) # 重试间隔时间（默认3秒） - - -def startswith(p: Path, start: Path) -> bool: - try: - p.relative_to(str(start.resolve())) - return True - except Exception as e: - return False - - -def get_project_root(p: Path) -> Path: - """获取 Project 根目录""" - while str(p.resolve()) != "/": - if is_project_dir(p): - return p - p = p.resolve().parent - return None - - -def get_konfig_root() -> Path: - """获取大库根目录""" - p = Path(os.getcwd()) - while str(p.resolve()) != "/": - if (p / "kcl.mod").is_file() and (p / HACK_DIR).is_dir(): - return p - p = p.resolve().parent - return Path(os.getcwd()) - - -def get_konfig_projects() -> List[Path]: - """获取大库所有 project 目录""" - result = [] - for project_dir, _, _ in os.walk(get_konfig_root()): - project_dir = Path(project_dir) - if is_project_dir(project_dir): - result.append(project_dir) - return result - - -def get_konfig_projects_relative() -> List[Path]: - """获取大库所有 project 相对于根目录的路径""" - project_dirs = get_konfig_projects() - konfig_root = get_konfig_root() - return [item.relative_to(konfig_root) for item in project_dirs] - - -def is_project_dir(p: Path) -> bool: - """当前目录是否为项目目录""" - project_file = p / PROJECT_FILE - return project_file.is_file() - - -def is_stack_dir(p: Path) -> bool: - """当前目录是否为 Stack 目录""" - stack_file = p / STACK_FILE - return stack_file.is_file() - - -def has_settings_file(path: Path) -> bool: - """当前目录是否包含 settings 文件""" - settings_file = path / SETTINGS_FILE - return settings_file.is_file() - - -def check_path_is_relative_to(path_a: Union[str, Path], path_b: Union[str, Path]): - """ - check if path_a is relative to path_b. - Here are some examples: - path_a: Path('/etc/passwd/') path_b: Path('/etc') True - path_a: Path('/etc/') path_b: Path('/etc') True - path_a: Path('/etc/a/b/c') path_b: Path('/etc') True - path_a: Path('/usr/') path_b: Path('/etc') False - - :param path_a: string type or pathlib.Path type. - :param path_b: string type or pathlib.Path type. - :return: if path_a is relative to path_b. - """ - return Path(path_b) in [p for p in Path(path_a).parents] + [Path(path_a)] - - -def get_affected_projects() -> List[str]: - affected_projects_str = os.getenv("AFFECTED_PROJECTS") or "" - return [project for project in affected_projects_str.split("\n") if project] - - -def get_affected_stacks() -> List[str]: - affected_stacks_str = os.getenv("AFFECTED_STACKS") or "" - return [stack for stack in affected_stacks_str.split("\n") if stack] - - -def get_stack_files_paths_from_change_paths(change_paths_str): - stack_path_list = [] - changed_filepath_list = [item[1:-1] for item in change_paths_str.split(" ") if item] - for filepath in changed_filepath_list: - if not filepath: - continue - elif not filepath.endswith("stdout.golden.yaml"): - continue - # find nearest stack.yaml - splits = filepath.split("/") - path = filepath - for index in range(len(splits) - 1): - path = path.rsplit("/", 1)[0] - if not path: - continue - if path and find_in_dirs(path, "stack.yaml"): - if path not in stack_path_list: - stack_path_list.append(path) - break - return stack_path_list - - -def find_in_dirs(path, file_name): - dir_path = Path(path) - if dir_path.exists(): - for filename in os.listdir(path): - if filename.lower() == file_name: - stack_path = os.path.join(path, filename) - print("find " + stack_path) - return stack_path - return "" diff --git a/hack/lib/util.py b/hack/lib/util.py index 53fe7cb8..5158ea21 100644 --- a/hack/lib/util.py +++ b/hack/lib/util.py @@ -1,5 +1,8 @@ import os +import subprocess +from pathlib import Path from typing import List +import yaml from .common import * @@ -35,3 +38,52 @@ def should_ignore_stack(stack_path: str) -> bool: if stack_path.startswith(project_path): return True return False + + +def create_workspaces(stack_paths: List[str]): + workspaces = detect_workspaces(stack_paths) + workspace_file_dir = get_workspace_file_dir() + for workspace in workspaces: + create_workspace(workspace, workspace_file_dir) + + +def create_workspace(workspace: str, workspace_file_dir: str): + workspace_file = workspace_file_path(workspace_file_dir, workspace) + cmd = [KUSION_CMD, WORKSPACE_CMD, CREATE_CMD, workspace, FILE_FLAG, workspace_file] + process = subprocess.run( + cmd, capture_output=True, env=dict(os.environ) + ) + if process.returncode != 0: + raise Exception(f"Create workspace {workspace} with file {workspace_file} failed", + f"stdout = {process.stdout.decode().strip()}", + f"stderr = {process.stderr.decode().strip()}", + f"returncode = {process.returncode}") + + +def detect_workspaces(stack_paths: List[str]) -> List[str]: + workspaces = [] + for stack_path in stack_paths: + workspaces.append(get_stack_name(Path(stack_path))) + return list(set(filter(None, workspaces))) + + +def get_stack_name(stack_dir: Path) -> str: + yaml_content = read_to_yaml(str(stack_dir / STACK_FILE)) + return yaml_content.get(NAME) or "" + + +def read_to_yaml(file_path): + with open(file_path) as file: + # The FullLoader parameter handles the conversion from YAML + # scalar values to Python the dictionary format + # See: https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation#how-to-disable-the-warning + return yaml.load(file, Loader=yaml.FullLoader) + + +def get_workspace_file_dir() -> str: + return os.getenv(WORKSPACE_FILE_DIR) or "workspaces" + + +def workspace_file_path(file_dir: str, name: str) -> str: + file = name + YAML_FILE_SUFFIX + return os.path.join(file_dir, file) diff --git a/hack/preview_changed_stacks.py b/hack/preview_changed_stacks.py index 98ba9503..9255c83c 100644 --- a/hack/preview_changed_stacks.py +++ b/hack/preview_changed_stacks.py @@ -62,6 +62,7 @@ def pack_result_files(): stack_dirs = util.get_changed_stacks() +util.create_workspaces(stack_dirs) success = preview_stacks(stack_dirs) if success: pack_result_files() diff --git a/hack/test_correctness.py b/hack/test_correctness.py index d21f4767..dbcc558c 100644 --- a/hack/test_correctness.py +++ b/hack/test_correctness.py @@ -6,6 +6,7 @@ from lib import util stack_dirs = util.get_changed_stacks() +util.create_workspaces(stack_dirs) @pytest.mark.parametrize("stack_dir", stack_dirs)