From 87e4c8d732e33e1fb0946a2a00dc50e02c9d5892 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Mon, 6 Jan 2025 17:00:02 +1100 Subject: [PATCH 01/40] Move jobs around so they can be part of a single parallel workflow (#203) * settings-1-update.yml: Validate settings in parallel based on target * deploy-2-start.yml: Output modules and spack location from workflow * deploy-2-start.yml: Capitalize inputs.type values to keep in line with callers * deploy-1-setup.yml: Update inputs/outputs to be suitable for a matrix job. This matrix job must have the required I/O to do pre-deploy checks, as well as a deployment. * Move check-spack-yaml and check-config jobs from ci.yml to deploy-1-setup.yml * deploy-1-setup.yml: Use new inputs as inputs to deploy-2-start.yml * deploy-1-start.yml: Upload 'outputs' of workflow as an artifact. This is due to dynamic matrix job outputs not being collected appropriately. See https://github.com/orgs/community/discussions/17245 * cd.yml: Run checks of target config in parallel * ci.yml: Dynamically generate deployment comment, start matrix of deployment jobs, misc changes * cd.yml: Update deployment matrix jq * ci.yml: Add newline to deployment comment to fix formatting * Update job names to include deploy target --- .github/workflows/cd.yml | 34 ++- .github/workflows/ci.yml | 378 +++++++++--------------- .github/workflows/deploy-1-setup.yml | 352 ++++++++++++++++++---- .github/workflows/deploy-2-start.yml | 23 +- .github/workflows/settings-1-update.yml | 29 +- 5 files changed, 511 insertions(+), 305 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 8bfeb93..d31356e 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -35,6 +35,7 @@ jobs: runs-on: ubuntu-latest outputs: root-sbd: ${{ steps.root-sbd.outputs.default }} + targets: ${{ steps.target.outputs.matrix }} steps: - name: root-sbd id: root-sbd @@ -45,9 +46,27 @@ jobs: echo "default=${{ inputs.root-sbd }}" >> $GITHUB_OUTPUT fi + - name: Get deployment settings.json + uses: actions/checkout@v4 + with: + repository: access-nri/build-cd + ref: main + + - name: Generate Deployment Target Matrix + id: target + run: | + targets=$(jq --compact-output --raw-output '.deployment | keys' config/settings.json) + echo "$targets" + echo "targets=$targets" >> $GITHUB_OUTPUT + verify-settings: name: Verify Deployment Settings runs-on: ubuntu-latest + needs: + - defaults + strategy: + matrix: + target: ${{ fromJson(needs.defaults.outputs.targets) }} steps: - uses: actions/checkout@v4 with: @@ -57,8 +76,7 @@ jobs: - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@main with: settings-path: ./config/settings.json - # TODO: Turn this into a matrix job of targets - target: Gadi + target: ${{ matrix.target }} error-level: error push-tag: @@ -103,11 +121,17 @@ jobs: needs: - defaults - push-tag + strategy: + matrix: + target: ${{ fromJson(needs.defaults.outputs.targets) }} uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@main with: - ref: ${{ github.ref_name }} - version: ${{ needs.push-tag.outputs.name }} - root-sbd: ${{ needs.defaults.outputs.root-sbd }} + deployment-target: ${{ matrix.target }} + deployment-ref: ${{ github.ref_name }} + deployment-type: Release + deployment-version: ${{ needs.push-tag.outputs.name }} + spack-manifest-path: ./spack.yaml + spack-manifest-root-sbd: ${{ needs.defaults.outputs.root-sbd }} secrets: inherit permissions: contents: write diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb929cc..bed929a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ on: # - created # - edited env: - SPACK_YAML_MODEL_YQ: .spack.specs[0] + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} jobs: defaults: name: Set Defaults @@ -54,7 +54,9 @@ jobs: head-ref: ${{ steps.pr.outputs.head }} head-sha: ${{ steps.pr.outputs.sha }} base-ref: ${{ steps.pr.outputs.base }} - next-deployment-number: ${{ steps.branch.outputs.next-deployment-number }} + next-deployment-number: ${{ steps.prerelease.outputs.next-deployment-number }} + prerelease-version: ${{ steps.prerelease.outputs.version }} + targets: ${{ steps.target.outputs.targets }} steps: - name: root-sbd default id: root-sbd @@ -93,7 +95,7 @@ jobs: echo "base=$base" >> $GITHUB_OUTPUT - name: Branch metadata - id: branch + id: prerelease # Essentially, count all the deployment entries that match the given branch, as well as # all the `!redeploy` comments, to get the next deployment number. # See https://docs.github.com/en/rest/deployments/deployments?apiVersion=2022-11-28#list-deployments @@ -109,6 +111,7 @@ jobs: --json comments \ --jq '[.comments[] | select(.body | startswith("!redeploy"))] | length' ) + # Since the number of $pr_deployments do not include the current deployment (yet), # but $comment_deployments do, we need to increment the next deployment number by one if it is a pr deployment. next_deployment_is_pr_deployment=${{ github.event_name == 'pull_request' && '1' || '0' }} @@ -116,6 +119,25 @@ jobs: echo "Next Deployment Number is $pr_deployments + $comment_deployments + $next_deployment_is_pr_deployment = $next_deployment_number" echo "next-deployment-number=$next_deployment_number" >> $GITHUB_OUTPUT + version="pr${{ inputs.pr }}-$next_deployment_number" + echo "Prerelease version will be $version" + echo "version=$version" >> $GITHUB_OUTPUT + + - name: Get deployment settings.json + uses: actions/checkout@v4 + with: + repository: access-nri/build-cd + ref: main + + - name: Generate Deployment Target Matrix + id: target + # These are used in the deploy job to determine the targets to deploy to - irrespective of the type of deployment. + run: | + targets=$(jq --raw-output --compact-output '.deployment | keys' config/settings.json) + echo "$targets" + echo "targets=$targets" >> $GITHUB_OUTPUT + + redeploy-pre: name: '!redeploy Pending' if: github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '!redeploy') @@ -148,7 +170,7 @@ jobs: - name: Exit if no write permissions if: steps.commenter.outputs.has-permission == 'false' run: | - echo "User ${{ github.event.comment.user.login }} doesn't have 'write' permission on ${{ github.repository }}, not allowing deployment" + echo "::error::User ${{ github.event.comment.user.login }} doesn't have 'write' permission on ${{ github.repository }}, not allowing deployment" exit 1 - name: Set Commit Status Args @@ -167,205 +189,31 @@ jobs: context: ${{ steps.commit-status-args.outputs.context }} description: ${{ steps.commit-status-args.outputs.description }} - check-config: - name: Check Config Fields + deploy: + name: Deploy needs: - defaults - runs-on: ubuntu-latest - outputs: - spack-version: ${{ steps.spack.outputs.version }} - spack-packages-version: ${{ steps.spack-packages.outputs.version }} - spack-config-version: ${{ steps.spack-config.outputs.version }} - config-settings-failures: ${{ steps.settings.outputs.failures }} - steps: - - name: Checkout ${{ github.repository }} Config - uses: actions/checkout@v4 - with: - path: model - ref: ${{ needs.defaults.outputs.head-ref }} - - - name: Validate ${{ github.repository }} config/versions.json - uses: access-nri/schema/.github/actions/validate-with-schema@main - with: - schema-version: ${{ vars.CONFIG_VERSIONS_SCHEMA_VERSION }} - schema-location: au.org.access-nri/model/deployment/config/versions - data-location: ./model/config/versions.json - - - name: Validate spack-packages version - id: spack-packages - uses: access-nri/build-cd/.github/actions/validate-repo-version@main - with: - repo-to-check: spack-packages - pr: ${{ needs.defaults.outputs.head-ref }} - - - name: Validate spack version - id: spack - uses: access-nri/build-cd/.github/actions/validate-repo-version@main - with: - repo-to-check: spack - pr: ${{ needs.defaults.outputs.head-ref }} - - - name: Checkout build-cd Config - uses: actions/checkout@v4 - with: - repository: ACCESS-NRI/build-cd - ref: main - path: cd - - - name: Get spack-config version - id: spack-config - # TODO: For future targets, we need to know which target we are using by this point - run: | - version=$(jq --compact-output --raw-output \ - --arg spack_version "${{ steps.spack.outputs.version }}" \ - '.deployment.Gadi.Prerelease[$spack_version]."spack-config"' cd/config/settings.json - ) - echo $version - echo "version=$version" >> $GITHUB_OUTPUT - - - name: Validate build-cd config/settings.json - id: settings - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@main - with: - settings-path: ./cd/config/settings.json - # TODO: Turn this into a matrix job of targets - target: Gadi - - check-spack-yaml: - name: Check spack.yaml - runs-on: ubuntu-latest - needs: - - defaults - permissions: - pull-requests: write - outputs: - release: ${{ steps.version.outputs.release }} - prerelease: ${{ steps.version.outputs.prerelease }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ needs.defaults.outputs.head-ref }} - - - name: Validate ACCESS-NRI spack.yaml Restrictions - uses: access-nri/schema/.github/actions/validate-with-schema@main - with: - schema-version: ${{ vars.SPACK_YAML_SCHEMA_VERSION }} - schema-location: au.org.access-nri/model/spack/environment/deployment - data-location: ./spack.yaml - - - name: Check Model Version Modified - # We don't want to fire off model deployment version checks if the PR is never going to be merged. - # We determine this by checking if either the pull request, or the pull request that a comment - # belongs to, is drafted. - # Yes, github.event.issue.draft refers to the pull request if the comment is made on a pull request! - if: >- - (github.event_name == 'pull_request' && !github.event.pull_request.draft) || - (github.event_name == 'issue_comment' && !github.event.issue.draft) - id: version-modified - run: | - git checkout ${{ needs.defaults.outputs.base-ref }} - - if [ ! -f spack.yaml ]; then - echo "::notice::There is no previous version of the spack.yaml to check at ${{ needs.defaults.outputs.base-ref }}, continuing..." - git checkout ${{ needs.defaults.outputs.head-ref }} - exit 0 - fi - - base_version=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) - - git checkout ${{ needs.defaults.outputs.head-ref }} - current_version=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) - echo "current=${current_version}" >> $GITHUB_OUTPUT - - if [[ "${base_version}" == "${current_version}" ]]; then - echo "::warning::The version string hasn't been modified in this PR, but needs to be before merging." - exit 1 - fi - - - name: Same Model Version Failure Notifier - if: failure() && steps.version-modified.outcome == 'failure' - uses: access-nri/actions/.github/actions/pr-comment@main - with: - pr: ${{ inputs.pr }} - comment: | - The model version in the `spack.yaml` has not been updated. - Either update it manually, or comment the following to have it updated and committed automatically: - * `!bump major` for feature releases - * `!bump minor` for bugfixes - - - name: Projection Version Matches - # this step checks that the versions of the packages themselves match with the - # names of the projections, if they're given. - # For example, access-om3@git.2023.12.12 matches with the - # modulefile access-om3/2023.12.12 (specifically, the version strings match) - # TODO: Move this into the `scripts` directory - it's getting unweildly. - run: | - FAILED="false" - - # Get all the defined projections (minus 'all') and make them suitable for a bash for loop - DEPS=$(yq '.spack.modules.default.tcl.projections | del(.all) | keys | join(" ")' spack.yaml) - - # for each of the modules - for DEP in $DEPS; do - DEP_VER='' - if [[ "$DEP" == "${{ needs.defaults.outputs.root-sbd }}" ]]; then - # The model version is the bit after '@git.', before any later, space-separated, optional variants. - # For example, in 'MODEL@git.VERSION type=ACCESS ~debug' the version is 'VERSION'. - DEP_VER=$(yq '.spack.specs[0] | capture(".+@git\\.(?[^ ]+).*") | .version' spack.yaml) - else - # Capture the section after '@git.' or '@' (if it's not a git-attributed version) and before a possible '=' for a given dependency. - # Ex. '@git.2024.02.11' -> '2024.02.11', '@access-esm1.5' -> 'access-esm1.5', '@git.2024.05.21=access-esm1.5' -> '2024.05.21' - DEP_VER=$(yq ".spack.packages.\"$DEP\".require[0] | match(\"^@(?:git.)?([^=]*)\").captures[0].string" spack.yaml) - fi - - # Get the version from the module projection, for comparison with DEP_VER - # Projections are of the form '{name}/VERSION[-{hash:7}]', in which we only care about VERSION. For example, '{name}/2024.11.11', or '{name}/2024.11.11-{hash:7}' - MODULE_NAME=$(yq ".spack.modules.default.tcl.projections.\"$DEP\"" spack.yaml) - MODULE_VER="${MODULE_NAME#*/}" # Strip '{name}/' from '{name}/VERSION' module, even if VERSION contains '/' - MODULE_VER="${MODULE_VER%%-\{hash:7\}}" # Strip a potential '-{hash:7}' appendix from the VERSION, since we won't have that in the DEP_VER - - if [[ "$DEP_VER" != "$MODULE_VER" ]]; then - echo "::error::$DEP: Version of dependency and projection do not match ($DEP_VER != $MODULE_VER)" - FAILED='true' - fi - done - if [[ "$FAILED" == "true" ]]; then - exit 1 - fi - - - name: Generate Versions - id: version - # This step generates the release and prerelease version numbers. - # The release is a general version number from the spack.yaml, looking the - # same as a regular release build, without optional variants. Ex. 'access-om2@git.2024.01.1 ~debug' -> '2024.01.1' - # The prerelease looks like: `pr-`. - # Ex. Pull Request #12 with 2 deployments on branch -> `pr12-2`. - run: | - echo "release=$(yq '${{ env.SPACK_YAML_MODEL_YQ }} | capture("@git\.(?[^ ~+]+)") | .version' spack.yaml)" >> $GITHUB_OUTPUT - echo "prerelease=pr${{ inputs.pr }}-${{ needs.defaults.outputs.next-deployment-number }}" >> $GITHUB_OUTPUT - - # ----------------------------- - # | PRERELEASE DEPLOYMENT JOB | - # ----------------------------- - prerelease-deploy: - name: Deploy to Prerelease - # This will create a `spack` environment with the name `-pr-`. - # For example, `access-om3-pr13-3` for the third deployment on the PR#13. - needs: - - defaults # so we can access `inputs.root-sbd` that could have defaulted to `inputs.model` - - check-spack-yaml # implies all the spack.yaml-related checks have passed, has appropriate version for the prerelease build - - check-config # implies all the json-related checks have passed + strategy: + fail-fast: false + matrix: + # Example: ['Gadi', 'Setonix', ...] + target: ${{ fromJson(needs.defaults.outputs.targets) }} uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@main with: - type: prerelease - ref: ${{ needs.defaults.outputs.head-ref }} - version: ${{ needs.check-spack-yaml.outputs.prerelease }} - root-sbd: ${{ needs.defaults.outputs.root-sbd }} + deployment-target: ${{ matrix.target }} + deployment-ref: ${{ needs.defaults.outputs.head-ref }} + deployment-type: Prerelease + # The prerelease looks like: `pr-`. + # Ex. Pull Request #12 with 2 deployments on branch -> `pr12-2`. + # Note that for multi-target prereleases, they share the same deployment number. + deployment-version: ${{ needs.defaults.outputs.prerelease-version }} + prerelease-compare-ref: ${{ needs.defaults.outputs.base-ref }} + spack-manifest-path: ./spack.yaml + spack-manifest-root-sbd: ${{ needs.defaults.outputs.root-sbd }} secrets: inherit redeploy-post: - name: '!redeploy Status ${{ needs.prerelease-deploy.result }}' + name: '!redeploy Status ${{ needs.deploy.result }}' # Always set the commit status after the redeploy job - don't want an always pending status! # successful redeploy = successful commit status # failed, skipped, cancelled redeploy = failure commit status @@ -373,7 +221,7 @@ jobs: needs: - defaults # to get access to the head-sha - redeploy-pre # to get the initial commit status context and description - - prerelease-deploy # to get the overall status of this workflow + - deploy # to get the overall status of this workflow runs-on: ubuntu-latest permissions: statuses: write # so we can set the commit status! @@ -383,61 +231,121 @@ jobs: uses: access-nri/actions/.github/actions/react-to-comment@main with: token: ${{ github.token }} - reaction: ${{ needs.prerelease-deploy.result == 'success' && '+1' || '-1' }} + reaction: ${{ needs.deploy.result == 'success' && '+1' || '-1' }} - - name: Set commit status from workflow ${{ needs.prerelease-deploy.result }} + - name: Set commit status from workflow ${{ needs.deploy.result }} uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: - status: ${{ needs.prerelease-deploy.result == 'success' && 'success' || 'failure' }} + status: ${{ needs.deploy.result == 'success' && 'success' || 'failure' }} sha: ${{ needs.defaults.outputs.head-sha }} context: ${{ needs.redeploy-pre.outputs.commit-status-context }} description: ${{ needs.redeploy-pre.outputs.commit-status-description }} - notifier: - name: Notifier + post-deploy-notifier: + name: Post-Deploy Notifier + if: always() needs: - defaults # so we can access `inputs.root-sbd` that could have defaulted to `inputs.model` - - check-spack-yaml # implies all the spack.yaml-related checks have passed, has appropriate version for the prerelease build - - check-config # so we can access potential failures from config/settings.json validation + - deploy # so we can access deployment information such as versions used, status of deployments, etc. runs-on: ubuntu-latest permissions: pull-requests: write + env: + OUTPUT_ARTIFACT_PATH: ./merged_outputs steps: + - name: Download matrix deployment outputs + uses: actions/download-artifact@v4 + with: + pattern: ${{ needs.deploy.outputs.general-outputs-artifact-glob }} + path: ${{ env.OUTPUT_ARTIFACT_PATH }} + merge-multiple: true + + - name: Create deployment message + id: matrix-output-parser + # Outputs defined in deploy-1-setup.yml under the outputs-upload job + # Creates a comment of the form: https://github.com/ACCESS-NRI/ACCESS-TEST/pull/15#issuecomment-2558675980 + run: | + echo "deployments-message<> $GITHUB_OUTPUT + + for file in ${{ env.OUTPUT_ARTIFACT_PATH }}/*; do + # Setting all the variables that would be needed for the PR comment creation... + filename=$(basename -- "$file") + target="${filename##*.}" + + # For brevity, '-cr' is '--compact-output --raw-output' + spack_version=$(jq -cr '.spack_version' "$file") + spack_config_version=$(jq -cr '.spack_config_version' "$file") + spack_packages_version=$(jq -cr '.spack_packages_version' "$file") + release_deployment_version=$(jq -cr '.release_deployment_version' "$file") + spack_environment_name=$(jq -cr '.spack_environment_name' "$file") + deployment_result=$(jq -cr '.deployment_result' "$file") + deployment_modules_location=$(jq -cr '.deployment_modules_location' "$file") + deployment_spack_location=$(jq -cr '.deployment_spack_location' "$file") + + # Create the message for the deployment + # Header + if [[ "$deployment_result" == "success" ]]; then + echo "### :white_check_mark: :desktop_computer: \`$target\` Deployment" >> $GITHUB_OUTPUT + else + echo "### :x: :desktop_computer: \`$target\` Deployment" >> $GITHUB_OUTPUT + continue + fi + + # Deployment Details + echo "
" >> $GITHUB_OUTPUT + echo "Usage Instructions" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT + echo "Deployed as: " >> $GITHUB_OUTPUT + echo "* \`$release_deployment_version\` as a Release (when merged)." >> $GITHUB_OUTPUT + echo "* \`${{ needs.defaults.outputs.prerelease-version }}\` as a Prerelease (during this PR)." >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT + echo "This Prerelease is accessible on \`$target\` using:" >> $GITHUB_OUTPUT + echo "\`\`\`bash" >> $GITHUB_OUTPUT + echo "module use $deployment_modules_location" >> $GITHUB_OUTPUT + echo "module load ${{ needs.defaults.outputs.root-sbd }}/${{ needs.defaults.outputs.prerelease-version }}" >> $GITHUB_OUTPUT + echo "\`\`\`" >> $GITHUB_OUTPUT + echo "where the binaries shall be on your \`PATH\`." >> $GITHUB_OUTPUT + echo "For advanced users, this Prerelease is also accessible on \`$target\` via \`$deployment_spack_location\` in the \`${{ needs.defaults.outputs.root-sbd }}-${{ needs.defaults.outputs.prerelease-version }}\` environment." >> $GITHUB_OUTPUT + echo "
" >> $GITHUB_OUTPUT + + # Configuration Details + echo "
" >> $GITHUB_OUTPUT + echo "Configuration Details" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT + echo "This Prelease was deployed using:" >> $GITHUB_OUTPUT + echo "* \`access-nri/spack\` on branch [$spack_version](https://github.com/ACCESS-NRI/spack/tree/releases/v${spack_version})" >> $GITHUB_OUTPUT + echo "* \`access-nri/spack-packages\` version [$spack_packages_version](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${spack_packages_version})" >> $GITHUB_OUTPUT + echo "* \`access-nri/spack-config\` version [$spack_config_version](https://github.com/ACCESS-NRI/spack-config/releases/tag/${spack_config_version})" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT + echo "If the above was not expected, please commit changes to \`config/versions.json\`." >> $GITHUB_OUTPUT + echo "
" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT + done + + echo "EOF" >> $GITHUB_OUTPUT + + - name: Config status roll up + id: config-rollup + # Roll up the config status booleans from each of the files, and return the overall result + run: | + echo "errors=$(jq --slurp 'map(.ci_configuration_check_failure | any)' ${{ env.OUTPUT_ARTIFACT_PATH }}/*)" >> $GITHUB_OUTPUT + + - name: Deployment status roll up + id: deployment-rollup + # Roll up the deployment status from each of the files, and return the overall result + # In other words, if any of the deployments are not a success (AKA, failure/skipped/cancelled), then the overall status is failure + run: | + echo "errors=$(jq --slurp 'map(.deployment_result == "success") | all | not' ${{ env.OUTPUT_ARTIFACT_PATH }}/*)" >> $GITHUB_OUTPUT + - name: PR Comment Notifier id: comment uses: access-nri/actions/.github/actions/pr-comment@main with: - pr: ${{ inputs.pr }} comment: | - :rocket: Deploying ${{ inputs.model }} `${{ needs.check-spack-yaml.outputs.release }}` as prerelease `${{ needs.check-spack-yaml.outputs.prerelease }}` with commit ${{ needs.defaults.outputs.head-sha }} - ${{ needs.check-config.outputs.config-settings-failures != '' && ':warning:There are issues with the `build-cd` deployment configuration. If this is unexpected, let @ACCESS-NRI/model-release know.' || '' }} -
- Details and usage instructions - - This `${{ inputs.model }}` model will be deployed as: - * `${{ needs.check-spack-yaml.outputs.release }}` as a Release (when merged). - * `${{ needs.check-spack-yaml.outputs.prerelease }}` as a Prerelease (during this PR). - - This Prerelease is accessible on Gadi using: - ```bash - module use /g/data/vk83/prerelease/modules - module load ${{ needs.defaults.outputs.root-sbd }}/${{ needs.check-spack-yaml.outputs.prerelease }} - ``` - where the binaries shall be on your `$PATH`. - This Prerelease is also accessible on Gadi via `/g/data/vk83/prerelease/apps/spack/${{ needs.check-config.outputs.spack-version }}/spack` in the `${{ needs.defaults.outputs.root-sbd }}-${{ needs.check-spack-yaml.outputs.prerelease }}` environment. -
- - :hammer_and_wrench: Using: spack `${{ needs.check-config.outputs.spack-version }}`, spack-packages `${{ needs.check-config.outputs.spack-packages-version}}`, spack-config `${{ needs.check-config.outputs.spack-config-version }}` -
- Details - - It will be deployed using: - * `access-nri/spack` on branch [`${{ needs.check-config.outputs.spack-version }}`](https://github.com/ACCESS-NRI/spack/tree/releases/v${{ needs.check-config.outputs.spack-version }}) - * `access-nri/spack-packages` version [`${{ needs.check-config.outputs.spack-packages-version }}`](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ needs.check-config.outputs.spack-packages-version }}) - * `access-nri/spack-config` version [`${{ needs.check-config.outputs.spack-config-version }}`](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ needs.check-config.outputs.spack-config-version }}) - - If this is not what was expected, commit changes to `config/versions.json`. -
+ :rocket: [${{ steps.deployment-rollup.outputs.errors == 'true' && 'Attempted to deploy' || 'Deployed' }}](${{ env.RUN_URL }}) `${{ inputs.model }}` as prerelease `${{ needs.defaults.outputs.root-sbd }}/${{ needs.defaults.outputs.prerelease-version }}` with commit ${{ needs.defaults.outputs.head-sha }} + ${{ steps.config-rollup.outputs.errors == 'true' && ':warning:There are issues with the `build-cd` deployment configuration. If this is unexpected, let @ACCESS-NRI/model-release know.' || '' }} + + ${{ steps.matrix-output-parser.outputs.deployments-message }} - name: PR Description Notifier env: @@ -445,7 +353,7 @@ jobs: PR_BODY_PATH: ./body.txt PR_BODY_PATH_UPDATED: ./updated.body.txt PRERELEASE_SECTION_REGEX: "^:rocket: .* :rocket:$" - PRERELEASE_SECTION: ":rocket: The latest prerelease `${{ needs.defaults.outputs.root-sbd }}/${{ needs.check-spack-yaml.outputs.prerelease }}` at ${{ needs.defaults.outputs.head-sha }} is here: ${{ steps.comment.outputs.comment-link }} :rocket:" + PRERELEASE_SECTION: ":rocket: The latest prerelease `${{ needs.defaults.outputs.root-sbd }}/${{ needs.defaults.outputs.prerelease-version }}` at ${{ needs.defaults.outputs.head-sha }} is here: ${{ steps.comment.outputs.comment-url }} :rocket:" run: | gh pr view ${{ inputs.pr }} --repo ${{ github.repository }} --json body --jq .body > ${{ env.PR_BODY_PATH }} diff --git a/.github/workflows/deploy-1-setup.yml b/.github/workflows/deploy-1-setup.yml index e4458c3..9f84f04 100644 --- a/.github/workflows/deploy-1-setup.yml +++ b/.github/workflows/deploy-1-setup.yml @@ -1,88 +1,324 @@ -name: Deployment Setup +name: Deploy on: workflow_call: inputs: - type: + deployment-target: type: string - required: false - default: release - description: The type of deployment - either 'release' or 'prerelease' - ref: + required: true + description: | + The target machine for the model being deployed. + Equivalent to the name of the GitHub Environment, minus a potential Prerelease Type. + deployment-type: + type: string + required: true + description: | + The type of model being deployed. + Can be one of: Release, Prerelease. + deployment-version: type: string required: true - description: The git commit-ish ref where the `spack.yaml` is located - version: + description: | + The version of the model being deployed. + Usually takes the form of either a git tag (2024.12.0), or a Prerelease version (pr12-2) + deployment-ref: + type: string + required: true + description: | + The git ref where the spack manifest that will be deployed is located. + In Prereleases, this is the HEAD of the source branch of the PR. + prerelease-compare-ref: + type: string + required: false + description: | + Optional git ref to compare against the current inputs.deployment-ref + Useful for checking that files have been modified appropriately. + spack-manifest-path: type: string required: true - description: The version for the model being deployed - root-sbd: + default: spack.yaml + description: | + Relative path in the Model Deployment Repository that contains the spack manifest file. + Usually a spack.yaml file. + spack-manifest-root-sbd: type: string required: true - description: The root SBD that is being used as the modulefile name -env: - CONFIG_SETTINGS_PATH: ./config/settings.json + description: | + Within the spack manifest, the overarching spack bundle that contains all other packages. + Usually the first entry in the .spack.specs section of the manifest. + outputs: + general-outputs-artifact-glob: + value: ${{ jobs.outputs-upload.outputs.general-artifact-name }} + description: | + General pattern for the artifact that contains the outputs for this invocation of the job. + Output files are of the form deploy-outputs.{{inputs.deployment-target}} + # TODO: This is a workaround for matrixed dynamic job outputs. See https://github.com/orgs/community/discussions/17245 + # The outputs in the file are below: + # spack-version: + # value: ${{ jobs.check-config.outputs.spack-version }} + # description: | + # Branch of 'access-nri/spack' that is used to deploy the model. + # The VERSION part of the 'releases/VERSION' branch. + # spack-config-version: + # value: ${{ jobs.check-config.outputs.spack-config-version }} + # description: | + # Git ref of 'access-nri/spack-config' that is used to configure spack. + # spack-packages-version: + # value: ${{ jobs.check-config.outputs.spack-packages-version }} + # description: | + # Git ref of 'access-nri/spack-packages' that is used to reference custom packages in spack. + # release-deployment-version: + # value: ${{ jobs.check-spack-yaml.outputs.release }} + # description: | + # Version of the model being deployed as if it were a release version. + # ci-configuration-check-failure: + # value: ${{ jobs.check-config.outputs.config-settings-failures }} + # description: | + # Boolean true/false for whether the configuration check failed for inputs.deployment-target + # spack-environment-name: + # value: ${{ jobs.check-spack-yaml.outputs.spack-env-name }} + # description: | + # Name for the spack environment that contains the deployed model + # deployment-result: + # value: ${{ jobs.deployment.result }} + # description: | + # Result of the deployment - skipped/cancelled/success/failure + # deployment-modules-location: + # value: ${{ jobs.deployment.outputs.module-location }} + # description: | + # Directory containing the modules of the deployed model on the inputs.deployment-target + # deployment-spack-location: + # value: ${{ jobs.deployment.outputs.spack-location }} + # description: | + # Directory containing the spack instance of the deployed model on the inputs.deployment-target jobs: - setup-spack-env: - name: Setup Spack Environment + check-config: + name: '${{ inputs.deployment-target }}: Check Config' runs-on: ubuntu-latest outputs: - # Model name inferred from repository name - model: ${{ steps.get-model.outputs.model }} - # Spack env name in form - - env-name: ${{ steps.get-env-name.outputs.env-name }} + spack-version: ${{ steps.spack.outputs.version }} + spack-packages-version: ${{ steps.spack-packages.outputs.version }} + spack-config-version: ${{ steps.spack-config.outputs.version }} + config-settings-failures: ${{ steps.settings.outputs.failures }} steps: - - name: Get Model - id: get-model - # for the cases where the repo name is in uppercase but the package name is lowercase (eg. access-nri/MOM5) - # and also the cases where there is a '.' in the repo name, which isn't spack-compliant (eg. access-nri/ACCESS-ESM1.5) - run: echo "model=$(echo ${{ github.event.repository.name }} | tr [:upper:] [:lower:] | tr '.' 'p')" >> $GITHUB_OUTPUT + - name: Checkout ${{ github.repository }} Config + uses: actions/checkout@v4 + with: + path: model + ref: ${{ inputs.deployment-ref }} - - name: Set Spack Env Name String - id: get-env-name - # replace occurences of '.' with '_' in environment name as spack doesn't support '.'. Ex: 'access-om2-v1.0.0' -> 'access-om2-v1_0_0'. - run: echo "env-name=$(echo '${{ steps.get-model.outputs.model }}-${{ inputs.version }}' | tr '.' '_')" >> $GITHUB_OUTPUT + - name: Validate ${{ github.repository }} config/versions.json + uses: access-nri/schema/.github/actions/validate-with-schema@main + with: + schema-version: ${{ vars.CONFIG_VERSIONS_SCHEMA_VERSION }} + schema-location: au.org.access-nri/model/deployment/config/versions + data-location: ./model/config/versions.json + + - name: Validate spack-packages version + id: spack-packages + uses: access-nri/build-cd/.github/actions/validate-repo-version@main + with: + repo-to-check: spack-packages + pr: ${{ inputs.deployment-ref }} + + - name: Validate spack version + id: spack + uses: access-nri/build-cd/.github/actions/validate-repo-version@main + with: + repo-to-check: spack + pr: ${{ inputs.deployment-ref }} + + - name: Checkout build-cd Config + uses: actions/checkout@v4 + with: + repository: ACCESS-NRI/build-cd + ref: main + path: cd + + - name: Get spack-config version + id: spack-config + run: | + version=$(jq --compact-output --raw-output \ + --arg spack_version "${{ steps.spack.outputs.version }}" \ + '.deployment.${{ inputs.deployment-target }}.${{ inputs.deployment-type }}[$spack_version]."spack-config"' cd/config/settings.json + ) + echo $version + echo "version=$version" >> $GITHUB_OUTPUT - setup-deployment-env: - name: Setup Deployment Environment + - name: Validate build-cd config/settings.json + id: settings + uses: access-nri/build-cd/.github/actions/validate-deployment-settings@main + with: + settings-path: ./cd/config/settings.json + target: ${{ inputs.deployment-target }} + + check-spack-yaml: + name: '${{ inputs.deployment-target }}: Check Spack Manifest' runs-on: ubuntu-latest + permissions: + pull-requests: write + env: + SPACK_YAML_MODEL_YQ: .spack.specs[0] outputs: - deployment-environments: ${{ steps.get-deployment-environment.outputs.deployment-environments }} + # Release version of the deployment. Inferred if not given in inputs.deployment-version + release: ${{ steps.version.outputs.release }} + # Spack env name in form - + spack-env-name: ${{ steps.get-env-name.outputs.env-name }} steps: - - name: Checkout Config - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: - repository: access-nri/build-cd + fetch-depth: 0 + ref: ${{ inputs.deployment-ref }} - - name: Get Environments - id: get-deployment-environment + - name: Validate ACCESS-NRI spack.yaml Restrictions + uses: access-nri/schema/.github/actions/validate-with-schema@main + with: + schema-version: ${{ vars.SPACK_YAML_SCHEMA_VERSION }} + schema-location: au.org.access-nri/model/spack/environment/deployment + data-location: ${{ inputs.spack-manifest-path }} + + - name: Check Model Version Modified + # We don't want to fire off model deployment version checks if the PR is never going to be merged. + # We determine this by checking if either the pull request, or the pull request that a comment + # belongs to, is drafted. + # Yes, github.event.issue.draft refers to the pull request if the comment is made on a pull request! + if: >- + inputs.deployment-type != 'Release' && + (github.event_name == 'pull_request' && !github.event.pull_request.draft) || + (github.event_name == 'issue_comment' && !github.event.issue.draft) + id: version-modified run: | - if [[ "${{ inputs.type }}" == "release" ]]; then - # Convention is to just have the name by itself for release (eg. 'Gadi') - echo "deployment-environments=$(jq --compact-output '.deployment | keys' ${{ env.CONFIG_SETTINGS_PATH }})" >> $GITHUB_OUTPUT - elif [[ "${{ inputs.type }}" == "prerelease" ]]; then - # Convention is to have the name + Prerelease (eg. 'Gadi Prerelease') - echo "deployment-environments=$(jq --compact-output '[.deployment | keys | "\(.[]) Prerelease"]' ${{ env.CONFIG_SETTINGS_PATH }})" >> $GITHUB_OUTPUT - else - echo "::error::The 'type' input was invalid. Check the inputs documentation." + git checkout ${{ inputs.prerelease-compare-ref }} + + if [ ! -f spack.yaml ]; then + echo "::notice::There is no previous version of the spack.yaml to check at ${{ inputs.prerelease-compare-ref }}, continuing..." + git checkout ${{ inputs.deployment-ref }} + exit 0 + fi + + base_version=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) + + git checkout ${{ inputs.deployment-ref }} + current_version=$(yq e '${{ env.SPACK_YAML_MODEL_YQ }}' spack.yaml) + echo "current=${current_version}" >> $GITHUB_OUTPUT + + if [[ "${base_version}" == "${current_version}" ]]; then + echo "::warning::The version string hasn't been modified in this PR, but needs to be before merging." + exit 1 + fi + + - name: Same Model Version Failure Notifier + if: failure() && steps.version-modified.outcome == 'failure' + uses: access-nri/actions/.github/actions/pr-comment@main + with: + comment: | + The model version in the `${{ inputs.spack-manifest-path }}` has not been updated. + Either update it manually, or comment the following to have it updated and committed automatically: + * `!bump major` for feature releases + * `!bump minor` for bugfixes + + - name: Projection Version Matches + # this step checks that the versions of the packages themselves match with the + # names of the projections, if they're given. + # For example, access-om3@git.2023.12.12 matches with the + # modulefile access-om3/2023.12.12 (specifically, the version strings match) + # TODO: Move this into the `scripts` directory - it's getting unweildly. + run: | + FAILED="false" + + # Get all the defined projections (minus 'all') and make them suitable for a bash for loop + DEPS=$(yq '.spack.modules.default.tcl.projections | del(.all) | keys | join(" ")' spack.yaml) + + # for each of the modules + for DEP in $DEPS; do + DEP_VER='' + if [[ "$DEP" == "${{ inputs.spack-manifest-root-sbd }}" ]]; then + # The model version is the bit after '@git.', before any later, space-separated, optional variants. + # For example, in 'MODEL@git.VERSION type=ACCESS ~debug' the version is 'VERSION'. + DEP_VER=$(yq '.spack.specs[0] | capture(".+@git\\.(?[^ ]+).*") | .version' spack.yaml) + else + # Capture the section after '@git.' or '@' (if it's not a git-attributed version) and before a possible '=' for a given dependency. + # Ex. '@git.2024.02.11' -> '2024.02.11', '@access-esm1.5' -> 'access-esm1.5', '@git.2024.05.21=access-esm1.5' -> '2024.05.21' + DEP_VER=$(yq ".spack.packages.\"$DEP\".require[0] | match(\"^@(?:git.)?([^=]*)\").captures[0].string" spack.yaml) + fi + + # Get the version from the module projection, for comparison with DEP_VER + # Projections are of the form '{name}/VERSION[-{hash:7}]', in which we only care about VERSION. For example, '{name}/2024.11.11', or '{name}/2024.11.11-{hash:7}' + MODULE_NAME=$(yq ".spack.modules.default.tcl.projections.\"$DEP\"" spack.yaml) + MODULE_VER="${MODULE_NAME#*/}" # Strip '{name}/' from '{name}/VERSION' module, even if VERSION contains '/' + MODULE_VER="${MODULE_VER%%-\{hash:7\}}" # Strip a potential '-{hash:7}' appendix from the VERSION, since we won't have that in the DEP_VER + + if [[ "$DEP_VER" != "$MODULE_VER" ]]; then + echo "::error::$DEP: Version of dependency and projection do not match ($DEP_VER != $MODULE_VER)" + FAILED='true' + fi + done + if [[ "$FAILED" == "true" ]]; then exit 1 fi + - name: Generate Versions + id: version + # This step generates the release version number, if it wasn't already given. + # The release is a general version number from the spack.yaml, looking the + # same as a regular release build, without optional variants. Ex. 'access-om2@git.2024.01.1 ~debug' -> '2024.01.1' + run: echo "release=$(yq '${{ env.SPACK_YAML_MODEL_YQ }} | capture("@git\.(?[^ ~+]+)") | .version' spack.yaml)" >> $GITHUB_OUTPUT + + - name: Set Spack Env Name String + id: get-env-name + # replace occurences of '.' with '_' in environment name as spack doesn't support '.'. Ex: 'access-om2-v1.0.0' -> 'access-om2-v1_0_0'. + run: echo "env-name=$(echo '${{ inputs.spack-manifest-root-sbd }}-${{ inputs.deployment-version }}' | tr '.' '_')" >> $GITHUB_OUTPUT + deployment: - name: Deployment + name: ${{ inputs.deployment-target }} needs: - - setup-spack-env - - setup-deployment-env - strategy: - fail-fast: false - matrix: - deployment-environment: ${{ fromJson(needs.setup-deployment-env.outputs.deployment-environments) }} + - check-config # Verify configuration information is correct + - check-spack-yaml # Verify spack manifest information is correct uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@main with: - type: ${{ inputs.type }} - model: ${{ needs.setup-spack-env.outputs.model }} - ref: ${{ inputs.ref }} - version: ${{ inputs.version }} - env-name: ${{ needs.setup-spack-env.outputs.env-name }} - deployment-environment: ${{ matrix.deployment-environment }} - root-sbd: ${{ inputs.root-sbd }} + type: ${{ inputs.deployment-type }} + model: ${{ inputs.spack-manifest-root-sbd }} + ref: ${{ inputs.deployment-ref }} + version: ${{ inputs.deployment-version }} + env-name: ${{ needs.check-spack-yaml.outputs.spack-env-name }} + deployment-environment: ${{ inputs.deployment-type == 'Prerelease' && format('{0} Prerelease', inputs.deployment-target) || inputs.deployment-target }} + root-sbd: ${{ inputs.spack-manifest-root-sbd }} secrets: inherit + + outputs-upload: + name: '${{ inputs.deployment-target }}:Outputs Upload' + # Creates a JSON artifact with information on the deployment created during this run. + # In workflows that call this as a matrix, one must do a deep merge with all files generated from the matrix, + # using the outputs.general-outputs-artifact-name artifact as part of the workflow run. + if: always() + # always, because we might want to interrogate some of the information even in a failed deployment. + needs: + - check-config + - check-spack-yaml + - deployment + runs-on: ubuntu-latest + env: + UPLOAD_FILE_NAME: deploy-outputs.${{ inputs.deployment-target }} + outputs: + general-artifact-name: deploy-outputs.* + steps: + - name: Create outputs file + # See the on.workflow_call.outputs section for details on the info created. + run: | + jq --null-input '{ + spack_version: "${{ needs.check-config.outputs.spack-version }}", + spack_config_version: "${{ needs.check-config.outputs.spack-config-version }}", + spack_packages_version: "${{ needs.check-config.outputs.spack-packages-version }}", + ci_configuration_check_failure: ${{ needs.check-config.outputs.config-settings-failures != '' }}, + release_deployment_version: "${{ needs.check-spack-yaml.outputs.release }}", + spack_environment_name: "${{ needs.check-spack-yaml.outputs.spack-env-name }}", + deployment_result: "${{ needs.deployment.result }}", + deployment_modules_location: "${{ needs.deployment.outputs.modules-location }}", + deployment_spack_location: "${{ needs.deployment.outputs.spack-location }}" + }' > ./${{ env.UPLOAD_FILE_NAME }} + + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: ${{ env.UPLOAD_FILE_NAME }} + path: ./${{ env.UPLOAD_FILE_NAME }} + if-no-files-found: error diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index 3ac3c7b..5eabdb0 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -6,7 +6,7 @@ on: type: type: string required: true - description: The type of deployment - either 'release' or 'prerelease' + description: The type of deployment - either 'Release' or 'Prerelease' model: type: string required: true @@ -31,6 +31,13 @@ on: type: string required: true description: The root SBD that is being used as the modulefile name + outputs: + modules-location: + value: ${{ jobs.deploy-to-environment.outputs.modules-location }} + description: The location of the modules directory on the deployment environment + spack-location: + value: ${{ jobs.deploy-to-environment.outputs.spack-location }} + description: The location of the spack directory on the deployment environment env: SPACK_YAML_SPEC_YQ: .spack.specs[0] SPACK_YAML_MODULEFILE_PROJECTION_YQ: .spack.modules.default.tcl.projections.${{ inputs.root-sbd }} @@ -43,6 +50,8 @@ jobs: outputs: packages-version: ${{ steps.versions.outputs.packages }} config-version: ${{ steps.versions.outputs.config }} + spack-location: ${{ steps.location.outputs.spack }} + modules-location: ${{ steps.location.outputs.modules }} steps: # Deployment - uses: actions/checkout@v4 @@ -73,7 +82,7 @@ jobs: ${{ secrets.HOST_DATA }} - name: Prerelease spack.yaml Modifications - if: inputs.type == 'prerelease' + if: inputs.type == 'Prerelease' # Modifies the name of the prerelease modulefile to the # `pr-` style. For example, `access-om3/pr12-2`. # Also removes the `@git.VERSION` specifier for Prereleases so @@ -132,6 +141,12 @@ jobs: spack module tcl refresh -y EOT + - name: Export deployment target locations + id: location + run: | + echo "spack=${{ steps.path.outputs.spack }}" >> $GITHUB_OUTPUT + echo "modules=${{ vars.DEPLOYED_MODULES_DIR }}" >> $GITHUB_OUTPUT + - name: Get metadata from ${{ inputs.deployment-environment }} env: SPACK_ENV_PATH: ${{ steps.path.outputs.spack }}/../environments/${{ inputs.env-name }} @@ -195,7 +210,7 @@ jobs: release: name: Create Release - if: inputs.type == 'release' + if: inputs.type == 'Release' needs: - deploy-to-environment runs-on: ubuntu-latest @@ -235,7 +250,7 @@ jobs: build-db: name: Build DB Metadata Upload - if: inputs.type == 'release' + if: inputs.type == 'Release' needs: - deploy-to-environment - release diff --git a/.github/workflows/settings-1-update.yml b/.github/workflows/settings-1-update.yml index 5d23fb6..018c7f2 100644 --- a/.github/workflows/settings-1-update.yml +++ b/.github/workflows/settings-1-update.yml @@ -14,8 +14,32 @@ env: CONFIG_SETTINGS_PATH: ./config/settings.json CONFIG_SETTINGS_SCHEMA_PATH: ./config/settings.schema.json jobs: + setup-settings-validation: + name: Setup Validate Deployment Settings + runs-on: ubuntu-latest + outputs: + targets: ${{ steps.target.outputs.matrix }} + steps: + - name: Get deployment settings.json + uses: actions/checkout@v4 + with: + repository: access-nri/build-cd + ref: main + + - name: Generate Deployment Target Matrix + id: target + run: | + targets=$(jq --compact-output '.deployments' config/settings.json) + echo "$targets" + echo "targets=$targets" >> $GITHUB_OUTPUT + settings-validation: name: Validate Deployment Settings + needs: + - setup-settings-validation + strategy: + matrix: + target: ${{ fromJson(needs.setup-settings-validation.outputs.targets) }} runs-on: ubuntu-latest steps: - name: Checkout settings.json @@ -33,15 +57,14 @@ jobs: uses: access-nri/build-cd/.github/actions/validate-deployment-settings@main with: settings-path: ${{ env.CONFIG_SETTINGS_PATH }} - # TODO: Turn this into a matrix job of targets - target: Gadi + target: ${{ matrix.target }} - name: Comment Validation Issues if: steps.validate.outputs.failures != '' && github.event_name == 'pull_request' uses: access-nri/actions/.github/actions/pr-comment@main with: comment: | - :warning: `${{ env.CONFIG_SETTINGS_PATH }}`: Inconsistencies detected with the configuration. See below. :warning: + :warning: `${{ env.CONFIG_SETTINGS_PATH }}`: Inconsistencies detected with the `${{ matrix.target }}` configuration. See below. :warning: ${{ steps.validate.outputs.failures }} setup-settings-update: From faa931eb7675951295e97dac18873cbb935fa1bb Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Tue, 7 Jan 2025 13:30:05 +1100 Subject: [PATCH 02/40] deploy-2-start.yml: Remove `release` and `build-db` jobs, output new general metadata artifact glob --- .github/workflows/deploy-2-start.yml | 92 +++------------------------- 1 file changed, 10 insertions(+), 82 deletions(-) diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index 5eabdb0..3230e49 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -38,10 +38,16 @@ on: spack-location: value: ${{ jobs.deploy-to-environment.outputs.spack-location }} description: The location of the spack directory on the deployment environment + general-metadata-artifact-glob: + value: ${{ jobs.deploy-to-environment.outputs.metadata-artifact-name }} + description: | + General pattern for the artifact that contains the deployment metadata files for this invocation of the job. + These files are of the form deploy-metadata.{{inputs.deployment-target}} env: SPACK_YAML_SPEC_YQ: .spack.specs[0] SPACK_YAML_MODULEFILE_PROJECTION_YQ: .spack.modules.default.tcl.projections.${{ inputs.root-sbd }} METADATA_PATH: /opt/metadata + ARTIFACT_NAME: deploy-metadata.${{ inputs.deployment-environment }} jobs: deploy-to-environment: name: Deploy to ${{ inputs.deployment-environment }} @@ -52,6 +58,7 @@ jobs: config-version: ${{ steps.versions.outputs.config }} spack-location: ${{ steps.location.outputs.spack }} modules-location: ${{ steps.location.outputs.modules }} + metadata-artifact-name: ${{ env.ARTIFACT_NAME }} steps: # Deployment - uses: actions/checkout@v4 @@ -199,89 +206,10 @@ jobs: run: | rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ '${{ secrets.USER}}@${{ secrets.HOST_DATA }}:${{ env.SPACK_ENV_PATH }}/spack.*' \ - ./${{ inputs.env-name }} + ./${{ env.ARTIFACT_NAME }} - name: Upload Metadata Artifact uses: actions/upload-artifact@v4 with: - name: ${{ inputs.env-name }} - path: ./${{ inputs.env-name }}/* - overwrite: true - - release: - name: Create Release - if: inputs.type == 'Release' - needs: - - deploy-to-environment - runs-on: ubuntu-latest - outputs: - url: ${{ steps.release.outputs.url }} - created-at: ${{ steps.metadata.outputs.created-at }} - steps: - - uses: actions/checkout@v4 - - - name: Download Metadata Artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.env-name }} - path: ${{ env.METADATA_PATH }} - - - name: Create Release - id: release - uses: softprops/action-gh-release@69320dbe05506a9a39fc8ae11030b214ec2d1f87 # v2.0.5 - with: - tag_name: ${{ inputs.version }} - name: ${{ inputs.model}} ${{ inputs.version }} - body: | - This release of ${{ inputs.model }} ${{ inputs.version }} uses [spack-packages ${{ needs.deploy-to-environment.outputs.packages-version }}](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ needs.deploy-to-environment.outputs.packages-version }}) and [spack-config ${{ needs.deploy-to-environment.outputs.config-version }}](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ needs.deploy-to-environment.outputs.config-version }}). - generate_release_notes: true - fail_on_unmatched_files: true - files: | - ${{ env.METADATA_PATH }}/spack.yaml - ${{ env.METADATA_PATH }}/spack.lock - ${{ env.METADATA_PATH }}/spack.location - ${{ env.METADATA_PATH }}/spack.location.json - - - name: Release Metadata - id: metadata - env: - GH_TOKEN: ${{ github.token }} - run: echo "created-at=$(gh release view --json createdAt --jq '.createdAt')" >> $GITHUB_OUTPUT - - build-db: - name: Build DB Metadata Upload - if: inputs.type == 'Release' - needs: - - deploy-to-environment - - release - runs-on: ubuntu-latest - steps: - - name: Download Metadata Artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.env-name }} - path: ${{ env.METADATA_PATH }} - - - name: Checkout Upload Script - uses: actions/checkout@v4 - with: - repository: access-nri/build-cd - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ vars.PYTHON_VERSION }} - cache: pip - - - name: Install Build Metadata Script Requirements - run: pip install -r tools/release_provenance/requirements.txt - - - name: Upload Build Metadata - env: - BUILD_DB_CONNECTION_STR: ${{ secrets.BUILD_DB_CONNECTION_STR }} - OUTPUT_PATH: ./metadata_output - run: | - ./scripts/generate-build-metadata.bash ${{ needs.release.outputs.url }} ${{ needs.release.outputs.created-at }} ${{ needs.deploy-to-environment.outputs.packages-version }} ${{ needs.deploy-to-environment.outputs.config-version }} ${{ env.METADATA_PATH }} ${{ env.OUTPUT_PATH }} ${{ inputs.root-sbd }} ${{ vars.BUILD_DB_PACKAGES }} - - echo "Attempting upload of build_metadata.json" - python ./tools/release_provenance/save_release.py "${{ env.OUTPUT_PATH }}/build_metadata.json" + name: ${{ env.ARTIFACT_NAME }} + path: ./${{ env.ARTIFACT_NAME }}/* From 0cd3f8ab5392a08998eed773eb34a36f7b531419 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Tue, 7 Jan 2025 13:30:56 +1100 Subject: [PATCH 03/40] deploy-1-setup.yml: Output `general-metadata-artifact-glob` from `deploy-2-start.yml` invocation --- .github/workflows/deploy-1-setup.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-1-setup.yml b/.github/workflows/deploy-1-setup.yml index 9f84f04..a4b071b 100644 --- a/.github/workflows/deploy-1-setup.yml +++ b/.github/workflows/deploy-1-setup.yml @@ -46,10 +46,15 @@ on: Within the spack manifest, the overarching spack bundle that contains all other packages. Usually the first entry in the .spack.specs section of the manifest. outputs: + general-metadata-artifact-glob: + value: ${{ jobs.deployment.outputs.general-metadata-artifact-glob }} + description: | + General pattern for the artifact that contains the deployment metadata files for this invocation of the job. + These files are of the form deploy-metadata.{{inputs.deployment-target}} general-outputs-artifact-glob: value: ${{ jobs.outputs-upload.outputs.general-artifact-name }} description: | - General pattern for the artifact that contains the outputs for this invocation of the job. + General pattern for the artifact that contains the text outputs for this invocation of the job. Output files are of the form deploy-outputs.{{inputs.deployment-target}} # TODO: This is a workaround for matrixed dynamic job outputs. See https://github.com/orgs/community/discussions/17245 # The outputs in the file are below: @@ -273,7 +278,7 @@ jobs: needs: - check-config # Verify configuration information is correct - check-spack-yaml # Verify spack manifest information is correct - uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@main + uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@v3 with: type: ${{ inputs.deployment-type }} model: ${{ inputs.spack-manifest-root-sbd }} From 9c812e41a2a5ebafd41e7e0081403209a853c506 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Tue, 7 Jan 2025 13:31:34 +1100 Subject: [PATCH 04/40] cd.yml: Add `release`, `build-db` jobs from `deploy-2-start.yml` --- .github/workflows/cd.yml | 90 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index d31356e..dcd921f 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -28,6 +28,8 @@ on: env: SPACK_YAML_MODEL_YQ: .spack.specs[0] + METADATA_PATH: /opt/metadata + OUTPUTS_PATH: /opt/outputs jobs: defaults: name: Set Defaults @@ -124,7 +126,7 @@ jobs: strategy: matrix: target: ${{ fromJson(needs.defaults.outputs.targets) }} - uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@main + uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v3 with: deployment-target: ${{ matrix.target }} deployment-ref: ${{ github.ref_name }} @@ -135,3 +137,89 @@ jobs: secrets: inherit permissions: contents: write + + release: + name: Create Release + needs: + - push-tag + - deploy-release + runs-on: ubuntu-latest + outputs: + url: ${{ steps.release.outputs.url }} + created-at: ${{ steps.metadata.outputs.created-at }} + steps: + - uses: actions/checkout@v4 + + - name: Download Metadata Artifacts + uses: actions/download-artifact@v4 + with: + pattern: ${{ needs.deploy-release.outputs.general-metadata-artifact-glob }} + path: ${{ env.METADATA_PATH }} + merge-multiple: true + + - name: Download Outputs Artifacts + uses: actions/download-artifact@v4 + with: + pattern: ${{ needs.deploy-release.outputs.general-outputs-artifact-glob }} + path: ${{ env.OUTPUTS_PATH}} + merge-multiple: true + + - name: Create Release + id: release + uses: softprops/action-gh-release@69320dbe05506a9a39fc8ae11030b214ec2d1f87 # v2.0.5 + with: + tag_name: ${{ needs.push-tag.outputs.name }} + name: ${{ inputs.model}} ${{ needs.push-tag.outputs.name }} + body: | + This release of ${{ inputs.model }} ${{ needs.push-tag.outputs.name }} uses [spack-packages ${{ needs.deploy-to-environment.outputs.packages-version }}](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ needs.deploy-to-environment.outputs.packages-version }}) and [spack-config ${{ needs.deploy-to-environment.outputs.config-version }}](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ needs.deploy-to-environment.outputs.config-version }}). + generate_release_notes: true + fail_on_unmatched_files: true + files: | + ${{ env.METADATA_PATH }}/spack.yaml + ${{ env.METADATA_PATH }}/spack.lock + ${{ env.METADATA_PATH }}/spack.location + ${{ env.METADATA_PATH }}/spack.location.json + + - name: Release Metadata + id: metadata + env: + GH_TOKEN: ${{ github.token }} + run: echo "created-at=$(gh release view --json createdAt --jq '.createdAt')" >> $GITHUB_OUTPUT + + # FIXME: This won't work until ACCESS-NRI/build-cd#201 is resolved + build-db: + name: Build DB Metadata Upload + needs: + - deploy-release + - release + runs-on: ubuntu-latest + steps: + - name: Download Metadata Artifact + uses: actions/download-artifact@v4 + with: + pattern: ${{ needs.deploy-release.outputs.general-metadata-artifact-glob }} + path: ${{ env.METADATA_PATH }} + + - name: Checkout Upload Script + uses: actions/checkout@v4 + with: + repository: access-nri/build-cd + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ vars.PYTHON_VERSION }} + cache: pip + + - name: Install Build Metadata Script Requirements + run: pip install -r tools/release_provenance/requirements.txt + + - name: Upload Build Metadata + env: + BUILD_DB_CONNECTION_STR: ${{ secrets.BUILD_DB_CONNECTION_STR }} + OUTPUT_PATH: ./metadata_output + run: | + ./scripts/generate-build-metadata.bash ${{ needs.release.outputs.url }} ${{ needs.release.outputs.created-at }} ${{ needs.deploy-to-environment.outputs.packages-version }} ${{ needs.deploy-to-environment.outputs.config-version }} ${{ env.METADATA_PATH }} ${{ env.OUTPUT_PATH }} ${{ inputs.root-sbd }} ${{ vars.BUILD_DB_PACKAGES }} + + echo "Attempting upload of build_metadata.json" + python ./tools/release_provenance/save_release.py "${{ env.OUTPUT_PATH }}/build_metadata.json" From f60b30dcc13ab3993855cf190991765f0af5e3f1 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Tue, 7 Jan 2025 14:19:53 +1100 Subject: [PATCH 05/40] deploy-2-start.yml: Prepend metadata artifacts with deployment environment --- .github/workflows/cd.yml | 8 ++++---- .github/workflows/deploy-2-start.yml | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index dcd921f..2684e45 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -175,10 +175,10 @@ jobs: generate_release_notes: true fail_on_unmatched_files: true files: | - ${{ env.METADATA_PATH }}/spack.yaml - ${{ env.METADATA_PATH }}/spack.lock - ${{ env.METADATA_PATH }}/spack.location - ${{ env.METADATA_PATH }}/spack.location.json + ${{ env.METADATA_PATH }}/*.spack.yaml + ${{ env.METADATA_PATH }}/*.spack.lock + ${{ env.METADATA_PATH }}/*.spack.location + ${{ env.METADATA_PATH }}/*.spack.location.json - name: Release Metadata id: metadata diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index 3230e49..b39a4ad 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -208,6 +208,13 @@ jobs: '${{ secrets.USER}}@${{ secrets.HOST_DATA }}:${{ env.SPACK_ENV_PATH }}/spack.*' \ ./${{ env.ARTIFACT_NAME }} + # Rename the files to include the deployment environment + cd ./${{ env.ARTIFACT_NAME }} + for file in *; do + mv "$file" "${{ inputs.deployment-environment }}.$file" + done + cd - + - name: Upload Metadata Artifact uses: actions/upload-artifact@v4 with: From de70856c90d0805a43d82928c7f46287df5a7431 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Tue, 7 Jan 2025 14:21:35 +1100 Subject: [PATCH 06/40] cd.yml: Generate release notes based on deployment targets --- .github/workflows/cd.yml | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 2684e45..5cd1c09 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -141,6 +141,7 @@ jobs: release: name: Create Release needs: + - defaults - push-tag - deploy-release runs-on: ubuntu-latest @@ -148,8 +149,6 @@ jobs: url: ${{ steps.release.outputs.url }} created-at: ${{ steps.metadata.outputs.created-at }} steps: - - uses: actions/checkout@v4 - - name: Download Metadata Artifacts uses: actions/download-artifact@v4 with: @@ -164,6 +163,36 @@ jobs: path: ${{ env.OUTPUTS_PATH}} merge-multiple: true + - name: Generate Release Notes + id: release-body + run: | + echo "body<> $GITHUB_OUTPUT + + echo "This is an official release of \`${{ inputs.model }}\` \`${{ needs.push-tag.outputs.name }}\`. " >> $GITHUB_OUTPUT + echo "## Deployment Information" >> $GITHUB_OUTPUT + + for file in ${{ env.OUTPUTS_PATH }}/*; do + # Setting all the variables that would be needed for the release body creation... + filename=$(basename -- "$file") + target="${filename##*.}" + + # For brevity, '-cr' is '--compact-output --raw-output' + spack_version=$(jq -cr '.spack_version' "$file") + spack_config_version=$(jq -cr '.spack_config_version' "$file") + spack_packages_version=$(jq -cr '.spack_packages_version' "$file") + deployment_modules_location=$(jq -cr '.deployment_modules_location' "$file") + + echo "### \`$target\`" >> $GITHUB_OUTPUT + echo "Deployed using [spack $spack_version](https://github.com/ACCESS-NRI/spack/tree/releases/$spack_version), [spack-packages $spack_packages_version](https://github.com/ACCESS-NRI/spack-packages/releases/tag/$spack_packages_version) and [spack-config $spack_config_version](https://github.com/ACCESS-NRI/spack-config/releases/tag/$spack_config_version)." >> $GITHUB_OUTPUT + echo "Deployed as module accessible using:" >> $GITHUB_OUTPUT + echo "\`\`\`bash" >> $GITHUB_OUTPUT + echo "module use $deployment_modules_location" >> $GITHUB_OUTPUT + echo "module load ${{ needs.defaults.outputs.root-sbd }}/${{ needs.push-tag.outputs.name }}" >> $GITHUB_OUTPUT + echo "\`\`\`" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT + done + echo "EOF" >> $GITHUB_OUTPUT + - name: Create Release id: release uses: softprops/action-gh-release@69320dbe05506a9a39fc8ae11030b214ec2d1f87 # v2.0.5 @@ -171,7 +200,7 @@ jobs: tag_name: ${{ needs.push-tag.outputs.name }} name: ${{ inputs.model}} ${{ needs.push-tag.outputs.name }} body: | - This release of ${{ inputs.model }} ${{ needs.push-tag.outputs.name }} uses [spack-packages ${{ needs.deploy-to-environment.outputs.packages-version }}](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ needs.deploy-to-environment.outputs.packages-version }}) and [spack-config ${{ needs.deploy-to-environment.outputs.config-version }}](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ needs.deploy-to-environment.outputs.config-version }}). + ${{ steps.release-body.outputs.body }} generate_release_notes: true fail_on_unmatched_files: true files: | @@ -184,7 +213,7 @@ jobs: id: metadata env: GH_TOKEN: ${{ github.token }} - run: echo "created-at=$(gh release view --json createdAt --jq '.createdAt')" >> $GITHUB_OUTPUT + run: echo "created-at=$(gh release view --json createdAt --jq '.createdAt' --repo ${{ github.repository }})" >> $GITHUB_OUTPUT # FIXME: This won't work until ACCESS-NRI/build-cd#201 is resolved build-db: From bcfb373bf80ee86c6a7e180dde234ff1a540be98 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Fri, 10 Jan 2025 09:56:44 +1100 Subject: [PATCH 07/40] .gitignore venv, pytest metadata --- .gitignore | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f5f340b..d2842b9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ # Files generated during generate-build-metadata tests -tests/scripts/generate-build-metadata/outputs \ No newline at end of file +tests/scripts/generate-build-metadata/outputs +# Python venvs +venv +# Testing files +*.pyc +__pycache__ \ No newline at end of file From 35cb667cfc7dda949aa38348c0a78e669b1d0bce Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Fri, 10 Jan 2025 10:55:00 +1100 Subject: [PATCH 08/40] Create j2 templates for release notes and deployment comments Co-authoured-by: Tom McAdam --- .../templates/deployment-comment-body.md.j2 | 45 +++++++++++++++++++ .../templates/release-body.md.j2 | 15 +++++++ 2 files changed, 60 insertions(+) create mode 100644 scripts/jinja_template/templates/deployment-comment-body.md.j2 create mode 100644 scripts/jinja_template/templates/release-body.md.j2 diff --git a/scripts/jinja_template/templates/deployment-comment-body.md.j2 b/scripts/jinja_template/templates/deployment-comment-body.md.j2 new file mode 100644 index 0000000..ca00879 --- /dev/null +++ b/scripts/jinja_template/templates/deployment-comment-body.md.j2 @@ -0,0 +1,45 @@ +:rocket: [{{ 'Attempted to deploy' if deployment_errors else 'Deployed' }}]({{ run_url }}) `{{ root_sbd }}` Prerelease `{{ prerelease_version }}` with commit {{ head_sha }} +{% if config_errors %} +:warning:There are issues with the `build-cd` deployment configuration. If this is unexpected, let @ACCESS-NRI/model-release know. +{% endif %} + +{% for deployment in deployments %} +{% if deployment.result == "success" %} +### :desktop_computer: `{{ deployment.name }}` Deployment :heavy_check_mark:' + +
+Usage Instructions + +This `{{ model }}` model will be deployed to `{{ deployment.name }}` as: +* `{{ deployment.release_version }}` as a Release (when merged). +* `{{ prerelease_version }}` as a Prerelease (during this PR). + +This Prerelease is accessible on `{{ deployment.name }}` using: +```bash +module use {{ deployment.modules_location }} +module load {{ root_sbd }}/{{ prerelease_version}} +``` + +When using the above modules, the binaries shall be on your `$PATH`. + +For advanced users, this Prerelease is also accessible on `{{ deployment.name }}` via `{{ deployment.spack_location }}` in the `{{ root_sbd }}-{{ prerelease_version }}` environment. + +
+ +
+Configuration Information + +This Prerelease is deployed using: +* `access-nri/spack` on branch [`{{ deployment.spack_version }}`](https://github.com/ACCESS-NRI/spack/tree/releases/v{{ deployment.spack_version }}) +* `access-nri/spack-packages` version [`{{ deployment.spack_packages_version }}`](https://github.com/ACCESS-NRI/spack-packages/releases/tag/{{ deployment.spack_packages_version }}) +* `access-nri/spack-config` version [`{{ deployment.spack_config_version }}`](https://github.com/ACCESS-NRI/spack-config/releases/tag/{{ deployment.spack_config_version }}) + +If the above was not what was expected, commit changes to `config/versions.json` in this PR. + +
+ +{% else %} +### :desktop_computer: `{{ deployment.name }}` Deployment :x: + +{% endif %} +{% endfor %} diff --git a/scripts/jinja_template/templates/release-body.md.j2 b/scripts/jinja_template/templates/release-body.md.j2 new file mode 100644 index 0000000..cd130e8 --- /dev/null +++ b/scripts/jinja_template/templates/release-body.md.j2 @@ -0,0 +1,15 @@ +This is an official release of `{{ model }}` `{{ version }}` to the HPC(s) {% for deployment in deployments %} `{{ deployment.name }}`{{"," if not loop.last else "" }}{% endfor %}. + +## Deployment Information +{% for deployment in deployments %} +### `{{ deployment.name }}` + +Deployed using [spack {{ deployment.spack_version }}](https://github.com/ACCESS-NRI/spack/tree/releases/{{ deployment.spack_version }}), [spack-packages {{ deployment.spack_packages_version }}](https://github.com/ACCESS-NRI/spack-packages/releases/tag/{{ deployment.spack_packages_version }}) and [spack-config {{ deployment.spack_config_version }}](https://github.com/ACCESS-NRI/spack-config/releases/tag/{{ deployment.spack_config_version }}). + +Deployed as a module accessible using: + +```bash +module use {{ deployment.modules_location }} +module load {{ root_sbd }}/{{ version }} +``` +{% endfor %} From 08fac8aba5e8c1da83fbe6837e0fa6f219c1774f Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Fri, 10 Jan 2025 10:57:17 +1100 Subject: [PATCH 09/40] Add scripts for jinja generation, requirements, created modules --- pytest.ini | 2 + scripts/__init__.py | 0 scripts/jinja_template/__init__.py | 0 scripts/jinja_template/render.py | 10 ++ .../jinja_template/render_deployment_info.py | 115 ++++++++++++++++++ scripts/jinja_template/requirements.txt | 1 + 6 files changed, 128 insertions(+) create mode 100644 pytest.ini create mode 100644 scripts/__init__.py create mode 100644 scripts/jinja_template/__init__.py create mode 100644 scripts/jinja_template/render.py create mode 100644 scripts/jinja_template/render_deployment_info.py create mode 100644 scripts/jinja_template/requirements.txt diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..03f586d --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +pythonpath = . \ No newline at end of file diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/jinja_template/__init__.py b/scripts/jinja_template/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/jinja_template/render.py b/scripts/jinja_template/render.py new file mode 100644 index 0000000..c2ebf37 --- /dev/null +++ b/scripts/jinja_template/render.py @@ -0,0 +1,10 @@ +import os +from jinja2 import Environment, FileSystemLoader + +env = Environment(loader=FileSystemLoader(".")) + + +def render_using_context(template_path: str, context: dict) -> str: + + template = env.get_template(template_path) + return template.render(context) diff --git a/scripts/jinja_template/render_deployment_info.py b/scripts/jinja_template/render_deployment_info.py new file mode 100644 index 0000000..abd550a --- /dev/null +++ b/scripts/jinja_template/render_deployment_info.py @@ -0,0 +1,115 @@ +import os +import json +import argparse +from typing import Any + +from scripts.jinja_template.render import render_using_context + + +def _build_deployments_context_from_folder( + deployment_outputs_folder: str, +) -> list[dict[str, Any]]: + """ + Builds a list of deployment contexts from deployment output files in the specified folder. + + Args: + deployment_outputs_folder (str): The path to the folder containing deployment output JSON files. + + Returns: + list[dict[str, Any]]: A list of dictionaries, each containing deployment information extracted from the JSON files. + """ + deployments: list[dict[str, Any]] = [] + + for f in sorted(os.listdir(deployment_outputs_folder)): + # f has the format 'deploy-outputs.DEPLOYMENT_TARGET' + with open(os.path.join(deployment_outputs_folder, f), "r") as f_in: + data = json.load(f_in) + deployment = { + "name": f.split(".")[1], + "result": data["deployment_result"], + "release_version": data["release_deployment_version"], + "spack_version": data["spack_version"], + "spack_config_version": data["spack_config_version"], + "spack_packages_version": data["spack_packages_version"], + "modules_location": data["deployment_modules_location"], + "spack_location": data["deployment_spack_location"], + } + deployments.append(deployment) + + return deployments + + +def build_deployment_context( + deployments_folder_path: str, template_prefix: str +) -> dict[str, Any]: + """ + Builds a deployment context dictionary by extracting environment variables + that start with a given prefix and combining them with deployment information + from a specified folder. + + Args: + deployments_folder_path (str): The path to the folder containing deployment information files. + template_prefix (str): The prefix used to filter template environment variables. + + Returns: + dict[str, Any]: A dictionary containing the combined deployment context. + """ + template_variables = { + k.removeprefix(template_prefix).lower(): v + for k, v in os.environ.items() + if k.startswith(template_prefix) + } + + context: dict[str, Any] = { + **template_variables, + "deployments": _build_deployments_context_from_folder(deployments_folder_path), + } + + print(context) + + return context + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Script for generating deployment-related rendered output" + ) + parser.add_argument( + "--template", type=str, required=True, help="Path to .j2 template" + ) + parser.add_argument( + "--template-var-prefix", + type=str, + default="J2_", + help="Environment variables with this prefix will be added to the template context", + ) + parser.add_argument( + "--deployment-outputs", + type=str, + required=True, + help="Path to folder containing deploy-outputs.* files", + ) + parser.add_argument("--output", type=str, help="Path to output file") + + return parser.parse_args() + + +def main(): + args = parse_args() + + context = build_deployment_context( + deployments_folder_path=args.deployment_outputs, + template_prefix=args.template_var_prefix, + ) + + rendered_output = render_using_context(template_path=args.template, context=context) + + print(rendered_output) + + if args.output is not None: + with open(args.output, "w") as f: + f.write(rendered_output) + + +if __name__ == "__main__": + main() diff --git a/scripts/jinja_template/requirements.txt b/scripts/jinja_template/requirements.txt new file mode 100644 index 0000000..518bb28 --- /dev/null +++ b/scripts/jinja_template/requirements.txt @@ -0,0 +1 @@ +Jinja2==3.1.5 \ No newline at end of file From 454206d61a2e037ffe4bb154106a2c3bb4cf76b8 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Mon, 13 Jan 2025 12:02:30 +1100 Subject: [PATCH 10/40] Update ci/cd to use templating script --- .github/workflows/cd.yml | 59 +++++++++++---------- .github/workflows/ci.yml | 112 +++++++++++++++------------------------ 2 files changed, 73 insertions(+), 98 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 5cd1c09..f4b99fd 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -145,10 +145,18 @@ jobs: - push-tag - deploy-release runs-on: ubuntu-latest + env: + TEMPLATED_RELEASE_BODY_PATH: /opt/release-body.md outputs: url: ${{ steps.release.outputs.url }} created-at: ${{ steps.metadata.outputs.created-at }} steps: + - name: Checkout build-cd + uses: actions/checkout@v4 + with: + repository: access-nri/build-cd + ref: v3 + - name: Download Metadata Artifacts uses: actions/download-artifact@v4 with: @@ -160,38 +168,32 @@ jobs: uses: actions/download-artifact@v4 with: pattern: ${{ needs.deploy-release.outputs.general-outputs-artifact-glob }} - path: ${{ env.OUTPUTS_PATH}} + path: ${{ env.OUTPUTS_PATH }} merge-multiple: true + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + cache: pip + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -q -q -r scripts/jinja_tempate/requirements.txt + - name: Generate Release Notes id: release-body + env: + J2_MODEL: ${{ inputs.model }} + J2_VERSION: ${{ needs.push-tag.outputs.name }} + J2_ROOT_SBD: ${{ needs.defaults.outputs.root-sbd }} run: | - echo "body<> $GITHUB_OUTPUT - - echo "This is an official release of \`${{ inputs.model }}\` \`${{ needs.push-tag.outputs.name }}\`. " >> $GITHUB_OUTPUT - echo "## Deployment Information" >> $GITHUB_OUTPUT - - for file in ${{ env.OUTPUTS_PATH }}/*; do - # Setting all the variables that would be needed for the release body creation... - filename=$(basename -- "$file") - target="${filename##*.}" - - # For brevity, '-cr' is '--compact-output --raw-output' - spack_version=$(jq -cr '.spack_version' "$file") - spack_config_version=$(jq -cr '.spack_config_version' "$file") - spack_packages_version=$(jq -cr '.spack_packages_version' "$file") - deployment_modules_location=$(jq -cr '.deployment_modules_location' "$file") - - echo "### \`$target\`" >> $GITHUB_OUTPUT - echo "Deployed using [spack $spack_version](https://github.com/ACCESS-NRI/spack/tree/releases/$spack_version), [spack-packages $spack_packages_version](https://github.com/ACCESS-NRI/spack-packages/releases/tag/$spack_packages_version) and [spack-config $spack_config_version](https://github.com/ACCESS-NRI/spack-config/releases/tag/$spack_config_version)." >> $GITHUB_OUTPUT - echo "Deployed as module accessible using:" >> $GITHUB_OUTPUT - echo "\`\`\`bash" >> $GITHUB_OUTPUT - echo "module use $deployment_modules_location" >> $GITHUB_OUTPUT - echo "module load ${{ needs.defaults.outputs.root-sbd }}/${{ needs.push-tag.outputs.name }}" >> $GITHUB_OUTPUT - echo "\`\`\`" >> $GITHUB_OUTPUT - echo "" >> $GITHUB_OUTPUT - done - echo "EOF" >> $GITHUB_OUTPUT + python -m scripts/jinja_template/render_deployment_info.py \ + --type release_notes + --template scripts/jinja_template/templates/release-body.md.j2 + --deployment-outputs ${{ env.OUTPUTS_PATH }} + --output ${{ env.TEMPLATED_RELEASE_BODY_PATH }} - name: Create Release id: release @@ -199,8 +201,7 @@ jobs: with: tag_name: ${{ needs.push-tag.outputs.name }} name: ${{ inputs.model}} ${{ needs.push-tag.outputs.name }} - body: | - ${{ steps.release-body.outputs.body }} + body_path: ${{ env.TEMPLATED_RELEASE_BODY_PATH }} generate_release_notes: true fail_on_unmatched_files: true files: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bed929a..5da4725 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -253,6 +253,12 @@ jobs: env: OUTPUT_ARTIFACT_PATH: ./merged_outputs steps: + - name: Checkout build-cd for scripts + uses: actions/checkout@v4 + with: + repository: access-nri/build-cd + ref: v3 + - name: Download matrix deployment outputs uses: actions/download-artifact@v4 with: @@ -260,70 +266,6 @@ jobs: path: ${{ env.OUTPUT_ARTIFACT_PATH }} merge-multiple: true - - name: Create deployment message - id: matrix-output-parser - # Outputs defined in deploy-1-setup.yml under the outputs-upload job - # Creates a comment of the form: https://github.com/ACCESS-NRI/ACCESS-TEST/pull/15#issuecomment-2558675980 - run: | - echo "deployments-message<> $GITHUB_OUTPUT - - for file in ${{ env.OUTPUT_ARTIFACT_PATH }}/*; do - # Setting all the variables that would be needed for the PR comment creation... - filename=$(basename -- "$file") - target="${filename##*.}" - - # For brevity, '-cr' is '--compact-output --raw-output' - spack_version=$(jq -cr '.spack_version' "$file") - spack_config_version=$(jq -cr '.spack_config_version' "$file") - spack_packages_version=$(jq -cr '.spack_packages_version' "$file") - release_deployment_version=$(jq -cr '.release_deployment_version' "$file") - spack_environment_name=$(jq -cr '.spack_environment_name' "$file") - deployment_result=$(jq -cr '.deployment_result' "$file") - deployment_modules_location=$(jq -cr '.deployment_modules_location' "$file") - deployment_spack_location=$(jq -cr '.deployment_spack_location' "$file") - - # Create the message for the deployment - # Header - if [[ "$deployment_result" == "success" ]]; then - echo "### :white_check_mark: :desktop_computer: \`$target\` Deployment" >> $GITHUB_OUTPUT - else - echo "### :x: :desktop_computer: \`$target\` Deployment" >> $GITHUB_OUTPUT - continue - fi - - # Deployment Details - echo "
" >> $GITHUB_OUTPUT - echo "Usage Instructions" >> $GITHUB_OUTPUT - echo "" >> $GITHUB_OUTPUT - echo "Deployed as: " >> $GITHUB_OUTPUT - echo "* \`$release_deployment_version\` as a Release (when merged)." >> $GITHUB_OUTPUT - echo "* \`${{ needs.defaults.outputs.prerelease-version }}\` as a Prerelease (during this PR)." >> $GITHUB_OUTPUT - echo "" >> $GITHUB_OUTPUT - echo "This Prerelease is accessible on \`$target\` using:" >> $GITHUB_OUTPUT - echo "\`\`\`bash" >> $GITHUB_OUTPUT - echo "module use $deployment_modules_location" >> $GITHUB_OUTPUT - echo "module load ${{ needs.defaults.outputs.root-sbd }}/${{ needs.defaults.outputs.prerelease-version }}" >> $GITHUB_OUTPUT - echo "\`\`\`" >> $GITHUB_OUTPUT - echo "where the binaries shall be on your \`PATH\`." >> $GITHUB_OUTPUT - echo "For advanced users, this Prerelease is also accessible on \`$target\` via \`$deployment_spack_location\` in the \`${{ needs.defaults.outputs.root-sbd }}-${{ needs.defaults.outputs.prerelease-version }}\` environment." >> $GITHUB_OUTPUT - echo "
" >> $GITHUB_OUTPUT - - # Configuration Details - echo "
" >> $GITHUB_OUTPUT - echo "Configuration Details" >> $GITHUB_OUTPUT - echo "" >> $GITHUB_OUTPUT - echo "This Prelease was deployed using:" >> $GITHUB_OUTPUT - echo "* \`access-nri/spack\` on branch [$spack_version](https://github.com/ACCESS-NRI/spack/tree/releases/v${spack_version})" >> $GITHUB_OUTPUT - echo "* \`access-nri/spack-packages\` version [$spack_packages_version](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${spack_packages_version})" >> $GITHUB_OUTPUT - echo "* \`access-nri/spack-config\` version [$spack_config_version](https://github.com/ACCESS-NRI/spack-config/releases/tag/${spack_config_version})" >> $GITHUB_OUTPUT - echo "" >> $GITHUB_OUTPUT - echo "If the above was not expected, please commit changes to \`config/versions.json\`." >> $GITHUB_OUTPUT - echo "
" >> $GITHUB_OUTPUT - echo "" >> $GITHUB_OUTPUT - done - - echo "EOF" >> $GITHUB_OUTPUT - - name: Config status roll up id: config-rollup # Roll up the config status booleans from each of the files, and return the overall result @@ -337,15 +279,47 @@ jobs: run: | echo "errors=$(jq --slurp 'map(.deployment_result == "success") | all | not' ${{ env.OUTPUT_ARTIFACT_PATH }}/*)" >> $GITHUB_OUTPUT + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + cache: pip + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -q -q -r scripts/jinja_tempate/requirements.txt + + - name: Create deployment message + id: matrix-output-parser + # Outputs defined in deploy-1-setup.yml under the outputs-upload job + # Creates a comment of the form: https://github.com/ACCESS-NRI/ACCESS-TEST/pull/15#issuecomment-2558675980 + env: + J2_MODEL: ${{ inputs.model }} + J2_PRERELEASE_VERSION: ${{ needs.defaults.outputs.prerelease-version }} + J2_ROOT_SBD: ${{ needs.defaults.outputs.root-sbd }} + J2_DEPLOYMENT_ERRORS: ${{ steps.deployment-rollup.outputs.errors }} + J2_CONFIG_ERRORS: ${{ steps.config-rollup.outputs.errors }} + J2_HEAD_SHA: ${{ needs.defaults.outputs.head-sha }} + J2_RUN_URL: ${{ env.RUN_URL }} + TEMPLATED_COMMENT_BODY_PATH: "/opt/comment-body.md" + run: | + python -m scripts/jinja_template/render_deployment_info.py \ + --type deployment_comment + --template scripts/jinja_template/templates/deployment-comment-body.md.j2 + --deployment-outputs ${{ env.OUTPUT_ARTIFACT_PATH }} + --output ${{ env.TEMPLATED_COMMENT_BODY_PATH }} + + echo "deployment-comment<> $(cat ${{ env.TEMPLATED_COMMENT_BODY_PATH }})" >> $GITHUB_OUTPUT + echo "$(cat ${{ env.TEMPLATED_COMMENT_BODY_PATH }})" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: PR Comment Notifier id: comment uses: access-nri/actions/.github/actions/pr-comment@main with: comment: | - :rocket: [${{ steps.deployment-rollup.outputs.errors == 'true' && 'Attempted to deploy' || 'Deployed' }}](${{ env.RUN_URL }}) `${{ inputs.model }}` as prerelease `${{ needs.defaults.outputs.root-sbd }}/${{ needs.defaults.outputs.prerelease-version }}` with commit ${{ needs.defaults.outputs.head-sha }} - ${{ steps.config-rollup.outputs.errors == 'true' && ':warning:There are issues with the `build-cd` deployment configuration. If this is unexpected, let @ACCESS-NRI/model-release know.' || '' }} - - ${{ steps.matrix-output-parser.outputs.deployments-message }} + ${{ steps.matrix-output-parser.outputs.deployment-comment }} - name: PR Description Notifier env: @@ -353,7 +327,7 @@ jobs: PR_BODY_PATH: ./body.txt PR_BODY_PATH_UPDATED: ./updated.body.txt PRERELEASE_SECTION_REGEX: "^:rocket: .* :rocket:$" - PRERELEASE_SECTION: ":rocket: The latest prerelease `${{ needs.defaults.outputs.root-sbd }}/${{ needs.defaults.outputs.prerelease-version }}` at ${{ needs.defaults.outputs.head-sha }} is here: ${{ steps.comment.outputs.comment-url }} :rocket:" + PRERELEASE_SECTION: ":rocket: The latest prerelease `${{ needs.defaults.outputs.root-sbd }}/${{ needs.defaults.outputs.prerelease-version }}` at ${{ needs.defaults.outputs.head-sha }} is here: ${{ steps.comment.outputs.comment-link }} :rocket:" run: | gh pr view ${{ inputs.pr }} --repo ${{ github.repository }} --json body --jq .body > ${{ env.PR_BODY_PATH }} From 44c7c593e8cd6bcd4852bcac51d938d0b5557770 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Mon, 13 Jan 2025 15:28:20 +1100 Subject: [PATCH 11/40] Add testing infrastructure --- scripts/jinja_template/requirements-dev.txt | 2 + .../jinja_template/inputs/deploy-outputs.Gadi | 11 ++++ .../inputs/deploy-outputs.Setonix | 11 ++++ .../jinja_template/outputs/release-body.md | 25 ++++++++ .../test_render_deployment_info.py | 60 +++++++++++++++++++ 5 files changed, 109 insertions(+) create mode 100644 scripts/jinja_template/requirements-dev.txt create mode 100644 tests/scripts/jinja_template/inputs/deploy-outputs.Gadi create mode 100644 tests/scripts/jinja_template/inputs/deploy-outputs.Setonix create mode 100644 tests/scripts/jinja_template/outputs/release-body.md create mode 100644 tests/scripts/jinja_template/test_render_deployment_info.py diff --git a/scripts/jinja_template/requirements-dev.txt b/scripts/jinja_template/requirements-dev.txt new file mode 100644 index 0000000..a7b33e7 --- /dev/null +++ b/scripts/jinja_template/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest==8.3.4 \ No newline at end of file diff --git a/tests/scripts/jinja_template/inputs/deploy-outputs.Gadi b/tests/scripts/jinja_template/inputs/deploy-outputs.Gadi new file mode 100644 index 0000000..410cf29 --- /dev/null +++ b/tests/scripts/jinja_template/inputs/deploy-outputs.Gadi @@ -0,0 +1,11 @@ +{ + "spack_version": "0.22", + "spack_config_version": "2024.11.27", + "spack_packages_version": "2024.09.20", + "ci_configuration_check_failure": false, + "release_deployment_version": "2024.11.16", + "spack_environment_name": "access-test-pr15-11", + "deployment_result": "success", + "deployment_modules_location": "/g/data/vk83/prerelease/modules", + "deployment_spack_location": "/g/data/vk83/prerelease/apps/spack/0.22/spack" +} diff --git a/tests/scripts/jinja_template/inputs/deploy-outputs.Setonix b/tests/scripts/jinja_template/inputs/deploy-outputs.Setonix new file mode 100644 index 0000000..e8a1458 --- /dev/null +++ b/tests/scripts/jinja_template/inputs/deploy-outputs.Setonix @@ -0,0 +1,11 @@ +{ + "spack_version": "0.22", + "spack_config_version": "2024.11.27", + "spack_packages_version": "2024.09.20", + "ci_configuration_check_failure": false, + "release_deployment_version": "2024.11.16", + "spack_environment_name": "access-test-pr15-11", + "deployment_result": "success", + "deployment_modules_location": "/some/dir/with/modules", + "deployment_spack_location": "/some/apps/spack/0.22/spack" +} diff --git a/tests/scripts/jinja_template/outputs/release-body.md b/tests/scripts/jinja_template/outputs/release-body.md new file mode 100644 index 0000000..6bb5cf4 --- /dev/null +++ b/tests/scripts/jinja_template/outputs/release-body.md @@ -0,0 +1,25 @@ +This is an official release of `access-om2` `2024.11.0`. + +## Deployment Information + +### `Gadi` + +Deployed using [spack 0.22](https://github.com/ACCESS-NRI/spack/tree/releases/0.22), [spack-packages 2024.09.20](https://github.com/ACCESS-NRI/spack-packages/releases/tag/2024.09.20) and [spack-config 2024.11.27](https://github.com/ACCESS-NRI/spack-config/releases/tag/2024.11.27). + +Deployed as module accessible using: + +```bash +module use /g/data/vk83/prerelease/modules +module load access-om2/2024.11.0 +``` + +### `Setonix` + +Deployed using [spack 0.22](https://github.com/ACCESS-NRI/spack/tree/releases/0.22), [spack-packages 2024.09.20](https://github.com/ACCESS-NRI/spack-packages/releases/tag/2024.09.20) and [spack-config 2024.11.27](https://github.com/ACCESS-NRI/spack-config/releases/tag/2024.11.27). + +Deployed as module accessible using: + +```bash +module use /some/dir/with/modules +module load access-om2/2024.11.0 +``` diff --git a/tests/scripts/jinja_template/test_render_deployment_info.py b/tests/scripts/jinja_template/test_render_deployment_info.py new file mode 100644 index 0000000..278ba76 --- /dev/null +++ b/tests/scripts/jinja_template/test_render_deployment_info.py @@ -0,0 +1,60 @@ +import pytest +from pathlib import Path + +from scripts.jinja_template.render_deployment_info import build_deployment_context, _build_deployments_context_from_folder + +@pytest.fixture +def deployments_folder(request): + return Path(request.path.parent) / "inputs" + +@pytest.fixture( + params=[ + [ + ("J2_MODEL", "model", "access-om2"), + ("J2_VERSION","version", "pr12-2"), + ("J2_ROOT_SBD","root_sbd","access-om2") + ], + [ + ("J2_MODEL", "model", "access-om2") + ], + ] +) +def env_vars(request, monkeypatch): + for env_name, _, value in request.param: + monkeypatch.setenv(env_name, value) + + yield request.param + +def test_build_deployment_context_envs(env_vars): + context = build_deployment_context( + template_prefix="J2_", + deployments_folder_path="tests/scripts/jinja_template/inputs", + ) + + for env_name, key, value in env_vars: + assert key in context, f"{key} (from {env_name}) does not exist in {context}" + assert value == context[f"{key}"], f"{value} does not equal expected {context[f"{key}"]}" + + +def test_build_deployments_context_from_folder(deployments_folder): + context = _build_deployments_context_from_folder(deployments_folder) + + assert len(context) == 2 + + assert context[0]["name"] == "Gadi" + assert context[0]["result"] == "success" + assert context[0]["release_version"] == "2024.11.16" + assert context[0]["spack_version"] == "0.22" + assert context[0]["spack_config_version"] == "2024.11.27" + assert context[0]["spack_packages_version"] == "2024.09.20" + assert context[0]["modules_location"] == "/g/data/vk83/prerelease/modules" + assert context[0]["spack_location"] == "/g/data/vk83/prerelease/apps/spack/0.22/spack" + + assert context[1]["name"] == "Setonix" + assert context[1]["result"] == "success" + assert context[1]["release_version"] == "2024.11.16" + assert context[1]["spack_version"] == "0.22" + assert context[1]["spack_config_version"] == "2024.11.27" + assert context[1]["spack_packages_version"] == "2024.09.20" + assert context[1]["modules_location"] == "/some/dir/with/modules" + assert context[1]["spack_location"] == "/some/apps/spack/0.22/spack" \ No newline at end of file From c0954837ff3d954aaa77dd906f290f06a194fc4b Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Tue, 14 Jan 2025 13:00:02 +1100 Subject: [PATCH 12/40] Updated ci/cd calls to use script notation, removed --type arg --- .github/workflows/cd.yml | 3 +-- .github/workflows/ci.yml | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index f4b99fd..743c933 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -189,8 +189,7 @@ jobs: J2_VERSION: ${{ needs.push-tag.outputs.name }} J2_ROOT_SBD: ${{ needs.defaults.outputs.root-sbd }} run: | - python -m scripts/jinja_template/render_deployment_info.py \ - --type release_notes + python -m scripts.jinja_template.render_deployment_info \ --template scripts/jinja_template/templates/release-body.md.j2 --deployment-outputs ${{ env.OUTPUTS_PATH }} --output ${{ env.TEMPLATED_RELEASE_BODY_PATH }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5da4725..5bf0545 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -304,13 +304,12 @@ jobs: J2_RUN_URL: ${{ env.RUN_URL }} TEMPLATED_COMMENT_BODY_PATH: "/opt/comment-body.md" run: | - python -m scripts/jinja_template/render_deployment_info.py \ - --type deployment_comment + python -m scripts.jinja_template.render_deployment_info \ --template scripts/jinja_template/templates/deployment-comment-body.md.j2 --deployment-outputs ${{ env.OUTPUT_ARTIFACT_PATH }} --output ${{ env.TEMPLATED_COMMENT_BODY_PATH }} - echo "deployment-comment<> $(cat ${{ env.TEMPLATED_COMMENT_BODY_PATH }})" >> $GITHUB_OUTPUT + echo "deployment-comment<> $GITHUB_OUTPUT echo "$(cat ${{ env.TEMPLATED_COMMENT_BODY_PATH }})" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT From 1d35de00e79cb81d3365d736c5a690ab7939854e Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 16 Jan 2025 11:30:30 +1100 Subject: [PATCH 13/40] ci.yml: Use output file as comment due to https://github.com/ACCESS-NRI/actions/pull/20 --- .github/workflows/ci.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bf0545..7447fee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -252,6 +252,7 @@ jobs: pull-requests: write env: OUTPUT_ARTIFACT_PATH: ./merged_outputs + TEMPLATED_COMMENT_BODY_PATH: /opt/comment-body.md steps: - name: Checkout build-cd for scripts uses: actions/checkout@v4 @@ -302,23 +303,17 @@ jobs: J2_CONFIG_ERRORS: ${{ steps.config-rollup.outputs.errors }} J2_HEAD_SHA: ${{ needs.defaults.outputs.head-sha }} J2_RUN_URL: ${{ env.RUN_URL }} - TEMPLATED_COMMENT_BODY_PATH: "/opt/comment-body.md" run: | python -m scripts.jinja_template.render_deployment_info \ --template scripts/jinja_template/templates/deployment-comment-body.md.j2 --deployment-outputs ${{ env.OUTPUT_ARTIFACT_PATH }} --output ${{ env.TEMPLATED_COMMENT_BODY_PATH }} - echo "deployment-comment<> $GITHUB_OUTPUT - echo "$(cat ${{ env.TEMPLATED_COMMENT_BODY_PATH }})" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - name: PR Comment Notifier id: comment uses: access-nri/actions/.github/actions/pr-comment@main with: - comment: | - ${{ steps.matrix-output-parser.outputs.deployment-comment }} + comment-file: ${{ env.TEMPLATED_COMMENT_BODY_PATH }} - name: PR Description Notifier env: From 2d4b4a1c0ec09ce99954dbd58872219b6238aceb Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 16 Jan 2025 11:50:57 +1100 Subject: [PATCH 14/40] Add test validating non-collection of variables with the wrong `template_prefix` --- .../jinja_template/test_render_deployment_info.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/scripts/jinja_template/test_render_deployment_info.py b/tests/scripts/jinja_template/test_render_deployment_info.py index 278ba76..8ac1616 100644 --- a/tests/scripts/jinja_template/test_render_deployment_info.py +++ b/tests/scripts/jinja_template/test_render_deployment_info.py @@ -25,7 +25,7 @@ def env_vars(request, monkeypatch): yield request.param -def test_build_deployment_context_envs(env_vars): +def test_build_deployment_context__valid_envs(env_vars): context = build_deployment_context( template_prefix="J2_", deployments_folder_path="tests/scripts/jinja_template/inputs", @@ -35,6 +35,14 @@ def test_build_deployment_context_envs(env_vars): assert key in context, f"{key} (from {env_name}) does not exist in {context}" assert value == context[f"{key}"], f"{value} does not equal expected {context[f"{key}"]}" +def test_build_deployment_context__invalid_envs(env_vars): + context = build_deployment_context( + template_prefix="SOMETHING_", + deployments_folder_path="tests/scripts/jinja_template/inputs" + ) + + assert len(context) == 1, "Environment variables were added with an incorrect template_prefix" + def test_build_deployments_context_from_folder(deployments_folder): context = _build_deployments_context_from_folder(deployments_folder) From 7ac981721fa2c233f18bbac97287d2b9e5fc8c27 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 16 Jan 2025 12:04:43 +1100 Subject: [PATCH 15/40] Remove outputs folder as it is unused --- .../jinja_template/outputs/release-body.md | 25 ------------------- 1 file changed, 25 deletions(-) delete mode 100644 tests/scripts/jinja_template/outputs/release-body.md diff --git a/tests/scripts/jinja_template/outputs/release-body.md b/tests/scripts/jinja_template/outputs/release-body.md deleted file mode 100644 index 6bb5cf4..0000000 --- a/tests/scripts/jinja_template/outputs/release-body.md +++ /dev/null @@ -1,25 +0,0 @@ -This is an official release of `access-om2` `2024.11.0`. - -## Deployment Information - -### `Gadi` - -Deployed using [spack 0.22](https://github.com/ACCESS-NRI/spack/tree/releases/0.22), [spack-packages 2024.09.20](https://github.com/ACCESS-NRI/spack-packages/releases/tag/2024.09.20) and [spack-config 2024.11.27](https://github.com/ACCESS-NRI/spack-config/releases/tag/2024.11.27). - -Deployed as module accessible using: - -```bash -module use /g/data/vk83/prerelease/modules -module load access-om2/2024.11.0 -``` - -### `Setonix` - -Deployed using [spack 0.22](https://github.com/ACCESS-NRI/spack/tree/releases/0.22), [spack-packages 2024.09.20](https://github.com/ACCESS-NRI/spack-packages/releases/tag/2024.09.20) and [spack-config 2024.11.27](https://github.com/ACCESS-NRI/spack-config/releases/tag/2024.11.27). - -Deployed as module accessible using: - -```bash -module use /some/dir/with/modules -module load access-om2/2024.11.0 -``` From 352d529d15a3e40d9786d5f338b393809bb621ee Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 16 Jan 2025 12:31:48 +1100 Subject: [PATCH 16/40] Update requirements.txt filepath --- .github/workflows/cd.yml | 2 +- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 743c933..b0f0bd8 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -180,7 +180,7 @@ jobs: - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install -q -q -r scripts/jinja_tempate/requirements.txt + pip install -q -q -r scripts/jinja_template/requirements.txt - name: Generate Release Notes id: release-body diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7447fee..60d73cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -289,7 +289,7 @@ jobs: - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install -q -q -r scripts/jinja_tempate/requirements.txt + pip install -q -q -r scripts/jinja_template/requirements.txt - name: Create deployment message id: matrix-output-parser From d0c00d632b37e7fcb6b828e5138c4b481d7dca38 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 16 Jan 2025 12:50:52 +1100 Subject: [PATCH 17/40] Added backslashes to multiline script invocation --- .github/workflows/cd.yml | 4 ++-- .github/workflows/ci.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index b0f0bd8..09e4507 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -190,8 +190,8 @@ jobs: J2_ROOT_SBD: ${{ needs.defaults.outputs.root-sbd }} run: | python -m scripts.jinja_template.render_deployment_info \ - --template scripts/jinja_template/templates/release-body.md.j2 - --deployment-outputs ${{ env.OUTPUTS_PATH }} + --template scripts/jinja_template/templates/release-body.md.j2 \ + --deployment-outputs ${{ env.OUTPUTS_PATH }} \ --output ${{ env.TEMPLATED_RELEASE_BODY_PATH }} - name: Create Release diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60d73cf..9af12c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -305,8 +305,8 @@ jobs: J2_RUN_URL: ${{ env.RUN_URL }} run: | python -m scripts.jinja_template.render_deployment_info \ - --template scripts/jinja_template/templates/deployment-comment-body.md.j2 - --deployment-outputs ${{ env.OUTPUT_ARTIFACT_PATH }} + --template scripts/jinja_template/templates/deployment-comment-body.md.j2 \ + --deployment-outputs ${{ env.OUTPUT_ARTIFACT_PATH }} \ --output ${{ env.TEMPLATED_COMMENT_BODY_PATH }} - name: PR Comment Notifier From 63f1981a3ee05b308be0e8c96147bc2231026d25 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 16 Jan 2025 13:09:47 +1100 Subject: [PATCH 18/40] Update template --- scripts/jinja_template/templates/deployment-comment-body.md.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/jinja_template/templates/deployment-comment-body.md.j2 b/scripts/jinja_template/templates/deployment-comment-body.md.j2 index ca00879..6964457 100644 --- a/scripts/jinja_template/templates/deployment-comment-body.md.j2 +++ b/scripts/jinja_template/templates/deployment-comment-body.md.j2 @@ -5,7 +5,7 @@ {% for deployment in deployments %} {% if deployment.result == "success" %} -### :desktop_computer: `{{ deployment.name }}` Deployment :heavy_check_mark:' +### :desktop_computer: `{{ deployment.name }}` Deployment :heavy_check_mark:
Usage Instructions From c8570dfd74cf4319117814de20fcba7e31070466 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Fri, 17 Jan 2025 14:50:20 +1100 Subject: [PATCH 19/40] Replace `env.SPACK_YAML_MODEL_YQ` to accomodate for multi-target-format `spack.yaml` --- .github/workflows/cd.yml | 4 +++- .github/workflows/ci-comment.yml | 6 ++++-- .github/workflows/deploy-1-setup.yml | 4 +++- .github/workflows/deploy-2-start.yml | 4 +++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 09e4507..3a07077 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -27,7 +27,9 @@ on: # - spack.yaml env: - SPACK_YAML_MODEL_YQ: .spack.specs[0] + # This attempts to get the spack.yaml model first via the multi-target + # .spack.definitions.root_package method, then the traditional '.spack.specs[0]'. + SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."root_package") | .[][]) // .spack.specs[0] METADATA_PATH: /opt/metadata OUTPUTS_PATH: /opt/outputs jobs: diff --git a/.github/workflows/ci-comment.yml b/.github/workflows/ci-comment.yml index b7f4bfe..b9d89ec 100644 --- a/.github/workflows/ci-comment.yml +++ b/.github/workflows/ci-comment.yml @@ -13,7 +13,7 @@ on: root-sbd: type: string required: false - # The equivalent default is set in the defaults job below. + # The equivalent default is set in the defaults job below. # default: ${{ inputs.model }} description: | The name of the root Spack Bundle Definition, if it is different from the model name. @@ -53,7 +53,9 @@ jobs: contents: write env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SPACK_YAML_MODEL_YQ: .spack.specs[0] + # This attempts to get the spack.yaml model first via the multi-target + # .spack.definitions.root_package method, then the traditional '.spack.specs[0]'. + SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."root_package") | .[][]) // .spack.specs[0] SPACK_YAML_MODEL_PROJECTION_YQ: .spack.modules.default.tcl.projections.${{ needs.defaults.outputs.root-sbd }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/deploy-1-setup.yml b/.github/workflows/deploy-1-setup.yml index a4b071b..b88560c 100644 --- a/.github/workflows/deploy-1-setup.yml +++ b/.github/workflows/deploy-1-setup.yml @@ -162,7 +162,9 @@ jobs: permissions: pull-requests: write env: - SPACK_YAML_MODEL_YQ: .spack.specs[0] + # This attempts to get the spack.yaml model first via the multi-target + # .spack.definitions.root_package method, then the traditional '.spack.specs[0]'. + SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."root_package") | .[][]) // .spack.specs[0] outputs: # Release version of the deployment. Inferred if not given in inputs.deployment-version release: ${{ steps.version.outputs.release }} diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index b39a4ad..7dccbd8 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -44,7 +44,9 @@ on: General pattern for the artifact that contains the deployment metadata files for this invocation of the job. These files are of the form deploy-metadata.{{inputs.deployment-target}} env: - SPACK_YAML_SPEC_YQ: .spack.specs[0] + # This attempts to get the spack.yaml model first via the multi-target + # .spack.definitions.root_package method, then the traditional '.spack.specs[0]'. + SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."root_package") | .[][]) // .spack.specs[0] SPACK_YAML_MODULEFILE_PROJECTION_YQ: .spack.modules.default.tcl.projections.${{ inputs.root-sbd }} METADATA_PATH: /opt/metadata ARTIFACT_NAME: deploy-metadata.${{ inputs.deployment-environment }} From 6ebc7c21d3d948c39fa45130a84646653cf29944 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Tue, 21 Jan 2025 09:41:59 +1100 Subject: [PATCH 20/40] Uppercase ROOT_PACKAGE --- .github/workflows/cd.yml | 2 +- .github/workflows/ci-comment.yml | 2 +- .github/workflows/deploy-1-setup.yml | 2 +- .github/workflows/deploy-2-start.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 3a07077..90db9c8 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -29,7 +29,7 @@ on: env: # This attempts to get the spack.yaml model first via the multi-target # .spack.definitions.root_package method, then the traditional '.spack.specs[0]'. - SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."root_package") | .[][]) // .spack.specs[0] + SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."ROOT_PACKAGE") | .[][]) // .spack.specs[0] METADATA_PATH: /opt/metadata OUTPUTS_PATH: /opt/outputs jobs: diff --git a/.github/workflows/ci-comment.yml b/.github/workflows/ci-comment.yml index b9d89ec..9c07ed4 100644 --- a/.github/workflows/ci-comment.yml +++ b/.github/workflows/ci-comment.yml @@ -55,7 +55,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This attempts to get the spack.yaml model first via the multi-target # .spack.definitions.root_package method, then the traditional '.spack.specs[0]'. - SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."root_package") | .[][]) // .spack.specs[0] + SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."ROOT_PACKAGE") | .[][]) // .spack.specs[0] SPACK_YAML_MODEL_PROJECTION_YQ: .spack.modules.default.tcl.projections.${{ needs.defaults.outputs.root-sbd }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/deploy-1-setup.yml b/.github/workflows/deploy-1-setup.yml index b88560c..757ecb5 100644 --- a/.github/workflows/deploy-1-setup.yml +++ b/.github/workflows/deploy-1-setup.yml @@ -164,7 +164,7 @@ jobs: env: # This attempts to get the spack.yaml model first via the multi-target # .spack.definitions.root_package method, then the traditional '.spack.specs[0]'. - SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."root_package") | .[][]) // .spack.specs[0] + SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."ROOT_PACKAGE") | .[][]) // .spack.specs[0] outputs: # Release version of the deployment. Inferred if not given in inputs.deployment-version release: ${{ steps.version.outputs.release }} diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index 7dccbd8..f986221 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -46,7 +46,7 @@ on: env: # This attempts to get the spack.yaml model first via the multi-target # .spack.definitions.root_package method, then the traditional '.spack.specs[0]'. - SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."root_package") | .[][]) // .spack.specs[0] + SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."ROOT_PACKAGE") | .[][]) // .spack.specs[0] SPACK_YAML_MODULEFILE_PROJECTION_YQ: .spack.modules.default.tcl.projections.${{ inputs.root-sbd }} METADATA_PATH: /opt/metadata ARTIFACT_NAME: deploy-metadata.${{ inputs.deployment-environment }} From 616dc3a16ea33421e467884307489869a7637557 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Wed, 22 Jan 2025 09:23:30 +1100 Subject: [PATCH 21/40] Update target matrix generation to take into account model repos deployment targets --- .github/workflows/cd.yml | 19 +++++++++++++++++-- .github/workflows/ci.yml | 19 ++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 90db9c8..3dc7ab7 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -58,9 +58,24 @@ jobs: - name: Generate Deployment Target Matrix id: target + # These are used in the deploy job to determine the targets to deploy to - irrespective of the type of deployment. + # We use the list of valid targets (from build-cd) and targets set in the caller repository to determine the final list. run: | - targets=$(jq --compact-output --raw-output '.deployment | keys' config/settings.json) - echo "$targets" + valid_targets=$(jq --raw-output --compact-output \ + '.deployment | keys' \ + config/settings.json + ) + repo_release_targets=$(jq --null-input --raw-output --compact-output \ + --arg valids "${{ vars.RELEASE_DEPLOYMENT_TARGETS }}" \ + '$valids | split(" ")' + ) + targets=$(jq --null-input --raw-output --compact-output\ + --argjson valid "$valid_targets" \ + --argjson ours "$repo_release_targets" \ + '$valid - ($valid - $ours)' + ) + + echo "From build-cd's valid targets: $valid_targets, and ${{ github.repository }}s chosen targets: $repo_release_targets, we will deploy to $targets" echo "targets=$targets" >> $GITHUB_OUTPUT verify-settings: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9af12c5..a702ea7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,11 +132,24 @@ jobs: - name: Generate Deployment Target Matrix id: target # These are used in the deploy job to determine the targets to deploy to - irrespective of the type of deployment. + # We use the list of valid targets (from build-cd) and targets set in the caller repository to determine the final list. run: | - targets=$(jq --raw-output --compact-output '.deployment | keys' config/settings.json) - echo "$targets" - echo "targets=$targets" >> $GITHUB_OUTPUT + valid_targets=$(jq --raw-output --compact-output \ + '.deployment | keys' \ + config/settings.json + ) + repo_prerelease_targets=$(jq --null-input --raw-output --compact-output \ + --arg valids "${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }}" \ + '$valids | split(" ")' + ) + targets=$(jq --null-input --raw-output --compact-output\ + --argjson valid "$valid_targets" \ + --argjson ours "$repo_prerelease_targets" \ + '$valid - ($valid - $ours)' + ) + echo "From build-cd's valid targets: $valid_targets, and ${{ github.repository }}s chosen targets: $repo_prerelease_targets, we will deploy to $targets" + echo "targets=$targets" >> $GITHUB_OUTPUT redeploy-pre: name: '!redeploy Pending' From 2f52e430125a9c2cde3a9cfae162e0c66084ab39 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Wed, 22 Jan 2025 09:32:40 +1100 Subject: [PATCH 22/40] Added comments --- .github/workflows/cd.yml | 3 ++- .github/workflows/ci.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 3dc7ab7..396966e 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -59,7 +59,7 @@ jobs: - name: Generate Deployment Target Matrix id: target # These are used in the deploy job to determine the targets to deploy to - irrespective of the type of deployment. - # We use the list of valid targets (from build-cd) and targets set in the caller repository to determine the final list. + # We use the list of valid targets (from build-cd) intersected with targets set in the caller repository to determine the final list. run: | valid_targets=$(jq --raw-output --compact-output \ '.deployment | keys' \ @@ -69,6 +69,7 @@ jobs: --arg valids "${{ vars.RELEASE_DEPLOYMENT_TARGETS }}" \ '$valids | split(" ")' ) + # This is the intersection between the valid and caller-defined targets targets=$(jq --null-input --raw-output --compact-output\ --argjson valid "$valid_targets" \ --argjson ours "$repo_release_targets" \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a702ea7..4649944 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,7 +132,7 @@ jobs: - name: Generate Deployment Target Matrix id: target # These are used in the deploy job to determine the targets to deploy to - irrespective of the type of deployment. - # We use the list of valid targets (from build-cd) and targets set in the caller repository to determine the final list. + # We use the list of valid targets (from build-cd) intersected with targets set in the caller repository to determine the final list. run: | valid_targets=$(jq --raw-output --compact-output \ '.deployment | keys' \ @@ -142,6 +142,7 @@ jobs: --arg valids "${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }}" \ '$valids | split(" ")' ) + # This is the intersection between the valid and caller-defined targets targets=$(jq --null-input --raw-output --compact-output\ --argjson valid "$valid_targets" \ --argjson ours "$repo_prerelease_targets" \ From 895fdd033c076956527897b9e091883d9f7464d1 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 23 Jan 2025 11:08:32 +1100 Subject: [PATCH 23/40] Added new get-target-matrix action --- .github/actions/get-target-matrix/README.md | 40 ++++++++++++++++++ .github/actions/get-target-matrix/action.yml | 44 ++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 .github/actions/get-target-matrix/README.md create mode 100644 .github/actions/get-target-matrix/action.yml diff --git a/.github/actions/get-target-matrix/README.md b/.github/actions/get-target-matrix/README.md new file mode 100644 index 0000000..4c9239c --- /dev/null +++ b/.github/actions/get-target-matrix/README.md @@ -0,0 +1,40 @@ +# Get Deployment Target Matrix Action + +Action that returns a matrix of deployment targets based on the caller's input and the valid targets in the build-cd repository. + +## Inputs + +| Name | Type | Description | Required | Default | Example | +| ---- | ---- | ----------- | -------- | ------- | ------- | +| `targets` | `string` (space-separated) | A space-separated list of deployment targets for the model deployment repository. | `true` | N/A | `"Gadi Setonix"` | + +## Outputs + +| Name | Type | Description | Example | +| ---- | ---- | ----------- | ------- | +| `valid-targets` | `string` | A JSON array of valid deployment targets, suitable for use as a matrix. | `"["Gadi", "Setonix"]"` | + +## Example + +```yaml +# ... +jobs: + generate-matrix: + runs-on: ubuntu-latest + outputs: + valid-targets: ${{ steps.generate.outputs.valid-targets }} + steps: + - id: generate + uses: access-nri/build-cd/.github/actions/get-target-matrix@main + with: + targets: ${{ vars.MODEL_REPO_TARGETS }} + + matrix: + runs-on: ubuntu-latest + needs: [generate-matrix] + strategy: + matrix: + target: ${{ fromJson(needs.generate-matrix.outputs.valid-targets) }} + steps: + - run: echo "Wow, this is running in ${{ matrix.target }}!" +``` diff --git a/.github/actions/get-target-matrix/action.yml b/.github/actions/get-target-matrix/action.yml new file mode 100644 index 0000000..fbf9bbf --- /dev/null +++ b/.github/actions/get-target-matrix/action.yml @@ -0,0 +1,44 @@ +name: Get Deployment Target Matrix +description: Action that returns a matrix of deployment targets based on the caller's input and the valid targets in the build-cd repository. +author: Tommy Gatti +inputs: + targets: + type: string + required: true + description: A space-separated list of deployment targets for the model deployment repository. +outputs: + valid-targets: + description: A JSON array of valid deployment targets, suitable for use as a matrix. + value: ${{ steps.generate.outputs.targets }} +runs: + using: composite + steps: + - name: Get deployment settings.json + uses: actions/checkout@v4 + with: + repository: access-nri/build-cd + ref: main + + - name: Generate Deployment Target Matrix + shell: bash + id: generate + # These are used in the deploy job to determine the targets to deploy to - irrespective of the type of deployment. + # We use the list of valid targets (from build-cd) intersected with targets set in the caller repository to determine the final list. + run: | + valid_targets=$(jq --raw-output --compact-output \ + '.deployment | keys' \ + config/settings.json + ) + repo_prerelease_targets=$(jq --null-input --raw-output --compact-output \ + --arg valids "${{ inputs.deployment-targets }}" \ + '$valids | split(" ")' + ) + # This is the intersection between the valid and caller-defined targets + targets=$(jq --null-input --raw-output --compact-output\ + --argjson valid "$valid_targets" \ + --argjson ours "$repo_prerelease_targets" \ + '$valid - ($valid - $ours)' + ) + + echo "From build-cd's valid targets: $valid_targets, and ${{ github.repository }}s chosen targets: $repo_prerelease_targets, we will deploy to $targets" + echo "targets=$targets" >> $GITHUB_OUTPUT \ No newline at end of file From cd03bf13ba92764b33ee207ac2173ae06d64dfe2 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 23 Jan 2025 11:09:38 +1100 Subject: [PATCH 24/40] Update ci.yml, cd.yml to use get-target-matrix --- .github/workflows/cd.yml | 29 +++-------------------------- .github/workflows/ci.yml | 31 ++++--------------------------- 2 files changed, 7 insertions(+), 53 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 396966e..5326d4e 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -50,34 +50,11 @@ jobs: echo "default=${{ inputs.root-sbd }}" >> $GITHUB_OUTPUT fi - - name: Get deployment settings.json - uses: actions/checkout@v4 - with: - repository: access-nri/build-cd - ref: main - - name: Generate Deployment Target Matrix id: target - # These are used in the deploy job to determine the targets to deploy to - irrespective of the type of deployment. - # We use the list of valid targets (from build-cd) intersected with targets set in the caller repository to determine the final list. - run: | - valid_targets=$(jq --raw-output --compact-output \ - '.deployment | keys' \ - config/settings.json - ) - repo_release_targets=$(jq --null-input --raw-output --compact-output \ - --arg valids "${{ vars.RELEASE_DEPLOYMENT_TARGETS }}" \ - '$valids | split(" ")' - ) - # This is the intersection between the valid and caller-defined targets - targets=$(jq --null-input --raw-output --compact-output\ - --argjson valid "$valid_targets" \ - --argjson ours "$repo_release_targets" \ - '$valid - ($valid - $ours)' - ) - - echo "From build-cd's valid targets: $valid_targets, and ${{ github.repository }}s chosen targets: $repo_release_targets, we will deploy to $targets" - echo "targets=$targets" >> $GITHUB_OUTPUT + uses: access-nri/build-cd/.github/actions/get-target-matrix@v3 + with: + targets: ${{ vars.RELEASE_DEPLOYMENT_TARGETS }} verify-settings: name: Verify Deployment Settings diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4649944..333d76c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: base-ref: ${{ steps.pr.outputs.base }} next-deployment-number: ${{ steps.prerelease.outputs.next-deployment-number }} prerelease-version: ${{ steps.prerelease.outputs.version }} - targets: ${{ steps.target.outputs.targets }} + targets: ${{ steps.target.outputs.valid-targets }} steps: - name: root-sbd default id: root-sbd @@ -123,34 +123,11 @@ jobs: echo "Prerelease version will be $version" echo "version=$version" >> $GITHUB_OUTPUT - - name: Get deployment settings.json - uses: actions/checkout@v4 - with: - repository: access-nri/build-cd - ref: main - - name: Generate Deployment Target Matrix id: target - # These are used in the deploy job to determine the targets to deploy to - irrespective of the type of deployment. - # We use the list of valid targets (from build-cd) intersected with targets set in the caller repository to determine the final list. - run: | - valid_targets=$(jq --raw-output --compact-output \ - '.deployment | keys' \ - config/settings.json - ) - repo_prerelease_targets=$(jq --null-input --raw-output --compact-output \ - --arg valids "${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }}" \ - '$valids | split(" ")' - ) - # This is the intersection between the valid and caller-defined targets - targets=$(jq --null-input --raw-output --compact-output\ - --argjson valid "$valid_targets" \ - --argjson ours "$repo_prerelease_targets" \ - '$valid - ($valid - $ours)' - ) - - echo "From build-cd's valid targets: $valid_targets, and ${{ github.repository }}s chosen targets: $repo_prerelease_targets, we will deploy to $targets" - echo "targets=$targets" >> $GITHUB_OUTPUT + uses: access-nri/build-cd/.github/actions/get-target-matrix@v3 + with: + targets: ${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }} redeploy-pre: name: '!redeploy Pending' From 4d8a7c8262e8744bb62c79387ad14ab2e322a791 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 23 Jan 2025 11:10:57 +1100 Subject: [PATCH 25/40] Simplify undeploy workflow --- .github/workflows/ci-closed.yml | 25 +++++-- .github/workflows/undeploy-1-setup.yml | 41 ------------ .github/workflows/undeploy-1-start.yml | 90 ++++++++++++++++++++++++++ .github/workflows/undeploy-2-start.yml | 66 ------------------- 4 files changed, 108 insertions(+), 114 deletions(-) delete mode 100644 .github/workflows/undeploy-1-setup.yml create mode 100644 .github/workflows/undeploy-1-start.yml delete mode 100644 .github/workflows/undeploy-2-start.yml diff --git a/.github/workflows/ci-closed.yml b/.github/workflows/ci-closed.yml index 6031c7a..5a3eab1 100644 --- a/.github/workflows/ci-closed.yml +++ b/.github/workflows/ci-closed.yml @@ -9,10 +9,10 @@ run-name: ${{ inputs.model }} PR Closed Cleanup on: workflow_call: inputs: - model: + root-sbd: type: string required: true - description: The model that is being tested and deployed + description: The model package name that is going to be removed # Callers usually have the trigger: # pull_request: # types: @@ -29,19 +29,30 @@ jobs: runs-on: ubuntu-latest outputs: version-pattern: ${{ steps.version.outputs.pattern }} + targets: ${{ steps.target.outputs.valid-targets }} steps: - - name: Version Pattern + - name: Get Version Pattern id: version # For example, `access-om3-pr12-*` - run: | - repo_name_sanitized=$(echo ${{ github.event.repository.name }} | tr [:upper:] [:lower:] | tr '.' 'p' ) - echo "pattern=${repo_name_sanitized}-pr${{ github.event.pull_request.number }}-*" >> $GITHUB_OUTPUT + run: echo "pattern=${{ inputs.root-sbd }}-pr${{ github.event.pull_request.number }}-*" >> $GITHUB_OUTPUT + + - name: Generate Deployment Target Matrix + id: target + uses: access-nri/build-cd/.github/actions/get-target-matrix@v3 + with: + targets: ${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }} undeploy-prereleases: name: Undeploy Prereleases Matching ${{ needs.setup.outputs.version-pattern }} needs: - setup + strategy: + matrix: + target: ${{ fromJson(needs.setup.outputs.targets) }} + fail-fast: false uses: access-nri/build-cd/.github/workflows/undeploy-1-setup.yml@main with: - version-pattern: ${{ needs.setup.outputs.version-pattern }} + version-pattern: ${{ inputs.root-sbd }}-pr${{ github.event.pull_request.number }}-* + target: ${{ matrix.target }} + type: Prerelease secrets: inherit diff --git a/.github/workflows/undeploy-1-setup.yml b/.github/workflows/undeploy-1-setup.yml deleted file mode 100644 index 5799329..0000000 --- a/.github/workflows/undeploy-1-setup.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Undeployment Setup -on: - workflow_call: - inputs: - version-pattern: - type: string - required: true - description: A wildcard-supported string of spack environments to remove -jobs: - get-prerelease-deployment-env: - name: Get Prerelease Deployment Environment - runs-on: ubuntu-latest - outputs: - deployment-environments: ${{ steps.get-deployment-environment.outputs.deployment-environments }} - steps: - - name: Checkout Config - uses: actions/checkout@v4 - with: - repository: access-nri/build-cd - - - name: Get Environments - id: get-deployment-environment - # Note: We run under the assumption that all Prerelease Environments have 'Prerelease' - # appended to their GitHub Environment name, which is set out in our deployment template - # https://github.com/ACCESS-NRI/model-deployment-template/blob/main/README.md?plain=1#L41 - run: echo "deployment-environments=$(jq --compact-output --raw-output '.deployment | [keys[] | "\(.) Prerelease"]' ./config/settings.json)" >> $GITHUB_OUTPUT - - - undeployment: - name: Remove Deployment - needs: - - get-prerelease-deployment-env - strategy: - fail-fast: false - matrix: - deployment-environment: ${{ fromJson(needs.get-prerelease-deployment-env.outputs.deployment-environments) }} - uses: access-nri/build-cd/.github/workflows/undeploy-2-start.yml@main - with: - deployment-environment: ${{ matrix.deployment-environment }} - env-name: ${{ inputs.version-pattern }} - secrets: inherit diff --git a/.github/workflows/undeploy-1-start.yml b/.github/workflows/undeploy-1-start.yml new file mode 100644 index 0000000..e2d6564 --- /dev/null +++ b/.github/workflows/undeploy-1-start.yml @@ -0,0 +1,90 @@ +name: Remove Deployment From Target +on: + workflow_call: + inputs: + version-pattern: + type: string + required: true + description: | + A glob of spack environments to remove. + For example, `access-om3-pr12-*`. + target: + type: string + required: true + description: | + The deployment target to remove the environments from. + This comes from the GitHub Environment Name - for example, Gadi. + type: + type: string + required: true + description: | + The type of deployment to remove the environments from. + Must be one of: Prerelease, Release. +jobs: + validate-inputs: + name: Validate Inputs + runs-on: ubuntu-latest + steps: + - name: Validate Inputs + id: validate + run: | + if [[ "${{ inputs.type }}" != "Prerelease" || "${{ inputs.type }}" != "Release" ]]; then + echo "::error::Invalid type: ${{ inputs.type }}. Must be one of: Prerelease, Release." + exit 1 + fi + echo "Validated inputs: ${{ inputs.version-pattern }} for ${{ inputs.target }} ${{ inputs.type }}" + + undeploy-from-environment: + name: Undeploy ${{ inputs.version-pattern }} from ${{ inputs.target }} ${{ inputs.type }} + needs: validate-inputs + runs-on: ubuntu-latest + environment: ${{ inputs.target }} ${{ inputs.type != 'Release' && inputs.type }} + steps: + - uses: actions/checkout@v4 + + - name: Get Spack Version From config/versions.json + id: versions + run: echo "spack=$(jq --compact-output --raw-output '.spack' ./config/versions.json)" >> $GITHUB_OUTPUT + + - name: Get ${{ inputs.target }} Remote Paths + id: path + uses: access-nri/build-cd/.github/actions/get-deploy-paths@main + with: + spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }} + spack-version: ${{ steps.versions.outputs.spack }} + deployment-environment: ${{ inputs.target }} ${{ inputs.type != 'Release' && inputs.type }} + + - name: Setup SSH + id: ssh + uses: access-nri/actions/.github/actions/setup-ssh@main + with: + private-key: ${{ secrets.SSH_KEY }} + hosts: | + ${{ secrets.HOST }} + ${{ secrets.HOST_DATA }} + + - name: Undeploy + # ssh into deployment environment, create and activate the env, remove all the unneeded environments + id: undeploy + run: | + ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + . ${{ steps.path.outputs.spack-config }}/spack-enable.bash + envs=$(find ${{ steps.path.outputs.spack }}/../environments -type d -name '${{ inputs.version-pattern }}' -printf '%f ') + + for env in $envs; do + spack env activate $env + spack uninstall --remove --dependents --yes-to-all --all + spack env deactivate + spack env rm $env --yes-to-all + done + spack gc --yes-to-all + EOT + + - name: Undeploy Status Notifier + if: always() + run: | + if [[ "${{ steps.undeploy.outcome }}" == "success" ]]; then + echo "::notice::Deployment ${{ inputs.version-pattern }} was successfully removed from ${{ inputs.target }} ${{ inputs.type }}, found at ${{ steps.path.outputs.spack }}" + else + echo "::error::Deployment ${{ inputs.version-pattern }} couldn't be removed from ${{ inputs.target }} ${{ inputs.type }}, found at ${{ steps.path.outputs.spack }}. Please check manually." + fi diff --git a/.github/workflows/undeploy-2-start.yml b/.github/workflows/undeploy-2-start.yml deleted file mode 100644 index ef9732f..0000000 --- a/.github/workflows/undeploy-2-start.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Undeployment Start -on: - workflow_call: - inputs: - env-name: - type: string - required: true - description: The spack-env-compliant model name to remove - deployment-environment: - type: string - required: true - description: The GitHub deployment environment name -jobs: - undeploy-from-environment: - name: Undeploy ${{ inputs.env-name }} from ${{ inputs.deployment-environment }} - runs-on: ubuntu-latest - environment: ${{ inputs.deployment-environment }} - steps: - - uses: actions/checkout@v4 - - - name: Get Spack Version From config/versions.json - id: versions - run: echo "spack=$(jq --compact-output --raw-output '.spack' ./config/versions.json)" >> $GITHUB_OUTPUT - - - name: Get ${{ inputs.deployment-environment }} Remote Paths - id: path - uses: access-nri/build-cd/.github/actions/get-deploy-paths@main - with: - spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }} - spack-version: ${{ steps.versions.outputs.spack }} - deployment-environment: ${{ inputs.deployment-environment }} - - - name: Setup SSH - id: ssh - uses: access-nri/actions/.github/actions/setup-ssh@main - with: - private-key: ${{ secrets.SSH_KEY }} - hosts: | - ${{ secrets.HOST }} - ${{ secrets.HOST_DATA }} - - - name: Undeploy - # ssh into deployment environment, create and activate the env, remove all the unneeded environments - id: undeploy - run: | - ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' - . ${{ steps.path.outputs.spack-config }}/spack-enable.bash - envs=$(find ${{ steps.path.outputs.spack }}/../environments -type d -name '${{ inputs.env-name }}' -printf '%f ') - - for env in $envs; do - spack env activate $env - spack uninstall --remove --dependents --yes-to-all --all - spack env deactivate - spack env rm $env --yes-to-all - done - spack gc --yes-to-all - EOT - - - name: Undeploy Status Notifier - if: always() - run: | - if [[ "${{ steps.undeploy.outcome }}" == "success" ]]; then - echo "::notice::Deployment ${{ inputs.env-name }} was successfully removed from ${{ inputs.deployment-environment }}, found at ${{ steps.path.outputs.spack }}" - else - echo "::error::Deployment ${{ inputs.env-name }} couldn't be removed from ${{ inputs.deployment-environment}}, found at ${{ steps.path.outputs.spack }}. Please check manually." - fi From 17bfea2630a63938cf20811f8fc5959f5d8fdbac Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 23 Jan 2025 11:19:13 +1100 Subject: [PATCH 26/40] cd.yml: Updated output variable --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 5326d4e..a1959e5 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -39,7 +39,7 @@ jobs: runs-on: ubuntu-latest outputs: root-sbd: ${{ steps.root-sbd.outputs.default }} - targets: ${{ steps.target.outputs.matrix }} + targets: ${{ steps.target.outputs.valid-targets }} steps: - name: root-sbd id: root-sbd From cf41a690e687813951f4ca8b9aabd4b0acc19cd1 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 23 Jan 2025 11:31:33 +1100 Subject: [PATCH 27/40] Updated workflow reference to `undeploy-1-start.yml` --- .github/workflows/ci-closed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-closed.yml b/.github/workflows/ci-closed.yml index 5a3eab1..1f1431b 100644 --- a/.github/workflows/ci-closed.yml +++ b/.github/workflows/ci-closed.yml @@ -50,7 +50,7 @@ jobs: matrix: target: ${{ fromJson(needs.setup.outputs.targets) }} fail-fast: false - uses: access-nri/build-cd/.github/workflows/undeploy-1-setup.yml@main + uses: access-nri/build-cd/.github/workflows/undeploy-1-start.yml@main with: version-pattern: ${{ inputs.root-sbd }}-pr${{ github.event.pull_request.number }}-* target: ${{ matrix.target }} From a106dabf266cf89415a7559f68a6fdd4dbb423af Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 23 Jan 2025 11:42:10 +1100 Subject: [PATCH 28/40] get-target-matrix: Made vars more generic --- .github/actions/get-target-matrix/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/get-target-matrix/action.yml b/.github/actions/get-target-matrix/action.yml index fbf9bbf..ca930e4 100644 --- a/.github/actions/get-target-matrix/action.yml +++ b/.github/actions/get-target-matrix/action.yml @@ -29,16 +29,16 @@ runs: '.deployment | keys' \ config/settings.json ) - repo_prerelease_targets=$(jq --null-input --raw-output --compact-output \ - --arg valids "${{ inputs.deployment-targets }}" \ + repo_targets=$(jq --null-input --raw-output --compact-output \ + --arg valids "${{ inputs.targets }}" \ '$valids | split(" ")' ) # This is the intersection between the valid and caller-defined targets targets=$(jq --null-input --raw-output --compact-output\ --argjson valid "$valid_targets" \ - --argjson ours "$repo_prerelease_targets" \ + --argjson ours "$repo_targets" \ '$valid - ($valid - $ours)' ) - echo "From build-cd's valid targets: $valid_targets, and ${{ github.repository }}s chosen targets: $repo_prerelease_targets, we will deploy to $targets" + echo "From build-cd's valid targets: $valid_targets, and ${{ github.repository }}s chosen targets: $repo_targets, we will deploy to $targets" echo "targets=$targets" >> $GITHUB_OUTPUT \ No newline at end of file From 270b4c5270a3d7f2ff82684f22a77396d57d0040 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Wed, 22 Jan 2025 15:12:36 +1100 Subject: [PATCH 29/40] generate-build-metadata.bash: Reference explicit Gadi.* files --- scripts/generate-build-metadata.bash | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/generate-build-metadata.bash b/scripts/generate-build-metadata.bash index abfd68d..f14ffbd 100755 --- a/scripts/generate-build-metadata.bash +++ b/scripts/generate-build-metadata.bash @@ -26,11 +26,12 @@ packages=( "$@" ) ### SCRIPT ### mkdir -p "$output_dir" +# FIXME: This script is Gadi-specific and will need to be updated for future targets spack=$(jq \ '{ version: .spack.version, commit: .spack.commit - }' "$json_dir/spack.lock") + }' "$json_dir/Gadi.spack.lock") model=$(jq \ --arg model "$model_name" \ @@ -49,7 +50,7 @@ model=$(jq \ spack_config: $spack_config_version, status: "active", spack_version: $spack - }' "$json_dir/spack.lock" + }' "$json_dir/Gadi.spack.lock" ) # construction of the initial build_metadata.json @@ -64,7 +65,7 @@ for pkg in "${packages[@]}"; do pkg_hash=$(jq --raw-output \ --arg pkg "$pkg" \ '.concrete_specs | to_entries[] | select(.value.name == $pkg) | .key' \ - "$json_dir/spack.lock" + "$json_dir/Gadi.spack.lock" ) echo "Hash of $pkg is $pkg_hash" @@ -72,13 +73,13 @@ for pkg in "${packages[@]}"; do install_path=$(jq --raw-output \ --arg pkg_hash "$pkg_hash" \ 'to_entries[] | select(.key == $pkg_hash) | .value' \ - "$json_dir/spack.location.json" + "$json_dir/Gadi.spack.location.json" ) release_url=$(jq --raw-output \ --arg pkg "$pkg" \ '.[$pkg]' \ - "$json_dir/build-db-pkgs.json" + "$json_dir/Gadi.build-db-pkgs.json" ) component=$(jq \ @@ -91,7 +92,7 @@ for pkg in "${packages[@]}"; do spec: (.value.name + "@" + .value.version), install_path: $install_path, release_url: $release_url - }' "$json_dir/spack.lock" + }' "$json_dir/Gadi.spack.lock" ) # piecewise construction of the entire build_metadata.json for each From 0b92cf3812d31ed1402163e93a1cdd0226fe5dbe Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Wed, 22 Jan 2025 15:13:29 +1100 Subject: [PATCH 30/40] tests/scripts/generate-build-metadata: Updated test data inputs/outputs --- .../generate-build-metadata/inputs/Gadi.build-db-pkgs.json | 4 ++++ .../{spack.location.json => Gadi.spack.location.json} | 0 .../inputs/{spack.lock => Gadi.spack.lock} | 0 .../generate-build-metadata/valid/build_metadata.json | 6 ++++-- 4 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 tests/scripts/generate-build-metadata/inputs/Gadi.build-db-pkgs.json rename tests/scripts/generate-build-metadata/inputs/{spack.location.json => Gadi.spack.location.json} (100%) rename tests/scripts/generate-build-metadata/inputs/{spack.lock => Gadi.spack.lock} (100%) diff --git a/tests/scripts/generate-build-metadata/inputs/Gadi.build-db-pkgs.json b/tests/scripts/generate-build-metadata/inputs/Gadi.build-db-pkgs.json new file mode 100644 index 0000000..dd8ce38 --- /dev/null +++ b/tests/scripts/generate-build-metadata/inputs/Gadi.build-db-pkgs.json @@ -0,0 +1,4 @@ +{ + "mom5": "https://github.com/ACCESS-NRI/MOM5/releases/tag/2023.11.09", + "cice5": "https://github.com/ACCESS-NRI/cice5/releases/tag/2023.10.19" +} \ No newline at end of file diff --git a/tests/scripts/generate-build-metadata/inputs/spack.location.json b/tests/scripts/generate-build-metadata/inputs/Gadi.spack.location.json similarity index 100% rename from tests/scripts/generate-build-metadata/inputs/spack.location.json rename to tests/scripts/generate-build-metadata/inputs/Gadi.spack.location.json diff --git a/tests/scripts/generate-build-metadata/inputs/spack.lock b/tests/scripts/generate-build-metadata/inputs/Gadi.spack.lock similarity index 100% rename from tests/scripts/generate-build-metadata/inputs/spack.lock rename to tests/scripts/generate-build-metadata/inputs/Gadi.spack.lock diff --git a/tests/scripts/generate-build-metadata/valid/build_metadata.json b/tests/scripts/generate-build-metadata/valid/build_metadata.json index 6c6413b..d33d42b 100644 --- a/tests/scripts/generate-build-metadata/valid/build_metadata.json +++ b/tests/scripts/generate-build-metadata/valid/build_metadata.json @@ -16,12 +16,14 @@ { "spack_hash": "qji4nlmr6utrribaiyhewe4je6mifguz", "spec": "mom5@git.2023.11.09=2023.11.09", - "install_path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/mom5-git.2023.11.09=2023.11.09-qji4nlmr6utrribaiyhewe4je6mifguz" + "install_path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/mom5-git.2023.11.09=2023.11.09-qji4nlmr6utrribaiyhewe4je6mifguz", + "release_url": "https://github.com/ACCESS-NRI/MOM5/releases/tag/2023.11.09" }, { "spack_hash": "v3zncpqjj2gyseudbwiudolcjq3k3leo", "spec": "cice5@git.2023.10.19=2023.10.19", - "install_path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/cice5-git.2023.10.19=2023.10.19-v3zncpqjj2gyseudbwiudolcjq3k3leo" + "install_path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/cice5-git.2023.10.19=2023.10.19-v3zncpqjj2gyseudbwiudolcjq3k3leo", + "release_url": "https://github.com/ACCESS-NRI/cice5/releases/tag/2023.10.19" } ] } From a0f0bbebb4b991a9fec182b69bc6dec1641cc33d Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Wed, 22 Jan 2025 15:16:15 +1100 Subject: [PATCH 31/40] cd.yml: Use Gadi-specific outputs/metadata for build-db upload --- .github/workflows/cd.yml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index a1959e5..7d52181 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -210,7 +210,7 @@ jobs: GH_TOKEN: ${{ github.token }} run: echo "created-at=$(gh release view --json createdAt --jq '.createdAt' --repo ${{ github.repository }})" >> $GITHUB_OUTPUT - # FIXME: This won't work until ACCESS-NRI/build-cd#201 is resolved + # TODO: This will only upload data related to the Gadi deployment build-db: name: Build DB Metadata Upload needs: @@ -223,6 +223,20 @@ jobs: with: pattern: ${{ needs.deploy-release.outputs.general-metadata-artifact-glob }} path: ${{ env.METADATA_PATH }} + merge-multiple: true + + - name: Download Outputs Artifact + uses: actions/download-artifact@v4 + with: + pattern: ${{ needs.deploy-release.outputs.general-outputs-artifact-glob }} + path: ${{ env.OUTPUTS_PATH }} + merge-multiple: true + + - name: 'Get Gadi spack-* Repo Versions' + id: spack-versions + run: | + echo "packages-version=$(jq --raw-output --compact-output '.spack_packages_version' ${{ env.OUTPUTS_PATH }}/deploy-outputs.Gadi)" >> $GITHUB_OUTPUT + echo "config-version=$(jq --raw-output --compact-output '.spack_config_version' ${{ env.OUTPUTS_PATH }}/deploy-outputs.Gadi)" >> $GITHUB_OUTPUT - name: Checkout Upload Script uses: actions/checkout@v4 @@ -241,9 +255,12 @@ jobs: - name: Upload Build Metadata env: BUILD_DB_CONNECTION_STR: ${{ secrets.BUILD_DB_CONNECTION_STR }} - OUTPUT_PATH: ./metadata_output run: | - ./scripts/generate-build-metadata.bash ${{ needs.release.outputs.url }} ${{ needs.release.outputs.created-at }} ${{ needs.deploy-to-environment.outputs.packages-version }} ${{ needs.deploy-to-environment.outputs.config-version }} ${{ env.METADATA_PATH }} ${{ env.OUTPUT_PATH }} ${{ inputs.root-sbd }} ${{ vars.BUILD_DB_PACKAGES }} + ./scripts/generate-build-metadata.bash \ + ${{ needs.release.outputs.url }} ${{ needs.release.outputs.created-at }} \ + ${{ steps.spack-versions.outputs.packages-version }} ${{ steps.spack-versions.outputs.config-version }} \ + ${{ env.METADATA_PATH }} ${{ env.OUTPUTS_PATH }} \ + ${{ inputs.root-sbd }} ${{ vars.BUILD_DB_PACKAGES }} echo "Attempting upload of build_metadata.json" - python ./tools/release_provenance/save_release.py "${{ env.OUTPUT_PATH }}/build_metadata.json" + python ./tools/release_provenance/save_release.py "${{ env.OUTPUTS_PATH }}/build_metadata.json" From dad7e3b4e499a5fb3f75611b8f0dd115e90c6eca Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 23 Jan 2025 13:24:07 +1100 Subject: [PATCH 32/40] Update README.md with information on this repository and it's workflows --- README.md | 170 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 107 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 9f693cc..2845157 100644 --- a/README.md +++ b/README.md @@ -1,104 +1,148 @@ # build-cd -This repository houses reusable workflows for the building and deployment of ACCESS-NRI models to different environments. +This repository houses reusable workflows, actions and scripts for the building and deployment of ACCESS-NRI Climate Models to different environments. It is considered an "umbrella repository" for all Model Deployment Repositories. + +## Repositories Serviced By `build-cd` + +These are the repositories with the `deployment` topic. + +To find the most up to date deployment repositories, run: + +```bash +gh search repos --owner access-nri --json name --jq '[.[].name] | @sh' -- topic:deployment -topic:template +``` + +### Model Deployment Repositories + +* [ACCESS-NRI/ACCESS-OM2](https://github.com/ACCESS-NRI/ACCESS-OM2) +* [ACCESS-NRI/ACCESS-OM2-BGC](https://github.com/ACCESS-NRI/ACCESS-OM2-BGC) +* [ACCESS-NRI/ACCESS-OM3](https://github.com/ACCESS-NRI/ACCESS-OM3) +* [ACCESS-NRI/ACCESS-ESM1.5](https://github.com/ACCESS-NRI/ACCESS-ESM1.5) +* [ACCESS-NRI/ACCESS-ESM1.6](https://github.com/ACCESS-NRI/ACCESS-OM2) +* [ACCESS-NRI/CABLE-standalone](https://github.com/ACCESS-NRI/CABLE-standalone) + +### Testing and Template Repositories + +* [ACCESS-NRI/ACCESS-TEST](https://github.com/ACCESS-NRI/ACCESS-TEST) +* [ACCESS-NRI/model-deployment-template](https://github.com/ACCESS-NRI/model-deployment-template) ## Overview -This repository is broken down into two folders: `config` and `.github`. +This repository is broken down into the following top-level folders: -`config` contains information on the deployment environments that the models deploy to, which is currently just the name of the deployment target. This is used by the aforementioned deployment workflows to gather secrets and configuration details from the associated GitHub Environment. +`config` contains CODEOWNER-locked information on the deployment environments that the models can deploy to. This is used by deployment workflows to gather secrets and configuration details from the associated GitHub Environment. -`.github/workflows` houses validation and reusable deployment workflows that are called by ACCESS-NRI model repositories. Currently, only [ACCESS-NRI/ACCESS-OM2](https://github.com/ACCESS-NRI/access-om2) is supported. +`scripts` contains independently-testable `python` and `bash` scripts used directly by `build-cd` deployment workflows. -`.github/actions` houses custom actions to do with convenience checks for deployment. More information on these actions can be found in `.github/actions/*/README.md`. +`tools` contains `python` and `bash` scripts used for tasks outside the main deployment workflows. -Below is a brief summary of the three pipelines in `.github/workflows`: `deploy-*`, `undeploy-*` and `validate-json`. +`tests` contains tests for the above scripts. -### `deploy-*` +`.github/workflows` houses validation and reusable deployment workflows that are called by ACCESS-NRI model deployment repositories, or within `build-cd` itself. -#### Inputs +`.github/actions` houses custom actions used by deployment workflows. More information on these actions can be found in `.github/actions/*/README.md`. -| Name | Type | Description | Required | Default | Example | -| ---- | ---- | ----------- | -------- | ------- | ------- | -| `type` | string | The type of deployment - either 'release' or 'prerelease' | true | `release` | `prerelease` | -| `version` | string | The version of the model being deployed | true | N/A | `2024.01.1` | +## Versioning in This Repository -#### Explanation +The [entrypoint workflows](#entrypoint-workflows) (and other reusable workflows) are versioned both via major version branches (of the form `vX`) and tags (of the form `vX.Y`). -This pipeline is responsible for the gathering of configuration information, building and deployment of whatever model repository calls it. It is split into two workflows. +Major versions are used to denote changes to any of the following: -##### `deploy-1-setup.yml` +* `build-cd` entrypoint workflow inputs are created, updated or deleted, requiring an update to model deployment repositories workflows. +* Changes to `build-cd` require new `vars`/`secrets` in model deployment repositories. +* Changes to `build-cd` are significant updates to existing features. -This workflow obtains the relevant spack and GitHub Environment information, and creates parallelized deployments based on the list of environments. +Minor versions are new features, or updates that don't create new `vars`/`secrets`, or updates that don't affect entrypoint workflow inputs. -##### `deploy-2-start.yml` +### Using Workflow Versions -Using the GitHub Environment, it `ssh`s into the deployment environments `spack` instance, and installs the model associated with the repository that called it. It then copies back relevant metadata and creates a versioned GitHub Release in the caller repository, if it is not a `prerelease` deployment. +Model Deployment Repositories can use `build-cd` workflows via: -#### Usage +* Branch references (`vX`): These can be used to ensure that existing Model Deployment Repository infrastructure will always work within a major version, without updates. Using this reference means you will still get updates to the workflow that don't modify existing infrastructure. +* Tag references (`vX.Y`) (or commit references): These can be used to have a single version of the infrastructure. -For supported `spack`-installable ACCESS-NRI models, simply call the `deploy-1-setup.yml` reusable workflow from the given repositories workflow file, as shown below. Don't forget to add required inputs! +## Entrypoint Workflows -```yml -deploy: - uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@main - with: - version: 1.2.3 - secrets: inherit - permissions: - contents: write -``` +These are called directly by Model Deployment Repositories - `ci.yml`, `ci-comment.yml`, `ci-closed.yml` and `cd.yml`. -### `undeploy-*` +### `ci.yml` - PR Prerelease Deployment Entrypoint -For given `spack` environments, we can also remove deployments. +This entrypoint is used to deploy (and [`!redeploy`](#redeploy)) Prereleases as part of Pull Requests into `main` or `backport/*.*` branches. -> [!NOTE] -> This workflow is not one that should be used directly. It is used within the automated deployment pipeline to remove prerelease builds of models. +It sets up configuring and parallelizing deployments based on HPC target. -#### Inputs +### `ci-comment.yml`- PR `!bump` Comment Command Entrypoint -| Name | Type | Description | Required | Default | Example | -| ---- | ---- | ----------- | -------- | ------- | ------- | -| `version-pattern` | string | A wildcard-supported string for version(s) of the model being removed | true | N/A | `2024.01.1-pre*` | +This entrypoint is used to handle the `!bump` Comment Command, which updates, commits and pushes the version of the model automatically. -#### Explanation +### `ci-closed.yml` - PR Deployment Cleanup Entrypoint -This pipeline is responsible for removing deployed models from a deployment environment. Particular use-cases include removing `prerelease` builds from a deployment environment once a PR is merged. +This entrypoint handles cleanup of existing Prerelease environments from the referenced PR. -##### `undeploy-1-setup.yml` +Similar to `ci.yml`, it parallelizes cleanups based on HPC target. -This workflow obtains the relevant spack and GitHub Environment information, and creates parallelized jobs removing the given `spack` environments based on the list of deployment environments. +### `cd.yml` - Release Deployment Entrypoint -##### `undeploy-2-start.yml` +This entrypoint is used to deploy Releases as part of merged Pull Requests into `main` or `backport/*.*` branches. -Using the GitHub Environment, it `ssh`s into the deployment environments `spack` instance, and installs the model associated with the repository that called it. It then copies back relevant metadata and creates a versioned GitHub Release in the caller repository, if it is not a `prerelease` deployment. +Similar to `ci.yml`, it parallelizes deployments based on HPC target. -#### Usage +## `deploy-*.yml` - Target Deployment Pipeline -```yml -remove-prereleases: - uses: access-nri/build-cd/.github/workflows/undeploy-2-setup.yml@main - with: - version-pattern: ${{ inputs.model }}-pre* - secrets: inherit -``` +This pipeline is responsible for deploying a given model, via [`spack`](https://spack.readthedocs.io/en/latest/), to a single HPC target. This pipeline is deployment-type-independent - it works for both Prereleases and Releases. + +### `deploy-1-setup.yml` - Checks and Configuration + +This workflow validates environment configuration information from both `build-cd` and the Model Deployment Repository's `config` directory; and also validates the Model Deployment Repository's `spack.yaml`. It then passes this validated information to the [next workflow](#deploy-2-startyml---deployment-and-metadata-retrieval) returning deployment information to [the caller](#deploy-yml---target-deployment-pipeline) via a target-specific file artifact. + +### `deploy-2-start.yml` - Deployment and Metadata Retrieval + +This workflow deploys the climate model via spack to the given deployment target. It also collects metadata relating to the spack install and returns it to [the previous workflow](#deploy-1-setupyml---checks-and-configuration). + +## `undeploy-*.yml` - Target Deployment Removal Pipeline -This will remove every `spack` environment from the deployment target that matches `-pre*`. +This pipeline is responsible for removing all spack environments associated with a closed Pull Request for a single HPC target. -### `validate-json.yml` +### `undeploy-1-start.yml` - Remove Prereleases from Target -This workflow is used to validate the `config` folders `*.json` files based on their associated `*.schema.json`. This is used for PR checks on the `build-cd` repo itself. +This workflow, currently being the single part of the pipeline, removes the spack environments given as a glob pattern, installed in a particular spack instance, on a particular HPC target. -### `create-deployment-spack.yml` +## `settings-*.yml` - `build-cd config` Update Pipeline -This workflow_dispatch-triggered workflow is used to create a version of `spack` on `Gadi`. +This pipeline is responsible for validating and deploying changes based on protected deployment information in `build-cd`s `config` directory. More information on this folder is found in [`config/README.md`](./config/README.md). + +### `settings-1-update.yml` - Validate Updated Settings + +This workflow is responsible for validating modifications made to `config/settings.json` on Pull Request or push to `build-cd`. Additionally, it will setup matrixing [the deployment workflow](#settings-2-deployyml---deploy-updated-settings) if the workflow trigger is `on.push`. + +### `settings-2-deploy.yml` - Deploy Updated Settings + +This workflow will update the repositories referenced in `config/settings.json` to the refs in the file for a HPC target. + +## (Legacy) JSON Validation Workflow - `validate-json.yml` + +This workflow is used to validate JSON data against JSON schemas that are housed within this repository (as opposed to workflows housed within `ACCESS-NRI/schema`). + +## Comment Commands Handled By `build-cd` + +Comment Commands are a ChatOps-style interface to repository functions in Model Deployment Repository Pull Requests. + +### `!bump` + +```txt +!bump [major|minor] +``` + +This Comment Command bumps a models release version, so one does not have to edit the `spack.yaml` themselves. + +It bumps the `spack.yaml` model version (of the form `YEAR.MONTH.MINOR`, where `YEAR.MONTH` is considered the `MAJOR` portion) and commits the result to the PR branch. + +### `!redeploy` + +```txt +!redeploy +``` -#### Inputs +This Comment Command deploys the current `HEAD` of the PR branch again. -| Name | Type | Description | Required | Default | Example | -| ---- | ---- | ----------- | -------- | ------- | ------- | -| `spack-version` | string | A version of `spack` | true | N/A | `0.20` | -| `spack-packages-version` | string | A version of ACCESS-NRI/spack-packages to be bundled with the install of `spack` | true | `main` | `2023.11.12` | -| `spack-config-version` | string | A version of ACCESS-NRI/spack-config to be bundled with the install of `spack` | true | `main` | `2024.01.01` | -| `deployment-location` | true | A path in the deployment environment where Spack should be created. For example, if it is `opt/spack`, spack will be installed under `opt/spack//` | true | N/A | `/opt/spack` | +This is most useful for models that are using `@git.BRANCH` references for versions of model dependencies. From 0eb22e87fec3a720b95b2d168ddeaf36964e4bc2 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 23 Jan 2025 13:50:30 +1100 Subject: [PATCH 33/40] Add config/README.md to explain files within the folder --- config/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 config/README.md diff --git a/config/README.md b/config/README.md new file mode 100644 index 0000000..91e4354 --- /dev/null +++ b/config/README.md @@ -0,0 +1,17 @@ +# Deployment Configuration Information + +This folder contains configuration information relevant to the orchestration of deployments across Model Deployment Repositories. + +Modifications to settings in this folder are deployed and validated via the `.github/workflows/settings-*.yml` pipeline. + +## `settings.json` + +This information is used as a single point of truth for the versions of repositories used for `spack` installs, across all valid HPC targets. + +Modifications to this file (when merged) update the versions of the repositories referenced on the HPC. + +This means that one should not modify the repositories versions on the HPC targets directly - open a PR in this repository into `main` with the changes required. + +## `settings.schema.json` + +This schema enforces the structure shown in `settings.json`. From 85c25cb8c657c0b0e0d9e21e4843d54f470d5b32 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 23 Jan 2025 16:06:57 +1100 Subject: [PATCH 34/40] Apply suggestions from code review Co-authored-by: jo-basevi Signed-off-by: Tommy Gatti --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2845157..924535b 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ gh search repos --owner access-nri --json name --jq '[.[].name] | @sh' -- topic: * [ACCESS-NRI/ACCESS-OM2-BGC](https://github.com/ACCESS-NRI/ACCESS-OM2-BGC) * [ACCESS-NRI/ACCESS-OM3](https://github.com/ACCESS-NRI/ACCESS-OM3) * [ACCESS-NRI/ACCESS-ESM1.5](https://github.com/ACCESS-NRI/ACCESS-ESM1.5) -* [ACCESS-NRI/ACCESS-ESM1.6](https://github.com/ACCESS-NRI/ACCESS-OM2) +* [ACCESS-NRI/ACCESS-ESM1.6](https://github.com/ACCESS-NRI/ACCESS-ESM1.6) * [ACCESS-NRI/CABLE-standalone](https://github.com/ACCESS-NRI/CABLE-standalone) ### Testing and Template Repositories From d84fa7ae0cc2918500ce80fbbfe92e0e4b86fd1d Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Thu, 23 Jan 2025 16:09:46 +1100 Subject: [PATCH 35/40] Add GitHub search link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 924535b..3cb067e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This repository houses reusable workflows, actions and scripts for the building These are the repositories with the `deployment` topic. -To find the most up to date deployment repositories, run: +To find the most up to date deployment repositories, use [this search URL](https://github.com/orgs/ACCESS-NRI/repositories?q=topic%3Adeployment+-topic%3Atemplate) or run: ```bash gh search repos --owner access-nri --json name --jq '[.[].name] | @sh' -- topic:deployment -topic:template From fa11ad3961248b590d6c5723ec0e3696338ffc86 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Tue, 28 Jan 2025 12:00:23 +1100 Subject: [PATCH 36/40] Small scale fixes from ACCESS-TEST E2E Testing --- .github/workflows/cd.yml | 6 +++++- .github/workflows/deploy-1-setup.yml | 3 ++- .github/workflows/deploy-2-start.yml | 12 ++++++++++-- .github/workflows/undeploy-1-start.yml | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 7d52181..7b4f227 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -132,6 +132,9 @@ jobs: secrets: inherit permissions: contents: write + # We require pull-requests:write despite not using it in this path, + # because it is needed by the pull_request invocation of the workflow. + pull-requests: write release: name: Create Release @@ -214,6 +217,7 @@ jobs: build-db: name: Build DB Metadata Upload needs: + - defaults - deploy-release - release runs-on: ubuntu-latest @@ -260,7 +264,7 @@ jobs: ${{ needs.release.outputs.url }} ${{ needs.release.outputs.created-at }} \ ${{ steps.spack-versions.outputs.packages-version }} ${{ steps.spack-versions.outputs.config-version }} \ ${{ env.METADATA_PATH }} ${{ env.OUTPUTS_PATH }} \ - ${{ inputs.root-sbd }} ${{ vars.BUILD_DB_PACKAGES }} + ${{ needs.defaults.outputs.root-sbd }} ${{ vars.BUILD_DB_PACKAGES }} echo "Attempting upload of build_metadata.json" python ./tools/release_provenance/save_release.py "${{ env.OUTPUTS_PATH }}/build_metadata.json" diff --git a/.github/workflows/deploy-1-setup.yml b/.github/workflows/deploy-1-setup.yml index 757ecb5..7939a08 100644 --- a/.github/workflows/deploy-1-setup.yml +++ b/.github/workflows/deploy-1-setup.yml @@ -1,5 +1,6 @@ name: Deploy on: + # This is invoked by both `on.pull_request` and `on.push` events. workflow_call: inputs: deployment-target: @@ -241,7 +242,7 @@ jobs: if [[ "$DEP" == "${{ inputs.spack-manifest-root-sbd }}" ]]; then # The model version is the bit after '@git.', before any later, space-separated, optional variants. # For example, in 'MODEL@git.VERSION type=ACCESS ~debug' the version is 'VERSION'. - DEP_VER=$(yq '.spack.specs[0] | capture(".+@git\\.(?[^ ]+).*") | .version' spack.yaml) + DEP_VER=$(yq '${{ env.SPACK_YAML_MODEL_YQ }} | capture(".+@git\\.(?[^ ]+).*") | .version' spack.yaml) else # Capture the section after '@git.' or '@' (if it's not a git-attributed version) and before a possible '=' for a given dependency. # Ex. '@git.2024.02.11' -> '2024.02.11', '@access-esm1.5' -> 'access-esm1.5', '@git.2024.05.21=access-esm1.5' -> '2024.05.21' diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index f986221..8b56db1 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -97,7 +97,7 @@ jobs: # Also removes the `@git.VERSION` specifier for Prereleases so # we don't have to shift tags around. run: | - yq -i '${{ env.SPACK_YAML_SPEC_YQ }} = (${{ env.SPACK_YAML_SPEC_YQ }} | split("@").[0])' spack.yaml + yq -i '${{ env.SPACK_YAML_MODEL_YQ }} = (${{ env.SPACK_YAML_MODEL_YQ }} | split("@").[0])' spack.yaml yq -i '${{ env.SPACK_YAML_MODULEFILE_PROJECTION_YQ }} = "{name}/${{ inputs.version }}"' spack.yaml echo '::notice::Prerelease accessible as module `${{ inputs.model}}/${{ inputs.version }}`' @@ -135,6 +135,10 @@ jobs: exit 1 fi + # Export vars.DEPLOYMENT_TARGET + export DEPLOYMENT_TARGET="${{ vars.DEPLOYMENT_TARGET }}" + echo "DEPLOYMENT_TARGET exported as $DEPLOYMENT_TARGET" + # Update spack-packages git -C ${{ steps.path.outputs.spack-packages }} fetch git -C ${{ steps.path.outputs.spack-packages }} checkout --force ${{ steps.versions.outputs.packages }} @@ -160,7 +164,11 @@ jobs: env: SPACK_ENV_PATH: ${{ steps.path.outputs.spack }}/../environments/${{ inputs.env-name }} run: | - ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + # Export vars.DEPLOYMENT_TARGET + export DEPLOYMENT_TARGET="${{ vars.DEPLOYMENT_TARGET }}" + echo "DEPLOYMENT_TARGET exported as $DEPLOYMENT_TARGET" + . ${{ steps.path.outputs.spack-config }}/spack-enable.bash spack env activate ${{ inputs.env-name }} diff --git a/.github/workflows/undeploy-1-start.yml b/.github/workflows/undeploy-1-start.yml index e2d6564..e2bfc42 100644 --- a/.github/workflows/undeploy-1-start.yml +++ b/.github/workflows/undeploy-1-start.yml @@ -28,7 +28,7 @@ jobs: - name: Validate Inputs id: validate run: | - if [[ "${{ inputs.type }}" != "Prerelease" || "${{ inputs.type }}" != "Release" ]]; then + if [[ "${{ inputs.type }}" != "Prerelease" && "${{ inputs.type }}" != "Release" ]]; then echo "::error::Invalid type: ${{ inputs.type }}. Must be one of: Prerelease, Release." exit 1 fi From beee2cf448850f4b03e4e2a01be9513b94ea5289 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Tue, 28 Jan 2025 15:10:59 +1100 Subject: [PATCH 37/40] Split `deployment-environment` input into `deployment-target` and `deployment-type` --- .github/workflows/deploy-1-setup.yml | 3 ++- .github/workflows/deploy-2-start.yml | 33 ++++++++++++++++++---------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/deploy-1-setup.yml b/.github/workflows/deploy-1-setup.yml index 7939a08..e431f4f 100644 --- a/.github/workflows/deploy-1-setup.yml +++ b/.github/workflows/deploy-1-setup.yml @@ -288,7 +288,8 @@ jobs: ref: ${{ inputs.deployment-ref }} version: ${{ inputs.deployment-version }} env-name: ${{ needs.check-spack-yaml.outputs.spack-env-name }} - deployment-environment: ${{ inputs.deployment-type == 'Prerelease' && format('{0} Prerelease', inputs.deployment-target) || inputs.deployment-target }} + deployment-target: ${{ inputs.deployment-target }} + deployment-type: ${{ inputs.deployment-type }} root-sbd: ${{ inputs.spack-manifest-root-sbd }} secrets: inherit diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index 8b56db1..6ae37d3 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -1,5 +1,5 @@ name: Deploy Start -concurrency: ${{ inputs.deployment-environment }} +concurrency: ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} on: workflow_call: inputs: @@ -23,10 +23,19 @@ on: type: string required: true description: The spack-env-compliant environment name for the model - deployment-environment: + deployment-target: type: string required: true - description: The GitHub deployment environment name + description: | + The GitHub Environment, minus a potential Type. + Combined with inputs.deployment-type to create the GitHub Environment name. + deployment-type: + type: string + required: true + description: | + The type of deployment. + Can be one of: Release, Prerelease. + Combined with inputs.deployment-type to create the GitHub Environment name. root-sbd: type: string required: true @@ -49,12 +58,12 @@ env: SPACK_YAML_MODEL_YQ: (.spack.definitions[] | select(."ROOT_PACKAGE") | .[][]) // .spack.specs[0] SPACK_YAML_MODULEFILE_PROJECTION_YQ: .spack.modules.default.tcl.projections.${{ inputs.root-sbd }} METADATA_PATH: /opt/metadata - ARTIFACT_NAME: deploy-metadata.${{ inputs.deployment-environment }} + ARTIFACT_NAME: deploy-metadata.${{ inputs.deployment-target }} jobs: deploy-to-environment: - name: Deploy to ${{ inputs.deployment-environment }} + name: Deploy to ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} runs-on: ubuntu-latest - environment: ${{ inputs.deployment-environment }} + environment: ${{ inputs.deployment-target }} ${{ inputs.deployment-type != 'Release' && inputs.deployment-type || '' }} outputs: packages-version: ${{ steps.versions.outputs.packages }} config-version: ${{ steps.versions.outputs.config }} @@ -73,13 +82,13 @@ jobs: echo "spack=$(jq --compact-output --raw-output '.spack' ./config/versions.json)" >> $GITHUB_OUTPUT echo "packages=$(jq --compact-output --raw-output '."spack-packages"' ./config/versions.json)" >> $GITHUB_OUTPUT - - name: Get ${{ inputs.deployment-environment }} Remote Paths + - name: Get ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} Remote Paths id: path uses: access-nri/build-cd/.github/actions/get-deploy-paths@main with: spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }} spack-version: ${{ steps.versions.outputs.spack }} - deployment-environment: ${{ inputs.deployment-environment }} + deployment-environment: ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} - name: Setup SSH id: ssh @@ -125,13 +134,13 @@ jobs: done EOT - - name: Deploy to ${{ inputs.deployment-environment }} + - name: Deploy to ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} # ssh into deployment environment, create and activate the env, install the spack.yaml. run: | ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' # Check that a suitable deployment location exists if [ ! -d "${{ steps.path.outputs.root }}" ]; then - echo '::error::A deployment of spack does not exist in `${{ steps.path.outputs.root }}` for `${{ inputs.deployment-environment }}`' + echo '::error::A deployment of spack does not exist in `${{ steps.path.outputs.root }}` for `${{ inputs.deployment-target }} ${{ inputs.deployment-type}}`' exit 1 fi @@ -160,7 +169,7 @@ jobs: echo "spack=${{ steps.path.outputs.spack }}" >> $GITHUB_OUTPUT echo "modules=${{ vars.DEPLOYED_MODULES_DIR }}" >> $GITHUB_OUTPUT - - name: Get metadata from ${{ inputs.deployment-environment }} + - name: Get metadata from ${{ inputs.deployment-target }} ${{ inputs.deployment-type}} env: SPACK_ENV_PATH: ${{ steps.path.outputs.spack }}/../environments/${{ inputs.env-name }} run: | @@ -221,7 +230,7 @@ jobs: # Rename the files to include the deployment environment cd ./${{ env.ARTIFACT_NAME }} for file in *; do - mv "$file" "${{ inputs.deployment-environment }}.$file" + mv "$file" "${{ inputs.deployment-target }}.$file" done cd - From 6299c48ec38d9faa58bce41c0d6f510c9204bf88 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Wed, 29 Jan 2025 09:57:42 +1100 Subject: [PATCH 38/40] Update all build-cd refs to latest `v4`, get latest default branch ref for non-action calls --- .github/actions/get-deploy-paths/README.md | 2 +- .github/actions/get-target-matrix/README.md | 2 +- .github/actions/get-target-matrix/action.yml | 7 ++++++- .../validate-deployment-settings/README.md | 2 +- .../actions/validate-repo-version/README.md | 2 +- .github/workflows/cd.yml | 20 +++++++++++++------ .github/workflows/ci-closed.yml | 4 ++-- .github/workflows/ci.yml | 10 +++++++--- .github/workflows/deploy-1-setup.yml | 16 +++++++++------ .github/workflows/deploy-2-start.yml | 2 +- .github/workflows/settings-1-update.yml | 11 +++++++--- .github/workflows/undeploy-1-start.yml | 2 +- 12 files changed, 53 insertions(+), 27 deletions(-) diff --git a/.github/actions/get-deploy-paths/README.md b/.github/actions/get-deploy-paths/README.md index 9e8df42..3cf5a49 100644 --- a/.github/actions/get-deploy-paths/README.md +++ b/.github/actions/get-deploy-paths/README.md @@ -30,7 +30,7 @@ jobs: steps: - name: Get Deployment Paths id: paths - uses: access-nri/build-cd/.github/actions/get-deploy-paths@main + uses: access-nri/build-cd/.github/actions/get-deploy-paths@vX with: spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_PATH }} spack-version: "0.21" diff --git a/.github/actions/get-target-matrix/README.md b/.github/actions/get-target-matrix/README.md index 4c9239c..3127974 100644 --- a/.github/actions/get-target-matrix/README.md +++ b/.github/actions/get-target-matrix/README.md @@ -25,7 +25,7 @@ jobs: valid-targets: ${{ steps.generate.outputs.valid-targets }} steps: - id: generate - uses: access-nri/build-cd/.github/actions/get-target-matrix@main + uses: access-nri/build-cd/.github/actions/get-target-matrix@vX with: targets: ${{ vars.MODEL_REPO_TARGETS }} diff --git a/.github/actions/get-target-matrix/action.yml b/.github/actions/get-target-matrix/action.yml index ca930e4..79b9835 100644 --- a/.github/actions/get-target-matrix/action.yml +++ b/.github/actions/get-target-matrix/action.yml @@ -13,11 +13,16 @@ outputs: runs: using: composite steps: + - name: Get default branch + shell: bash + id: default + run: echo "branch=$(gh repo view access-nri/build-cd --json defaultBranchRef --jq '.[].name')" >> $GITHUB_OUTPUT + - name: Get deployment settings.json uses: actions/checkout@v4 with: repository: access-nri/build-cd - ref: main + ref: ${{ steps.default.outputs.branch }} - name: Generate Deployment Target Matrix shell: bash diff --git a/.github/actions/validate-deployment-settings/README.md b/.github/actions/validate-deployment-settings/README.md index 767eb9e..ccd292b 100644 --- a/.github/actions/validate-deployment-settings/README.md +++ b/.github/actions/validate-deployment-settings/README.md @@ -22,7 +22,7 @@ This action validates various `ACCESS-NRI/build-cd` deployment settings. # ... - name: Validate settings id: settings - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@main + uses: access-nri/build-cd/.github/actions/validate-deployment-settings@vX with: settings-path: ./some/settings.json target: Supercomputer diff --git a/.github/actions/validate-repo-version/README.md b/.github/actions/validate-repo-version/README.md index b010359..cd8848d 100644 --- a/.github/actions/validate-repo-version/README.md +++ b/.github/actions/validate-repo-version/README.md @@ -20,7 +20,7 @@ This action checks that the tags specified in a models `config/versions.json` is ```yaml # ... - id: validate - uses: access-nri/build-cd/.github/actions/validate-repo-version@main + uses: access-nri/build-cd/.github/actions/validate-repo-version@vX with: repo-to-check: spack-packages pr: 12 diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 7b4f227..7d5e939 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -52,7 +52,7 @@ jobs: - name: Generate Deployment Target Matrix id: target - uses: access-nri/build-cd/.github/actions/get-target-matrix@v3 + uses: access-nri/build-cd/.github/actions/get-target-matrix@v4 with: targets: ${{ vars.RELEASE_DEPLOYMENT_TARGETS }} @@ -65,12 +65,16 @@ jobs: matrix: target: ${{ fromJson(needs.defaults.outputs.targets) }} steps: + - name: Get default branch + id: default + run: echo "branch=$(gh repo view access-nri/build-cd --json defaultBranchRef --jq '.[].name')" >> $GITHUB_OUTPUT + - uses: actions/checkout@v4 with: - repository: ACCESS-NRI/build-cd - ref: main + repository: access-nri/build-cd + ref: ${{ steps.default.outputs.branch }} - - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@main + - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v4 with: settings-path: ./config/settings.json target: ${{ matrix.target }} @@ -121,7 +125,7 @@ jobs: strategy: matrix: target: ${{ fromJson(needs.defaults.outputs.targets) }} - uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v3 + uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v4 with: deployment-target: ${{ matrix.target }} deployment-ref: ${{ github.ref_name }} @@ -149,11 +153,15 @@ jobs: url: ${{ steps.release.outputs.url }} created-at: ${{ steps.metadata.outputs.created-at }} steps: + - name: Get default branch + id: default + run: echo "branch=$(gh repo view access-nri/build-cd --json defaultBranchRef --jq '.[].name')" >> $GITHUB_OUTPUT + - name: Checkout build-cd uses: actions/checkout@v4 with: repository: access-nri/build-cd - ref: v3 + ref: ${{ steps.default.outputs.branch }} - name: Download Metadata Artifacts uses: actions/download-artifact@v4 diff --git a/.github/workflows/ci-closed.yml b/.github/workflows/ci-closed.yml index 1f1431b..5943fdd 100644 --- a/.github/workflows/ci-closed.yml +++ b/.github/workflows/ci-closed.yml @@ -38,7 +38,7 @@ jobs: - name: Generate Deployment Target Matrix id: target - uses: access-nri/build-cd/.github/actions/get-target-matrix@v3 + uses: access-nri/build-cd/.github/actions/get-target-matrix@v4 with: targets: ${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }} @@ -50,7 +50,7 @@ jobs: matrix: target: ${{ fromJson(needs.setup.outputs.targets) }} fail-fast: false - uses: access-nri/build-cd/.github/workflows/undeploy-1-start.yml@main + uses: access-nri/build-cd/.github/workflows/undeploy-1-start.yml@v4 with: version-pattern: ${{ inputs.root-sbd }}-pr${{ github.event.pull_request.number }}-* target: ${{ matrix.target }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 333d76c..615ca89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,7 @@ jobs: - name: Generate Deployment Target Matrix id: target - uses: access-nri/build-cd/.github/actions/get-target-matrix@v3 + uses: access-nri/build-cd/.github/actions/get-target-matrix@v4 with: targets: ${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }} @@ -189,7 +189,7 @@ jobs: matrix: # Example: ['Gadi', 'Setonix', ...] target: ${{ fromJson(needs.defaults.outputs.targets) }} - uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@main + uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v4 with: deployment-target: ${{ matrix.target }} deployment-ref: ${{ needs.defaults.outputs.head-ref }} @@ -245,11 +245,15 @@ jobs: OUTPUT_ARTIFACT_PATH: ./merged_outputs TEMPLATED_COMMENT_BODY_PATH: /opt/comment-body.md steps: + - name: Get default branch + id: default + run: echo "branch=$(gh repo view access-nri/build-cd --json defaultBranchRef --jq '.[].name')" >> $GITHUB_OUTPUT + - name: Checkout build-cd for scripts uses: actions/checkout@v4 with: repository: access-nri/build-cd - ref: v3 + ref: ${{ steps.default.outputs.branch }} - name: Download matrix deployment outputs uses: actions/download-artifact@v4 diff --git a/.github/workflows/deploy-1-setup.yml b/.github/workflows/deploy-1-setup.yml index e431f4f..1a6debc 100644 --- a/.github/workflows/deploy-1-setup.yml +++ b/.github/workflows/deploy-1-setup.yml @@ -121,23 +121,27 @@ jobs: - name: Validate spack-packages version id: spack-packages - uses: access-nri/build-cd/.github/actions/validate-repo-version@main + uses: access-nri/build-cd/.github/actions/validate-repo-version@v4 with: repo-to-check: spack-packages pr: ${{ inputs.deployment-ref }} - name: Validate spack version id: spack - uses: access-nri/build-cd/.github/actions/validate-repo-version@main + uses: access-nri/build-cd/.github/actions/validate-repo-version@v4 with: repo-to-check: spack pr: ${{ inputs.deployment-ref }} + - name: Get default branch + id: default + run: echo "branch=$(gh repo view access-nri/build-cd --json defaultBranchRef --jq '.[].name')" >> $GITHUB_OUTPUT + - name: Checkout build-cd Config uses: actions/checkout@v4 with: - repository: ACCESS-NRI/build-cd - ref: main + repository: access-nri/build-cd + ref: ${{ steps.default.outputs.branch }} path: cd - name: Get spack-config version @@ -152,7 +156,7 @@ jobs: - name: Validate build-cd config/settings.json id: settings - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@main + uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v4 with: settings-path: ./cd/config/settings.json target: ${{ inputs.deployment-target }} @@ -281,7 +285,7 @@ jobs: needs: - check-config # Verify configuration information is correct - check-spack-yaml # Verify spack manifest information is correct - uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@v3 + uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@v4 with: type: ${{ inputs.deployment-type }} model: ${{ inputs.spack-manifest-root-sbd }} diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index 6ae37d3..2fb780a 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -84,7 +84,7 @@ jobs: - name: Get ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} Remote Paths id: path - uses: access-nri/build-cd/.github/actions/get-deploy-paths@main + uses: access-nri/build-cd/.github/actions/get-deploy-paths@v4 with: spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }} spack-version: ${{ steps.versions.outputs.spack }} diff --git a/.github/workflows/settings-1-update.yml b/.github/workflows/settings-1-update.yml index 018c7f2..934c144 100644 --- a/.github/workflows/settings-1-update.yml +++ b/.github/workflows/settings-1-update.yml @@ -20,11 +20,16 @@ jobs: outputs: targets: ${{ steps.target.outputs.matrix }} steps: + - name: Get default branch + shell: bash + id: default + run: echo "branch=$(gh repo view access-nri/build-cd --json defaultBranchRef --jq '.[].name')" >> $GITHUB_OUTPUT + - name: Get deployment settings.json uses: actions/checkout@v4 with: repository: access-nri/build-cd - ref: main + ref: ${{ steps.default.outputs.branch }} - name: Generate Deployment Target Matrix id: target @@ -54,7 +59,7 @@ jobs: - name: Validate Deployment Settings id: validate - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@main + uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v4 with: settings-path: ${{ env.CONFIG_SETTINGS_PATH }} target: ${{ matrix.target }} @@ -147,7 +152,7 @@ jobs: # - deployment-environment: Gadi # type: Prerelease # etc ... - uses: access-nri/build-cd/.github/workflows/settings-2-deploy.yml@main + uses: access-nri/build-cd/.github/workflows/settings-2-deploy.yml@v4 with: deployment-environment: ${{ matrix.update.deployment-environment }} spack-type: ${{ matrix.update.type }} diff --git a/.github/workflows/undeploy-1-start.yml b/.github/workflows/undeploy-1-start.yml index e2bfc42..3d514bf 100644 --- a/.github/workflows/undeploy-1-start.yml +++ b/.github/workflows/undeploy-1-start.yml @@ -48,7 +48,7 @@ jobs: - name: Get ${{ inputs.target }} Remote Paths id: path - uses: access-nri/build-cd/.github/actions/get-deploy-paths@main + uses: access-nri/build-cd/.github/actions/get-deploy-paths@v4 with: spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }} spack-version: ${{ steps.versions.outputs.spack }} From 2b635fbe942ce9d42c6b9782ed71488e0b272c9e Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Fri, 31 Jan 2025 09:39:26 +1100 Subject: [PATCH 39/40] Use default `inputs.ref` for `checkout` action to get default branch --- .github/actions/get-target-matrix/action.yml | 6 ------ .github/workflows/cd.yml | 10 ---------- .github/workflows/ci.yml | 5 ----- .github/workflows/deploy-1-setup.yml | 5 ----- .github/workflows/settings-1-update.yml | 6 ------ 5 files changed, 32 deletions(-) diff --git a/.github/actions/get-target-matrix/action.yml b/.github/actions/get-target-matrix/action.yml index 79b9835..576b971 100644 --- a/.github/actions/get-target-matrix/action.yml +++ b/.github/actions/get-target-matrix/action.yml @@ -13,16 +13,10 @@ outputs: runs: using: composite steps: - - name: Get default branch - shell: bash - id: default - run: echo "branch=$(gh repo view access-nri/build-cd --json defaultBranchRef --jq '.[].name')" >> $GITHUB_OUTPUT - - name: Get deployment settings.json uses: actions/checkout@v4 with: repository: access-nri/build-cd - ref: ${{ steps.default.outputs.branch }} - name: Generate Deployment Target Matrix shell: bash diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 7d5e939..7685dd3 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -65,14 +65,9 @@ jobs: matrix: target: ${{ fromJson(needs.defaults.outputs.targets) }} steps: - - name: Get default branch - id: default - run: echo "branch=$(gh repo view access-nri/build-cd --json defaultBranchRef --jq '.[].name')" >> $GITHUB_OUTPUT - - uses: actions/checkout@v4 with: repository: access-nri/build-cd - ref: ${{ steps.default.outputs.branch }} - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v4 with: @@ -153,15 +148,10 @@ jobs: url: ${{ steps.release.outputs.url }} created-at: ${{ steps.metadata.outputs.created-at }} steps: - - name: Get default branch - id: default - run: echo "branch=$(gh repo view access-nri/build-cd --json defaultBranchRef --jq '.[].name')" >> $GITHUB_OUTPUT - - name: Checkout build-cd uses: actions/checkout@v4 with: repository: access-nri/build-cd - ref: ${{ steps.default.outputs.branch }} - name: Download Metadata Artifacts uses: actions/download-artifact@v4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 615ca89..6fa8fbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,15 +245,10 @@ jobs: OUTPUT_ARTIFACT_PATH: ./merged_outputs TEMPLATED_COMMENT_BODY_PATH: /opt/comment-body.md steps: - - name: Get default branch - id: default - run: echo "branch=$(gh repo view access-nri/build-cd --json defaultBranchRef --jq '.[].name')" >> $GITHUB_OUTPUT - - name: Checkout build-cd for scripts uses: actions/checkout@v4 with: repository: access-nri/build-cd - ref: ${{ steps.default.outputs.branch }} - name: Download matrix deployment outputs uses: actions/download-artifact@v4 diff --git a/.github/workflows/deploy-1-setup.yml b/.github/workflows/deploy-1-setup.yml index 1a6debc..0ff38b2 100644 --- a/.github/workflows/deploy-1-setup.yml +++ b/.github/workflows/deploy-1-setup.yml @@ -133,15 +133,10 @@ jobs: repo-to-check: spack pr: ${{ inputs.deployment-ref }} - - name: Get default branch - id: default - run: echo "branch=$(gh repo view access-nri/build-cd --json defaultBranchRef --jq '.[].name')" >> $GITHUB_OUTPUT - - name: Checkout build-cd Config uses: actions/checkout@v4 with: repository: access-nri/build-cd - ref: ${{ steps.default.outputs.branch }} path: cd - name: Get spack-config version diff --git a/.github/workflows/settings-1-update.yml b/.github/workflows/settings-1-update.yml index 934c144..9c4ac0d 100644 --- a/.github/workflows/settings-1-update.yml +++ b/.github/workflows/settings-1-update.yml @@ -20,16 +20,10 @@ jobs: outputs: targets: ${{ steps.target.outputs.matrix }} steps: - - name: Get default branch - shell: bash - id: default - run: echo "branch=$(gh repo view access-nri/build-cd --json defaultBranchRef --jq '.[].name')" >> $GITHUB_OUTPUT - - name: Get deployment settings.json uses: actions/checkout@v4 with: repository: access-nri/build-cd - ref: ${{ steps.default.outputs.branch }} - name: Generate Deployment Target Matrix id: target From 23bf067321ea109629b2a21aca8648b7ab8694c4 Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Mon, 3 Feb 2025 14:19:19 +1100 Subject: [PATCH 40/40] Remove conditional logic for `deployment-type`s This is because all environment are now of the form SUPERCOMPUTER TYPE --- .github/workflows/deploy-1-setup.yml | 3 +-- .github/workflows/deploy-2-start.yml | 12 ++++-------- .github/workflows/undeploy-1-start.yml | 4 ++-- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/deploy-1-setup.yml b/.github/workflows/deploy-1-setup.yml index 0ff38b2..7ceef77 100644 --- a/.github/workflows/deploy-1-setup.yml +++ b/.github/workflows/deploy-1-setup.yml @@ -8,7 +8,7 @@ on: required: true description: | The target machine for the model being deployed. - Equivalent to the name of the GitHub Environment, minus a potential Prerelease Type. + Equivalent to the name of the GitHub Environment, minus a Release/Prerelease Type. deployment-type: type: string required: true @@ -282,7 +282,6 @@ jobs: - check-spack-yaml # Verify spack manifest information is correct uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@v4 with: - type: ${{ inputs.deployment-type }} model: ${{ inputs.spack-manifest-root-sbd }} ref: ${{ inputs.deployment-ref }} version: ${{ inputs.deployment-version }} diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index 2fb780a..d3d3615 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -3,10 +3,6 @@ concurrency: ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} on: workflow_call: inputs: - type: - type: string - required: true - description: The type of deployment - either 'Release' or 'Prerelease' model: type: string required: true @@ -27,7 +23,7 @@ on: type: string required: true description: | - The GitHub Environment, minus a potential Type. + The GitHub Environment, minus a Type. Combined with inputs.deployment-type to create the GitHub Environment name. deployment-type: type: string @@ -35,7 +31,7 @@ on: description: | The type of deployment. Can be one of: Release, Prerelease. - Combined with inputs.deployment-type to create the GitHub Environment name. + Combined with inputs.deployment-target to create the GitHub Environment name. root-sbd: type: string required: true @@ -63,7 +59,7 @@ jobs: deploy-to-environment: name: Deploy to ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} runs-on: ubuntu-latest - environment: ${{ inputs.deployment-target }} ${{ inputs.deployment-type != 'Release' && inputs.deployment-type || '' }} + environment: ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} outputs: packages-version: ${{ steps.versions.outputs.packages }} config-version: ${{ steps.versions.outputs.config }} @@ -100,7 +96,7 @@ jobs: ${{ secrets.HOST_DATA }} - name: Prerelease spack.yaml Modifications - if: inputs.type == 'Prerelease' + if: inputs.deployment-type == 'Prerelease' # Modifies the name of the prerelease modulefile to the # `pr-` style. For example, `access-om3/pr12-2`. # Also removes the `@git.VERSION` specifier for Prereleases so diff --git a/.github/workflows/undeploy-1-start.yml b/.github/workflows/undeploy-1-start.yml index 3d514bf..c7fbbf1 100644 --- a/.github/workflows/undeploy-1-start.yml +++ b/.github/workflows/undeploy-1-start.yml @@ -38,7 +38,7 @@ jobs: name: Undeploy ${{ inputs.version-pattern }} from ${{ inputs.target }} ${{ inputs.type }} needs: validate-inputs runs-on: ubuntu-latest - environment: ${{ inputs.target }} ${{ inputs.type != 'Release' && inputs.type }} + environment: ${{ inputs.target }} ${{ inputs.type }} steps: - uses: actions/checkout@v4 @@ -52,7 +52,7 @@ jobs: with: spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }} spack-version: ${{ steps.versions.outputs.spack }} - deployment-environment: ${{ inputs.target }} ${{ inputs.type != 'Release' && inputs.type }} + deployment-environment: ${{ inputs.target }} ${{ inputs.type }} - name: Setup SSH id: ssh