Skip to content

Commit 2df1b30

Browse files
CodeGatCodeGat
authored
Merge pull request #34 from ACCESS-NRI/32-prerelease
Generic `Prerelease` Functionality
2 parents dba15e8 + 79f8341 commit 2df1b30

8 files changed

+233
-23
lines changed

‎.github/workflows/create-deployment-spack.yml

+7-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@ on:
1616
required: true
1717
default: main
1818
description: A version of ACCESS-NRI/spack-config
19+
deployment-location:
20+
type: string
21+
required: true
22+
description: |
23+
A path in the deployment environment where Spack should be created.
24+
For example, if it is `opt/spack`, spack will be installed under `opt/spack/<spack-version>/`
1925
jobs:
2026
create-spack:
2127
name: Spack
@@ -39,7 +45,7 @@ jobs:
3945

4046
- name: Install
4147
env:
42-
ROOT_VERSION_LOCATION: ${{ vars.ROOT_SPACK_LOCATION }}/${{ steps.strip.outputs.version-dir }}
48+
ROOT_VERSION_LOCATION: ${{ inputs.deployment-location }}/${{ steps.strip.outputs.version-dir }}
4349
# This step will fail if `mkdir` fails to create a version directory (e.g. `0.20` is already deployed and exists)
4450
run: |
4551
ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'

‎.github/workflows/deploy-1-setup.yml

+27-7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
name: Deployment Setup
22
on:
33
workflow_call:
4+
inputs:
5+
type:
6+
type: string
7+
required: false
8+
default: release
9+
description: The type of deployment - either 'release' or 'prerelease'
10+
version:
11+
type: string
12+
required: true
13+
description: The version of the model being deployed
414
jobs:
515
setup-spack-env:
616
name: Setup Spack Environment
@@ -15,22 +25,31 @@ jobs:
1525
run: echo "model=$(echo ${{ github.event.repository.name }} | tr [:upper:] [:lower:])" >> $GITHUB_OUTPUT
1626
- name: Set Spack Env Name String
1727
id: get-env-name
18-
# replace occurences of '.' with '_' in environment name as spack doesn't support '.'. Ex: 'access-om2-v1.0.0' -> 'access-om2-v1_0_0'.
19-
run: echo "env-name=$(echo '${{ steps.get-model.outputs.model }}-${{ github.ref_name }}' | tr '.' '_')" >> $GITHUB_OUTPUT
20-
28+
# replace occurences of '.' with '_' in environment name as spack doesn't support '.'. Ex: 'access-om2-v1.0.0' -> 'access-om2-v1_0_0'.
29+
run: echo "env-name=$(echo '${{ steps.get-model.outputs.model }}-${{ inputs.version }}' | tr '.' '_')" >> $GITHUB_OUTPUT
30+
2131
setup-deployment-env:
2232
name: Setup Deployment Environment
2333
runs-on: ubuntu-latest
2434
outputs:
2535
deployment-environments: ${{ steps.get-deployment-environment.outputs.deployment-environments }}
2636
steps:
27-
- name: Checkout config
37+
- name: Checkout Config
2838
uses: actions/checkout@v4
2939
with:
3040
repository: access-nri/build-cd
31-
- name: Get environments
41+
42+
- name: Get Environments
3243
id: get-deployment-environment
33-
run: echo "deployment-environments=$(jq --compact-output '.environments' ./config/deployment-environment.json)" >> $GITHUB_OUTPUT
44+
run: |
45+
if [[ "${{ inputs.type }}" == "release" ]]; then
46+
echo "deployment-environments=$(jq --compact-output '.environments' ./config/deployment-environment.json)" >> $GITHUB_OUTPUT
47+
else if [[ "${{ inputs.type }}" == "prerelease" ]]; then
48+
echo "deployment-environments=$(jq --compact-output '.prerelease-environments' ./config/deployment-environment.json)" >> $GITHUB_OUTPUT
49+
else
50+
echo "::error::The 'type' input was invalid. Check the inputs documentation."
51+
exit 1
52+
fi
3453
3554
deployment:
3655
name: Deployment
@@ -43,8 +62,9 @@ jobs:
4362
deployment-environment: ${{ fromJson(needs.setup-deployment-env.outputs.deployment-environments) }}
4463
uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@main
4564
with:
65+
type: ${{ inputs.type }}
4666
model: ${{ needs.setup-spack-env.outputs.model }}
47-
version: ${{ github.ref_name }}
67+
version: ${{ inputs.version }}
4868
env-name: ${{ needs.setup-spack-env.outputs.env-name }}
4969
deployment-environment: ${{ matrix.deployment-environment }}
5070
secrets: inherit

‎.github/workflows/deploy-2-start.yml

+7
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ concurrency: ${{ inputs.deployment-environment }}
33
on:
44
workflow_call:
55
inputs:
6+
type:
7+
type: string
8+
required: true
9+
description: The type of deployment - either 'release' or 'prerelease'
610
model:
711
type: string
812
required: true
@@ -72,11 +76,14 @@ jobs:
7276
7377
# Release
7478
- name: Get Release Metadata
79+
if: inputs.type == 'release'
7580
run: |
7681
rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
7782
'${{ secrets.USER}}@${{ secrets.HOST_DATA }}:${{ vars.SPACK_LOCATION }}/var/spack/environments/${{ inputs.env-name }}/spack.*' \
7883
./${{ inputs.env-name }}
84+
7985
- name: Create Release
86+
if: inputs.type == 'release'
8087
uses: softprops/[email protected]
8188
with:
8289
tag_name: ${{ inputs.version }}

‎.github/workflows/undeploy-1-setup.yml

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: Undeployment Setup
2+
on:
3+
workflow_call:
4+
inputs:
5+
version-pattern:
6+
type: string
7+
required: true
8+
description: A wildcard-supported string for version(s) of the model being removed
9+
jobs:
10+
get-spack-env:
11+
name: Get Spack Environment
12+
runs-on: ubuntu-latest
13+
outputs:
14+
model: ${{ steps.get-model.outputs.model }}
15+
env-name: ${{ steps.get-env-name.outputs.env-name }}
16+
steps:
17+
- name: Get Model
18+
id: get-model
19+
# for the cases where the repo name is in uppercase but the package name is lowercase (eg. access-nri/MOM5)
20+
run: echo "model=$(echo ${{ github.event.repository.name }} | tr [:upper:] [:lower:])" >> $GITHUB_OUTPUT
21+
- name: Set Spack Env Name String
22+
id: get-env-name
23+
# replace occurences of '.' with '_' in environment name as spack doesn't support '.'. Ex: 'access-om2-v1.0.0' -> 'access-om2-v1_0_0'.
24+
run: echo "env-name=$(echo '${{ steps.get-model.outputs.model }}-${{ inputs.version-pattern }}' | tr '.' '_')" >> $GITHUB_OUTPUT
25+
26+
get-prerelease-deployment-env:
27+
name: Get Prerelease Deployment Environment
28+
runs-on: ubuntu-latest
29+
outputs:
30+
deployment-environments: ${{ steps.get-deployment-environment.outputs.deployment-environments }}
31+
steps:
32+
- name: Checkout Config
33+
uses: actions/checkout@v4
34+
with:
35+
repository: access-nri/build-cd
36+
37+
- name: Get Environments
38+
id: get-deployment-environment
39+
run: echo "deployment-environments=$(jq --compact-output '.prerelease-environments' ./config/deployment-environment.json)" >> $GITHUB_OUTPUT
40+
41+
42+
undeployment:
43+
name: Remove Deployment
44+
needs:
45+
- get-spack-env
46+
- get-prerelease-deployment-env
47+
strategy:
48+
fail-fast: false
49+
matrix:
50+
deployment-environment: ${{ fromJson(needs.get-prerelease-deployment-env.outputs.deployment-environments) }}
51+
uses: access-nri/build-cd/.github/workflows/undeploy-2-start.yml@main
52+
with:
53+
deployment-environment: ${{ matrix.deployment-environment }}
54+
env-name: ${{ needs.get-spack-env.outputs.env-name }}
55+
secrets: inherit

‎.github/workflows/undeploy-2-start.yml

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: Undeployment Start
2+
on:
3+
workflow_call:
4+
inputs:
5+
env-name:
6+
type: string
7+
required: true
8+
description: The spack-env-compliant model name to remove
9+
deployment-environment:
10+
type: string
11+
required: true
12+
description: The GitHub deployment environment name
13+
jobs:
14+
undeploy-from-environment:
15+
name: Undeploy ${{ inputs.env-name }} from ${{ inputs.deployment-environment }}
16+
runs-on: ubuntu-latest
17+
environment: ${{ inputs.deployment-environment }}
18+
steps:
19+
- name: Setup SSH
20+
id: ssh
21+
uses: access-nri/actions/.github/actions/setup-ssh@main
22+
with:
23+
private-key: ${{ secrets.SSH_KEY }}
24+
hosts: |
25+
${{ secrets.HOST }}
26+
${{ secrets.HOST_DATA }}
27+
28+
- name: Undeploy
29+
# ssh into deployment environment, create and activate the env, remove all the unneeded environments
30+
id: undeploy
31+
run: |
32+
ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
33+
. ${{ vars.SPACK_CONFIG_LOCATION }}/spack-enable.bash
34+
envs=$(find ${{ vars.SPACK_LOCATION }}/var/spack/environments -type d -name '${{ inputs.env-name }}' -printf '%f ')
35+
36+
for env in $envs; do
37+
spack env activate $env
38+
spack uninstall --remove --dependents --yes-to-all --all
39+
spack env deactivate $env
40+
spack env rm $env --yes-to-all
41+
done
42+
spack gc --yes-to-all
43+
EOT
44+
45+
- name: Undeploy Status Notifier
46+
if: always()
47+
run: |
48+
if [[ "${{ steps.undeploy.outcome }}" == "success" ]]; then
49+
echo "::notice::Deployment ${{ inputs.env-name }} was successfully removed from ${{ inputs.deployment-environment }}, found at ${{ vars.SPACK_LOCATION }}"
50+
else
51+
echo "::error::Deployment ${{ inputs.env-name }} couldn't be removed from ${{ inputs.deployment-environment}}, found at ${{ vars.SPACK_LOCATION }}. Please check manually."
52+
fi

‎README.md

+75-14
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,102 @@
11
# build-cd
22

3-
This repository houses reusable workflows for the building and deployment of ACCESS-NRI models to different environments.
3+
This repository houses reusable workflows for the building and deployment of ACCESS-NRI models to different environments.
44

55
## Overview
66

7-
This repository is broken down into two folders: `config` and `.github/workflows`.
7+
This repository is broken down into two folders: `config` and `.github/workflows`.
88

9-
`config` contains information on the deployment environments that the models deploy to, which is currently just the name of the deployment target. This is used by the aforementioned deployment workflows to gather secrets and configuration details from the associated GitHub Environment.
9+
`config` contains information on the deployment environments that the models deploy to, which is currently just the name of the deployment target. This is used by the aforementioned deployment workflows to gather secrets and configuration details from the associated GitHub Environment.
1010

1111
`.github/workflows` houses validation and reusable deployment workflows that are called by ACCESS-NRI model repositories. Currently, only [ACCESS-NRI/ACCESS-OM2](https://github.com/ACCESS-NRI/access-om2) is supported.
1212

13-
Below is a brief summary of the two pipelines, `deploy-*` and `validate-json`.
13+
Below is a brief summary of the three pipelines, `deploy-*`, `undeploy-*` and `validate-json`.
1414

15-
### deploy-*
15+
### `deploy-*`
1616

17-
This pipeline is responsible for the gathering of configuration information, building and deployment of whatever model repository calls it. It is split into two workflows, as noted below.
17+
#### Inputs
1818

19-
#### deploy-1-setup.yml
19+
| Name | Type | Description | Required | Default | Example |
20+
| ---- | ---- | ----------- | -------- | ------- | ------- |
21+
| `type` | string | The type of deployment - either 'release' or 'prerelease' | true | `release` | `prerelease` |
22+
| `version` | string | The version of the model being deployed | true | N/A | `2024.01.1` |
2023

21-
This workflow obtains the relevant spack and GitHub Environment information, and creates parallelized deployments based on the list of environments.
24+
#### Explanation
2225

23-
#### deploy-2-start.yml
26+
This pipeline is responsible for the gathering of configuration information, building and deployment of whatever model repository calls it. It is split into two workflows.
2427

25-
Using the GitHub Environment, it `ssh`s into the deployment environments `spack` instance, and installs the model associated with the repository that called it. It then copies back relevant metadata and creates a versioned GitHub Release in the caller repository.
28+
##### `deploy-1-setup.yml`
2629

27-
### validate-json
30+
This workflow obtains the relevant spack and GitHub Environment information, and creates parallelized deployments based on the list of environments.
2831

29-
This workflow is used to validate the `config` folders `*.json` files based on their associated `*.schema.json`. This is used for PR checks on the `build-cd` repo itself.
32+
##### `deploy-2-start.yml`
3033

31-
## Usage
34+
Using the GitHub Environment, it `ssh`s into the deployment environments `spack` instance, and installs the model associated with the repository that called it. It then copies back relevant metadata and creates a versioned GitHub Release in the caller repository, if it is not a `prerelease` deployment.
3235

33-
For supported `spack`-installable ACCESS-NRI models, simply call the `deploy-1-setup.yml` reusable workflow from the given repositories workflow file, like so:
36+
#### Usage
37+
38+
For supported `spack`-installable ACCESS-NRI models, simply call the `deploy-1-setup.yml` reusable workflow from the given repositories workflow file, as shown below. Don't forget to add required inputs!
3439

3540
```yml
3641
deploy:
3742
uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@main
43+
with:
44+
version: 1.2.3
3845
secrets: inherit
3946
permissions:
4047
contents: write
4148
```
49+
50+
### `undeploy-*`
51+
52+
For given `spack` environments, we can also remove deployments.
53+
54+
> [!NOTE]
55+
> This workflow is not one that should be used directly. It is used within the automated deployment pipeline to remove prerelease builds of models.
56+
57+
#### Inputs
58+
59+
| Name | Type | Description | Required | Default | Example |
60+
| ---- | ---- | ----------- | -------- | ------- | ------- |
61+
| `version-pattern` | string | A wildcard-supported string for version(s) of the model being removed | true | N/A | `2024.01.1-pre*` |
62+
63+
#### Explanation
64+
65+
This pipeline is responsible for removing deployed models from a deployment environment. Particular use-cases include removing `prerelease` builds from a deployment environment once a PR is merged.
66+
67+
##### `undeploy-1-setup.yml`
68+
69+
This workflow obtains the relevant spack and GitHub Environment information, and creates parallelized jobs removing the given `spack` environments based on the list of deployment environments.
70+
71+
##### `undeploy-2-start.yml`
72+
73+
Using the GitHub Environment, it `ssh`s into the deployment environments `spack` instance, and installs the model associated with the repository that called it. It then copies back relevant metadata and creates a versioned GitHub Release in the caller repository, if it is not a `prerelease` deployment.
74+
75+
#### Usage
76+
77+
```yml
78+
remove-prereleases:
79+
uses: access-nri/build-cd/.github/workflows/undeploy-2-setup.yml@main
80+
with:
81+
version-pattern: ${{ inputs.model }}-pre*
82+
secrets: inherit
83+
```
84+
85+
This will remove every `spack` environment from the deployment target that matches `<model>-pre*`.
86+
87+
### `validate-json.yml`
88+
89+
This workflow is used to validate the `config` folders `*.json` files based on their associated `*.schema.json`. This is used for PR checks on the `build-cd` repo itself.
90+
91+
### `create-deployment-spack.yml`
92+
93+
This workflow_dispatch-triggered workflow is used to create a version of `spack` on `Gadi`.
94+
95+
#### Inputs
96+
97+
| Name | Type | Description | Required | Default | Example |
98+
| ---- | ---- | ----------- | -------- | ------- | ------- |
99+
| `spack-version` | string | A version of `spack` | true | N/A | `0.20` |
100+
| `spack-packages-version` | string | A version of ACCESS-NRI/spack-packages to be bundled with the install of `spack` | true | `main` | `2023.11.12` |
101+
| `spack-config-version` | string | A version of ACCESS-NRI/spack-config to be bundled with the install of `spack` | true | `main` | `2024.01.01` |
102+
| `deployment-location` | true | A path in the deployment environment where Spack should be created. For example, if it is `opt/spack`, spack will be installed under `opt/spack/<spack-version>/` | true | N/A | `/opt/spack` |

‎config/deployment-environment.json

+3
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,8 @@
22
"$schema": "./deployment-environment.schema.json",
33
"environments": [
44
"Gadi"
5+
],
6+
"prerelease-environments": [
7+
"Gadi Prerelease"
58
]
69
}

‎config/deployment-environment.schema.json

+7-1
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,14 @@
1212
"items": {
1313
"type": "string"
1414
}
15+
},
16+
"prerelease-environments": {
17+
"type": "array",
18+
"items": {
19+
"type": "string"
20+
}
1521
}
1622
},
17-
"required": ["environments"],
23+
"required": ["environments", "prerelease-environments"],
1824
"additionalProperties": false
1925
}

0 commit comments

Comments
 (0)