Skip to content

Commit 0c65447

Browse files
CodeGatCodeGat
authored
Move spack-config into config/settings.json (#128)
* validate-repo-version: update README instructions * Remove reading/modification of `spack-config` from `[un]deploy-2-*.yml` workflows * Rework settings.json to include `spack-config`, updated schema * validate-deployment-settings: Added spack-config check, added comments * validate-deployment-settings: Update checks to work with new settings.json schema * settings-*.yml: Updated to deploy updates to `spack-config` * ci.yml: Added newline to comment to aid code formatting
1 parent 9bc6d7c commit 0c65447

File tree

9 files changed

+139
-44
lines changed

9 files changed

+139
-44
lines changed

‎.github/actions/validate-deployment-settings/action.yml

+55-6
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,15 @@ outputs:
2121
runs:
2222
using: composite
2323
steps:
24+
################
25+
# SPACK CHECKS #
26+
################
2427
- name: Check All Spack Major Versions in Prerelease are in Release
2528
id: spack-maj-all-pre-in-rel
2629
shell: bash
2730
run: |
2831
majors_only_in_prerelease=$(jq --compact-output --raw-output \
29-
'.deployment.${{ inputs.target }} | (.Prerelease.spack | keys) - (.Release.spack | keys) | @sh' \
32+
'.deployment.${{ inputs.target }} | (.Prerelease | keys) - (.Release | keys) | @sh' \
3033
${{ inputs.settings-path }}
3134
)
3235
@@ -43,8 +46,21 @@ runs:
4346
# It then merges them together based on major version (Ex. [[{"0.20": "u298rer3", "0.20": "r98u349u"}], ...]).
4447
# And then finally compares the hashes for each major version, and returns the major versions where they differ.
4548
run: |
46-
release_version_hashes=$(jq '.deployment.${{ inputs.target }}.Release.spack | to_entries' ${{ inputs.settings-path }})
47-
prerelease_version_hashes=$(jq '.deployment.${{ inputs.target }}.Prerelease.spack | to_entries' ${{ inputs.settings-path }})
49+
release_version_hashes=$(jq \
50+
'.deployment.${{ inputs.target }}.Release
51+
| [
52+
keys[] as $major | .[$major].spack as $hash
53+
| {($major): $hash} | to_entries[]
54+
]' ${{ inputs.settings-path }}
55+
)
56+
57+
prerelease_version_hashes=$(jq \
58+
'.deployment.${{ inputs.target }}.Prerelease
59+
| [
60+
keys[] as $major | .[$major].spack as $hash
61+
| {($major): $hash} | to_entries[]
62+
]' ${{ inputs.settings-path }}
63+
)
4864
4965
unequal_hashes_for_version=$(jq --compact-output --raw-output --null-input \
5066
--argjson r "$release_version_hashes" \
@@ -76,23 +92,56 @@ runs:
7692
run: |
7793
gh repo clone access-nri/spack ${{ env.CHECKOUT_DIR }} -- --no-checkout --bare --filter=blob:none
7894
79-
# Essentially, pull out all the spack 'version: hash' sections and iterate
80-
jq -cr '.deployment.${{ inputs.target }}[].spack | to_entries[] | "\(.key) \(.value)"' ${{ inputs.settings-path }} | while read -ra line; do
95+
# Essentially, pull out all the spack 'major: hash' sections and iterate
96+
version_hashes=$(jq --compact-output --raw-output \
97+
'.deployment.${{ inputs.target }}[]
98+
| keys[] as $major | .[$major].spack as $hash
99+
| "\($major) \($hash)"' \
100+
${{ inputs.settings-path }}
101+
)
102+
103+
# For each of the version hashes, check if $hash is in releases/v$major
104+
while read -ra line; do
81105
version=${line[0]}
82106
hash=${line[1]}
83107
echo "Checking if $hash is in $version"
84108
if ! git -C ${{ env.CHECKOUT_DIR }} merge-base --is-ancestor $hash releases/v$version; then
85109
echo "::${{ inputs.error-level }}::Commit $hash does not exist on branch releases/v$version"
86110
failed=true
87111
fi
88-
done
112+
done <<< "$version_hashes"
89113
90114
if [ -n "$failed" ]; then
91115
msg="Some commits referenced do not exist in access-nri/spack. Check the workflow logs."
92116
echo "::${{ inputs.error-level }}::$msg"
93117
echo "msg=$msg" >> $GITHUB_OUTPUT
94118
fi
95119
120+
rm -rf ${{ env.CHECKOUT_DIR }}
121+
122+
#######################
123+
# SPACK-CONFIG CHECKS #
124+
#######################
125+
- name: Verify spack-config Tag Exists
126+
id: spack-config-tag-exists
127+
shell: bash
128+
env:
129+
GH_TOKEN: ${{ github.token }}
130+
CHECKOUT_DIR: ./spack-config-metadata
131+
run: |
132+
gh repo clone access-nri/spack-config ${{ env.CHECKOUT_DIR }} -- --no-checkout --bare --filter=blob:none
133+
134+
if ! git -C ${{ env.CHECKOUT_DIR }} tag -l $spack_config_tag; then
135+
msg="Tag $spack_config_tag does not exist in access-nri/spack-config"
136+
echo "::${{ inputs.error-level }}::$msg"
137+
echo "msg=$msg" >> $GITHUB_OUTPUT
138+
fi
139+
140+
rm -rf ${{ env.CHECKOUT_DIR }}
141+
142+
#######################
143+
# END OF ACTION STEPS #
144+
#######################
96145
- name: Warning/Error Collector
97146
id: collector
98147
shell: bash

‎.github/actions/validate-repo-version/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ This action checks that the tags specified in a models `config/versions.json` is
66

77
| Name | Type | Description | Required | Default | Example |
88
| ---- | ---- | ----------- | -------- | ------- | ------- |
9-
| `repo-to-check` | `string` | ACCESS-NRI repositories to validate associated version in `config/versions.json` | `true` | N/A | `spack-packages spack-config` |
9+
| `repo-to-check` | `string` | ACCESS-NRI repository to validate associated version in `config/versions.json` | `true` | N/A | `spack-packages` |
1010

1111
## Outputs
1212

‎.github/workflows/ci.yml

+1
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,7 @@ jobs:
241241
where the binaries shall be on your `$PATH`.
242242
This Prerelease is also accessible on Gadi via `/g/data/vk83/prerelease/apps/spack/0.22/spack` in the `${{ needs.defaults.outputs.root-sbd }}-${{ needs.check-spack-yaml.outputs.prerelease }}` environment.
243243
</details>
244+
244245
:hammer_and_wrench: Using: spack-packages `${{ needs.check-config.outputs.spack-packages-version}}`, spack-config `${{ needs.check-config.outputs.spack-config-version }}`
245246
<details>
246247
<summary>Details</summary>

‎.github/workflows/deploy-2-start.yml

+1-4
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@ jobs:
5454
run: |
5555
echo "spack=$(jq --compact-output --raw-output '.spack' ./config/versions.json)" >> $GITHUB_OUTPUT
5656
echo "packages=$(jq --compact-output --raw-output '."spack-packages"' ./config/versions.json)" >> $GITHUB_OUTPUT
57-
echo "config=$(jq --compact-output --raw-output '."spack-config"' ./config/versions.json)" >> $GITHUB_OUTPUT
5857
5958
- name: Get ${{ inputs.deployment-environment }} Remote Paths
6059
id: path
@@ -100,11 +99,9 @@ jobs:
10099
exit 1
101100
fi
102101
103-
# Update spack-packages/config
102+
# Update spack-packages
104103
git -C ${{ steps.path.outputs.spack-packages }} fetch
105104
git -C ${{ steps.path.outputs.spack-packages }} checkout --force ${{ steps.versions.outputs.packages }}
106-
git -C ${{ steps.path.outputs.spack-config }} fetch
107-
git -C ${{ steps.path.outputs.spack-config }} checkout --force ${{ steps.versions.outputs.config }}
108105
109106
# Enable spack
110107
. ${{ steps.path.outputs.spack-config }}/spack-enable.bash

‎.github/workflows/settings-1-update.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ jobs:
6969
update=$(jq --raw-output --compact-output \
7070
'[.deployment | to_entries[] | .key as $d | .value | keys[] | {
7171
"deployment-environment": $d,
72-
"type": .key
72+
"type": .
7373
}]' ${{ env.CONFIG_SETTINGS_PATH }}
7474
)
7575
echo "updates=$update" >> $GITHUB_OUTPUT

‎.github/workflows/settings-2-deploy.yml

+47-4
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,27 @@ jobs:
2222

2323
- name: Setup spack updates
2424
id: spack
25-
# TODO: Since we can't format this json as an input for this job (see the earlier workflow) we need to do it here
25+
# TODO: Since we can't format any of this later json as an input for this job (see the earlier workflow) we need to do it here
2626
# Create a newline-separated list of strings of the form "MAJOR_VERSION COMMIT_HASH" so we
2727
# update the remotes MAJOR_VERSION/spack to COMMIT_HASH. Ex: "0.20 y7834gtbf3jf3434rr34r34ru"
2828
run: |
2929
updates=$(jq --compact-output --raw-output \
3030
--arg env "${{ inputs.deployment-environment }}" \
3131
--arg type "${{ inputs.spack-type }}" \
32-
'.deployment[$env][$type].spack | to_entries[] | "\(.key) \(.value)"' \
32+
'.deployment[$env][$type] | to_entries[] | "\(.key) \(.value.spack)"' \
33+
${{ env.CONFIG_SETTINGS_PATH }}
34+
)
35+
36+
echo "$updates"
37+
echo "updates=$updates" >> $GITHUB_OUTPUT
38+
39+
- name: Setup spack-config updates
40+
id: spack-config
41+
run: |
42+
updates=$(jq --compact-output --raw-output \
43+
--arg env "${{ inputs.deployment-environment }}" \
44+
--arg type "${{ inputs.spack-type }}" \
45+
'.deployment[$env][$type] | to_entries[] | "\(.key) \(.value.spack-config"' \
3346
${{ env.CONFIG_SETTINGS_PATH }}
3447
)
3548
@@ -43,16 +56,16 @@ jobs:
4356
hosts: ${{ secrets.HOST }}
4457
private-key: ${{ secrets.SSH_KEY }}
4558

46-
- name: Update
59+
- name: Update Spack
4760
continue-on-error: true
4861
run: |
4962
ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
5063
set +e
5164
while read -ra update; do
5265
version=${update[0]}
5366
new_commit=${update[1]}
54-
5567
current_head_commit=$(git -C ${{ secrets.SPACK_INSTALLS_ROOT_LOCATION }}/$version/spack rev-parse HEAD)
68+
5669
if [ $? -eq 128 ]; then
5770
# FIXME: Deploy spack instances in this job too.
5871
echo "::error::Error: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack does not exist. Deploy it via build-cds Create Deployment Spack workflow first."
@@ -73,3 +86,33 @@ jobs:
7386
fi
7487
done <<< "${{ steps.spack.outputs.updates }}"
7588
EOT
89+
90+
- name: Update spack-config
91+
continue-on-error: true
92+
run: |
93+
ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
94+
set +e
95+
while read -ra update; do
96+
version=${update[0]}
97+
new_commit=${update[1]}
98+
current_head_commit=$(git -C ${{ secrets.SPACK_INSTALLS_ROOT_LOCATION }}/$version/spack-config rev-parse HEAD)
99+
100+
if [ $? -eq 128 ]; then
101+
echo "::error::Error: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack-config does not exist."
102+
continue
103+
fi
104+
105+
git -C ${{ secrets.SPACK_INSTALLS_ROOT_LOCATION }}/$version/spack-config fetch
106+
107+
if [[ "$current_head_commit" != "$new_commit" ]]; then
108+
git -C ${{ secrets.SPACK_INSTALLS_ROOT_LOCATION }}/$version/spack-config checkout $new_commit
109+
if [ $? -ne 0 ]; then
110+
echo "::error::Error: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack-config failed checkout from $current_head_commit to $new_commit"
111+
else
112+
echo "::notice::Changed: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack-config changed from $current_head_commit to $new_commit"
113+
fi
114+
else
115+
echo "::notice::Unchanged: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack-config left at $current_head_commit"
116+
fi
117+
done <<< "${{ steps.spack-config.outputs.updates }}"
118+
EOT

‎.github/workflows/undeploy-2-start.yml

+2-4
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,9 @@ jobs:
1818
steps:
1919
- uses: actions/checkout@v4
2020

21-
- name: Get Versions From config/versions.json
21+
- name: Get Spack Version From config/versions.json
2222
id: versions
23-
run: |
24-
echo "spack=$(jq --compact-output --raw-output '.spack' ./config/versions.json)" >> $GITHUB_OUTPUT
25-
echo "config=$(jq --compact-output --raw-output '."spack-config"' ./config/versions.json)" >> $GITHUB_OUTPUT
23+
run: echo "spack=$(jq --compact-output --raw-output '.spack' ./config/versions.json)" >> $GITHUB_OUTPUT
2624

2725
- name: Get ${{ inputs.deployment-environment }} Remote Paths
2826
id: path

‎config/settings.json

+11-6
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,21 @@
33
"deployment": {
44
"Gadi": {
55
"Release": {
6-
"spack": {
7-
"0.20": "6812713cf470b473a607f0de0e8e1cf53f804fb7",
8-
"0.22": "21da7d7e2b5e2680cd9d2e0a2fb4a7d13d8baa9d"
6+
"0.20": {
7+
"spack": "6812713cf470b473a607f0de0e8e1cf53f804fb7",
8+
"spack-config": "2024.03.22"
9+
},
10+
"0.22": {
11+
"spack": "21da7d7e2b5e2680cd9d2e0a2fb4a7d13d8baa9d",
12+
"spack-config": "2024.07.05"
913
}
1014
},
1115
"Prerelease": {
12-
"spack": {
13-
"0.22": "21da7d7e2b5e2680cd9d2e0a2fb4a7d13d8baa9d"
16+
"0.22": {
17+
"spack": "21da7d7e2b5e2680cd9d2e0a2fb4a7d13d8baa9d",
18+
"spack-config": "2024.07.05"
1419
}
1520
}
1621
}
1722
}
18-
}
23+
}

‎config/settings.schema.json

+20-18
Original file line numberDiff line numberDiff line change
@@ -15,39 +15,41 @@
1515
"properties": {
1616
"Release": {
1717
"type": "object",
18-
"properties": {
19-
"spack": {
18+
"patternProperties": {
19+
"^.+$": {
2020
"type": "object",
21-
"patternProperties": {
22-
"^.+$": {
21+
"properties": {
22+
"spack": {
23+
"type": "string"
24+
},
25+
"spack-config": {
2326
"type": "string"
2427
}
2528
},
26-
"additionalProperties": true
29+
"additionalProperties": true,
30+
"required": ["spack", "spack-config"]
2731
}
2832
},
29-
"additionalProperties": true,
30-
"required": [
31-
"spack"
32-
]
33+
"additionalProperties": true
3334
},
3435
"Prerelease": {
3536
"type": "object",
36-
"properties": {
37-
"spack": {
37+
"patternProperties": {
38+
"^.+$": {
3839
"type": "object",
39-
"patternProperties": {
40-
"": {
40+
"properties": {
41+
"spack": {
42+
"type": "string"
43+
},
44+
"spack-config": {
4145
"type": "string"
4246
}
4347
},
44-
"additionalProperties": true
48+
"additionalProperties": true,
49+
"required": ["spack", "spack-config"]
4550
}
4651
},
47-
"additionalProperties": true,
48-
"required": [
49-
"spack"
50-
]
52+
"additionalProperties": true
5153
}
5254
},
5355
"additionalProperties": true,

0 commit comments

Comments
 (0)