Skip to content

Commit 9274bd6

Browse files
CodeGatCodeGat
authored
Upload Build Provenance Metadata (#63)
* deploy-2-start.yml: Moved release into it's own job, added build db upload job * Added scripts/upload-build-metadata.bash * deploy-2-start.yml: Removed hash from spack.location.json * deploy-2-start.yml: Added fixes from org testing * deploy-2-start.yml: Added connection string secret * upload-build-metadata.bash: Updated json section names * upload-build-metadata.bash: modifications to the json structure based on 0f110c4 * deploy-2-start.yml: Split requirements.txt install into own step * deploy-2-start.yml: Reverted spack.location.json creation logic * Renamed upload-build-metadata.bash -> generate-build-metadata.bash * deploy-2-start.yml: Ported upload logic from generate-build-metadata.bash to inline script * deploy-2-start.yml: Update spack.location.json to be {hash: path} object * generate-build-metadata.bash: Added command getting the package hash from the lockfile, used to get the path * Added testing infrastructure * Added .gitignore * generate-build-metadata.bash: Moved release_url and created_at to model_build * generate-build-metadata.bash: Moved component builds into an array rather than multiple files * Updated test script and refactored outputs as a single file * deploy-2-start.yml: Removed looping since output is a single file
1 parent af75fdc commit 9274bd6

File tree

7 files changed

+1114
-9
lines changed

7 files changed

+1114
-9
lines changed

‎.github/workflows/deploy-2-start.yml

+80-9
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,15 @@ on:
3434
description: The root SBD that is being used as the modulefile name
3535
env:
3636
SPACK_YAML_MODULEFILE_PROJECTION_YQ: .spack.modules.default.tcl.projections.${{ inputs.root-sbd }}
37+
METADATA_PATH: /opt/metadata
3738
jobs:
3839
deploy-to-environment:
3940
name: Deploy to ${{ inputs.deployment-environment }}
4041
runs-on: ubuntu-latest
4142
environment: ${{ inputs.deployment-environment }}
43+
outputs:
44+
packages-version: ${{ steps.versions.outputs.packages }}
45+
config-version: ${{ steps.versions.outputs.config }}
4246
steps:
4347
# Deployment
4448
- uses: actions/checkout@v4
@@ -95,32 +99,99 @@ jobs:
9599
96100
# Obtain metadata
97101
spack find --paths > ${{ vars.SPACK_LOCATION }}/var/spack/environments/${{ inputs.env-name }}/spack.location
98-
spack find --format '{name}@{version} {prefix}' | jq --raw-input --null-input '[inputs | split(" ") | {(.[0]):(.[1])}] | add' > ${{ vars.SPACK_LOCATION }}/var/spack/environments/${{ inputs.env-name }}/spack.location.json
102+
spack find --format '{hash} {prefix}' | jq --raw-input --null-input '[inputs | split(" ") | {(.[0]): (.[1])}] | add' > ${{ vars.SPACK_LOCATION }}/var/spack/environments/${{ inputs.env-name }}/spack.location.json
99103
100104
spack env deactivate
101105
echo "$(date): Deployed ${{ inputs.model }} ${{ inputs.version }} with spack-packages ${{ steps.versions.outputs.packages }}, spack-config ${{ steps.versions.outputs.config }}" >> ${{ vars.SPACK_RELEASE_LOCATION }}/release.log
102106
EOT
103107
104108
# Release
105109
- name: Get Release Metadata
106-
if: inputs.type == 'release'
107110
run: |
108111
rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
109112
'${{ secrets.USER}}@${{ secrets.HOST_DATA }}:${{ vars.SPACK_LOCATION }}/var/spack/environments/${{ inputs.env-name }}/spack.*' \
110113
./${{ inputs.env-name }}
111114
115+
- name: Upload Metadata Artifact
116+
uses: actions/upload-artifact@v4
117+
with:
118+
name: ${{ inputs.env-name }}
119+
path: ./${{ inputs.env-name }}/*
120+
overwrite: true
121+
122+
release:
123+
name: Create Release
124+
if: inputs.type == 'release'
125+
needs:
126+
- deploy-to-environment
127+
runs-on: ubuntu-latest
128+
outputs:
129+
url: ${{ steps.release.outputs.url }}
130+
created-at: ${{ steps.metadata.outputs.created-at }}
131+
steps:
132+
- uses: actions/checkout@v4
133+
134+
- name: Download Metadata Artifact
135+
uses: actions/download-artifact@v4
136+
with:
137+
name: ${{ inputs.env-name }}
138+
path: ${{ env.METADATA_PATH }}
139+
112140
- name: Create Release
113-
if: inputs.type == 'release'
114-
uses: softprops/action-gh-release@v0.1.15
141+
id: release
142+
uses: softprops/action-gh-release@69320dbe05506a9a39fc8ae11030b214ec2d1f87 # v2.0.5
115143
with:
116144
tag_name: ${{ inputs.version }}
117145
name: ${{ inputs.model}} ${{ inputs.version }}
118146
body: |
119-
This release of ${{ inputs.model }} ${{ inputs.version }} uses [spack-packages ${{ steps.versions.outputs.packages }}](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ steps.versions.outputs.packages }}) and [spack-config ${{ steps.versions.outputs.config }}](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ steps.versions.outputs.config }}).
147+
This release of ${{ inputs.model }} ${{ inputs.version }} uses [spack-packages ${{ needs.deploy-to-environment.outputs.packages-version }}](https://github.com/ACCESS-NRI/spack-packages/releases/tag/${{ needs.deploy-to-environment.outputs.packages-version }}) and [spack-config ${{ needs.deploy-to-environment.outputs.config-version }}](https://github.com/ACCESS-NRI/spack-config/releases/tag/${{ needs.deploy-to-environment.outputs.config-version }}).
120148
generate_release_notes: true
121149
fail_on_unmatched_files: true
122150
files: |
123-
./${{ inputs.env-name }}/spack.yaml
124-
./${{ inputs.env-name }}/spack.lock
125-
./${{ inputs.env-name }}/spack.location
126-
./${{ inputs.env-name }}/spack.location.json
151+
./${{ env.METADATA_PATH }}/spack.yaml
152+
./${{ env.METADATA_PATH }}/spack.lock
153+
./${{ env.METADATA_PATH }}/spack.location
154+
./${{ env.METADATA_PATH }}/spack.location.json
155+
156+
- name: Release Metadata
157+
id: metadata
158+
env:
159+
GH_TOKEN: ${{ github.token }}
160+
run: echo "created-at=$(gh release view --json createdAt --jq '.createdAt')" >> $GITHUB_OUTPUT
161+
162+
build-db:
163+
name: Build DB Metadata Upload
164+
if: inputs.type == 'release'
165+
needs:
166+
- release
167+
runs-on: ubuntu-latest
168+
steps:
169+
- name: Download Metadata Artifact
170+
uses: actions/download-artifact@v4
171+
with:
172+
name: ${{ inputs.env-name }}
173+
path: ${{ env.METADATA_PATH }}
174+
175+
- name: Checkout Upload Script
176+
uses: actions/checkout@v4
177+
with:
178+
repository: access-nri/build-cd
179+
180+
- name: Setup Python
181+
uses: actions/setup-python@v5
182+
with:
183+
python-version: "3.10"
184+
cache: pip
185+
186+
- name: Install Build Metadata Script Requirements
187+
run: pip install -r tools/release_provenance/requirements.txt
188+
189+
- name: Upload Build Metadata
190+
env:
191+
BUILD_DB_CONNECTION_STR: ${{ secrets.BUILD_DB_CONNECTION_STR }}
192+
OUTPUT_PATH: ./metadata_output
193+
run: |
194+
./scripts/generate-build-metadata.bash ${{ needs.release.outputs.url }} ${{ needs.release.outputs.created-at }} ${{ env.METADATA_PATH }} ${{ env.OUTPUT_PATH }} ${{ inputs.root-sbd }} ${{ vars.BUILD_DB_PACKAGES }}
195+
196+
echo "Attempting upload of build_metadata.json"
197+
python ./tools/release_provenance/save_release.py "${{ env.OUTPUT_PATH }}/build_metadata.json"

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Files generated during generate-build-metadata tests
2+
tests/scripts/generate-build-metadata/outputs

‎scripts/generate-build-metadata.bash

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/bin/bash
2+
set -x
3+
set -e
4+
5+
### INPUTS ###
6+
# URL for the associated GitHub Release of $model_name.
7+
release_url=$1
8+
# Timestamp for the creation of the $release_url.
9+
release_time=$2
10+
# Path to the dir containing the spack.{lock,location.json}.
11+
json_dir=$3
12+
# directory that contains the <package>.json files
13+
output_dir=$4
14+
# Name of the model (or root-sbd) (eg. access-om2)
15+
model_name=$5
16+
17+
# the rest of the model components for the given $model_name
18+
# (eg. for access-om2 there would be mom5, cice5, etc...)
19+
shift 5
20+
packages=( "$@" )
21+
22+
### SCRIPT ###
23+
mkdir -p "$output_dir"
24+
25+
spack=$(jq \
26+
'{
27+
version: .spack.version,
28+
commit: .spack.commit
29+
}' "$json_dir/spack.lock")
30+
31+
model=$(jq \
32+
--arg model "$model_name" \
33+
--arg release_url "$release_url" \
34+
--arg release_time "$release_time" \
35+
--argjson spack "$spack" \
36+
'.concrete_specs | to_entries[] | select(.value.name == $model)
37+
| {
38+
spack_hash: .key,
39+
spec: (.value.name + "@" + .value.version),
40+
created_at: $release_time,
41+
release_url: $release_url,
42+
spack_version: $spack
43+
}' "$json_dir/spack.lock"
44+
)
45+
46+
# construction of the initial build_metadata.json
47+
jq --null-input \
48+
--argjson model "$model" \
49+
'{
50+
model_build: $model,
51+
component_build: [],
52+
}' > "$output_dir/build_metadata.json"
53+
54+
for pkg in "${packages[@]}"; do
55+
pkg_hash=$(jq --raw-output \
56+
--arg pkg "$pkg" \
57+
'.concrete_specs | to_entries[] | select(.value.name == $pkg) | .key' \
58+
"$json_dir/spack.lock"
59+
)
60+
61+
echo "Hash of $pkg is $pkg_hash"
62+
63+
install_path=$(jq --raw-output \
64+
--arg pkg_hash "$pkg_hash" \
65+
'to_entries[] | select(.key == $pkg_hash) | .value.path' \
66+
"$json_dir/spack.location.json"
67+
)
68+
69+
component=$(jq \
70+
--arg pkg "$pkg" \
71+
--arg install_path "$install_path" \
72+
'.concrete_specs | to_entries[] | select(.value.name == $pkg)
73+
| {
74+
spack_hash: .key,
75+
spec: (.value.name + "@" + .value.version),
76+
install_path: $install_path
77+
}' "$json_dir/spack.lock"
78+
)
79+
80+
# piecewise construction of the entire build_metadata.json for each
81+
# build_component
82+
jq \
83+
--argjson component "$component" \
84+
'.component_build += [$component]' \
85+
"$output_dir/build_metadata.json" > "$output_dir/build_metadata.json.tmp"
86+
87+
mv "$output_dir/build_metadata.json.tmp" "$output_dir/build_metadata.json"
88+
89+
cat "$output_dir/build_metadata.json"
90+
done

‎tests/scripts/generate-build-metadata/inputs/spack.location.json

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"ucmkt2gy72vvikhstgvoft65wnskr7rd": {
3+
"path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/access-om2-git.2024.03.0=2024.03.0-ucmkt2gy72vvikhstgvoft65wnskr7rd"
4+
},
5+
"v3zncpqjj2gyseudbwiudolcjq3k3leo": {
6+
"path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/cice5-git.2023.10.19=2023.10.19-v3zncpqjj2gyseudbwiudolcjq3k3leo"
7+
},
8+
"qji4nlmr6utrribaiyhewe4je6mifguz": {
9+
"path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/mom5-git.2023.11.09=2023.11.09-qji4nlmr6utrribaiyhewe4je6mifguz"
10+
},
11+
"i3inxzaihefr3rqljoovhyevwai6bsff": {
12+
"path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.281/netcdf-c-4.7.4-i3inxzaihefr3rqljoovhyevwai6bsff"
13+
}
14+
}

0 commit comments

Comments
 (0)