added release_url to component build (#113)

* added release_url to component build

* Send git attributes of vars.BUILD_DB_PACKAGES to generate-build-metadata.bash

* Added /releases section to git url

* deploy-2-start.yml: Added tag to the end of the pkg_release_url

* deploy-2-start.yml: Enable and activate spack

* deploy-2-start.yml: Get version from spack.lock

* deploy-2-start.yml: Added further comments

---------

Co-authored-by: Tommy Gatti <[email protected]>

Author: utkarshgupta95

.github/workflows/deploy-2-start.yml

@@ -114,20 +114,61 @@ jobs:
           spack env activate ${{ inputs.env-name }}
           spack --debug install --fresh ${{ vars.SPACK_INSTALL_PARALLEL_JOBS }} || exit $?
           spack module tcl refresh -y
+          EOT
+
+      - name: Get metadata from ${{ inputs.deployment-environment }}
+        env:
+          SPACK_ENV_PATH: ${{ steps.path.outputs.spack }}/var/spack/environments/${{ inputs.env-name }}
+        run: |
+          ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
+          . ${{ steps.path.outputs.spack-config }}/spack-enable.bash
+          spack env activate ${{ inputs.env-name }}
+
+          spack find --paths > ${{ env.SPACK_ENV_PATH }}/spack.location
+          spack find --format '{hash} {prefix}' | jq --raw-input --null-input '[inputs | split(" ") | {(.[0]): (.[1])}] | add' > ${{ env.SPACK_ENV_PATH }}/spack.location.json
+
+          # Get the repos associated with the packages for the build database
+          jq -n '{}' > ${{ env.SPACK_ENV_PATH }}/build-db-pkgs.json
+          for pkg in ${{ vars.BUILD_DB_PACKAGES }}; do
+            # TODO: Is there a way to get the git attribute without concretizing?
+            pkg_repo_url=$(spack python -c "import spack.spec; print(spack.spec.Spec('$pkg').concretized().package.git)")
 
-          # Obtain metadata
-          spack find --paths > ${{ steps.path.outputs.spack }}/var/spack/environments/${{ inputs.env-name }}/spack.location
-          spack find --format '{hash} {prefix}' | jq --raw-input --null-input '[inputs | split(" ") | {(.[0]): (.[1])}] | add' > ${{ steps.path.outputs.spack }}/var/spack/environments/${{ inputs.env-name }}/spack.location.json
+            # We get the version of $pkg from spack.lock, and then strip
+            # potential 'git.' and '=VERSION' parts from 'git.TAG=VERSION'
+            version=$(jq --compact-output --raw-output \
+              --arg p "$pkg" \
+              '.concrete_specs | to_entries[].value | select(.name == $p) | .version
+                | match("^(?:git.)?([^=]*)")
+                | .captures[0].string' \
+              ${{ env.SPACK_ENV_PATH}}/spack.lock
+            )
+
+            # Example:
+            # pkg_repo_url = https://github.com/ACCESS-NRI/MOM5.git, which is then stripped of the '.git'.
+            # version = 2024.08.11, giving
+            # pkg_release_url = https://github.com/ACCESS-NRI/MOM5/releases/tag/2024.08.11
+            pkg_release_url="${pkg_repo_url%.*}/releases/tag/${version}"
+
+            echo "$pkg pkg_repo_url is $pkg_repo_url, pkg_release_url is $pkg_release_url, version is $version"
+            jq \
+              --arg p "$pkg" \
+              --arg r "$pkg_release_url" \
+              '. += {($p): ($r)}' \
+              ${{ env.SPACK_ENV_PATH }}/build-db-pkgs.json > ${{ env.SPACK_ENV_PATH }}/build-db-pkgs.json.tmp
+            mv -f ${{ env.SPACK_ENV_PATH }}/build-db-pkgs.json.tmp ${{ env.SPACK_ENV_PATH }}/build-db-pkgs.json
+          done
 
           spack env deactivate
-          echo "$(date): Deployed ${{ inputs.model }} ${{ inputs.version }} with spack-packages ${{ steps.versions.outputs.packages }}, spack-config ${{ steps.versions.outputs.config }}" >> ${{ steps.path.outputs.root }}/release.log
           EOT
 
       # Release
       - name: Get Release Metadata
+        env:
+          # TODO: Can we put both envs above in a $GITHUB_ENV file instead?
+          SPACK_ENV_PATH: ${{ steps.path.outputs.spack }}/var/spack/environments/${{ inputs.env-name }}
         run: |
           rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
-            '${{ secrets.USER}}@${{ secrets.HOST_DATA }}:${{ steps.path.outputs.spack }}/var/spack/environments/${{ inputs.env-name }}/spack.*' \
+            '${{ secrets.USER}}@${{ secrets.HOST_DATA }}:${{ env.SPACK_ENV_PATH }}/spack.*' \
             ./${{ inputs.env-name }}
 
       - name: Upload Metadata Artifact

scripts/generate-build-metadata.bash

@@ -75,14 +75,22 @@ for pkg in "${packages[@]}"; do
     "$json_dir/spack.location.json"
   )
 
+  release_url=$(jq --raw-output \
+    --arg pkg "$pkg" \
+    '.[$pkg]' \
+    "$json_dir/build-db-pkgs.json"
+  )
+
   component=$(jq \
     --arg pkg "$pkg" \
     --arg install_path "$install_path" \
+    --arg release_url "$release_url" \
     '.concrete_specs | to_entries[] | select(.value.name == $pkg)
     | {
       spack_hash: .key,
       spec: (.value.name + "@" + .value.version),
-      install_path: $install_path
+      install_path: $install_path,
+      release_url: $release_url
     }' "$json_dir/spack.lock"
   )
 

tools/release_provenance/models.py

@@ -26,6 +26,7 @@ class ComponentBuild(Base):
     spack_hash = Column(String, primary_key=True, index=True)
     spec = Column(String, nullable=False)
     install_path = Column(String, nullable=False, unique=True)
+    release_url = Column(Text, nullable=False, unique=True)
     model_build = relationship('ModelBuild', secondary="model_component", back_populates='component_build')
 
 class ModelStatusEnum(enum.Enum):

tools/release_provenance/save_release.py

@@ -19,6 +19,7 @@ def get_component_build(component_build_data_list, model_build):
             component_build.install_path = component_build_data["install_path"]
             component_build.spack_hash = component_build_data["spack_hash"]
             component_build.spec = component_build_data["spec"]
+            component_build.release_url = component_build_data["release_url"]
             component_build.model_build.append(model_build)
             component_build_list.append(component_build)
         else:

tools/release_provenance/test_release_data.json

@@ -3,12 +3,14 @@
       {
         "spack_hash": "ewcdbrfukblyjxpkhd3mfkj4yxfolal8",
         "spec": "[email protected]=2023.11.09",
-        "install_path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.280/"
+        "install_path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.0.5.280/",
+        "release_url": "https://github.com/ACCESS-NRI/mom5/releases"
       },
       {
         "spack_hash": "ewcdbrfukblyjxpkhd3mfkj4yxfolal9",
         "spec": "[email protected]=2023.11.09",
-        "install_path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.1.5.283/"
+        "install_path": "/g/data/vk83/apps/spack/0.20/release/linux-rocky8-x86_64/intel-19.1.5.283/",
+        "release_url": "https://github.com/ACCESS-NRI/mom5-example/releases"
       }
     ],
     "model_build": {