!test Comment Command (#73)

* Add config-comment-repro.yml workflow for repro tests on any branch!

* Replace `against COMMITISH` with `compare COMMITISH`

* Added ability to commit the result of the repro test to the PR

* Added RUN_URL, removed some of the branch name stuff

* Add additional artifact content paths section for repro-ci

* Removed custom `[compare COMMITISH]` logic

* Update artifact structure to be in line with #74

* Renamed file, made it a more generic `!test` command

* Changed artifact paths back to initial version, removed `inputs.additional-artifact-content-paths` input

* Removed issue property from `github.event.comment.body`

* Reworked permission checking, bug fixes, comment updates

* Further changes from testing

* Added error-checking for `!test` command

* Made comments more informative

* Update README.md to account for new `!test repro` syntax

* Update .github/workflows/config-comment-test.yml

Co-authored-by: Aidan Heerdegen <[email protected]>

* README.md: Note the case-sensitivity of the command

* Add step in prepare-command to check the command starts with `!test`

---------

Co-authored-by: Aidan Heerdegen <[email protected]>

Author: CodeGat

.github/workflows/config-comment-test.yml

@@ -0,0 +1,375 @@
+name: 'Comment Command: !test'
+run-name: '!test on ${{ github.repository }}'
+on:
+  workflow_call:
+  # Triggered in calling workflow by:
+  # on:
+  #   issue_comment:
+  #     - edited
+  #     - created
+env:
+  USAGE: '!test TYPE [commit]'
+  USAGE_TYPES: repro
+  RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+jobs:
+  permission-check:
+    name: Permission Check
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Determine if commenter has permission to run the command
+        id: commenter
+        # TODO: The permissions-checking section seems ripe for turning into an action!
+        run: |
+          commenter_permissions=$(gh api \
+              repos/${{ github.repository }}/collaborators/${{ github.event.comment.user.login }}/permission \
+              --jq '.permission'
+          )
+          if [[ "$commenter_permissions" == "admin" || "$commenter_permissions" == "write" ]]; then
+            echo "Commenter has at least write permission"
+            echo "permission=true" >> $GITHUB_OUTPUT
+          else
+            echo "Commenter does not have at least write permission"
+            echo "permission=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: React to '!test'
+        uses: access-nri/actions/.github/actions/react-to-comment@main
+        with:
+          reaction: ${{ steps.commenter.outputs.permission == 'true' && 'rocket' || 'confused' }}
+          token: ${{ github.token }}
+
+      - name: Fail if no permissions
+        if: steps.commenter.outputs.permission == 'false'
+        run: exit 1
+
+  ci-config:
+    name: Read CI Testing Configuration
+    needs:
+      - permission-check
+    runs-on: ubuntu-latest
+    outputs:
+      markers: ${{ steps.repro-config.outputs.markers }}
+      payu-version: ${{ steps.repro-config.outputs.payu-version }}
+      model-config-tests-version: ${{ steps.repro-config.outputs.model-config-tests-version }}
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+        with:
+          ref: main
+
+      - name: Validate `config/ci.json`
+        uses: access-nri/schema/.github/actions/validate-with-schema@main
+        with:
+          # As with a lot of the `vars`/`secrets` in this repo, this will be defined in the caller repo
+          schema-version: ${{ vars.CI_JSON_SCHEMA_VERSION }}
+          meta-schema-version: draft-2020-12
+          schema-location: au.org.access-nri/model/configuration/ci
+          data-location: config/ci.json
+
+      - name: Get base branch for PR
+        id: base
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh pr checkout ${{ github.event.issue.number }}
+          echo "branch=$(gh pr view --json baseRefName --jq '.baseRefName')"
+
+      - name: Read reproducibility tests config
+        id: repro-config
+        uses: access-nri/model-config-tests/.github/actions/parse-ci-config@main
+        with:
+          check: reproducibility
+          branch-or-tag: ${{ steps.base.outputs.branch }}
+          config-filepath: "config/ci.json"
+
+  prepare-command:
+    name: Prepare Command
+    needs:
+      - permission-check
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+    outputs:
+      test-type: ${{ steps.test.outputs.type }}
+      # the full git hash of the configuration being tested
+      config-hash: ${{ steps.pr.outputs.hash }}
+      # the short git hash of the configuration being tested
+      config-short-hash: ${{ steps.pr.outputs.short-hash }}
+      # the git ref (branch or tag) of the configuration being tested
+      config-ref: ${{ steps.pr.outputs.ref }}
+      # the full git hash of the configuration being compared against
+      compared-config-hash: ${{ steps.compared.outputs.hash }}
+      # the short git hash of the configuration being compared against
+      compared-config-short-hash: ${{ steps.compared.outputs.short-hash }}
+      # the git ref (branch or tag) of the configuration being compared against
+      compared-config-ref: ${{ steps.compared.outputs.ref }}
+      # whether the commenter can commit to the repo (either 'true', 'false' or '' (when no 'commit' option given))
+      commit-requested: ${{ steps.commit.outputs.requested }}
+      # TODO: Make this an input to the command when we start deploying to multiple targets
+      environment-name: Gadi
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Verify !test
+        run: |
+          if [[ "${{ startsWith(github.event.comment.body, '!test') }}" == "true" ]]; then
+            echo 'Command starts with !test'
+          else
+            echo "::error::Usage: ${{ env.USAGE }}"
+            echo "::error::Command must start with !test to invoke model-config-tests' config-comment-test.yml."
+            exit 1
+
+      - name: Get !test type
+        id: test
+        run: |
+          command="${{ github.event.comment.body }}"
+          read -ra command_tokens <<< "$command"
+          type_in_comment="${command_tokens[1]}"
+          type_in_comment_valid=false
+
+          for type in ${{ env.USAGE_TYPES }}; do
+            if [[ "$type_in_comment" == "$type" ]]; then
+              type_in_comment_valid=true
+              break
+            fi
+          done
+
+          if [[ "$type_in_comment_valid" == "false" ]]; then
+            echo "::error::Usage: ${{ env.USAGE }}"
+            echo "::error::The command '${{ github.event.comment.body }}' doesn't have a valid TYPE. Was given '$type_in_comment', but needed to be one of: ${{ env.USAGE_TYPES }}."
+            exit 1
+          fi
+
+          echo "type=$type_in_comment" >> $GITHUB_OUTPUT
+
+      - name: Get config ref from PR comment
+        id: pr
+        run: |
+          gh pr checkout ${{ github.event.issue.number }}
+          echo "hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+          echo "short-hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "ref=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Get compared config ref
+        id: compared
+        # Set up a default ref
+        # We need to do this roundabout way to get the source branch commit because
+        # We can't access github.base_ref as this trigger is on.issue_comment
+        run: |
+          ref=$(gh pr view ${{ github.event.issue.number }} --json baseRefName --jq '.baseRefName')
+          echo "Using $ref"
+          hash=$(git show-branch --merge-base origin/$ref origin/${{ steps.pr.outputs.ref }})
+          short_hash=$(git rev-parse --short $hash)
+          echo "AKA $hash (shortened as $short_hash)"
+
+          echo "hash=$hash" >> $GITHUB_OUTPUT
+          echo "short-hash=$short_hash" >> $GITHUB_OUTPUT
+          echo "ref=$ref" >> $GITHUB_OUTPUT
+
+      - name: Determine whether to commit
+        id: commit
+        # Determine whether the commenter wants to commit something to the repository
+        run: |
+          command="${{ github.event.comment.body }}"
+          read -ra command_tokens <<< "$command"
+          potential_commit_token="${command_tokens[2]}"
+
+          if [[ "$potential_commit_token" == "commit" ]]; then
+            echo "'commit' option given"
+            echo "requested=true" >> $GITHUB_OUTPUT
+          elif [ -z "$potential_commit_token" ]; then
+            echo "'commit' option not given"
+            echo "requested=false" >> $GITHUB_OUTPUT
+          else
+            echo "::error::Non-commit option '$potential_commit_token' given. Usage: ${{ env.USAGE }}"
+            exit 1
+          fi
+
+      - name: Erroneous tokens check
+        # If commit is given, any tokens after the 3rd are considered erroneous - otherwise it is after the 2nd
+        # TODO: This will need to be made more robust if there are other options added
+        run: |
+          command="${{ github.event.comment.body }}"
+          read -ra command_tokens <<< "$command"
+
+          max_tokens_allowed=$([[ "${{ steps.commit.outputs.requested }}" == "true" ]] && echo "3" || echo "2")
+
+          if [ "${#command_tokens[@]}" -gt "$max_tokens_allowed" ]; then
+            echo "::error::Erroneous tokens given. Usage: ${{ env.USAGE }}"
+            exit 1
+          fi
+
+  repro:
+    name: Compare ${{ needs.prepare-command.outputs.config-ref }} against ${{ needs.prepare-command.outputs.compared-config-ref }}
+    needs:
+      - prepare-command
+      - ci-config
+    if: needs.prepare-command.outputs.test-type == 'repro'
+    uses: access-nri/model-config-tests/.github/workflows/test-repro.yml@main
+    with:
+      config-ref: ${{ needs.prepare-command.outputs.config-hash }}
+      compared-config-ref: ${{ needs.prepare-command.outputs.compared-config-hash }}
+      environment-name: ${{ needs.prepare-command.outputs.environment-name }}
+      payu-version: ${{ needs.ci-config.outputs.payu-version }}
+      model-config-tests-version: ${{ needs.ci-config.outputs.model-config-tests-version }}
+      test-markers: ${{ needs.ci-config.outputs.markers }}
+    secrets: inherit
+    permissions:
+      contents: write
+
+  check-repro:
+    # Parse the test report and return pass/fail result
+    name: Results
+    needs:
+      - prepare-command
+      - repro
+    runs-on: ubuntu-latest
+    env:
+      TESTING_LOCAL_LOCATION: /opt/testing
+    permissions:
+      pull-requests: write
+      checks: write
+    outputs:
+      result: ${{ steps.results.outputs.result }}
+    steps:
+      - name: Download Newly Created Checksum
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ needs.repro.outputs.artifact-name }}
+          path: ${{ env.TESTING_LOCAL_LOCATION }}
+
+      - name: Parse Test Report
+        id: tests
+        uses: EnricoMi/publish-unit-test-result-action/composite@82082dac68ad6a19d980f8ce817e108b9f496c2a  #v2.17.1
+        with:
+          files: ${{ env.TESTING_LOCAL_LOCATION }}/checksum/test_report.xml
+          comment_mode: off
+          check_run: true
+          compare_to_earlier_commit: false
+          report_individual_runs: true
+          report_suite_logs: any
+
+      - name: Repro results
+        id: results
+        run: |
+          echo "check-url=${{ fromJson(steps.tests.outputs.json).check_url }}" >> $GITHUB_OUTPUT
+
+          if (( ${{ fromJson(steps.tests.outputs.json).stats.tests_fail }} > 0 )); then
+            echo "result=fail" >> $GITHUB_OUTPUT
+          else
+            echo "result=pass" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Comment result
+        env:
+          RESULT: |-
+            ${{ steps.results.outputs.result == 'pass' && ':white_check_mark: The Bitwise Reproducibility Check Succeeded :white_check_mark:' || ':x: The Bitwise Reproducibility Check Failed :x:' }}
+          CONFIG_REF_URL: '[${{ needs.prepare-command.outputs.config-short-hash }}](${{ github.server_url}}/${{ github.repository }}/tree/${{ needs.prepare-command.outputs.config-hash }})'
+          COMPARED_CONFIG_REF_URL: '[${{ needs.prepare-command.outputs.compared-config-short-hash }}](${{ github.server_url }}/${{ github.repository }}/tree/${{ needs.prepare-command.outputs.compared-config-hash }})'
+        uses: access-nri/actions/.github/actions/pr-comment@main
+        with:
+          comment: |
+            ${{ env.RESULT }}
+
+            When comparing:
+
+            - `${{ needs.prepare-command.outputs.config-ref }}` (checksums created using commit ${{ env.CONFIG_REF_URL }}), against
+            - `${{ needs.prepare-command.outputs.compared-config-ref }}` (checksums in commit ${{ env.COMPARED_CONFIG_REF_URL }})
+
+            ${{ (needs.prepare-command.outputs.commit-requested == 'true' && steps.results.outputs.result == 'fail') && ':wrench: The checksums will be committed to this PR, as they differ.' || '' }}
+
+            <details>
+            <summary> Further information</summary>
+
+            The experiment can be found on Gadi at `${{ needs.repro.outputs.experiment-location }}`, and the test results at ${{ steps.results.outputs.check-run-url }}.
+
+            The checksums generated by this `!test` command are found in the `testing/checksum` directory of ${{ needs.repro.outputs.artifact-url }}.
+
+            The checksums compared against are found here ${{ github.server_url }}/${{ github.repository }}/tree/${{ needs.prepare-command.outputs.compared-config-hash }}/testing/checksum
+
+            </details>
+
+  commit:
+    name: Commit Result
+    if: needs.prepare-command.outputs.commit-requested == 'true' && needs.check-repro.outputs.result == 'fail'
+    needs:
+      - prepare-command
+      - repro
+      - check-repro
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    env:
+      ARTIFACT_LOCAL_LOCATION: /opt/artifact
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GH_COMMIT_CHECK_TOKEN }}
+
+      - name: Checkout Associated PR ${{ github.event.issue.number }}
+        # Since the trigger for this workflow was on.issue_comment, we need
+        # to do a bit more wrangling to checkout the pull request
+        run: gh pr checkout ${{ github.event.issue.number }}
+
+      - name: Download Newly Created Checksum
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ needs.repro.outputs.artifact-name }}
+          path: ${{ env.ARTIFACT_LOCAL_LOCATION }}
+
+      - name: Update files
+        # This will copy checksums from the artifact to the repo
+        run: |
+          mkdir testing
+          cp --recursive --verbose ${{ env.ARTIFACT_LOCAL_LOCATION }}/*/* testing
+
+      - name: Import Commit-Signing Key
+        uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4  # v6.1.0
+        with:
+          gpg_private_key: ${{ secrets.GH_ACTIONS_BOT_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.GH_ACTIONS_BOT_GPG_PASSPHRASE }}
+          git_config_global: true
+          git_committer_name: ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }}
+          git_committer_email: ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+          git_tag_gpgsign: true
+
+      - name: Commit and Push Updates
+        run: |
+          git add .
+          git commit -m "Updated checksums as part of ${{ env.RUN_URL }}"
+          git push
+
+  failure-notifier:
+    name: Notify PR of Workflow Failure
+    # We need the last jobs as 'needs' on the failure notifier so
+    # any of the dependent jobs that fail are covered here
+    needs:
+      - permission-check
+      - prepare-command
+      - check-repro
+      - commit
+    if: failure()
+    runs-on: ubuntu-latest
+    steps:
+      - uses: access-nri/actions/.github/actions/pr-comment@main
+        with:
+          comment: >-
+            :x: `!test` Command Failed :x:
+            ${{ needs.prepare-command.result == 'failure' && format('The command given could not be parsed correctly. Usage: {0}', env.USAGE) || '' }}
+            ${{ needs.permission-check.result == 'failure' && 'You do not have at least write permissions on this repository.' || '' }}
+            ${{ needs.commit.result == 'failure' && 'There was a problem committing the result of the reproducibility run.' || '' }}
+            See ${{ env.RUN_URL }}