[Deployment] Dynamic Versioning, Release Workflow Split, and CI/CD Hardening #412
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # .github/workflows/secure-evals.yaml | |
| name: Run Agent Evals | |
| on: | |
| # This workflow ONLY runs on pull_request_target for security. | |
| # It runs the workflow defined in the 'main' branch, not the one from the PR. | |
| pull_request_target: | |
| # We include 'closed' to trigger the concurrency group cancellation. | |
| # If a PR is merged or closed, we want to cancel any pending eval runs immediately. | |
| # The job itself has an 'if' check to skip execution on 'closed' events. | |
| types: [opened, synchronize, reopened, closed] | |
| branches: | |
| - main | |
| paths: | |
| # Run evals if code in the datacommons-mcp package or evals directory changed | |
| - "packages/datacommons-mcp/datacommons_mcp/**" | |
| - "packages/datacommons-mcp/evals/**" | |
| # Allows maintainers to manually re-run evals on a specific PR number. | |
| workflow_dispatch: | |
| inputs: | |
| pr: | |
| description: "PR number to run tests for" | |
| required: true | |
| concurrency: | |
| # When a new run starts (e.g., from a new push), automatically | |
| # cancel any runs that are already in-progress or queued for the current PR. | |
| cancel-in-progress: true | |
| # The group key is unique per PR, handling both 'pull_request_target' and 'workflow_dispatch' triggers. | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.inputs.pr }} | |
| jobs: | |
| agent-evals: | |
| if: github.event.action != 'closed' | |
| runs-on: ubuntu-latest | |
| environment: evals-and-secrets | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| steps: | |
| - name: Get PR's HEAD SHA for manual run | |
| if: github.event_name == 'workflow_dispatch' | |
| id: get_pr_sha | |
| run: | | |
| PR_SHA=$(gh pr view ${{ github.event.inputs.pr }} --json headRefOid -q '.headRefOid') | |
| echo "sha=$PR_SHA" >> "$GITHUB_OUTPUT" | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Checkout PR code | |
| uses: actions/checkout@v4 | |
| with: | |
| # This explicitly checks out the head commit of the PR | |
| ref: ${{ github.event_name == 'workflow_dispatch' && steps.get_pr_sha.outputs.sha || github.event.pull_request.head.sha }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install uv | |
| run: pip install uv | |
| - name: Run agent evals with pytest | |
| run: uv run --extra test pytest -k "eval" | |
| env: | |
| DC_API_KEY: ${{ secrets.DC_API_KEY }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| - name: Upload evaluation reports | |
| # This condition ensures the step runs even if the pytest step failed, | |
| # but only if the tests were meant to run in the first place. | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| # A more general name for the artifact | |
| name: agent-eval-reports | |
| # Add the CSV pattern on a new line | |
| path: | | |
| reports/evaluation-report-*.html | |
| reports/evaluation-report-*.csv | |
| if-no-files-found: error |