Nightly Regression Analysis #193
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Regression Analysis | |
| on: | |
| workflow_dispatch: | |
| workflow_run: | |
| workflows: ["Nightly tests"] | |
| types: | |
| - completed | |
| jobs: | |
| analyze: | |
| runs-on: ubuntu-latest | |
| # Only run if nightly tests completed successfully or failed (not skipped/cancelled) | |
| if: >- | |
| ${{ | |
| github.event_name == 'workflow_dispatch' || | |
| github.event.workflow_run.conclusion == 'success' || | |
| github.event.workflow_run.conclusion == 'failure' | |
| }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install dependencies | |
| run: pip install pandas | |
| - name: Run regression analysis | |
| id: analysis | |
| env: | |
| # Pass GITHUB_TOKEN so analyze_nightly.py's api.github.com calls | |
| # are authenticated. Unauthenticated requests hit a 60 req/hour | |
| # IP-shared rate limit on Actions runners and fall into 403s. | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| # Run analysis and capture stdout (report) separately from stderr (logs) | |
| set +e | |
| python ./scripts/analyze_nightly.py --regression-threshold 2 > analysis_report.md | |
| EXIT_CODE=$? | |
| set -e | |
| # Set outputs for later steps | |
| echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT | |
| # Save report as output (using delimiter for multiline) | |
| { | |
| echo "report<<EOF" | |
| cat analysis_report.md | |
| echo "EOF" | |
| } >> $GITHUB_OUTPUT | |
| # Print report to logs as well | |
| cat analysis_report.md | |
| - name: Generate job summary | |
| run: cat analysis_report.md >> $GITHUB_STEP_SUMMARY | |
| - name: Check for regressions | |
| if: ${{ steps.analysis.outputs.exit_code == '1' }} | |
| run: echo "::warning::Performance regressions detected! See job summary for details." | |
| - name: Check for errors | |
| if: ${{ steps.analysis.outputs.exit_code == '2' }} | |
| run: echo "::warning::Errors occurred during analysis. See job summary for details." | |
| - name: Send report to Matrix | |
| uses: fadenb/matrix-chat-message@v0.0.6 | |
| with: | |
| homeserver: ${{ secrets.MATRIX_HOMESERVER }} | |
| token: ${{ secrets.MATRIX_ACCESS_TOKEN }} | |
| channel: ${{ secrets.MATRIX_ROOM_ID }} | |
| message: | | |
| ${{ steps.analysis.outputs.report }} | |
| [View workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) |