ci: add automatic performance benchmark CI for pull requests #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Performance Benchmark CI Workflow | |
| # | |
| # This workflow automatically runs performance benchmarks on pull requests | |
| # and posts a comparison comment showing speed differences between the PR | |
| # and the base branch (master). | |
| # | |
| # The workflow: | |
| # 1. Runs benchmarks on the PR branch | |
| # 2. Runs benchmarks on the base branch (master) | |
| # 3. Compares the results and identifies performance regressions/improvements | |
| # 4. Posts a detailed comparison table as a PR comment | |
| # | |
| # Performance changes > 10% are flagged as significant to help maintainers | |
| # catch performance regressions before merging. | |
| name: Performance Benchmark | |
| on: | |
| pull_request: | |
| branches: [master] | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| env: | |
| BENCHMARK_PATH: tests/benchmarks/ | |
| steps: | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: | | |
| pip install -r requirements.txt | |
| pip install -r requirements_dev.txt | |
| - name: Run benchmarks on PR branch | |
| run: | | |
| python3 -m pytest \ | |
| --benchmark-only \ | |
| --benchmark-json=pr-benchmark.json \ | |
| --benchmark-columns=mean,stddev,iqr,ops,rounds \ | |
| ${{ env.BENCHMARK_PATH }} | |
| - name: Store PR benchmark result | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pr-benchmark | |
| path: pr-benchmark.json | |
| - name: Checkout base branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.base.ref }} | |
| - name: Install dependencies for base branch | |
| run: | | |
| pip install -r requirements.txt | |
| pip install -r requirements_dev.txt | |
| - name: Run benchmarks on base branch | |
| run: | | |
| python3 -m pytest \ | |
| --benchmark-only \ | |
| --benchmark-json=base-benchmark.json \ | |
| --benchmark-columns=mean,stddev,iqr,ops,rounds \ | |
| ${{ env.BENCHMARK_PATH }} | |
| - name: Store base benchmark result | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: base-benchmark | |
| path: base-benchmark.json | |
| # Checkout PR branch again to ensure artifact downloads happen in correct context | |
| - name: Checkout PR branch again | |
| uses: actions/checkout@v4 | |
| - name: Download PR benchmark result | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: pr-benchmark | |
| - name: Download base benchmark result | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: base-benchmark | |
| - name: Compare benchmarks and comment | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| // Read benchmark results | |
| const prBenchmark = JSON.parse(fs.readFileSync('pr-benchmark.json', 'utf8')); | |
| const baseBenchmark = JSON.parse(fs.readFileSync('base-benchmark.json', 'utf8')); | |
| // Create a map of base benchmarks for easy lookup | |
| const baseMap = {}; | |
| baseBenchmark.benchmarks.forEach(bench => { | |
| baseMap[bench.name] = bench; | |
| }); | |
| // Build comparison table | |
| let tableRows = []; | |
| let significantChanges = []; | |
| prBenchmark.benchmarks.forEach(prBench => { | |
| const baseBench = baseMap[prBench.name]; | |
| if (!baseBench) { | |
| tableRows.push({ | |
| name: prBench.name, | |
| status: '🆕 NEW', | |
| prMean: prBench.stats.mean, | |
| baseMean: '-', | |
| change: '-' | |
| }); | |
| return; | |
| } | |
| const prMean = prBench.stats.mean; | |
| const baseMean = baseBench.stats.mean; | |
| const changePercent = ((prMean - baseMean) / baseMean) * 100; | |
| let status = '✅'; | |
| let changeStr = changePercent.toFixed(2) + '%'; | |
| // Flag significant changes (> 10% regression) | |
| if (changePercent > 10) { | |
| status = '⚠️ SLOWER'; | |
| significantChanges.push(`${prBench.name}: ${changePercent.toFixed(2)}% slower`); | |
| } else if (changePercent < -10) { | |
| status = '🚀 FASTER'; | |
| } | |
| tableRows.push({ | |
| name: prBench.name, | |
| status: status, | |
| prMean: prMean, | |
| baseMean: baseMean, | |
| change: changeStr | |
| }); | |
| }); | |
| // Format table | |
| let comment = '## 📊 Performance Benchmark Results\n\n'; | |
| if (significantChanges.length > 0) { | |
| comment += '### ⚠️ Significant Performance Changes Detected\n\n'; | |
| significantChanges.forEach(change => { | |
| comment += `- ${change}\n`; | |
| }); | |
| comment += '\n'; | |
| } | |
| comment += '### Detailed Comparison\n\n'; | |
| comment += '| Benchmark | Status | PR Mean (s) | Base Mean (s) | Change |\n'; | |
| comment += '|-----------|--------|-------------|---------------|--------|\n'; | |
| tableRows.forEach(row => { | |
| const prMeanStr = typeof row.prMean === 'number' ? row.prMean.toExponential(4) : row.prMean; | |
| const baseMeanStr = typeof row.baseMean === 'number' ? row.baseMean.toExponential(4) : row.baseMean; | |
| comment += `| ${row.name} | ${row.status} | ${prMeanStr} | ${baseMeanStr} | ${row.change} |\n`; | |
| }); | |
| comment += '\n---\n'; | |
| comment += `**Base Branch:** \`${context.payload.pull_request.base.ref}\` (${context.payload.pull_request.base.sha.substring(0, 7)})\n`; | |
| comment += `**PR Branch:** \`${context.payload.pull_request.head.ref}\` (${context.payload.pull_request.head.sha.substring(0, 7)})\n`; | |
| if (prBenchmark.machine_info) { | |
| if (prBenchmark.machine_info.python_version) { | |
| comment += `**Python Version:** ${prBenchmark.machine_info.python_version}\n`; | |
| } | |
| if (prBenchmark.machine_info.system && prBenchmark.machine_info.release) { | |
| comment += `**Platform:** ${prBenchmark.machine_info.system} ${prBenchmark.machine_info.release}\n`; | |
| } | |
| } | |
| // Post comment | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: comment | |
| }); |