CI: Run benchmarks correctly on free GitHub hosted runners #23
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark Comparison | ||
| on: | ||
| pull_request: | ||
| branches: | ||
| - "*" | ||
| workflow_dispatch: | ||
| defaults: | ||
| run: | ||
| shell: bash | ||
| env: | ||
| GOCACHE: /home/runner/work/go/pkg/build | ||
| GOPATH: /home/runner/work/go | ||
| GOBIN: /home/runner/work/go/bin | ||
| GO111MODULE: on | ||
| GO_VERSION: 1.24.6 | ||
| REGRESSION_THRESHOLD: 10 | ||
| jobs: | ||
| benchmark: | ||
| name: Run Benchmarks and Compare | ||
| runs-on: ubuntu-latest | ||
| if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' | ||
| permissions: | ||
| contents: read | ||
| pull-requests: write | ||
| issues: write | ||
| steps: | ||
| - name: git checkout | ||
| uses: actions/checkout@v5 | ||
| with: | ||
| fetch-depth: 0 | ||
| - name: Clean up runner space | ||
| uses: ./.github/actions/cleanup-space | ||
| - name: setup go ${{ env.GO_VERSION }} | ||
| uses: actions/setup-go@v5 | ||
| with: | ||
| go-version: '${{ env.GO_VERSION }}' | ||
| - name: Install benchstat | ||
| run: go install golang.org/x/perf/cmd/benchstat@latest | ||
| - name: Run benchmarks on base branch | ||
| run: | | ||
| git fetch origin ${{ github.base_ref }} | ||
| git checkout origin/${{ github.base_ref }} | ||
| make unit-bench pkg=wallet bench=BenchmarkLabelTxAPI benchmem=1 log=error 2>&1 | tee base-bench.txt | ||
| echo "Base branch benchmarks completed" | ||
| - name: Run benchmarks on PR branch | ||
| run: | | ||
| # The PR head is already checked out by actions/checkout, just use it | ||
| git checkout ${{ github.event.pull_request.head.sha }} | ||
| make unit-bench pkg=wallet bench=BenchmarkLabelTxAPI benchmem=1 log=error 2>&1 | tee pr-bench.txt | ||
| echo "PR branch benchmarks completed" | ||
| - name: Compare benchmarks with benchstat | ||
| id: benchstat | ||
| run: | | ||
| echo "## Benchmark Comparison Results" > benchstat-output.txt | ||
| echo "" >> benchstat-output.txt | ||
| echo "Comparing \`${{ github.base_ref }}\` (base) vs \`${{ github.head_ref }}\` (PR)" >> benchstat-output.txt | ||
| echo "" >> benchstat-output.txt | ||
| echo '```' >> benchstat-output.txt | ||
| benchstat base-bench.txt pr-bench.txt | tee benchstat-raw.txt >> benchstat-output.txt || true | ||
| echo '```' >> benchstat-output.txt | ||
| # Save for PR comment | ||
| cat benchstat-output.txt > $GITHUB_STEP_SUMMARY | ||
| - name: Check for performance regressions (threshold: ${{ env.REGRESSION_THRESHOLD }}%) | ||
| run: | | ||
| # Performance regression thresholds from environment | ||
| MAX_TIME_REGRESSION=${{ env.REGRESSION_THRESHOLD }} | ||
| MAX_ALLOC_BYTES_REGRESSION=${{ env.REGRESSION_THRESHOLD }} | ||
| MAX_ALLOC_COUNT_REGRESSION=${{ env.REGRESSION_THRESHOLD }} | ||
| echo "Checking for performance regressions..." | ||
| echo "Thresholds: ${MAX_TIME_REGRESSION}% for time, ${MAX_ALLOC_BYTES_REGRESSION}% for bytes, ${MAX_ALLOC_COUNT_REGRESSION}% for alloc count" | ||
| # Parse benchstat output for regressions | ||
| # Format: "name old time/op new time/op delta" | ||
| # A "+" delta means regression (slower), "-" means improvement (faster) | ||
| REGRESSIONS_FOUND=false | ||
| # Check time regressions (lines with "ns/op" and positive delta > threshold) | ||
| while IFS= read -r line; do | ||
| # Skip header and non-benchmark lines | ||
| if [[ $line =~ ^name ]] || [[ ! $line =~ ns/op ]]; then | ||
| continue | ||
| fi | ||
| # Extract delta percentage (e.g., "+12.5%" -> "12.5") | ||
| if [[ $line =~ \+([0-9]+\.[0-9]+)% ]]; then | ||
| delta="${BASH_REMATCH[1]}" | ||
| benchmark_name=$(echo "$line" | awk '{print $1}') | ||
| # Compare with threshold (use bc for floating point) | ||
| if (( $(echo "$delta > $MAX_TIME_REGRESSION" | bc -l) )); then | ||
| echo "❌ TIME REGRESSION: $benchmark_name regressed by ${delta}% (threshold: ${MAX_TIME_REGRESSION}%)" | ||
| REGRESSIONS_FOUND=true | ||
| fi | ||
| fi | ||
| done < benchstat-raw.txt | ||
| # Check memory bytes allocation regressions (lines with "B/op" and positive delta > threshold) | ||
| while IFS= read -r line; do | ||
| # Skip header and non-benchmark lines | ||
| if [[ $line =~ ^name ]] || [[ ! $line =~ B/op ]]; then | ||
| continue | ||
| fi | ||
| # Extract delta percentage | ||
| if [[ $line =~ \+([0-9]+\.[0-9]+)% ]]; then | ||
| delta="${BASH_REMATCH[1]}" | ||
| benchmark_name=$(echo "$line" | awk '{print $1}') | ||
| # Compare with threshold | ||
| if (( $(echo "$delta > $MAX_ALLOC_BYTES_REGRESSION" | bc -l) )); then | ||
| echo "❌ MEMORY BYTES REGRESSION: $benchmark_name memory usage increased by ${delta}% (threshold: ${MAX_ALLOC_BYTES_REGRESSION}%)" | ||
| REGRESSIONS_FOUND=true | ||
| fi | ||
| fi | ||
| done < benchstat-raw.txt | ||
| # Check allocation count regressions (lines with "allocs/op" and positive delta > threshold) | ||
| while IFS= read -r line; do | ||
| # Skip header and non-benchmark lines | ||
| if [[ $line =~ ^name ]] || [[ ! $line =~ allocs/op ]]; then | ||
| continue | ||
| fi | ||
| # Extract delta percentage | ||
| if [[ $line =~ \+([0-9]+\.[0-9]+)% ]]; then | ||
| delta="${BASH_REMATCH[1]}" | ||
| benchmark_name=$(echo "$line" | awk '{print $1}') | ||
| # Compare with threshold | ||
| if (( $(echo "$delta > $MAX_ALLOC_COUNT_REGRESSION" | bc -l) )); then | ||
| echo "❌ ALLOCATION COUNT REGRESSION: $benchmark_name allocation count increased by ${delta}% (threshold: ${MAX_ALLOC_COUNT_REGRESSION}%)" | ||
| REGRESSIONS_FOUND=true | ||
| fi | ||
| fi | ||
| done < benchstat-raw.txt | ||
| if [ "$REGRESSIONS_FOUND" = true ]; then | ||
| echo "" | ||
| echo "⚠️ Performance regressions detected above threshold!" | ||
| echo "Review the benchmark results and consider optimizing before merging." | ||
| exit 1 | ||
| else | ||
| echo "✅ No significant performance regressions detected." | ||
| fi | ||
| - name: Upload benchmark results as artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: benchmark-results | ||
| path: | | ||
| base-bench.txt | ||
| pr-bench.txt | ||
| benchstat-output.txt | ||
| retention-days: 30 | ||
