benchmark.yml

name: Benchmark

on:
  push:
    branches: ['main', 'maint/*']
  pull_request:
    branches: ['main', 'maint/*']
  # Allow job to be triggered manually from GitHub interface
  workflow_dispatch:

defaults:
  run:
    shell: bash

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: write

env:
  BENCHMARK_ARTIFACT_PREFIX: benchmark-results
  BENCHMARK_MAIN_ARTIFACT: benchmark-results-main-latest
  BENCHMARK_RESULTS_PATH: benchmarks/results

jobs:
  benchmark:
    name: Linux
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    strategy:
      fail-fast: false
      matrix:
        python-version: [ '3.12' ]

    steps:
    - name: Set up system
      uses: actions/checkout@v5
      with:
        fetch-depth: 0

    - name: Determine commit SHAs
      run: |
        git fetch origin

        MAIN_SHA=$(git rev-parse origin/main)
        HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
        MERGE_BASE_SHA=$(git merge-base HEAD origin/main)

        echo "Main SHA: $MAIN_SHA"
        echo "Head SHA: $HEAD_SHA"
        echo "Merge base SHA: $MERGE_BASE_SHA"

        echo "MAIN_SHA=$MAIN_SHA" >> $GITHUB_ENV
        echo "HEAD_SHA=$HEAD_SHA" >> $GITHUB_ENV
        echo "MERGE_BASE_SHA=$MERGE_BASE_SHA" >> $GITHUB_ENV

    # Check staleness against main tip (PRs only)
    - name: Check staleness against main tip
      if: github.event_name == 'pull_request'
      id: staleness
      run: |
        # Count commits behind
        COMMITS_BEHIND=$(git rev-list --count ${MERGE_BASE_SHA}..${MAIN_SHA})
        echo "commits-behind=$COMMITS_BEHIND" >> $GITHUB_OUTPUT
        
        # Check if stale (more than 1 commit behind)
        if [ $COMMITS_BEHIND -gt 1 ]; then
          echo "::warning::PR is $COMMITS_BEHIND commits behind main tip. Benchmarks may not reflect true merge impact."
          echo "::warning::Consider rebasing to ensure benchmarks reflect true merge impact."
          exit 1
        fi

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install .[antsopt,benchmark]

    - name: Set threading parameters for reliable benchmarking
      run: |
        echo "OPENBLAS_NUM_THREADS=1" >> $GITHUB_ENV
        echo "MKL_NUM_THREADS=1" >> $GITHUB_ENV
        echo "OMP_NUM_THREADS=1" >> $GITHUB_ENV

    - name: Configure ASV
      run: |
        CONFIG_FILE="$(pwd)/benchmarks/asv.conf.json"

        # Force a stable machine name for GitHub Actions so baseline + PR results match
        ASV_MACHINE="github-actions-ubuntu-latest-py${{ matrix.python-version }}"

        asv machine --yes --config "$CONFIG_FILE" --machine "$ASV_MACHINE"

        echo "ASV_CONFIG=$CONFIG_FILE" >> $GITHUB_ENV
        echo "ASV_MACHINE=$ASV_MACHINE" >> $GITHUB_ENV
        echo "ASV_RESULTS_DIR=$(pwd)/$BENCHMARK_RESULTS_PATH" >> $GITHUB_ENV

    # For PRs: Download baseline from main tip
    - name: Download baseline benchmarks from main
      if: github.event_name == 'pull_request'
      id: download-main
      uses: dawidd6/action-download-artifact@v19
      continue-on-error: true
      with:
        workflow: benchmark.yml
        branch: main
        name: ${{ env.BENCHMARK_MAIN_ARTIFACT }}
        path: ${{ env.BENCHMARK_RESULTS_PATH }}
        repo: nipreps/nifreeze
        github_token: ${{ secrets.GITHUB_TOKEN }}
        search_artifacts: true
        if_no_artifact_found: ignore

    # Report download status
    - name: Check baseline download status
      if: github.event_name == 'pull_request'
      run: |
        if [ -d "$BENCHMARK_RESULTS_PATH" ] && [ -n "$(ls -A $BENCHMARK_RESULTS_PATH 2>/dev/null)" ]; then
          echo "✅ Baseline benchmarks downloaded successfully from main"
          ls -lah $BENCHMARK_RESULTS_PATH
        else
          echo "::warning::️No baseline benchmarks found - will run benchmarks on main tip $MAIN_SHA"
        fi

    # Check if we actually got results
    - name: Check if baseline exists
      if: github.event_name == 'pull_request'
      id: baseline-check
      run: |
        if [ -d "$BENCHMARK_RESULTS_PATH" ] && [ -n "$(ls -A $BENCHMARK_RESULTS_PATH 2>/dev/null)" ]; then
          echo "found=true" >> $GITHUB_OUTPUT
        else
          echo "found=false" >> $GITHUB_OUTPUT
        fi

    # If artifact download failed, run benchmarks on main tip
    - name: Run benchmarks on main tip (if needed)
      if: github.event_name == 'pull_request' && steps.baseline-check.outputs.found == 'false'
      run: |
        echo "Running benchmarks on main tip $MAIN_SHA..."
        asv run --config "$ASV_CONFIG" --show-stderr --quick "$MAIN_SHA^!"

    # Run benchmarks on current commit
    - name: Run benchmarks on head commit
      run: |
        echo "Running benchmarks on head commit $HEAD_SHA..."
        asv run --config "$ASV_CONFIG" --show-stderr --quick "$HEAD_SHA^!"

    # Compare PR benchmarks to main tip
    - name: Compare benchmarks
      if: github.event_name == 'pull_request'
      id: compare
      run: |
        echo "Comparing $HEAD_SHA against main tip $MAIN_SHA..."

        FACTOR=$(python -c "import json; print(json.load(open('$ASV_CONFIG')).get('regression_factor', 1.1))")
        echo "FACTOR=$FACTOR" >> "$GITHUB_ENV"


        asv compare --config "$ASV_CONFIG" --machine "$ASV_MACHINE" \
          --factor $FACTOR --split "$MAIN_SHA" "$HEAD_SHA" | tee comparison.txt

        # Check for regressions
        if grep -q "got worse" comparison.txt; then
          echo "regression=true" >> $GITHUB_OUTPUT
          echo "::error::Performance regression detected compared to main tip!"
        else
          echo "regression=false" >> $GITHUB_OUTPUT
          echo "✅ No significant regressions detected"
        fi

    # Fail CI if has regression
    - name: Fail on performance regression
      if: |
        github.event_name == 'pull_request' && steps.compare.outputs.regression == 'true'
      run: |
        echo "::error::Performance regression detected. Benchmarks are $FACTOR times slower than main tip."
        echo "See comparison output above for details."
        exit 1

    # Upload benchmark results as artifact (for main branch)
    - name: Upload benchmark results (main only)
      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
      uses: actions/upload-artifact@v7
      with:
        name: ${{ env.BENCHMARK_MAIN_ARTIFACT }}
        path: ${{ env.BENCHMARK_RESULTS_PATH }}
        compression-level: 9

    # Also save per-commit for debugging
    - name: Save head commit benchmarks
      if: always()
      uses: actions/upload-artifact@v7
      with:
        name: ${{ env.BENCHMARK_ARTIFACT_PREFIX }}-${{ github.sha }}
        path: ${{ env.BENCHMARK_RESULTS_PATH }}
        compression-level: 9