Implement MultiShellKernel #1942
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| push: | |
| branches: ['main', 'maint/*'] | |
| pull_request: | |
| branches: ['main', 'maint/*'] | |
| # Allow job to be triggered manually from GitHub interface | |
| workflow_dispatch: | |
| defaults: | |
| run: | |
| shell: bash | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: write | |
| env: | |
| BENCHMARK_ARTIFACT_PREFIX: benchmark-results | |
| BENCHMARK_MAIN_ARTIFACT: benchmark-results-main-latest | |
| BENCHMARK_RESULTS_PATH: benchmarks/results | |
| jobs: | |
| benchmark: | |
| name: Linux | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [ '3.12' ] | |
| steps: | |
| - name: Set up system | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Determine commit SHAs | |
| run: | | |
| git fetch origin | |
| MAIN_SHA=$(git rev-parse origin/main) | |
| HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}" | |
| MERGE_BASE_SHA=$(git merge-base HEAD origin/main) | |
| echo "Main SHA: $MAIN_SHA" | |
| echo "Head SHA: $HEAD_SHA" | |
| echo "Merge base SHA: $MERGE_BASE_SHA" | |
| echo "MAIN_SHA=$MAIN_SHA" >> $GITHUB_ENV | |
| echo "HEAD_SHA=$HEAD_SHA" >> $GITHUB_ENV | |
| echo "MERGE_BASE_SHA=$MERGE_BASE_SHA" >> $GITHUB_ENV | |
| # Check staleness against main tip (PRs only) | |
| - name: Check staleness against main tip | |
| if: github.event_name == 'pull_request' | |
| id: staleness | |
| run: | | |
| # Count commits behind | |
| COMMITS_BEHIND=$(git rev-list --count ${MERGE_BASE_SHA}..${MAIN_SHA}) | |
| echo "commits-behind=$COMMITS_BEHIND" >> $GITHUB_OUTPUT | |
| # Check if stale (more than 1 commit behind) | |
| if [ $COMMITS_BEHIND -gt 1 ]; then | |
| echo "::warning::PR is $COMMITS_BEHIND commits behind main tip. Benchmarks may not reflect true merge impact." | |
| echo "::warning::Consider rebasing to ensure benchmarks reflect true merge impact." | |
| exit 1 | |
| fi | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install .[antsopt,benchmark] | |
| - name: Set threading parameters for reliable benchmarking | |
| run: | | |
| echo "OPENBLAS_NUM_THREADS=1" >> $GITHUB_ENV | |
| echo "MKL_NUM_THREADS=1" >> $GITHUB_ENV | |
| echo "OMP_NUM_THREADS=1" >> $GITHUB_ENV | |
| - name: Configure ASV | |
| run: | | |
| CONFIG_FILE="$(pwd)/benchmarks/asv.conf.json" | |
| # Force a stable machine name for GitHub Actions so baseline + PR results match | |
| ASV_MACHINE="github-actions-ubuntu-latest-py${{ matrix.python-version }}" | |
| asv machine --yes --config "$CONFIG_FILE" --machine "$ASV_MACHINE" | |
| echo "ASV_CONFIG=$CONFIG_FILE" >> $GITHUB_ENV | |
| echo "ASV_MACHINE=$ASV_MACHINE" >> $GITHUB_ENV | |
| echo "ASV_RESULTS_DIR=$(pwd)/$BENCHMARK_RESULTS_PATH" >> $GITHUB_ENV | |
| # For PRs: Download baseline from main tip | |
| - name: Download baseline benchmarks from main | |
| if: github.event_name == 'pull_request' | |
| id: download-main | |
| uses: dawidd6/action-download-artifact@v19 | |
| continue-on-error: true | |
| with: | |
| workflow: benchmark.yml | |
| branch: main | |
| name: ${{ env.BENCHMARK_MAIN_ARTIFACT }} | |
| path: ${{ env.BENCHMARK_RESULTS_PATH }} | |
| repo: nipreps/nifreeze | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| search_artifacts: true | |
| if_no_artifact_found: ignore | |
| # Report download status | |
| - name: Check baseline download status | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| if [ -d "$BENCHMARK_RESULTS_PATH" ] && [ -n "$(ls -A $BENCHMARK_RESULTS_PATH 2>/dev/null)" ]; then | |
| echo "✅ Baseline benchmarks downloaded successfully from main" | |
| ls -lah $BENCHMARK_RESULTS_PATH | |
| else | |
| echo "::warning::️No baseline benchmarks found - will run benchmarks on main tip $MAIN_SHA" | |
| fi | |
| # Check if we actually got results | |
| - name: Check if baseline exists | |
| if: github.event_name == 'pull_request' | |
| id: baseline-check | |
| run: | | |
| if [ -d "$BENCHMARK_RESULTS_PATH" ] && [ -n "$(ls -A $BENCHMARK_RESULTS_PATH 2>/dev/null)" ]; then | |
| echo "found=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "found=false" >> $GITHUB_OUTPUT | |
| fi | |
| # If artifact download failed, run benchmarks on main tip | |
| - name: Run benchmarks on main tip (if needed) | |
| if: github.event_name == 'pull_request' && steps.baseline-check.outputs.found == 'false' | |
| run: | | |
| echo "Running benchmarks on main tip $MAIN_SHA..." | |
| asv run --config "$ASV_CONFIG" --show-stderr --quick "$MAIN_SHA^!" | |
| # Run benchmarks on current commit | |
| - name: Run benchmarks on head commit | |
| run: | | |
| echo "Running benchmarks on head commit $HEAD_SHA..." | |
| asv run --config "$ASV_CONFIG" --show-stderr --quick "$HEAD_SHA^!" | |
| # Compare PR benchmarks to main tip | |
| - name: Compare benchmarks | |
| if: github.event_name == 'pull_request' | |
| id: compare | |
| run: | | |
| echo "Comparing $HEAD_SHA against main tip $MAIN_SHA..." | |
| FACTOR=$(python -c "import json; print(json.load(open('$ASV_CONFIG')).get('regression_factor', 1.1))") | |
| echo "FACTOR=$FACTOR" >> "$GITHUB_ENV" | |
| asv compare --config "$ASV_CONFIG" --machine "$ASV_MACHINE" \ | |
| --factor $FACTOR --split "$MAIN_SHA" "$HEAD_SHA" | tee comparison.txt | |
| # Check for regressions | |
| if grep -q "got worse" comparison.txt; then | |
| echo "regression=true" >> $GITHUB_OUTPUT | |
| echo "::error::Performance regression detected compared to main tip!" | |
| else | |
| echo "regression=false" >> $GITHUB_OUTPUT | |
| echo "✅ No significant regressions detected" | |
| fi | |
| # Fail CI if has regression | |
| - name: Fail on performance regression | |
| if: | | |
| github.event_name == 'pull_request' && steps.compare.outputs.regression == 'true' | |
| run: | | |
| echo "::error::Performance regression detected. Benchmarks are $FACTOR times slower than main tip." | |
| echo "See comparison output above for details." | |
| exit 1 | |
| # Upload benchmark results as artifact (for main branch) | |
| - name: Upload benchmark results (main only) | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ${{ env.BENCHMARK_MAIN_ARTIFACT }} | |
| path: ${{ env.BENCHMARK_RESULTS_PATH }} | |
| compression-level: 9 | |
| # Also save per-commit for debugging | |
| - name: Save head commit benchmarks | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ${{ env.BENCHMARK_ARTIFACT_PREFIX }}-${{ github.sha }} | |
| path: ${{ env.BENCHMARK_RESULTS_PATH }} | |
| compression-level: 9 |