chore: Single-shot benchmarking + continuous benchmarking (#183) #576
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous integration | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| merge_group: | |
| types: [checks_requested] | |
| workflow_dispatch: {} | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_INCREMENTAL: 0 | |
| MIRIFLAGS: '-Zmiri-permissive-provenance' # Required due to warnings in bitvec 1.0.1 | |
| jobs: | |
| check: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install stable toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt, clippy | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| prefix-key: v0 | |
| - name: Check formatting | |
| run: cargo fmt -- --check | |
| - name: Run clippy | |
| run: cargo clippy --all-targets --all-features -- -D warnings | |
| - name: Build docs | |
| run: cargo doc --no-deps --all-features | |
| env: | |
| RUSTDOCFLAGS: "-Dwarnings" | |
| miri: | |
| # Not required, we can ignore it for the merge queue check. | |
| if: github.event_name != 'merge_group' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install latest nightly | |
| uses: dtolnay/rust-toolchain@nightly | |
| with: | |
| components: miri | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| prefix-key: v0-nightly | |
| - name: Run miri | |
| run: cargo miri test | |
| benches: | |
| name: continuous Benchmarking | |
| # Not required, we can ignore it for the merge queue check. | |
| if: github.event_name != 'merge_group' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| checks: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install stable toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| prefix-key: v0 | |
| # The installed iai-callgrind-runner version must match the | |
| # version of iai-callgrind in the Cargo.toml | |
| - uses: cargo-bins/cargo-binstall@main | |
| - name: Install iai-callgrind-runner | |
| run: | | |
| version=$(cargo metadata --format-version=1 |\ | |
| jq '.packages[] | select(.name == "iai-callgrind").version' |\ | |
| tr -d '"' | |
| ) | |
| cargo binstall --no-confirm iai-callgrind-runner --version $version --force | |
| - uses: bencherdev/bencher@main | |
| - name: Install valgrind | |
| run: sudo apt update && sudo apt install -y valgrind | |
| - name: Track base branch IAI benchmarks | |
| if: github.event_name == 'push' | |
| run: | | |
| bencher run \ | |
| --project portgraph \ | |
| --token '${{ secrets.BENCHER_API_TOKEN }}' \ | |
| --branch main \ | |
| --testbed ubuntu-latest \ | |
| --threshold-measure instructions \ | |
| --threshold-test t_test \ | |
| --threshold-max-sample-size 64 \ | |
| --threshold-upper-boundary 0.99 \ | |
| --thresholds-reset \ | |
| --err \ | |
| --github-actions '${{ secrets.HUGRBOT_PAT }}' \ | |
| --adapter rust_iai_callgrind \ | |
| "cargo bench --bench iai_benches" | |
| - name: Track PR IAI benchmarks | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| bencher run \ | |
| --project portgraph \ | |
| --token '${{ secrets.BENCHER_API_TOKEN }}' \ | |
| --branch "${{ github.event.pull_request.head.ref }}" \ | |
| --testbed ubuntu-latest \ | |
| --start-point "${{ github.event.pull_request.base.ref }}" \ | |
| --start-point-clone-thresholds \ | |
| --start-point-reset \ | |
| --err \ | |
| --github-actions '${{ secrets.HUGRBOT_PAT }}' \ | |
| --adapter rust_iai_callgrind \ | |
| "cargo bench --bench iai_benches" | |
| # --start-point-hash '${{ github.event.pull_request.base.sha }}' \ | |
| tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| rust: ['1.75', stable, beta, nightly] | |
| # workaround to ignore non-stable tests when running the merge queue checks | |
| # see: https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6 | |
| isMerge: | |
| - ${{ github.event_name == 'merge_group' }} | |
| exclude: | |
| - rust: '1.75' | |
| isMerge: true | |
| - rust: beta | |
| isMerge: true | |
| - rust: nightly | |
| isMerge: true | |
| name: tests (Rust ${{ matrix.rust }}) | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - id: toolchain | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: ${{ matrix.rust }} | |
| - name: Configure default rust toolchain | |
| run: rustup override set ${{steps.toolchain.outputs.name}} | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| prefix-key: v0-rust-${{ matrix.rust }} | |
| - name: Build with no features | |
| run: cargo test --verbose --no-default-features --no-run | |
| - name: Tests with no features | |
| run: cargo test --verbose --no-default-features | |
| - name: Build with all features | |
| run: cargo test --verbose --all-features --no-run | |
| - name: Tests with all features | |
| run: cargo test --verbose --all-features | |
| rs-semver-checks: | |
| needs: [check] | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: CQCL/hugrverse-actions/.github/workflows/rs-semver-checks.yml@main | |
| secrets: | |
| GITHUB_PAT: ${{ secrets.HUGRBOT_PAT }} | |
| coverage: | |
| if: github.event_name != 'merge_group' | |
| needs: [tests, check] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: mozilla-actions/[email protected] | |
| - uses: dtolnay/rust-toolchain@nightly | |
| with: | |
| components: llvm-tools-preview | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Run tests with coverage instrumentation | |
| run: | | |
| cargo llvm-cov clean --workspace | |
| cargo llvm-cov --doctests | |
| - name: Generate coverage report | |
| run: cargo llvm-cov report --codecov --output-path coverage.json | |
| - name: Upload coverage to codecov.io | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| files: coverage.json | |
| name: ubuntu | |
| token: ${{ secrets.CODECOV_TOKEN }} |