benchmark_gpu_coprocessor.yml

# Run all fhevm coprocessor benchmarks on a GPU instance on Hyperstack and return parsed results to Slab CI bot.
name: benchmark_gpu_coprocessor

on:
  workflow_dispatch:
    inputs:
      profile:
        description: "Instance type"
        required: true
        type: choice
        options:
          - "hyperstack::l40 (n3-L40x1)"
          - "hyperstack::4-l40 (n3-L40x4)"
          - "hyperstack::multi-a100-nvlink (n3-A100x8-NVLink)"
          - "hyperstack::single-h100 (n3-H100x1)"
          - "hyperstack::2-h100 (n3-H100x2)"
          - "hyperstack::4-h100 (n3-H100x4)"
          - "hyperstack::multi-h100 (n3-H100x8)"
          - "hyperstack::multi-h100-nvlink (n3-H100x8-NVLink)"
          - "hyperstack::multi-h100-sxm5 (n3-H100-SXM5x8)"
          - "scaleway::multi-h100-sxm5 (H100-SXM-8-80G)"
          - "scaleway::4-h100-sxm5 (H100-SXM-4-80G)"
          - "scaleway::2-h100-sxm5 (H100-SXM-2-80G)"
          - "scaleway::single-h100 (H100-1-80G)"
  schedule:
    # Weekly tests @ 1AM
    - cron: "0 1 * * 6"

permissions:
  contents: read

# zizmor: ignore[concurrency-limits] concurrency is managed after instance setup to ensure safe provisioning

env:
  CARGO_TERM_COLOR: always
  RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json
  ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
  RUST_BACKTRACE: "full"
  RUST_MIN_STACK: "8388608"
  CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }}
  PROFILE_SCHEDULED_RUN: "hyperstack::multi-h100-sxm5 (n3-H100-SXM5x8)"
  PROFILE_MANUAL_RUN: ${{ inputs.profile }}
  IS_MANUAL_RUN: ${{ github.event_name == 'workflow_dispatch' }}
  BENCHMARK_TYPE: "ALL"
  OPTIMIZATION_TARGET: "throughput"
  BATCH_SIZE: "5000"
  SCHEDULING_POLICY: "MAX_PARALLELISM"
  BENCHMARKS: "erc20"
  BRANCH_NAME: ${{ github.ref_name }}
  COMMIT_SHA: ${{ github.sha }}
  SLAB_SECRET: ${{ secrets.JOB_SECRET }}

jobs:
  parse-inputs:
    name: benchmark_gpu_coprocessor/parse-inputs
    if: github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs')
    runs-on: ubuntu-latest
    permissions:
      contents: 'read'
    outputs:
      backend: ${{ steps.parse_profile.outputs.backend }}
      profile: ${{ steps.parse_profile.outputs.profile }}
      hardware_name: ${{ steps.parse_profile.outputs.hardware }}
      cloud_provider: ${{ steps.parse_profile.outputs.cloud_provider }}
    steps:
      - name: Checkout tfhe-rs repo
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
        with:
          persist-credentials: 'false'
          token: ${{ env.CHECKOUT_TOKEN }}

      - name: Parse profile and hardware name
        id: parse_profile
        run: |
          if [[ ${IS_MANUAL_RUN} == true ]]; then
            PROFILE_RAW="${PROFILE_MANUAL_RUN}"
          else
            PROFILE_RAW="${PROFILE_SCHEDULED_RUN}"
          fi
          python3 ci/parse_benchmark_profile.py "$PROFILE_RAW" "$GITHUB_OUTPUT"

  setup-instance:
    name: benchmark_gpu_coprocessor/setup-instance
    needs: parse-inputs
    runs-on: ubuntu-latest
    permissions:
      contents: 'read'
    outputs:
      runner-name: ${{ steps.start-remote-instance.outputs.label }}
    steps:
      - name: Start remote instance
        id: start-remote-instance
        uses: zama-ai/slab-github-runner@d4a5bb6e2a4b8aa19a10f596d7f142006a7e2c21 # v1.6.3
        with:
          mode: start
          github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
          slab-url: ${{ secrets.SLAB_BASE_URL }}
          job-secret: ${{ secrets.JOB_SECRET }}
          backend: ${{ needs.parse-inputs.outputs.backend }}
          profile: ${{ needs.parse-inputs.outputs.profile }}

  benchmark-gpu:
    name: benchmark_gpu_coprocessor/benchmark-gpu (bpr)
    needs: [ parse-inputs, setup-instance ]
    runs-on: ${{ needs.setup-instance.outputs.runner-name }}
    continue-on-error: true
    timeout-minutes: 720  # 12 hours
    permissions:
      contents: 'read' # Needed to read repositories contents
      packages: 'read' # Needed to get fhevm packages
    strategy:
      fail-fast: false
      # explicit include-based build matrix, of known valid options
      matrix:
        include:
          - os: ubuntu-22.04
            cuda: "12.8 12.2"
            gcc: 12
    steps:
      - name: Install git LFS
        run: |
          bash ci/wait_for_apt_lock.sh

          sudo apt-get update
          sudo apt-get install -y git-lfs protobuf-compiler
          git lfs install

      - name: Checkout tfhe-rs
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Check fhEVM and TFHE-rs repos
        run: |
          pwd
          ls

      - name: Checkout fhevm
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: zama-ai/fhevm
          persist-credentials: 'false'
          fetch-depth: 0
          lfs: true
          ref: antoniu/use-tfhe-main-benches
          path: fhevm

      - name: Get benchmark details
        run: |
          COMMIT_DATE_ENV=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${COMMIT_SHA}")
          {
            echo "BENCH_DATE=$(date --iso-8601=seconds)";
            echo "COMMIT_DATE=$COMMIT_DATE_ENV";
            echo "COMMIT_HASH=$(git rev-parse HEAD)";
          } >> "${GITHUB_ENV}"

      - name: Setup cloud GPU machine dependencies
        uses: ./.github/actions/gpu_setup
        with:
          cuda-version: ${{ matrix.cuda }}
          gcc-version: ${{ matrix.gcc }}
          cloud-provider: ${{ needs.parse-inputs.outputs.cloud_provider }}

      - name: Resolve GPU hardware name
        uses: ./.github/actions/resolve_gpu_hardware
        with:
          requested-hardware-name: ${{ needs.parse-inputs.outputs.hardware_name }}

      - name: Checkout tfhe-rs in fhevm coprocessor
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          path: fhevm/coprocessor/tfhe-rs
          persist-credentials: false

      - name: Checkout LFS objects
        run: git lfs checkout
        working-directory: fhevm/

      - name: Install rust
        uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # zizmor: ignore[stale-action-refs] this action doesn't create releases
        with:
          toolchain: nightly

      - name: Install Safe-chain
        run: |
          make install_aikido_safe_chain_ci
        working-directory: fhevm/coprocessor/tfhe-rs

      - name: Install cargo dependencies
        run: |
          sudo apt-get install -y protobuf-compiler pkg-config libssl-dev \
                                  libclang-dev acl
          if ! command -v docker >/dev/null 2>&1; then
            sudo apt-get install -y docker.io
          fi
          if ! docker compose version >/dev/null 2>&1; then
            sudo apt-get install -y docker-compose-v2
          fi
          sudo usermod -aG docker "$USER"
          newgrp docker
          sudo setfacl --modify user:"$USER":rw /var/run/docker.sock
          cargo install sqlx-cli --version=0.8.6 --locked

      - name: Install foundry
        uses: foundry-rs/foundry-toolchain@908c540300062bd5a7e473851cdb4282204cee09

      - name: Cache cargo
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: ${{ runner.os }}-cargo-

      - name: Login to GitHub Container Registry
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Login to Chainguard Registry
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
        with:
          registry: cgr.dev
          username: ${{ secrets.CGR_USERNAME }}
          password: ${{ secrets.CGR_PASSWORD }}

      - name: Init database
        run: make init_db
        working-directory: fhevm/coprocessor/fhevm-engine/tfhe-worker

      - name: Use Node.js
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: 20.x

      - name: Build contracts
        env:
          HARDHAT_NETWORK: hardhat
        run: |
          ls
          pwd
          cp ./host-contracts/.env.example ./host-contracts/.env
          cd ./host-contracts 
          npm ci --include=optional
          npm install && npm run deploy:emptyProxies && npx hardhat compile
        working-directory: fhevm/

      - name: Profile erc20 no-cmux benchmark on GPU
        run: |
          export DATABASE_URL="postgres://postgres:postgres@127.0.0.1:5432/coprocessor"
          export RUSTFLAGS="-C target-cpu=native"
          export BENCHMARK_BATCH_SIZE="${BATCH_SIZE}"
          export FHEVM_DF_SCHEDULE="${SCHEDULING_POLICY}"
          export BENCHMARK_TYPE="LATENCY"
          export OPTIMIZATION_TARGET="latency"
          export FHEVM_TEST_NUM_SAMPLES=20
          cargo +nightly bench --no-run --bench erc20 --features=gpu-profile,bench,"${OPTIMIZATION_TARGET}" --
          nsys profile --trace=cuda,osrt,nvtx \
            cargo +nightly bench --bench erc20 --features=gpu-profile,bench,"${OPTIMIZATION_TARGET}" -- 'latency::no_cmux'
        working-directory: fhevm/coprocessor/fhevm-engine/tfhe-worker

      - name: Get nsys profile name
        id: nsys_profile_name
        run: echo "profile=coprocessor_profile_$(date +"%Y-%m-%d-%Hh").nsys-rep" >> "$GITHUB_OUTPUT"

      - name: Timestamp nsys profile # zizmor: ignore[template-injection]
        env:
          REPORT_NAME: ${{ steps.nsys_profile_name.outputs.profile }}
        run: |
          mv report1.nsys-rep ${{ env.REPORT_NAME }}
        working-directory: fhevm/coprocessor/fhevm-engine/tfhe-worker

      - name: Upload profile artifact
        env:
          REPORT_NAME: ${{ steps.nsys_profile_name.outputs.profile }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: ${{ env.REPORT_NAME }}
          path: fhevm/coprocessor/fhevm-engine/tfhe-worker/${{ env.REPORT_NAME }}

      - name: Run latency benchmark on GPU
        run: |
          BENCHMARK_BATCH_SIZE="${BATCH_SIZE}" FHEVM_DF_SCHEDULE="${SCHEDULING_POLICY}" BENCHMARK_TYPE="LATENCY" OPTIMIZATION_TARGET="${OPTIMIZATION_TARGET}" make -e "benchmark_${BENCHMARKS}_gpu"
        working-directory: fhevm/coprocessor/fhevm-engine/tfhe-worker

      - name: Run throughput benchmarks on GPU
        run: |
          BENCHMARK_BATCH_SIZE="${BATCH_SIZE}" FHEVM_DF_SCHEDULE="${SCHEDULING_POLICY}" BENCHMARK_TYPE="THROUGHPUT_200" OPTIMIZATION_TARGET="${OPTIMIZATION_TARGET}" make -e "benchmark_${BENCHMARKS}_gpu"
        working-directory: fhevm/coprocessor/fhevm-engine/tfhe-worker

      - name: Parse results
        run: |
          cargo run --release -p tfhe-benchmark-parser -- -i "${GITHUB_WORKSPACE}/fhevm/coprocessor/fhevm-engine/target/criterion" -o "${RESULTS_FILENAME}" \
          --database coprocessor \
          --hardware "${HARDWARE_NAME}" \
          --backend cuda \
          --project-version "${COMMIT_HASH}" \
          --branch "${BRANCH_NAME}" \
          --commit-date "${COMMIT_DATE}" \
          --bench-date "${BENCH_DATE}" \
          --walk-subdirs \
          --params-dir "${GITHUB_WORKSPACE}/fhevm/coprocessor/fhevm-engine/tfhe-worker" \
          --name-suffix "operation_batch_size_${BATCH_SIZE}-schedule_${SCHEDULING_POLICY}-optimization_target_${OPTIMIZATION_TARGET}"
        working-directory: fhevm/coprocessor/tfhe-rs

      - name: Upload parsed results artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: ${{ env.COMMIT_SHA }}_${{ env.BENCHMARKS }}_${{ needs.parse-inputs.outputs.profile }}
          path: fhevm/coprocessor/tfhe-rs/${{ env.RESULTS_FILENAME }}

      - name: Checkout Slab repo
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
        with:
          repository: zama-ai/slab
          path: slab
          persist-credentials: 'false'
          token: ${{ secrets.REPO_CHECKOUT_TOKEN }}

      - name: Send data to Slab
        shell: bash
        env:
          SLAB_URL: ${{ secrets.SLAB_URL }}
        run: |
          python3 slab/scripts/data_sender.py fhevm/coprocessor/tfhe-rs/"${RESULTS_FILENAME}" "${SLAB_SECRET}" \
          --slab-url "${SLAB_URL}"

  teardown-instance:
    name: benchmark_gpu_coprocessor/teardown-instance
    if: ${{ always() && needs.setup-instance.result == 'success' }}
    needs: [ setup-instance, benchmark-gpu ]
    runs-on: ubuntu-latest
    permissions:
      contents: 'read'
    steps:
      - name: Stop remote instance
        id: stop-instance
        uses: zama-ai/slab-github-runner@d4a5bb6e2a4b8aa19a10f596d7f142006a7e2c21 # v1.6.3
        with:
          mode: stop
          github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
          slab-url: ${{ secrets.SLAB_BASE_URL }}
          job-secret: ${{ secrets.JOB_SECRET }}
          label: ${{ needs.setup-instance.outputs.runner-name }}