Benchmark runtime weights #174
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark runtime weights | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| rebuild-docker: | |
| type: boolean | |
| description: rebuild-docker | |
| required: true | |
| default: true | |
| litentry: | |
| type: boolean | |
| description: litentry | |
| required: true | |
| default: true | |
| paseo: | |
| type: boolean | |
| description: paseo | |
| required: true | |
| default: true | |
| pallets: | |
| description: pallets to benchmark, * for all, or comma listed (e.g. frame-system,pallet-proxy) | |
| default: "*" | |
| required: true | |
| env: | |
| INSTANCE_NAME: "stress-test-server" | |
| INSTANCE_ZONE: "asia-southeast1-b" | |
| GOOGLE_PROJECT_ID: ${{ secrets.GOOGLE_PROJECT_ID }} | |
| GOOGLE_CREDENTIALS_JSON: ${{ secrets.GOOGLE_CREDENTIALS_JSON }} | |
| BENCHMARK_SSH_USER: "ubuntu" | |
| DOCKER_BUILDKIT: 1 | |
| jobs: | |
| ## build docker image with runtime-benchmarks feature and push it to the hub | |
| build-docker: | |
| if: ${{ github.event.inputs.rebuild-docker == 'true' }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout codes on ${{ github.ref }} | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Free up disk space | |
| if: startsWith(runner.name, 'GitHub Actions') | |
| uses: ./.github/actions/disk-cleanup | |
| - name: Set Swap Space | |
| uses: pierotofy/set-swap-space@master | |
| with: | |
| swap-size-gb: 10 | |
| - name: Build docker image | |
| run: | | |
| ./parachain/scripts/build-docker.sh production runtime-benchmarks --features=runtime-benchmarks | |
| - name: Dockerhub login | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - name: Push docker image | |
| run: | | |
| docker push litentry/litentry-parachain:runtime-benchmarks | |
| ## run the benchmarking remotely | |
| benchmark: | |
| runs-on: ubuntu-22.04 | |
| needs: build-docker | |
| if: | | |
| always() && | |
| (needs.build-docker.result == 'success' || needs.build-docker.result == 'skipped') | |
| steps: | |
| - name: Set env | |
| run: | | |
| chain="" | |
| if [ "${{ github.event.inputs.litentry }}" = "true" ]; then | |
| chain="$chain litentry" | |
| fi | |
| if [ "${{ github.event.inputs.paseo }}" = "true" ]; then | |
| chain="$chain paseo" | |
| fi | |
| if [ "$chain" = "" ]; then | |
| echo "::error::Please select at least one chain." | |
| exit 1 | |
| fi | |
| echo "CHAIN=$chain" >> $GITHUB_ENV | |
| - name: Checkout codes on ${{ github.ref }} | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Pull docker image | |
| run: | | |
| docker pull litentry/litentry-parachain:runtime-benchmarks | |
| - name: Authenticate with Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ env.GOOGLE_CREDENTIALS_JSON }} | |
| - name: Set up Google Cloud CLI | |
| run: | | |
| gcloud auth configure-docker | |
| gcloud config set project ${{ env.GOOGLE_PROJECT_ID }} | |
| - name: Start remote instance on GCE | |
| timeout-minutes: 10 | |
| id: start_instance | |
| run: | | |
| gcloud compute instances start ${{ env.INSTANCE_NAME }} --zone=${{ env.INSTANCE_ZONE }} | |
| sleep 5 | |
| SECONDS=0 | |
| while : ; do | |
| status=$(gcloud compute instances describe ${{ env.INSTANCE_NAME }} --zone=${{ env.INSTANCE_ZONE }} --format="get(status)") | |
| if [ "$status" = "RUNNING" ]; then | |
| break | |
| else | |
| sleep 10 | |
| SECONDS=$((SECONDS + 10)) | |
| fi | |
| done | |
| echo "Instance is running after $SECONDS seconds" | |
| remote_ip=$(gcloud compute instances describe ${{ env.INSTANCE_NAME }} --zone=${{ env.INSTANCE_ZONE }} --format="get(networkInterfaces[0].accessConfigs[0].natIP)") | |
| echo "remote_ip=$remote_ip" >> $GITHUB_ENV | |
| - name: Remotely benchmark pallets ${{ github.event.inputs.pallets }} for ${{ env.CHAIN }} | |
| timeout-minutes: 240 | |
| run: | | |
| echo "Remote instance IP address: ${{ env.remote_ip }}" | |
| arg="${{ github.event.inputs.pallets }}" | |
| chain="${{ env.CHAIN }}" | |
| if [ "$arg" = "*" ]; then | |
| arg="\\$arg"; | |
| fi | |
| for c in $chain; do | |
| gcloud compute scp --zone=${{ env.INSTANCE_ZONE }} --recurse parachain/scripts/benchmark-weight-remote.sh ${{ env.INSTANCE_NAME }}:~ | |
| gcloud compute ssh ${{ env.BENCHMARK_SSH_USER }}@${{ env.remote_ip }} --zone=${{ env.INSTANCE_ZONE }} --command="bash benchmark-weight-remote.sh \"$c\" \"${GITHUB_REF#refs/heads/}\" \"$arg\"" | |
| echo "Copying generated weights files back ..." | |
| gcloud compute scp --zone=${{ env.INSTANCE_ZONE }} ${{ env.INSTANCE_NAME }}:/tmp/litentry-parachain/parachain/runtime/$c/src/weights/*.rs parachain/runtime/$c/src/weights/ | |
| done | |
| - name: Stop remote instance on GCE | |
| if: always() | |
| run: | | |
| gcloud compute instances stop ${{ env.INSTANCE_NAME }} --zone=${{ env.INSTANCE_ZONE }} | |
| - name: Create auto PR | |
| uses: peter-evans/create-pull-request@v7 | |
| with: | |
| commit-message: "[benchmarking bot] Auto commit generated weights files" | |
| committer: benchmarking bot <[email protected]> | |
| author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> | |
| branch: benchmarking-bot-${{ github.run_id }} | |
| delete-branch: true | |
| title: "[benchmarking bot] Update generated weights files" | |
| body: | | |
| This is an automatically created PR. | |
| It updates the weights files under `runtime/*/src/weights/*.rs` after running benchmarks on the remote machine: ${{ env.INSTANCE_NAME }} | |
| Pallets: "${{ github.event.inputs.pallets }}" | |
| Chain: "${{ env.CHAIN }}" | |
| Github action run: https://github.com/litentry/litentry-parachain/actions/runs/${{ github.run_id }} | |
| labels: | | |
| automated-pr | |
| assignees: ${{ github.actor }} | |
| reviewers: ${{ github.actor }} | |
| draft: false |