scroll stable

.cargo/config.toml

@@ -1,2 +1,6 @@
 [target.'cfg(target_os="macos")']
 rustflags = ["-C", "link-args=-framework CoreFoundation -framework Security"]
+[net]
+git-fetch-with-cli = true
+[env]
+RUST_MIN_STACK = "16777216"

.github/ciChecksScripts/wakeUpRunner.sh

@@ -1,20 +1,21 @@
 #!/bin/bash
 
 profile="cirunner"
-runner_vpc_id="vpc-08ac0db4d8a7cce9e"
+runner_vpc_id="vpc-8bdf97ec"
+region_opt="--region us-west-2"
 
 # Get runner status
-runner=$(aws ec2 describe-instances --profile $profile --filters Name=tag:Name,Values=[ci-checks-runner] Name=network-interface.vpc-id,Values=[$runner_vpc_id] --query "Reservations[*].Instances[*][InstanceId]" --output text | xargs)
+runner=$(aws ec2 describe-instances --profile $profile --filters Name=tag:Name,Values=[jenkins] Name=network-interface.vpc-id,Values=[$runner_vpc_id] --query "Reservations[*].Instances[*][InstanceId]" ${region_opt} --output text | xargs)
 
 while true; do
-    runner_status=$(aws ec2 describe-instances --profile $profile --instance-ids $runner --query "Reservations[*].Instances[*].State.[Name]" --output text)
-    if [ $runner_status = "stopped" ]; then
-        aws ec2 start-instances --profile $profile --instance-ids $runner
+    runner_status=$(aws ec2 describe-instances --profile $profile --instance-ids $runner --query "Reservations[*].Instances[*].State.[Name]" ${region_opt} --output text)
+    if [[ $runner_status = "stopped" ]]; then
+        aws ec2 start-instances --profile $profile --instance-ids $runner ${region_opt}
         exit 0
-    elif [ $runner_status = "running" ]; then
+    elif [[ $runner_status = "running" ]]; then
         sleep 120
-        runner_status=$(aws ec2 describe-instances --profile $profile --instance-ids $runner --query "Reservations[*].Instances[*].State.[Name]" --output text)
-        if [ $runner_status = "running" ]; then
+        runner_status=$(aws ec2 describe-instances --profile $profile --instance-ids $runner --query "Reservations[*].Instances[*].State.[Name]" ${region_opt} --output text)
+        if [[ $runner_status = "running" ]]; then
             exit 0
         fi
     else

.github/proverCiScripts/benchmarks_result_reporting/reporting_main.py

@@ -2,9 +2,9 @@
 from pprint import pprint
 import reporting_modules as rmod
 
-env = json.load(open('/home/ubuntu/env.json'))
-cstats = '/home/ubuntu/cpu.stats'
-mstats = '/home/ubuntu/mem.stats'
+env = json.load(open('/home/CI/env.json'))
+cstats = '/home/CI/cpu.stats'
+mstats = '/home/CI/mem.stats'
 
 def main():
     parser = argparse.ArgumentParser(

.github/proverCiScripts/benchmarks_result_reporting/reporting_modules.py

@@ -94,8 +94,8 @@ def log_processor(pull_request,circuit, degree):
     SETUP_PREFIX     = "[Setup generation]"
     PROOFGEN_PREFIX  = "[Proof generation]"
     PROOFVER_PREFIX  = "[Proof verification]"
-    logfile = [i for i in os.listdir('/home/ubuntu/') if 'proverlog' in i][0]
-    f = open(f'/home/ubuntu/{logfile}', 'r')
+    logfile = [i for i in os.listdir('/home/CI/') if 'proverlog' in i][0]
+    f = open(f'/home/CI/{logfile}', 'r')
     logdata = f.read()
     logdata = logdata.split("\n")
     running = [i for i in logdata if 'running' in i and 'test' in i][0].split()[1]

.github/proverCiScripts/deployToProver.sh

@@ -3,7 +3,7 @@
 #set -x
 
 prnumber=$1
-base_dir="/home/ubuntu/CI_Prover_Benches/"
+base_dir="/home/CI/CI_Prover_Benches/"
 target_dir="$base_dir"PR"$1"
 
 source_dir=$2

.github/proverCiScripts/execBench.sh

@@ -3,7 +3,7 @@ set -e
 #set -x
 
 prnumber=$1
-base_dir="/home/ubuntu/CI_Prover_Benches/"
+base_dir="/home/CI/CI_Prover_Benches/"
 target_dir="$base_dir"PR"$1"
 k=$2
 circuit=$(echo $3 | awk '{ print $2 }' | tr '[:upper:]' '[:lower:]')

.github/proverCiScripts/getSysstat.sh

@@ -3,7 +3,7 @@ set -e
 #set -x
 
 prnumber=$1
-base_dir="/home/ubuntu/CI_Prover_Benches/"
+base_dir="/home/CI/CI_Prover_Benches/"
 target_dir="$base_dir"PR"$prnumber"
 
 logfile=$(ls $target_dir | grep proverlog | xargs -n 1 basename)

.github/proverCiScripts/injectHalo2Revision.sh

@@ -3,7 +3,7 @@
 #set -x
 
 prnumber=$1
-base_dir="/home/ubuntu/CI_Prover_Benches/"
+base_dir="/home/CI/CI_Prover_Benches/"
 target_dir="$base_dir"PR"$1"
 
 _revision=$2

.github/proverCiScripts/prepareProver.sh

@@ -3,7 +3,7 @@
 #set -x
 
 prnumber=$1
-base_dir="/home/ubuntu/CI_Prover_Benches/"
+base_dir="/home/CI/CI_Prover_Benches/"
 target_dir="$base_dir"PR"$1"
 source_dir=$2
 

.github/proverCiScripts/processResults.sh

@@ -5,7 +5,7 @@ set -e
 prnumber=$1
 label=$2
 degree=$3
-base_dir="/home/ubuntu/CI_Prover_Benches/"
+base_dir="/home/CI/CI_Prover_Benches/"
 target_dir="$base_dir"PR"$prnumber"
 
 rm -f ~/*.stats

.github/proverCiScripts/shutdownProver.sh

@@ -3,7 +3,7 @@
 #set -x
 
 profile="cirunner"
-provers_vpc_id="vpc-09fb44da782f32abb"
+provers_vpc_id="vpc-1176d875"
 
 running_prover=$(aws ec2 describe-instances --profile $profile --filters Name=tag:Name,Values=[proverbench-multiAZ] Name=instance-state-name,Values=[running] Name=network-interface.vpc-id,Values=[$provers_vpc_id] --query "Reservations[*].Instances[*][InstanceId]" --output text)
 

.github/proverCiScripts/wakeUpProver.sh

@@ -3,23 +3,23 @@
 #set -x
 
 profile="cirunner"
-provers_vpc_id="vpc-09fb44da782f32abb"
+provers_vpc_id="vpc-1176d875"
 dns_ipaddr=$(dig prover.cirunners.internal +short)
 zone_id=$(aws route53 --profile $profile list-hosted-zones --query 'HostedZones[?Name==`cirunners.internal.`].[Id]' --output text | awk -F \/ '{ print $3 }')
 route53_dir=".github/proverCiScripts/misc"
 
 sshprover () {
-    ssh -o ConnectTimeout=5 prover
+    ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no prover "uptime"
 }
 
 # Get running provers IDs
 provers_running=$(aws ec2 describe-instances --profile $profile --filters Name=tag:Name,Values=[proverbench-multiAZ] Name=instance-state-name,Values=[running] Name=network-interface.vpc-id,Values=[$provers_vpc_id] --query "Reservations[*].Instances[*][InstanceId]" --output text)
 provers_running_num=$(echo $provers_running | wc -w)
 
-if [ $provers_running_num -gt 1 ]; then
+if [[ $provers_running_num -gt 1 ]]; then
     echo "More than 1 provers running"
     exit 1
-elif [ $provers_running_num -eq 0 ]; then
+elif [[ $provers_running_num -eq 0 ]]; then
     # Get provers IDs and start first available
     provers=$(aws ec2 describe-instances --profile $profile --filters Name=tag:Name,Values=[proverbench-multiAZ] Name=network-interface.vpc-id,Values=[$provers_vpc_id] --query "Reservations[*].Instances[*][InstanceId]" --output text | xargs)
 else
@@ -28,7 +28,7 @@ else
 fi
 
 for prover in $provers; do
-    [ ! -z $single_prover_running ] || aws ec2 start-instances --profile $profile --instance-ids $prover
+    [[ ! -z $single_prover_running ]] || aws ec2 start-instances --profile $profile --instance-ids $prover
     if [ $? -eq 0 ]; then
         ipaddr=$(aws ec2 describe-instances --profile $profile --instance-ids $prover --query "Reservations[*].Instances[*].NetworkInterfaces[*].[PrivateIpAddress]" --output text)
         if [ $ipaddr != $dns_ipaddr ]; then
@@ -38,7 +38,7 @@ for prover in $provers; do
             aws route53 change-resource-record-sets --profile $profile --hosted-zone-id $zone_id --change-batch file://$route53_dir/route53.template.json
             sleep 61
             rm $route53_dir/route53.template.json
-	    ssh-keygen -f "/home/ubuntu/.ssh/known_hosts" -R "prover.cirunners.internal"
+	    ssh-keygen -f "/home/CI/.ssh/known_hosts" -R "prover.cirunners.internal"
         fi
         break
     fi

.github/workflows/ci.yml

@@ -1,16 +1,19 @@
 name: CI checks
 
 on:
-  merge_group:
   pull_request:
     types: [synchronize, opened, reopened, ready_for_review]
+    # Ignore when target branch is `main` and source branch is scroll-stable.
+    # It could avoid running twice. Since scroll-stable is set as `push` below.
+    branches-ignore:
+      - main
   push:
     branches:
-      - main
+      - scroll-stable
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+env:
+  CARGO_NET_GIT_FETCH_WITH_CLI: true
+  RUST_MIN_STACK: 16777216
 
 ## `actions-rs/toolchain@v1` overwrite set to false so that
 ## `rust-toolchain` is always used and the only source of truth.
@@ -28,23 +31,36 @@ jobs:
           concurrent_skipping: 'same_content_newer'
           paths_ignore: '["**/README.md"]'
 
-  test:
+  runner:
     needs: [skip_check]
     if: |
       github.event.pull_request.draft == false &&
       (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
+    uses: ./.github/workflows/select-runner.yml
 
+  test:
     name: Test
-    runs-on: ["${{github.run_id}}", self-hosted, c5.9xlarge]
+    needs: [runner]
+    runs-on: ${{ matrix.os }}
+    concurrency: ${{ needs.runner.outputs.concurrency-group }}
+    strategy:
+      matrix:
+        # We don't need to test across multiple platforms yet
+        # os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: ${{ fromJSON(needs.runner.outputs.runner-matrix) }}
 
     steps:
       - uses: actions/checkout@v2
+      - name: set HOME env
+        run: |
+          echo "HOME=/home/CI" >> $GITHUB_ENV
       - uses: actions-rs/toolchain@v1
         with:
           override: false
       - name: Setup golang
         uses: actions/setup-go@v3
         with:
+          cache: false
           go-version: ~1.18
       # Go cache for building geth-utils
       - name: Go cache
@@ -56,22 +72,18 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
+        # https://github.com/actions/cache/issues/810
+        env:
+          SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
       - name: Cargo cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+        uses: Swatinem/rust-cache@v2
       - name: Run light tests # light tests are run in parallel
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --verbose --release --all --all-features --exclude integration-tests --exclude circuit-benchmarks
+          args: --verbose --release --all --exclude integration-tests --exclude circuit-benchmarks
       - name: Run heavy tests # heavy tests are run serially to avoid OOM
+        if: false
         uses: actions-rs/cargo@v1
         with:
           command: test
@@ -80,6 +92,7 @@ jobs:
   build:
     needs: [skip_check]
     if: |
+      false &&
       github.event.pull_request.draft == false &&
       (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
 
@@ -108,16 +121,11 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
+        # https://github.com/actions/cache/issues/810
+        env:
+          SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
       - name: Cargo cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+        uses: Swatinem/rust-cache@v2
       - name: cargo build
         uses: actions-rs/cargo@v1
         with:
@@ -154,16 +162,11 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
+        # https://github.com/actions/cache/issues/810
+        env:
+          SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
       - name: Cargo cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+        uses: Swatinem/rust-cache@v2
       # Build benchmarks to prevent bitrot
       - name: Build benchmarks
         uses: actions-rs/cargo@v1
@@ -195,16 +198,11 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
+        # https://github.com/actions/cache/issues/810
+        env:
+          SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
       - name: Cargo cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+        uses: Swatinem/rust-cache@v2
       - name: cargo fetch
         uses: actions-rs/cargo@v1
         with:
@@ -242,16 +240,11 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
+        # https://github.com/actions/cache/issues/810
+        env:
+          SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
       - name: Cargo cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+        uses: Swatinem/rust-cache@v2
       - name: cargo check
         uses: actions-rs/cargo@v1
         with:

.github/workflows/draft.yml

@@ -3,7 +3,7 @@ name: Release Drafter
 on:
   push:
     branches:
-      - main
+      - scroll-stable
 
 jobs:
   update_release_draft:

.github/workflows/github-ops.yml

@@ -14,6 +14,7 @@ concurrency:
 
 jobs:
   ping:
+    if: false
     runs-on: ubuntu-latest
     timeout-minutes: 1
     steps: