feat: use updated SP1 (risc0#2)

Author: mattstam

Diff for: Cargo.lock

@@ -1,6 +1,5 @@
 [workspace]
-
-members = ["eval/cli"]
+members = ["eval"]
 
 resolver = "2"
 
@@ -11,14 +10,20 @@ opt-level = 3
 opt-level = 3
 
 [profile.release]
-debug = 1
+opt-level = 3
 lto = true
 
+[profile.fast]
+inherits = "release"
+debug = true
+debug-assertions = true
+
 [profile.release.build-override]
 opt-level = 3
 
-# Note: Need to add this patch until the Jolt repo adds an override which automatically adds this patch.
 [patch.crates-io]
 ark-ff = { git = "https://github.com/a16z/arkworks-algebra", branch = "optimize/field-from-u64" }
 ark-ec = { git = "https://github.com/a16z/arkworks-algebra", branch = "optimize/field-from-u64" }
 ark-serialize = { git = "https://github.com/a16z/arkworks-algebra", branch = "optimize/field-from-u64" }
+revm-interpreter = { git = "https://github.com/sp1-patches/revm-new", branch = "john/update-for-v1" }
+revm-primitives = { git = "https://github.com/sp1-patches/revm-new", branch = "john/update-for-v1" }

Diff for: Cargo.toml

@@ -1,113 +1,106 @@
-# zkVM benchmarks
+# SP1 Benchmarks
 
-A benchmarking suite for zkVMs.
+A suite of benchmarks to evaluate SP1's performance.
 
 ## Setup
 
 Install Rust:
-```
+
+```sh
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 rustup install nightly
 ```
 
-You might need to do `sudo sh` if the first command doesn't work.
+Install the [SP1 toolchain](https://docs.succinct.xyz/getting-started/install.html):
 
-Install the Succinct toolchain which is used for building the ELF files: https://succinctlabs.github.io/sp1/getting-started/install.html.
-```
+```sh
 curl -L https://sp1.succinct.xyz | bash
+source ~/.bashrc
 sp1up
 cargo prove --version
 ```
 
-Make sure that `go` is installed on your system, to run the gnark wrapper.
-
-Install the `jolt` [toolchain](https://jolt.a16zcrypto.com/usage/quickstart.html).
-
-```
-cargo install --git https://github.com/a16z/jolt --rev 845d39af373de078ee2616cf36a255f36f38334a --force --bins jolt 
-jolt install-toolchain
-```
+Install the [Risc0 toolchain](https://dev.risczero.com/api/zkvm/install):
 
-You might have to install these libraries for JOLT:
-```
-sudo apt-get update
-sudo apt-get install libc6
+```sh
+curl -L https://risczero.com/install | bash
+source ~/.bashrc
+rzup
+cargo risczero --version
 ```
 
-Install the Risc0 toolchain:
-```
-cargo install cargo-binstall
-cargo binstall cargo-risczero
-cargo risczero install
-```
-
-Install docker (needed for running R0's STARK -> SNARK):
+Install [Docker](https://docs.docker.com/engine/install/ubuntu/).
 * https://docs.docker.com/engine/install/ubuntu/
+* https://docs.docker.com/engine/install/linux-postinstall/
 
-Then, make sure you can run docker not in sudo mode:
-https://docs.docker.com/engine/install/linux-postinstall/
+Install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) if you are using NVIDIA GPUs.
+- https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
 
 **Note:** When benchmarking, make sure that you run 1 round of Fibonacci (or some other small program) to download the R0 docker image to make sure it doesn't contribute to benchmarking time.
 
 **Note:** On Ubuntu 22.04, you might need to install libssl1.0 to use the Risc0 toolchain. You can follow these instructions (from [here](https://stackoverflow.com/questions/72133316/libssl-so-1-1-cannot-open-shared-object-file-no-such-file-or-directory/73604364#73604364)).
 
-## Machine Requirements
-
-We ran our benchmarks on a AWS r7i.16xlarge machine with 64 vCPUs and 512 GB of memory. Smaller programs require significantly less memory. Note that the machine has to support AVX-512 instructions, otherwise the command in `eval.sh` will not run. If your machine doesn't support AVX-512, then modify this command to remove those flags.
-
 ## Usage
 
 You can conduct a sweep of the benchmarks by running either of the following commands:
 
-```
+```sh
 python3 sweep.py
 ```
 
-or running the following command for the JOLT sweep:
-
-```
-python3 sweep_jolt.py
-```
-
 To run a single benchmark, you can run:
 
-```
+```sh
 ./eval.sh (loop|fibonacci|ssz_withdrawals|tendermint) (sp1|risc0|jolt-zkvm) (poseidon|sha256|blake3|...)
 ```
 
 Note that right now only poseidion is supported for all zkVMs, since we are interested in also profiling recursion.
 
 Example SP1:
-```
+
+```sh
 ./eval.sh fibonacci sp1 poseidon 22 benchmark
 ```
 
 Example JOLT:
-```
+
+```sh
 ./eval.sh fibonacci jolt-zkvm poseidon 22 benchmark
 ```
 
 Example Risc0:
-```
+
+```sh
 ./eval.sh fibonacci risc0 poseidon 22 benchmark
 ```
 
-## Debugging
+Note for benchmarking the Reth program, you must also pass in a block number:
+
+```sh
+./eval.sh reth sp1 poseidon 22 benchmark 19409768
+```
+
+The inputs for these blocks have already been generated [here](./eval/cli/blocks/). You can add more
+blocks by using the [SP1-Reth](https://github.com/succinctlabs/sp1-reth) script.
+
+## Common Issues
 
 Because Risc0 uses C++ for their prover, you may need to install the C++ compiler and libraries.
 
 For Ubuntu/Debian-based systems:
-```
+
+```sh
 # Update system repositories and installed packages
 sudo apt update
 sudo apt upgrade
 
 # Install the latest version of GCC and libraries
-sudo apt install build-essential
+sudo apt install build-essential libc6
 ```
 
 For CentOS/RHEL-based systems:
-```
+
+```sh
 # Update system repositories and installed packages
 sudo yum update
 
@@ -119,10 +112,18 @@ sudo yum install -y gcc-c++
 ```
 
 For Fedora:
-```
+
+```sh
 # Update system repositories and installed packages
 sudo dnf update
 
 # Install the Development Tools group which includes GCC
 sudo dnf groupinstall "Development Tools"
+```
+
+Setting up NVIDIA:
+
+```sh
+sudo nvidia-ctk runtime configure --runtime=docker
+sudo systemctl restart docker
 ```

Diff for: README.md

@@ -1,6 +1,5 @@
 #!/bin/bash
 set -e
-
 echo "Running $1, $2, $3, $4, $5"
 
 # Get program directory name as $1 and append "-$2" to it if $1 == "tendermint"
@@ -15,7 +14,7 @@ echo "Building program"
 # cd to program directory computed above
 cd "programs/$program_directory"
 
-# If the prover is not jolt-zkvm, then build the program.
+# If the prover is sp1, then build the program.
 if [ "$2" == "sp1" ]; then
     # The reason we don't just use `cargo prove build` from the SP1 CLI is we need to pass a --features ...
     # flag to select between sp1 and risc0.
@@ -34,22 +33,42 @@ if [ "$2" == "risc0" ]; then
         cargo build --release --ignore-rust-version --features $2
 fi
 
-
-cd ../../eval
+cd ../../
 
 echo "Running eval script"
 
-# Runs the eval for the specified program, prover & hash function with conditional compilation based on the feature flag.
-
-RUST_LOG=info RUSTFLAGS='-C target-cpu=native -C target_feature=+avx512ifma,+avx512vl' cargo run -p eval --release --no-default-features --features $2 -- --program $1 --prover $2 --hashfn $3 --shard-size $4 --filename $5
+# Detect whether we're on an instance with a GPU.
+if nvidia-smi > /dev/null 2>&1; then
+  GPU_EXISTS=true
+else
+  GPU_EXISTS=false
+fi
 
-# TODO: conditionally run with avx flags based on the machine's architecture.
-#RUST_LOG=info RUSTFLAGS='-C target-cpu=native' cargo run -p eval --release --no-default-features --features $2 -- --program $1 --prover $2 --hashfn $3 --shard-size $4
+# Set the compilation flags.
+if [ "$GPU_EXISTS" = false ]; then
+  export RUSTFLAGS='-C target-cpu=native -C target_feature=+avx512ifma,+avx512vl'
+fi
 
-cd ../
+# Set the logging level.
+export RUST_LOG=info
 
-# Get the current commit hash
-commit_hash=$(git rev-parse HEAD)
+# Determine the features based on GPU existence.
+if [ "$GPU_EXISTS" = true ]; then
+  FEATURES="cuda"
+else
+  FEATURES="default"
+fi
 
-# Write the commit hash to COMMIT_HASH file
-echo "$commit_hash" > COMMIT_HASH
+# Run the benchmark.
+cargo run \
+    -p sp1-benchmarks-eval \
+    --release \
+    --no-default-features \
+    --features $FEATURES \
+    -- \
+    --program $1 \
+    --prover $2 \
+    --hashfn $3 \
+    --shard-size $4 \
+    --filename $5 \
+    ${6:+--block-number $6}