format

Author: adria0

bin/mpt-test/.gitignore

@@ -0,0 +1 @@
+web3_rpc_cache.bin

bin/mpt-test/README.md

@@ -4,7 +4,7 @@ This tool aims to verify mainnet blocks for the MPT circuit.
 
 ## Running tests
 
-Just run `cargo run --release`
+Just run `./test_mainnet_blocks.sh`
 
 ## Adding new blocks to prove
 
@@ -13,7 +13,11 @@ In order to add more blocks to prove you have to:
 - Add new entry in the `access-lists` folder
 - Set the environment variable `WEB3_SERVICE_PROVIDER` to a mainnet JSON-RPC provider
 - Run the tests again
+- You will have to upload the cache file again (web3_rpc_cache.bin) and update the `test_mainnet_blocks.sh` file
 
+## How can get an access list for other blocks?
 
-## How
+There's a [modified version of geth](https://github.com/adria0/go-ethereum/tree/zklight) that [tracks access lists](https://github.com/adria0/go-ethereum/commit/fd2d7cea3747e1003a817cd18e200bf2b00be03c#diff-c3757dc9e9d868f63bc84a0cc67159c1d5c22cc5d8c9468757098f0492e0658cR1306) and allows to retrieve them via [RPC eth_accessListByNumber call](https://github.com/adria0/go-ethereum/commit/fd2d7cea3747e1003a817cd18e200bf2b00be03c#diff-c426ecd2f7d247753b9ea8c1cc003f21fa412661c1f967d203d4edf8163da344R1587), so you can deploy this version and grab some access lists there.
+
+Note: of course this is just a method for testing , do not use in production.
 

bin/mpt-test/src/cache.rs

@@ -1,12 +1,15 @@
+//! A simple cache for web3 rpc requests
+
 use eth_types::keccak256;
-use eyre::{ensure, Result};
+use eyre::{ensure, eyre, Result};
 use flate2::{read::ZlibDecoder, write::ZlibEncoder, Compression};
 use hyper::{
     body::{self},
     service::{make_service_fn, service_fn},
     Body, Client, Request, Response, Server,
 };
 use serde::Deserialize;
+use serde_json::Value;
 use std::{
     collections::HashMap,
     convert::Infallible,
@@ -20,6 +23,10 @@ lazy_static! {
     static ref CACHE: tokio::sync::Mutex<CacheFile> = tokio::sync::Mutex::new(CacheFile::new());
 }
 
+/// Cache file format is a consecutive list of entries, each entry is:
+/// - 32 bytes: key (keccak256 of the request)
+/// - 4 bytes: length of the compressed response
+/// - N bytes: compressed response
 struct CacheFile {
     offsets: HashMap<[u8; 32], u64>,
 }
@@ -29,6 +36,7 @@ impl CacheFile {
             offsets: HashMap::new(),
         }
     }
+    /// Load all existing entries from the cache file
     pub async fn load(&mut self) -> Result<()> {
         if let Ok(mut f) = File::open(CACHE_PATH) {
             let mut hash = [0u8; 32];
@@ -44,7 +52,7 @@ impl CacheFile {
         }
         Ok(())
     }
-
+    /// Write a new entry
     async fn write(&mut self, key: [u8; 32], value: &[u8]) -> Result<()> {
         let mut encoder = ZlibEncoder::new(Vec::new(), Compression::best());
         encoder.write_all(value)?;
@@ -60,11 +68,12 @@ impl CacheFile {
 
         Ok(())
     }
-
+    /// Read an entry, returns Ok(None) if not found
     async fn read(&self, key: [u8; 32]) -> Result<Option<Vec<u8>>> {
         let offset = self.offsets.get(&key).cloned();
         if let Some(offset) = offset {
-            let mut f = File::open(CACHE_PATH).unwrap();
+            let mut f =
+                File::open(CACHE_PATH).expect("since offset exists, file should exist. qed.");
             f.seek(std::io::SeekFrom::Start(offset))?;
 
             let mut len_buf = [0u8; 4];
@@ -84,10 +93,8 @@ impl CacheFile {
     }
 }
 
-#[allow(dead_code)]
 #[derive(Deserialize, Debug)]
 struct RequestBody {
-    jsonrpc: String,
     method: String,
     params: Option<Vec<Param>>,
     id: u32,
@@ -100,25 +107,27 @@ enum Param {
     Bool(bool),
     StringVec(Vec<String>),
 }
+
 impl RequestBody {
-    fn name(&self) -> String {
+    fn hash(&self) -> [u8; 32] {
         let params = if let Some(params) = &self.params {
             params
                 .iter()
                 .map(|s| match s {
                     Param::String(s) => s.to_owned(),
                     Param::Bool(b) => format!("{}", b),
-                    Param::StringVec(v) => v.join(""),
+                    Param::StringVec(v) => v.join("-"),
                 })
                 .collect::<Vec<_>>()
                 .join("-")
         } else {
             "".to_owned()
         };
-        format!("{}_{}", self.method, params)
+        keccak256(format!("{}-{}", self.method, params).as_bytes())
     }
 }
 
+/// Handle a web3 rpc request with error handling
 async fn infallible_web3_proxy(req: Request<Body>) -> Result<Response<Body>, Infallible> {
     match web3_proxy(req).await {
         Ok(res) => Ok(res),
@@ -132,32 +141,40 @@ async fn infallible_web3_proxy(req: Request<Body>) -> Result<Response<Body>, Inf
     }
 }
 
+/// Handle a web3 rpc request, return cached result or call the env!("WEB3_PROVIDER_URL") RPC server
 async fn web3_proxy(req: Request<Body>) -> Result<Response<Body>> {
     let method = req.method().clone();
     let headers = req.headers().clone();
+
+    // try to read the result from the cache
     let request_body_bytes = hyper::body::to_bytes(req.into_body()).await?.to_vec();
     let request_body_json: RequestBody = serde_json::from_slice(&request_body_bytes)?;
-
-    let key = keccak256(request_body_json.name().as_bytes());
-
-    let response_body = CACHE.lock().await.read(key).await?;
-
-    let response_body = if let Some(response_body) = response_body {
-        response_body
+    let key = request_body_json.hash();
+    let response_result = CACHE.lock().await.read(key).await?;
+
+    let response_body = if let Some(response_result) = response_result {
+        // return cached result
+        format!(
+            "{{\"id\":{}, \"jsonrpc\":\"2.0\", \"result\":{}}}",
+            request_body_json.id,
+            String::from_utf8(response_result).unwrap()
+        )
+        .into_bytes()
     } else {
         println!(
             "=>{}",
             String::from_utf8(request_body_bytes.clone()).unwrap()
         );
 
+        // call RPC server, copying headers and method from the original request
         let connector = hyper_rustls::HttpsConnectorBuilder::new()
             .with_native_roots()
             .https_or_http()
             .enable_http1()
             .build();
 
         let client = Client::builder().build(connector);
-        let provider_url = std::env::var("PROVIDER_URL")?;
+        let provider_url = std::env::var("WEB3_PROVIDER_URL")?;
 
         let mut req = Request::builder()
             .method(method)
@@ -176,23 +193,33 @@ async fn web3_proxy(req: Request<Body>) -> Result<Response<Body>> {
         let (head, response_body) = resp.into_parts();
         ensure!(head.status.is_success(), "Provider does not return 200");
 
+        // parse response and cache it
         let response_bytes = body::to_bytes(response_body).await?.to_vec();
-        CACHE.lock().await.write(key, &response_bytes).await?;
+
+        let root: Value = serde_json::from_slice(&response_bytes)?;
+        let Some(result) = root.get("result") else {
+            return Err(eyre!("Provider does not return result"));
+        };
+        let result_bytes = serde_json::to_vec(result)?;
+        CACHE.lock().await.write(key, &result_bytes).await?;
 
         response_bytes
     };
 
+    // return HTTP 200 response
     let response = Response::builder()
         .status(200)
         .body(Body::from(response_body))?;
 
     Ok(response)
 }
 
+/// Load the cache file
 pub async fn load() -> Result<()> {
     CACHE.lock().await.load().await
 }
 
+/// Start the cache server at localhost:3000
 pub async fn start() -> Result<()> {
     let make_svc =
         make_service_fn(|_conn| async { Ok::<_, Infallible>(service_fn(infallible_web3_proxy)) });

bin/mpt-test/src/circuit/equal_words.rs

@@ -1,3 +1,5 @@
+//! A simple chip to compare two words for equality.
+
 use eth_types::Field;
 use eyre::Result;
 use gadgets::{

bin/mpt-test/src/circuit/state_update.rs

@@ -1,3 +1,5 @@
+//! A circuit to verify chained proofs of state updates in a block.
+
 use eth_types::Field;
 use eyre::Result;
 use gadgets::{
@@ -25,7 +27,7 @@ use zkevm_circuits::{
 
 use crate::circuit::{
     equal_words::EqualWordsConfig,
-    witness::{FieldTrieModification, FieldTrieModifications, Transforms, Witness},
+    witness::{FieldTrieModification, FieldTrieModifications, Witness},
 };
 
 #[cfg(not(feature = "disable-keccak"))]
@@ -39,36 +41,69 @@ pub fn xnif<F: Field>(a: Expression<F>, b: Expression<F>) -> Expression<F> {
     and::expr([a, not::expr(b)])
 }
 
-///
+/// This
 #[derive(Clone)]
 pub struct StateUpdateCircuitConfig<F: Field> {
     #[cfg(not(feature = "disable-keccak"))]
+    /// If enabled by feature, the verification of keccaks
+    /// Please note that the keccak circuit will dramatically increase the time to generate the
+    /// proof this is the reason why it is disabled by default (check the `disable-keccak`
+    /// feature)
     pub keccak_config: KeccakCircuitConfig<F>,
+
+    /// The MPT configuration
     pub mpt_config: MPTConfig<F>,
 
+    /// The MPT table, where the state updates are stored
     pub pi_mpt: MptTable,
-    pub pi_instance: Column<Instance>,
 
+    /// Intance column, used to check public inputs
+    pub instance: Column<Instance>,
+
+    /// ONE if the first row, ZERO otherwise
     pub is_first: Column<Fixed>,
+
+    /// ONE if row is paddding, ZERO otherwise
     pub is_padding: IsZeroConfig<F>,
+
+    /// ONE is the last used row, ZERO otherwise
     pub is_last: IsZeroConfig<F>,
 
+    /// Check if new_root is propagated
     pub new_root_propagation: EqualWordsConfig<F>,
+
+    /// Check if previous old_root, and new_root are equal
     pub root_chained: EqualWordsConfig<F>,
 
+    /// A down counter, that is decreased by one in each row
     pub count: Column<Advice>,
+
+    /// Check if the counter is monotonic decreasing
     pub count_decrement: IsZeroConfig<F>,
 
     pub q_enable: Selector,
 }
 
-/// MPT Circuit for proving the storage modification is valid.
+/// MPT Circuit for proving that a from a given root, a set of state updates are valid.
+/// Public inputs:
+///     old_root_lo
+///     old_root_hi
+///     new_root_lo
+///     new_root_hi
+///     number of MPT changes
+///     for each change
+///     | change_type
+///     | address
+///     | value_lo
+///     | value_hi
+///     | key_lo
+///     | key_hi
 #[derive(Default)]
 pub struct StateUpdateCircuit<F: Field> {
-    pub transforms: Transforms,
     #[cfg(not(feature = "disable-keccak"))]
     pub keccak_circuit: KeccakCircuit<F>,
     pub mpt_circuit: MPTCircuit<F>,
+
     pub lc_witness: FieldTrieModifications<F>,
     pub degree: usize,
     pub max_proof_count: usize,
@@ -110,7 +145,7 @@ impl<F: Field> Circuit<F> for StateUpdateCircuit<F> {
         let is_first = meta.fixed_column();
         let count = meta.advice_column();
         let q_enable = meta.complex_selector();
-        let pi_instance = meta.instance_column();
+        let instance = meta.instance_column();
         let pi_mpt = MptTable {
             address: meta.advice_column(),
             storage_key: word::Word::new([meta.advice_column(), meta.advice_column()]),
@@ -140,7 +175,7 @@ impl<F: Field> Circuit<F> for StateUpdateCircuit<F> {
             meta.enable_equality(*col);
         }
 
-        meta.enable_equality(pi_instance);
+        meta.enable_equality(instance);
         meta.enable_equality(count);
 
         let is_padding_inv = meta.advice_column();
@@ -279,7 +314,7 @@ impl<F: Field> Circuit<F> for StateUpdateCircuit<F> {
             new_root_propagation,
             root_chained,
             q_enable,
-            pi_instance,
+            instance,
             pi_mpt,
         };
 
@@ -297,7 +332,7 @@ impl<F: Field> Circuit<F> for StateUpdateCircuit<F> {
     ) -> Result<(), Error> {
         let challenges = _challenges.values(&mut layouter);
 
-        // assign MPT witness
+        // MPT witness
 
         config
             .mpt_config
@@ -309,6 +344,8 @@ impl<F: Field> Circuit<F> for StateUpdateCircuit<F> {
             self.mpt_circuit.max_nodes,
         )?;
 
+        // Keccak witness ( if apply )
+
         #[cfg(feature = "disable-keccak")]
         config.mpt_config.keccak_table.dev_load(
             &mut layouter,
@@ -320,7 +357,7 @@ impl<F: Field> Circuit<F> for StateUpdateCircuit<F> {
         self.keccak_circuit
             .synthesize_sub(&config.keccak_config, &challenges, &mut layouter)?;
 
-        // assign LC witness
+        // Circuit witness, returns the public inputs for the state update
 
         let pi = layouter.assign_region(
             || "lc witness",
@@ -342,7 +379,7 @@ impl<F: Field> Circuit<F> for StateUpdateCircuit<F> {
                     || "LC_count_monodec_inv",
                     config.count_decrement.value_inv,
                 );
-                region.name_column(|| "LC_pi_instance", config.pi_instance);
+                region.name_column(|| "LC_instance", config.instance);
 
                 region.assign_fixed(|| "", config.is_first, 0, || Value::known(F::ONE))?;
 
@@ -454,7 +491,7 @@ impl<F: Field> Circuit<F> for StateUpdateCircuit<F> {
 
         // check that state updates to lookup are the same that the specified in the public inputs
         for (n, value) in pi.into_iter().enumerate() {
-            layouter.constrain_instance(value.unwrap().cell(), config.pi_instance, n)?;
+            layouter.constrain_instance(value.unwrap().cell(), config.instance, n)?;
         }
 
         Ok(())
@@ -470,7 +507,6 @@ impl StateUpdateCircuit<Fr> {
     ) -> Result<StateUpdateCircuit<Fr>> {
         let Witness {
             mpt_witness,
-            transforms,
             lc_witness,
         } = witness;
 
@@ -498,7 +534,6 @@ impl StateUpdateCircuit<Fr> {
         let keccak_circuit = KeccakCircuit::<Fr>::new(2usize.pow(degree as u32), keccak_data);
 
         let lc_circuit = StateUpdateCircuit::<Fr> {
-            transforms,
             #[cfg(not(feature = "disable-keccak"))]
             keccak_circuit,
             mpt_circuit,

bin/mpt-test/src/circuit/utils.rs

@@ -1,3 +1,5 @@
+//! Some utilities.
+
 use crate::circuit::witness::FieldTrieModifications;
 use eth_types::Field;
 use halo2_proofs::{dev::MockProver, halo2curves::bn256::Fr};