add eip1559/2930 support to json/yaml parser

Author: lightsing

eth-types/src/lib.rs

@@ -41,7 +41,10 @@ use ethers_core::types;
 pub use ethers_core::{
     abi::ethereum_types::{BigEndianHash, U512},
     types::{
-        transaction::{eip2930::AccessList, response::Transaction},
+        transaction::{
+            eip2930::{AccessList, AccessListItem},
+            response::Transaction,
+        },
         Address, Block, Bytes, Signature, H160, H256, H64, U256, U64,
     },
 };

testool/src/statetest/executor.rs

@@ -138,6 +138,8 @@ fn check_post(
 }
 
 fn into_traceconfig(st: StateTest) -> (String, TraceConfig, StateTestResult) {
+    let tx_type = st.tx_type();
+
     let chain_id = 1;
     let wallet = LocalWallet::from_str(&hex::encode(st.secret_key.0)).unwrap();
     let mut tx = TransactionRequest::new()

testool/src/statetest/json.rs

@@ -71,8 +71,11 @@ struct JsonStateTest {
 #[derive(Debug, Clone, Deserialize)]
 #[serde(rename_all = "camelCase")]
 struct Transaction {
+    access_list: Option<parse::RawAccessList>,
     data: Vec<String>,
     gas_limit: Vec<String>,
+    max_priority_fee_per_gas: Option<String>,
+    max_fee_per_gas: Option<String>,
     gas_price: String,
     nonce: String,
     secret_key: String,
@@ -109,8 +112,7 @@ impl<'a> JsonStateTestBuilder<'a> {
     /// generates `StateTest` vectors from a ethereum josn test specification
     pub fn load_json(&mut self, path: &str, source: &str) -> Result<Vec<StateTest>> {
         let mut state_tests = Vec::new();
-        let tests: HashMap<String, JsonStateTest> =
-            serde_json::from_str(&strip_json_comments(source))?;
+        let tests: HashMap<String, JsonStateTest> = serde_json::from_str(source)?;
 
         for (test_name, test) in tests {
             let env = Self::parse_env(&test.env)?;
@@ -120,13 +122,32 @@ impl<'a> JsonStateTestBuilder<'a> {
             let secret_key = parse::parse_bytes(&test.transaction.secret_key)?;
             let from = secret_key_to_address(&SigningKey::from_slice(&secret_key)?);
             let nonce = parse::parse_u64(&test.transaction.nonce)?;
-            let gas_price = parse::parse_u256(&test.transaction.gas_price)?;
+
+            let max_priority_fee_per_gas = test
+                .transaction
+                .max_priority_fee_per_gas
+                .map_or(Ok(None), |s| parse::parse_u256(&s).map(Some))?;
+            let max_fee_per_gas = test
+                .transaction
+                .max_fee_per_gas
+                .map_or(Ok(None), |s| parse::parse_u256(&s).map(Some))?;
+
+            // Set gas price to `min(max_priority_fee_per_gas + base_fee, max_fee_per_gas)` for
+            // EIP-1559 transaction.
+            // <https://github.com/ethereum/go-ethereum/blob/1485814f89d8206bb4a1c8e10a4a2893920f683a/core/state_transition.go#L167>
+            let gas_price = parse::parse_u256(&test.transaction.gas_price).unwrap_or_else(|_| {
+                max_fee_per_gas
+                    .unwrap()
+                    .min(max_priority_fee_per_gas.unwrap() + env.current_base_fee)
+            });
+
+            let access_list = &test.transaction.access_list;
 
             let data_s: Vec<_> = test
                 .transaction
                 .data
                 .iter()
-                .map(|item| parse::parse_calldata(self.compiler, item))
+                .map(|item| parse::parse_calldata(self.compiler, item, access_list))
                 .collect::<Result<_>>()?;
 
             let gas_limit_s: Vec<_> = test
@@ -167,7 +188,7 @@ impl<'a> JsonStateTestBuilder<'a> {
                 }
             }
 
-            for (idx_data, data) in data_s.iter().enumerate() {
+            for (idx_data, calldata) in data_s.iter().enumerate() {
                 for (idx_gas, gas_limit) in gas_limit_s.iter().enumerate() {
                     for (idx_value, value) in value_s.iter().enumerate() {
                         for (data_refs, gas_refs, value_refs, result) in &expects {
@@ -193,10 +214,13 @@ impl<'a> JsonStateTestBuilder<'a> {
                                 to,
                                 secret_key: secret_key.clone(),
                                 nonce,
+                                max_priority_fee_per_gas,
+                                max_fee_per_gas,
                                 gas_price,
                                 gas_limit: *gas_limit,
                                 value: *value,
-                                data: data.0.clone(),
+                                data: calldata.data.clone(),
+                                access_list: calldata.access_list.clone(),
                                 exception: false,
                             });
                         }
@@ -313,29 +337,10 @@ impl<'a> JsonStateTestBuilder<'a> {
     }
 }
 
-fn strip_json_comments(json: &str) -> String {
-    fn strip(value: Value) -> Value {
-        use Value::*;
-        match value {
-            Array(vec) => Array(vec.into_iter().map(strip).collect()),
-            Object(map) => Object(
-                map.into_iter()
-                    .filter(|(k, _)| !k.starts_with("//"))
-                    .map(|(k, v)| (k, strip(v)))
-                    .collect(),
-            ),
-            _ => value,
-        }
-    }
-
-    let value: Value = serde_json::from_str(json).unwrap();
-    strip(value).to_string()
-}
-
 #[cfg(test)]
 mod test {
     use super::*;
-    use eth_types::{Bytes, H256};
+    use eth_types::{address, AccessList, AccessListItem, Bytes, H256};
     use std::{collections::HashMap, str::FromStr};
 
     const JSON: &str = r#"
@@ -381,6 +386,15 @@ mod test {
             }
         },
         "transaction" : {
+            "accessList" : [
+                {
+                    "address" : "0x009e7baea6a6c7c4c2dfeb977efac326af552d87",
+                    "storageKeys" : [
+                        "0x0000000000000000000000000000000000000000000000000000000000000000",
+                        "0x0000000000000000000000000000000000000000000000000000000000000001"
+                    ]
+                }
+            ],
             "data" : [
                 "0x6001",
                 "0x6002"
@@ -430,9 +444,24 @@ mod test {
             )?),
             gas_limit: 400000,
             gas_price: U256::from(10u64),
+            max_fee_per_gas: None,
+            max_priority_fee_per_gas: None,
             nonce: 0,
             value: U256::from(100000u64),
             data: Bytes::from(hex::decode("6001")?),
+            access_list: Some(AccessList(vec![AccessListItem {
+                address: address!("0x009e7baea6a6c7c4c2dfeb977efac326af552d87"),
+                storage_keys: vec![
+                    H256::from_str(
+                        "0x0000000000000000000000000000000000000000000000000000000000000000",
+                    )
+                    .unwrap(),
+                    H256::from_str(
+                        "0x0000000000000000000000000000000000000000000000000000000000000001",
+                    )
+                    .unwrap(),
+                ],
+            }])),
             pre: BTreeMap::from([(
                 acc095e,
                 Account {
@@ -460,13 +489,4 @@ mod test {
 
         Ok(())
     }
-
-    #[test]
-    fn test_strip() {
-        let original = r#"{"//a":"a1","b":[{"c":"c1","//d":"d1"}]}"#;
-        let expected = r#"{"b":[{"c":"c1"}]}"#;
-
-        let stripped = strip_json_comments(original);
-        assert_eq!(expected, stripped);
-    }
 }

testool/src/statetest/parse.rs

@@ -1,15 +1,52 @@
-use std::collections::HashMap;
-
 use crate::{abi, Compiler};
-
 use anyhow::{bail, Context, Result};
-use eth_types::{Address, Bytes, H256, U256};
+use eth_types::{address, AccessList, AccessListItem, Address, Bytes, H256, U256};
 use log::debug;
 use once_cell::sync::Lazy;
 use regex::Regex;
+use serde::Deserialize;
+use std::{collections::HashMap, str::FromStr};
 
 type Label = String;
 
+/// Raw access list to parse
+pub type RawAccessList = Vec<RawAccessListItem>;
+
+/// Raw access list item to parse
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RawAccessListItem {
+    address: String,
+    storage_keys: Vec<String>,
+}
+
+impl RawAccessListItem {
+    pub fn new(address: String, storage_keys: Vec<String>) -> Self {
+        Self {
+            address,
+            storage_keys,
+        }
+    }
+}
+
+/// parsed calldata
+#[derive(Debug)]
+pub struct Calldata {
+    pub data: Bytes,
+    pub label: Option<Label>,
+    pub access_list: Option<AccessList>,
+}
+
+impl Calldata {
+    fn new(data: Bytes, label: Option<Label>, access_list: Option<AccessList>) -> Self {
+        Self {
+            data,
+            label,
+            access_list,
+        }
+    }
+}
+
 static YUL_FRAGMENT_PARSER: Lazy<Regex> =
     Lazy::new(|| Regex::new(r#"\s*(?P<version>\w+)?\s*(?P<code>\{[\S\s]*)"#).unwrap());
 
@@ -66,47 +103,17 @@ fn decompose_tags(expr: &str) -> HashMap<String, String> {
 
 /// returns the element as calldata bytes, supports 0x, :raw, :abi, :yul and
 /// { LLL }
-pub fn parse_calldata(compiler: &Compiler, as_str: &str) -> Result<(Bytes, Option<Label>)> {
-    let tags = decompose_tags(as_str);
+pub fn parse_calldata(
+    compiler: &Compiler,
+    data: &str,
+    raw_access_list: &Option<RawAccessList>,
+) -> Result<Calldata> {
+    let tags = decompose_tags(data);
     let label = tags.get(":label").cloned();
+    let bytes = parse_call_bytes(compiler, tags)?;
+    let access_list = parse_access_list(raw_access_list)?;
 
-    if let Some(notag) = tags.get("") {
-        let notag = notag.trim();
-        if notag.is_empty() {
-            Ok((Bytes::default(), label))
-        } else if notag.starts_with('{') {
-            Ok((compiler.lll(notag)?, label))
-        } else if let Some(hex) = notag.strip_prefix("0x") {
-            Ok((Bytes::from(hex::decode(hex)?), label))
-        } else {
-            bail!("do not know what to do with calldata (1): '{:?}'", as_str);
-        }
-    } else if let Some(raw) = tags.get(":raw") {
-        if let Some(hex) = raw.strip_prefix("0x") {
-            Ok((Bytes::from(hex::decode(hex)?), label))
-        } else {
-            bail!("bad encoded calldata (3) {:?}", as_str)
-        }
-    } else if let Some(abi) = tags.get(":abi") {
-        Ok((abi::encode_funccall(abi)?, label))
-    } else if let Some(yul) = tags.get(":yul") {
-        let caps = YUL_FRAGMENT_PARSER
-            .captures(yul)
-            .ok_or_else(|| anyhow::anyhow!("do not know what to do with code(4) '{:?}'", as_str))?;
-        Ok((
-            compiler.yul(
-                caps.name("code").unwrap().as_str(),
-                caps.name("version").map(|m| m.as_str()),
-            )?,
-            label,
-        ))
-    } else {
-        bail!(
-            "do not know what to do with calldata: (2) {:?} '{:?}'",
-            tags,
-            as_str
-        )
-    }
+    Ok(Calldata::new(bytes, label, access_list))
 }
 
 /// parse entry as code, can be 0x, :raw or { LLL }
@@ -182,3 +189,60 @@ pub fn parse_u64(as_str: &str) -> Result<u64> {
         Ok(U256::from_str_radix(as_str, 10)?.as_u64())
     }
 }
+
+// Parse calldata to bytes
+fn parse_call_bytes(compiler: &Compiler, tags: HashMap<String, String>) -> Result<Bytes> {
+    if let Some(notag) = tags.get("") {
+        let notag = notag.trim();
+        if notag.is_empty() {
+            Ok(Bytes::default())
+        } else if notag.starts_with('{') {
+            Ok(compiler.lll(notag)?)
+        } else if let Some(hex) = notag.strip_prefix("0x") {
+            Ok(Bytes::from(hex::decode(hex)?))
+        } else {
+            bail!("do not know what to do with calldata (1): '{tags:?}'");
+        }
+    } else if let Some(raw) = tags.get(":raw") {
+        if let Some(hex) = raw.strip_prefix("0x") {
+            Ok(Bytes::from(hex::decode(hex)?))
+        } else {
+            bail!("bad encoded calldata (3) {:?}", tags)
+        }
+    } else if let Some(abi) = tags.get(":abi") {
+        Ok(abi::encode_funccall(abi)?)
+    } else if let Some(yul) = tags.get(":yul") {
+        let caps = YUL_FRAGMENT_PARSER
+            .captures(yul)
+            .ok_or_else(|| anyhow::anyhow!("do not know what to do with code(4) '{:?}'", tags))?;
+        Ok(compiler.yul(
+            caps.name("code").unwrap().as_str(),
+            caps.name("version").map(|m| m.as_str()),
+        )?)
+    } else {
+        bail!("do not know what to do with calldata: (2) '{:?}'", tags,)
+    }
+}
+
+// Parse access list
+fn parse_access_list(raw_access_list: &Option<RawAccessList>) -> Result<Option<AccessList>> {
+    if let Some(raw_access_list) = raw_access_list {
+        let mut items = Vec::with_capacity(raw_access_list.len());
+        for raw in raw_access_list {
+            let storage_keys = raw
+                .storage_keys
+                .iter()
+                .map(|key| H256::from_str(key))
+                .collect::<Result<_, _>>()?;
+
+            items.push(AccessListItem {
+                address: address!(raw.address),
+                storage_keys,
+            });
+        }
+
+        return Ok(Some(AccessList(items)));
+    }
+
+    Ok(None)
+}

testool/src/statetest/spec.rs

@@ -89,14 +89,19 @@ impl std::fmt::Display for StateTest {
                 format!("{k} :")
             };
             let max_len = max_len - k.len();
+            let v = v.chars().collect::<Vec<_>>();
             for i in 0..=v.len() / max_len {
                 if i == 0 && !k.is_empty() {
                     text.push_str(&k);
                 } else {
                     let padding: String = " ".repeat(k.len());
                     text.push_str(&padding);
                 }
-                text.push_str(&v[i * max_len..std::cmp::min((i + 1) * max_len, v.len())]);
+                text.push_str(
+                    &v[i * max_len..std::cmp::min((i + 1) * max_len, v.len())]
+                        .iter()
+                        .collect::<String>(),
+                );
                 text.push('\n');
             }
             text