diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index ed53f52da3ce2..c198fb400408e 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -117,6 +117,7 @@ pub struct BenchmarkingState { read_write_tracker: RefCell, whitelist: RefCell>, proof_recorder: Option>, + proof_recorder_root: Cell, } impl BenchmarkingState { @@ -129,7 +130,7 @@ impl BenchmarkingState { let mut state = BenchmarkingState { state: RefCell::new(None), db: Cell::new(None), - root: Cell::new(root), + root: Cell::new(root.clone()), genesis: Default::default(), genesis_root: Default::default(), record: Default::default(), @@ -139,6 +140,7 @@ impl BenchmarkingState { read_write_tracker: Default::default(), whitelist: Default::default(), proof_recorder: record_proof.then(Default::default), + proof_recorder_root: Cell::new(root.clone()), }; state.add_whitelist_to_tracker(); @@ -166,7 +168,10 @@ impl BenchmarkingState { None => Arc::new(kvdb_memorydb::create(1)), }; self.db.set(Some(db.clone())); - self.proof_recorder.as_ref().map(|r| r.reset()); + if let Some(recorder) = &self.proof_recorder { + recorder.reset(); + self.proof_recorder_root.set(self.root.get()); + } let storage_db = Arc::new(StorageDb:: { db, proof_recorder: self.proof_recorder.clone(), @@ -516,7 +521,27 @@ impl StateBackend> for BenchmarkingState { } fn proof_size(&self) -> Option { - self.proof_recorder.as_ref().map(|recorder| recorder.estimate_encoded_size() as u32) + self.proof_recorder.as_ref().map(|recorder| { + let proof_size = recorder.estimate_encoded_size() as u32; + let proof = recorder.to_storage_proof(); + let proof_recorder_root = self.proof_recorder_root.get(); + if proof_recorder_root == Default::default() || proof_size == 1 { + // empty trie + proof_size + } else { + if let Some(size) = proof.encoded_compact_size::>(proof_recorder_root) { + size as u32 + } else { + panic!( + "proof rec root {:?}, root {:?}, genesis {:?}, rec_len {:?}", + self.proof_recorder_root.get(), + self.root.get(), + self.genesis_root, + proof_size, + ); + } + } + }) } } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 479184b4b9905..0508bfb780929 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1402,14 +1402,22 @@ mod tests { } } + fn test_compact(remote_proof: StorageProof, remote_root: &sp_core::H256) -> StorageProof { + let compact_remote_proof = remote_proof.into_compact_proof::( + remote_root.clone(), + ).unwrap(); + compact_remote_proof.to_storage_proof::(Some(remote_root)).unwrap().0 + } + #[test] fn prove_read_and_proof_check_works() { let child_info = ChildInfo::new_default(b"sub1"); let child_info = &child_info; // fetch read proof from 'remote' full node let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(::std::iter::empty()).0; + let remote_root = remote_backend.storage_root(std::iter::empty()).0; let remote_proof = prove_read(remote_backend, &[b"value2"]).unwrap(); + let remote_proof = test_compact(remote_proof, &remote_root); // check proof locally let local_result1 = read_proof_check::( remote_root, @@ -1429,12 +1437,13 @@ mod tests { assert_eq!(local_result2, false); // on child trie let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(::std::iter::empty()).0; + let remote_root = remote_backend.storage_root(std::iter::empty()).0; let remote_proof = prove_child_read( remote_backend, child_info, &[b"value3"], ).unwrap(); + let remote_proof = test_compact(remote_proof, &remote_root); let local_result1 = read_child_proof_check::( remote_root, remote_proof.clone(), @@ -1457,6 +1466,50 @@ mod tests { ); } + #[test] + fn compact_multiple_child_trie() { + // this root will be queried + let child_info1 = ChildInfo::new_default(b"sub1"); + // this root will not be include in proof + let child_info2 = ChildInfo::new_default(b"sub2"); + // this root will be include in proof + let child_info3 = ChildInfo::new_default(b"sub"); + let mut remote_backend = trie_backend::tests::test_trie(); + let (remote_root, transaction) = remote_backend.full_storage_root( + std::iter::empty(), + vec![ + (&child_info1, vec![ + (&b"key1"[..], Some(&b"val2"[..])), + (&b"key2"[..], Some(&b"val3"[..])), + ].into_iter()), + (&child_info2, vec![ + (&b"key3"[..], Some(&b"val4"[..])), + (&b"key4"[..], Some(&b"val5"[..])), + ].into_iter()), + (&child_info3, vec![ + (&b"key5"[..], Some(&b"val6"[..])), + (&b"key6"[..], Some(&b"val7"[..])), + ].into_iter()), + ].into_iter(), + ); + remote_backend.backend_storage_mut().consolidate(transaction); + remote_backend.essence.set_root(remote_root.clone()); + let remote_proof = prove_child_read( + remote_backend, + &child_info1, + &[b"key1"], + ).unwrap(); + let remote_proof = test_compact(remote_proof, &remote_root); + let local_result1 = read_child_proof_check::( + remote_root, + remote_proof.clone(), + &child_info1, + &[b"key1"], + ).unwrap(); + assert_eq!(local_result1.len(), 1); + assert_eq!(local_result1.get(&b"key1"[..]), Some(&Some(b"val2".to_vec()))); + } + #[test] fn child_storage_uuid() { diff --git a/primitives/trie/Cargo.toml b/primitives/trie/Cargo.toml index 4396550a48a8f..9584ae678d409 100644 --- a/primitives/trie/Cargo.toml +++ b/primitives/trie/Cargo.toml @@ -21,7 +21,7 @@ harness = false codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } sp-std = { version = "3.0.0", default-features = false, path = "../std" } hash-db = { version = "0.15.2", default-features = false } -trie-db = { version = "0.22.2", default-features = false } +trie-db = { version = "0.22.3", default-features = false } trie-root = { version = "0.16.0", default-features = false } memory-db = { version = "0.26.0", default-features = false } sp-core = { version = "3.0.0", default-features = false, path = "../core" } diff --git a/primitives/trie/src/error.rs b/primitives/trie/src/error.rs index 8e1d9b974ffd5..bdaa49b1156f7 100644 --- a/primitives/trie/src/error.rs +++ b/primitives/trie/src/error.rs @@ -26,7 +26,7 @@ pub enum Error { /// Bad format. BadFormat, /// Decoding error. - Decode(codec::Error) + Decode(codec::Error), } impl From for Error { diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 572283f1c027e..89bef715ba99a 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -23,6 +23,7 @@ mod error; mod node_header; mod node_codec; mod storage_proof; +mod trie_codec; mod trie_stream; use sp_std::{boxed::Box, marker::PhantomData, vec::Vec, borrow::Borrow}; @@ -35,7 +36,7 @@ pub use error::Error; pub use trie_stream::TrieStream; /// The Substrate format implementation of `NodeCodec`. pub use node_codec::NodeCodec; -pub use storage_proof::StorageProof; +pub use storage_proof::{StorageProof, CompactProof}; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, nibble_ops, TrieDBIterator, @@ -45,6 +46,9 @@ pub use memory_db::KeyFunction; pub use memory_db::prefixed_key; /// Various re-exports from the `hash-db` crate. pub use hash_db::{HashDB as HashDBT, EMPTY_PREFIX}; +/// Trie codec reexport, mainly child trie support +/// for trie compact proof. +pub use trie_codec::{decode_compact, encode_compact, Error as CompactProofError}; #[derive(Default)] /// substrate trie layout diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index f0b2bfd4bc3d3..1e62d942bba31 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -31,6 +31,12 @@ pub struct StorageProof { trie_nodes: Vec>, } +/// Storage proof in compact form. +#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] +pub struct CompactProof { + pub encoded_nodes: Vec>, +} + impl StorageProof { /// Constructs a storage proof from a subset of encoded trie nodes in a storage backend. pub fn new(trie_nodes: Vec>) -> Self { @@ -75,6 +81,56 @@ impl StorageProof { Self { trie_nodes } } + + /// Encode as a compact proof with default + /// trie layout. + pub fn into_compact_proof( + self, + root: H::Out, + ) -> Result>> { + crate::encode_compact::>(self, root) + } + + /// Returns the estimated encoded size of the compact proof. + /// + /// Runing this operation is a slow operation (build the whole compact proof) and should only be + /// in non sensitive path. + /// Return `None` on error. + pub fn encoded_compact_size(self, root: H::Out) -> Option { + let compact_proof = self.into_compact_proof::(root); + compact_proof.ok().map(|p| p.encoded_size()) + } + +} + +impl CompactProof { + /// Return an iterator on the compact encoded nodes. + pub fn iter_compact_encoded_nodes(&self) -> impl Iterator { + self.encoded_nodes.iter().map(Vec::as_slice) + } + + /// Decode to a full storage_proof. + /// + /// Method use a temporary `HashDB`, and `sp_trie::decode_compact` + /// is often better. + pub fn to_storage_proof( + &self, + expected_root: Option<&H::Out>, + ) -> Result<(StorageProof, H::Out), crate::CompactProofError>> { + let mut db = crate::MemoryDB::::new(&[]); + let root = crate::decode_compact::, _, _>( + &mut db, + self.iter_compact_encoded_nodes(), + expected_root, + )?; + Ok((StorageProof::new(db.drain().into_iter().filter_map(|kv| + if (kv.1).1 > 0 { + Some((kv.1).0) + } else { + None + } + ).collect()), root)) + } } /// An iterator over trie nodes constructed from a storage proof. The nodes are not guaranteed to diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs new file mode 100644 index 0000000000000..efe3223580f3f --- /dev/null +++ b/primitives/trie/src/trie_codec.rs @@ -0,0 +1,259 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Compact proof support. +//! +//! This uses compact proof from trie crate and extends +//! it to substrate specific layout and child trie system. + +use crate::{ + EMPTY_PREFIX, HashDBT, TrieHash, TrieError, TrieConfiguration, + CompactProof, StorageProof, +}; +use sp_std::boxed::Box; +use sp_std::vec::Vec; +use trie_db::Trie; +#[cfg(feature="std")] +use std::fmt; +#[cfg(feature="std")] +use std::error::Error as StdError; + + +/// Error for trie node decoding. +pub enum Error { + /// Verification failed due to root mismatch. + RootMismatch(TrieHash, TrieHash), + /// Missing nodes in proof. + IncompleteProof, + /// Compact node is not needed. + ExtraneousChildNode, + /// Child content with root not in proof. + ExtraneousChildProof(TrieHash), + /// Bad child trie root. + InvalidChildRoot(Vec, Vec), + /// Errors from trie crate. + TrieError(Box>), +} + +impl From>> for Error { + fn from(error: Box>) -> Self { + Error::TrieError(error) + } +} + +#[cfg(feature="std")] +impl StdError for Error { + fn description(&self) -> &str { + match self { + Error::InvalidChildRoot(..) => "Invalid child root error", + Error::TrieError(..) => "Trie db error", + Error::RootMismatch(..) => "Trie db error", + Error::IncompleteProof => "Incomplete proof", + Error::ExtraneousChildNode => "Extraneous child node", + Error::ExtraneousChildProof(..) => "Extraneous child proof", + } + } +} + +#[cfg(feature="std")] +impl fmt::Debug for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + ::fmt(&self, f) + } +} + +#[cfg(feature="std")] +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::InvalidChildRoot(k, v) => write!(f, "InvalidChildRoot at {:x?}: {:x?}", k, v), + Error::TrieError(e) => write!(f, "Trie error: {}", e), + Error::IncompleteProof => write!(f, "Incomplete proof"), + Error::ExtraneousChildNode => write!(f, "Child node content with no root in proof"), + Error::ExtraneousChildProof(root) => write!(f, "Proof of child trie {:x?} not in parent proof", root.as_ref()), + Error::RootMismatch(root, expected) => write!( + f, + "Verification error, root is {:x?}, expected: {:x?}", + root.as_ref(), + expected.as_ref(), + ), + } + } +} + +/// Decode a compact proof. +/// +/// Takes as input a destination `db` for decoded node and `encoded` +/// an iterator of compact encoded nodes. +/// +/// Child trie are decoded in order of child trie root present +/// in the top trie. +pub fn decode_compact<'a, L, DB, I>( + db: &mut DB, + encoded: I, + expected_root: Option<&TrieHash>, +) -> Result, Error> + where + L: TrieConfiguration, + DB: HashDBT + hash_db::HashDBRef, + I: IntoIterator, +{ + let mut nodes_iter = encoded.into_iter(); + let (top_root, _nb_used) = trie_db::decode_compact_from_iter::( + db, + &mut nodes_iter, + )?; + + // Only check root if expected root is passed as argument. + if let Some(expected_root) = expected_root { + if expected_root != &top_root { + return Err(Error::RootMismatch(top_root.clone(), expected_root.clone())); + } + } + + let mut child_tries = Vec::new(); + { + // fetch child trie roots + let trie = crate::TrieDB::::new(db, &top_root)?; + + let mut iter = trie.iter()?; + + let childtrie_roots = sp_core::storage::well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX; + if iter.seek(childtrie_roots).is_ok() { + loop { + match iter.next() { + Some(Ok((key, value))) if key.starts_with(childtrie_roots) => { + // we expect all default child trie root to be correctly encoded. + // see other child trie functions. + let mut root = TrieHash::::default(); + // still in a proof so prevent panic + if root.as_mut().len() != value.as_slice().len() { + return Err(Error::InvalidChildRoot(key, value)); + } + root.as_mut().copy_from_slice(value.as_ref()); + child_tries.push(root); + }, + // allow incomplete database error: we only + // require access to data in the proof. + Some(Err(error)) => match *error { + trie_db::TrieError::IncompleteDatabase(..) => (), + e => return Err(Box::new(e).into()), + }, + _ => break, + } + } + } + } + + if !HashDBT::::contains(db, &top_root, EMPTY_PREFIX) { + return Err(Error::IncompleteProof); + } + + let mut previous_extracted_child_trie = None; + for child_root in child_tries.into_iter() { + if previous_extracted_child_trie.is_none() { + let (top_root, _) = trie_db::decode_compact_from_iter::( + db, + &mut nodes_iter, + )?; + previous_extracted_child_trie = Some(top_root); + } + + // we do not early exit on root mismatch but try the + // other read from proof (some child root may be + // in proof without actual child content). + if Some(child_root) == previous_extracted_child_trie { + previous_extracted_child_trie = None; + } + } + + if let Some(child_root) = previous_extracted_child_trie { + // A child root was read from proof but is not present + // in top trie. + return Err(Error::ExtraneousChildProof(child_root)); + } + + if nodes_iter.next().is_some() { + return Err(Error::ExtraneousChildNode); + } + + Ok(top_root) +} + +/// Encode a compact proof. +/// +/// Takes as input all full encoded node from the proof, and +/// the root. +/// Then parse all child trie root and compress main trie content first +/// then all child trie contents. +/// Child trie are ordered by the order of their roots in the top trie. +pub fn encode_compact( + proof: StorageProof, + root: TrieHash, +) -> Result> + where + L: TrieConfiguration, +{ + let mut child_tries = Vec::new(); + let partial_db = proof.into_memory_db(); + let mut compact_proof = { + let trie = crate::TrieDB::::new(&partial_db, &root)?; + + let mut iter = trie.iter()?; + + let childtrie_roots = sp_core::storage::well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX; + if iter.seek(childtrie_roots).is_ok() { + loop { + match iter.next() { + Some(Ok((key, value))) if key.starts_with(childtrie_roots) => { + let mut root = TrieHash::::default(); + if root.as_mut().len() != value.as_slice().len() { + // some child trie root in top trie are not an encoded hash. + return Err(Error::InvalidChildRoot(key.to_vec(), value.to_vec())); + } + root.as_mut().copy_from_slice(value.as_ref()); + child_tries.push(root); + }, + // allow incomplete database error: we only + // require access to data in the proof. + Some(Err(error)) => match *error { + trie_db::TrieError::IncompleteDatabase(..) => (), + e => return Err(Box::new(e).into()), + }, + _ => break, + } + } + } + + trie_db::encode_compact::(&trie)? + }; + + for child_root in child_tries { + if !HashDBT::::contains(&partial_db, &child_root, EMPTY_PREFIX) { + // child proof are allowed to be missing (unused root can be included + // due to trie structure modification). + continue; + } + + let trie = crate::TrieDB::::new(&partial_db, &child_root)?; + let child_proof = trie_db::encode_compact::(&trie)?; + + compact_proof.extend(child_proof); + } + + Ok(CompactProof { encoded_nodes: compact_proof }) +} diff --git a/utils/wasm-builder/src/wasm_project.rs b/utils/wasm-builder/src/wasm_project.rs index 58161f53113fe..466c2145e6cee 100644 --- a/utils/wasm-builder/src/wasm_project.rs +++ b/utils/wasm-builder/src/wasm_project.rs @@ -232,7 +232,8 @@ fn create_project_cargo_toml( wasm_workspace_toml.insert("profile".into(), profile.into()); // Add patch section from the project root `Cargo.toml` - if let Some(mut patch) = workspace_toml.remove("patch").and_then(|p| p.try_into::().ok()) { + while let Some(mut patch) = workspace_toml.remove("patch") + .and_then(|p| p.try_into::
().ok()) { // Iterate over all patches and make the patch path absolute from the workspace root path. patch.iter_mut() .filter_map(|p|