diff --git a/Cargo.lock b/Cargo.lock index b29242964799..9e769a2b2b4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5497,6 +5497,7 @@ dependencies = [ "sp-externalities", "sp-io", "sp-runtime", + "sp-runtime-interface", "sp-std", "sp-wasm-interface", "thiserror", @@ -5652,7 +5653,6 @@ dependencies = [ "static_assertions", "substrate-wasm-builder", "tiny-keccak", - "trie-db", ] [[package]] @@ -9744,9 +9744,9 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-db" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc176c377eb24d652c9c69c832c832019011b6106182bf84276c66b66d5c9a6" +checksum = "ec051edf7f0fc9499a2cb0947652cab2148b9d7f61cee7605e312e9f970dacaf" dependencies = [ "hash-db", "hashbrown", diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 9ff4a7ed5a42..56a309cd66ef 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -178,7 +178,7 @@ async fn runtime_api_request( #[derive(Debug)] enum AssumptionCheckOutcome { - Matches(PersistedValidationData, ValidationCode), + Matches(PersistedValidationData, Arc), DoesNotMatch, BadRequest, } @@ -227,7 +227,7 @@ async fn check_assumption_validation_data( match validation_code { Ok(None) | Err(_) => AssumptionCheckOutcome::BadRequest, - Ok(Some(v)) => AssumptionCheckOutcome::Matches(validation_data, v), + Ok(Some(v)) => AssumptionCheckOutcome::Matches(validation_data, Arc::new(v)), } } else { AssumptionCheckOutcome::DoesNotMatch @@ -333,7 +333,7 @@ async fn spawn_validate_exhaustive( ctx: &mut impl SubsystemContext, isolation_strategy: IsolationStrategy, persisted_validation_data: PersistedValidationData, - validation_code: ValidationCode, + validation_code: Arc, descriptor: CandidateDescriptor, pov: Arc, spawn: impl SpawnNamed + 'static, @@ -390,7 +390,7 @@ trait ValidationBackend { fn validate( arg: Self::Arg, - validation_code: &ValidationCode, + validation_code: Arc, params: ValidationParams, spawn: S, ) -> Result; @@ -403,12 +403,14 @@ impl ValidationBackend for RealValidationBackend { fn validate( isolation_strategy: IsolationStrategy, - validation_code: &ValidationCode, + validation_code: Arc, params: ValidationParams, spawn: S, ) -> Result { + let ext = validation_code.clone(); wasm_executor::validate_candidate( - &validation_code.0, + validation_code.0.as_slice(), + ext, params, &isolation_strategy, spawn, @@ -423,7 +425,7 @@ impl ValidationBackend for RealValidationBackend { fn validate_candidate_exhaustive( backend_arg: B::Arg, persisted_validation_data: PersistedValidationData, - validation_code: ValidationCode, + validation_code: Arc, descriptor: CandidateDescriptor, pov: Arc, spawn: S, @@ -442,7 +444,7 @@ fn validate_candidate_exhaustive( relay_parent_storage_root: persisted_validation_data.relay_parent_storage_root, }; - match B::validate(backend_arg, &validation_code, params, spawn) { + match B::validate(backend_arg, validation_code, params, spawn) { Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::Timeout)) => Ok(ValidationResult::Invalid(InvalidCandidate::Timeout)), Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::ParamsTooLarge(l))) => @@ -573,6 +575,7 @@ mod tests { use futures::executor; use assert_matches::assert_matches; use sp_keyring::Sr25519Keyring; + use std::sync::Arc; struct MockValidationBackend; @@ -585,7 +588,7 @@ mod tests { fn validate( arg: Self::Arg, - _validation_code: &ValidationCode, + _validation_code: Arc, _params: ValidationParams, _spawn: S, ) -> Result { @@ -662,7 +665,7 @@ mod tests { assert_matches!(check_result.await.unwrap(), AssumptionCheckOutcome::Matches(o, v) => { assert_eq!(o, validation_data); - assert_eq!(v, validation_code); + assert_eq!(v.as_ref(), &validation_code); }); }; @@ -726,7 +729,7 @@ mod tests { assert_matches!(check_result.await.unwrap(), AssumptionCheckOutcome::Matches(o, v) => { assert_eq!(o, validation_data); - assert_eq!(v, validation_code); + assert_eq!(v.as_ref(), &validation_code); }); }; @@ -910,7 +913,7 @@ mod tests { let v = validate_candidate_exhaustive::( MockValidationArg { result: Ok(validation_result) }, validation_data.clone(), - vec![1, 2, 3].into(), + Arc::new(vec![1, 2, 3].into()), descriptor, Arc::new(pov), TaskExecutor::new(), @@ -946,7 +949,7 @@ mod tests { )) }, validation_data, - vec![1, 2, 3].into(), + Arc::new(vec![1, 2, 3].into()), descriptor, Arc::new(pov), TaskExecutor::new(), @@ -975,7 +978,7 @@ mod tests { )) }, validation_data, - vec![1, 2, 3].into(), + Arc::new(vec![1, 2, 3].into()), descriptor, Arc::new(pov), TaskExecutor::new(), diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index f289c557e3e8..6a7a628bad6c 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -78,28 +78,40 @@ native_executor_instance!( pub PolkadotExecutor, polkadot_runtime::api::dispatch, polkadot_runtime::native_version, - frame_benchmarking::benchmarking::HostFunctions, + ( + frame_benchmarking::benchmarking::HostFunctions, + polkadot_parachain::validation::HostFunctions, + ), ); native_executor_instance!( pub KusamaExecutor, kusama_runtime::api::dispatch, kusama_runtime::native_version, - frame_benchmarking::benchmarking::HostFunctions, + ( + frame_benchmarking::benchmarking::HostFunctions, + polkadot_parachain::validation::HostFunctions, + ), ); native_executor_instance!( pub WestendExecutor, westend_runtime::api::dispatch, westend_runtime::native_version, - frame_benchmarking::benchmarking::HostFunctions, + ( + frame_benchmarking::benchmarking::HostFunctions, + polkadot_parachain::validation::HostFunctions, + ), ); native_executor_instance!( pub RococoExecutor, rococo_runtime::api::dispatch, rococo_runtime::native_version, - frame_benchmarking::benchmarking::HostFunctions, + ( + frame_benchmarking::benchmarking::HostFunctions, + polkadot_parachain::validation::HostFunctions, + ), ); #[derive(thiserror::Error, Debug)] diff --git a/node/subsystem/src/messages.rs b/node/subsystem/src/messages.rs index daff6dc4c136..042a76b8972f 100644 --- a/node/subsystem/src/messages.rs +++ b/node/subsystem/src/messages.rs @@ -139,7 +139,7 @@ pub enum CandidateValidationMessage { /// performed by the relay-chain. ValidateFromExhaustive( PersistedValidationData, - ValidationCode, + Arc, CandidateDescriptor, Arc, oneshot::Sender>, diff --git a/node/test/service/src/lib.rs b/node/test/service/src/lib.rs index 6fd0d167908d..ccc7d20f6ccd 100644 --- a/node/test/service/src/lib.rs +++ b/node/test/service/src/lib.rs @@ -60,7 +60,10 @@ native_executor_instance!( pub PolkadotTestExecutor, polkadot_test_runtime::api::dispatch, polkadot_test_runtime::native_version, - frame_benchmarking::benchmarking::HostFunctions, + ( + frame_benchmarking::benchmarking::HostFunctions, + polkadot_parachain::validation::HostFunctions, + ), ); /// The client type being used by the test service. diff --git a/parachain/Cargo.toml b/parachain/Cargo.toml index 232d2d322a37..e1ee5a52e896 100644 --- a/parachain/Cargo.toml +++ b/parachain/Cargo.toml @@ -15,13 +15,14 @@ sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", d sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-wasm-interface = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } +sp-runtime-interface = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } polkadot-core-primitives = { path = "../core-primitives", default-features = false } derive_more = "0.99.11" +sp-externalities = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } # all optional crates. thiserror = { version = "1.0.22", optional = true } serde = { version = "1.0.117", default-features = false, features = [ "derive" ], optional = true } -sp-externalities = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } sp-io = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } parking_lot = { version = "0.11.1", optional = true } @@ -46,7 +47,7 @@ std = [ "parking_lot", "log", "parity-util-mem", - "sp-externalities", + "sp-externalities/std", "sc-executor", "sp-io", "polkadot-core-primitives/std", diff --git a/parachain/src/lib.rs b/parachain/src/lib.rs index 67a5a6839245..dede48b3c60e 100644 --- a/parachain/src/lib.rs +++ b/parachain/src/lib.rs @@ -51,3 +51,55 @@ mod wasm_api; #[cfg(all(not(feature = "std"), feature = "wasm-api"))] pub use wasm_api::*; + +use sp_std::{vec::Vec, boxed::Box}; +use crate::primitives::ValidationCode; + +use sp_runtime_interface::runtime_interface; + +/// Validation specific host functions. +#[runtime_interface] +pub trait Validation { + /// Get validation wasm bytecode. + fn validation_code(&mut self) -> Vec { + use sp_externalities::ExternalitiesExt; + let extension = self.extension::() + .expect("Cannot get validation code without dynamic runtime dispatcher (ValidationExt)"); + extension.validation_code() + } +} + +#[cfg(feature = "std")] +sp_externalities::decl_extension! { + /// executor extension. + pub struct ValidationExt(Box); +} + +#[cfg(feature = "std")] +impl ValidationExt { + /// New instance of task executor extension. + pub fn new(validation_ext: impl Validation) -> Self { + Self(Box::new(validation_ext)) + } +} + +/// Base methods to implement validation extension. +pub trait Validation: Send + 'static { + /// Get the validation code currently running. + /// This can be use to check validity or to complete + /// proofs. + fn validation_code(&self) -> Vec; +} + +#[cfg(feature = "std")] +impl Validation for std::sync::Arc { + fn validation_code(&self) -> Vec { + self.0.clone() + } +} + +impl<'a> Validation for &'static [u8] { + fn validation_code(&self) -> Vec { + self.to_vec() + } +} diff --git a/parachain/src/wasm_executor/mod.rs b/parachain/src/wasm_executor/mod.rs index 5192bf115de0..e85b725ce9a1 100644 --- a/parachain/src/wasm_executor/mod.rs +++ b/parachain/src/wasm_executor/mod.rs @@ -26,6 +26,7 @@ use parity_scale_codec::{Decode, Encode}; use sp_core::{storage::{ChildInfo, TrackedStorageKey}, traits::{CallInWasm, SpawnNamed}}; use sp_externalities::Extensions; use sp_wasm_interface::HostFunctions as _; +use crate::{Validation, ValidationExt}; #[cfg(not(any(target_os = "android", target_os = "unknown")))] pub use validation_host::{run_worker, ValidationPool, EXECUTION_TIMEOUT_SEC, WORKER_ARGS}; @@ -167,6 +168,8 @@ pub struct ExecutorCache(sc_executor::WasmExecutor); impl Default for ExecutorCache { fn default() -> Self { + let mut host_functions = HostFunctions::host_functions(); + host_functions.extend(crate::validation::HostFunctions::host_functions().into_iter()); ExecutorCache(sc_executor::WasmExecutor::new( #[cfg(all(feature = "wasmtime", not(any(target_os = "android", target_os = "unknown"))))] sc_executor::WasmExecutionMethod::Compiled, @@ -174,7 +177,7 @@ impl Default for ExecutorCache { sc_executor::WasmExecutionMethod::Interpreted, // TODO: Make sure we don't use more than 1GB: https://github.com/paritytech/polkadot/issues/699 Some(1024), - HostFunctions::host_functions(), + host_functions, 8 )) } @@ -185,6 +188,7 @@ impl Default for ExecutorCache { /// This will fail if the validation code is not a proper parachain validation module. pub fn validate_candidate( validation_code: &[u8], + validation_ext: impl Validation, params: ValidationParams, isolation_strategy: &IsolationStrategy, spawner: impl SpawnNamed + 'static, @@ -196,6 +200,7 @@ pub fn validate_candidate( validation_code, ¶ms.encode(), spawner, + validation_ext, ) }, #[cfg(not(any(target_os = "android", target_os = "unknown")))] @@ -221,10 +226,12 @@ pub fn validate_candidate_internal( validation_code: &[u8], encoded_call_data: &[u8], spawner: impl SpawnNamed + 'static, + validation_ext: impl Validation, ) -> Result { let executor = &executor.0; let mut extensions = Extensions::new(); + extensions.register(ValidationExt::new(validation_ext)); extensions.register(sp_core::traits::TaskExecutorExt::new(spawner)); extensions.register(sp_core::traits::CallInWasmExt::new(executor.clone())); diff --git a/parachain/src/wasm_executor/validation_host.rs b/parachain/src/wasm_executor/validation_host.rs index 2db89b4bbaa1..dc1b1726d874 100644 --- a/parachain/src/wasm_executor/validation_host.rs +++ b/parachain/src/wasm_executor/validation_host.rs @@ -18,7 +18,7 @@ use std::{process, env, sync::Arc, sync::atomic, path::PathBuf}; use parity_scale_codec::{Decode, Encode}; -use crate::primitives::{ValidationParams, ValidationResult}; +use crate::primitives::{ValidationParams, ValidationResult, ValidationCode}; use super::{ validate_candidate_internal, ValidationError, InvalidCandidate, InternalError, MAX_CODE_MEM, MAX_RUNTIME_MEM, MAX_VALIDATION_RESULT_HEADER_MEM, @@ -185,8 +185,16 @@ pub fn run_worker(mem_id: &str) -> Result<(), String> { let (code, _) = code.split_at_mut(header.code_size as usize); let (call_data, _) = rest.split_at_mut(MAX_RUNTIME_MEM); let (call_data, _) = call_data.split_at_mut(header.params_size as usize); - - let result = validate_candidate_internal(&executor, code, call_data, task_executor.clone()); + // Costy code clone, could also use an unsafe implementation of ValidationExt. + let validation_ext = Arc::new(ValidationCode(code.to_vec())); + + let result = validate_candidate_internal( + &executor, + code, + call_data, + task_executor.clone(), + validation_ext, + ); debug!(target: LOG_TARGET, "{} Candidate validated: {:?}", process::id(), result); match result { diff --git a/parachain/test-parachains/adder/collator/src/lib.rs b/parachain/test-parachains/adder/collator/src/lib.rs index d8d8d3c1bb46..a6ad11d7b843 100644 --- a/parachain/test-parachains/adder/collator/src/lib.rs +++ b/parachain/test-parachains/adder/collator/src/lib.rs @@ -196,7 +196,7 @@ mod tests { use futures::executor::block_on; use polkadot_parachain::{primitives::ValidationParams, wasm_executor::IsolationStrategy}; - use polkadot_primitives::v1::PersistedValidationData; + use polkadot_primitives::v1::{PersistedValidationData, ValidationCode}; #[test] fn collator_works() { @@ -225,8 +225,10 @@ mod tests { } fn validate_collation(collator: &Collator, parent_head: HeadData, collation: Collation) { + let validation_ext = Arc::new(ValidationCode(collator.validation_code().to_vec())); let ret = polkadot_parachain::wasm_executor::validate_candidate( collator.validation_code(), + validation_ext, ValidationParams { parent_head: parent_head.encode().into(), block_data: collation.proof_of_validity.block_data, diff --git a/parachain/test-parachains/tests/adder/mod.rs b/parachain/test-parachains/tests/adder/mod.rs index 2dc581208106..5146c6a71ef2 100644 --- a/parachain/test-parachains/tests/adder/mod.rs +++ b/parachain/test-parachains/tests/adder/mod.rs @@ -63,6 +63,7 @@ fn execute_good_on_parent(isolation_strategy: IsolationStrategy) { }; let ret = parachain::wasm_executor::validate_candidate( + adder::wasm_binary_unwrap(), adder::wasm_binary_unwrap(), ValidationParams { parent_head: GenericHeadData(parent_head.encode()), @@ -101,6 +102,7 @@ fn execute_good_chain_on_parent() { }; let ret = parachain::wasm_executor::validate_candidate( + adder::wasm_binary_unwrap(), adder::wasm_binary_unwrap(), ValidationParams { parent_head: GenericHeadData(parent_head.encode()), @@ -140,6 +142,7 @@ fn execute_bad_on_parent() { }; let _ret = parachain::wasm_executor::validate_candidate( + adder::wasm_binary_unwrap(), adder::wasm_binary_unwrap(), ValidationParams { parent_head: GenericHeadData(parent_head.encode()), diff --git a/parachain/test-parachains/tests/wasm_executor/mod.rs b/parachain/test-parachains/tests/wasm_executor/mod.rs index a034ae5908f0..cdb13f094c77 100644 --- a/parachain/test-parachains/tests/wasm_executor/mod.rs +++ b/parachain/test-parachains/tests/wasm_executor/mod.rs @@ -37,6 +37,7 @@ fn terminates_on_timeout() { let isolation_strategy = isolation_strategy(); let result = parachain::wasm_executor::validate_candidate( + halt::wasm_binary_unwrap(), halt::wasm_binary_unwrap(), ValidationParams { block_data: BlockData(Vec::new()), @@ -65,6 +66,7 @@ fn parallel_execution() { let thread = std::thread::spawn(move || parachain::wasm_executor::validate_candidate( halt::wasm_binary_unwrap(), + halt::wasm_binary_unwrap(), ValidationParams { block_data: BlockData(Vec::new()), parent_head: Default::default(), @@ -75,6 +77,7 @@ fn parallel_execution() { sp_core::testing::TaskExecutor::new(), ).ok()); let _ = parachain::wasm_executor::validate_candidate( + halt::wasm_binary_unwrap(), halt::wasm_binary_unwrap(), ValidationParams { block_data: BlockData(Vec::new()), diff --git a/roadmap/implementers-guide/src/types/overseer-protocol.md b/roadmap/implementers-guide/src/types/overseer-protocol.md index 5b298472141f..073e8bdb2dd4 100644 --- a/roadmap/implementers-guide/src/types/overseer-protocol.md +++ b/roadmap/implementers-guide/src/types/overseer-protocol.md @@ -592,7 +592,7 @@ pub enum CandidateValidationMessage { /// performed by the relay-chain. ValidateFromExhaustive( PersistedValidationData, - ValidationCode, + Arc, CandidateDescriptor, Arc, oneshot::Sender>, diff --git a/runtime/common/Cargo.toml b/runtime/common/Cargo.toml index 50d61a2099b1..e3b24178729f 100644 --- a/runtime/common/Cargo.toml +++ b/runtime/common/Cargo.toml @@ -50,7 +50,7 @@ sp-application-crypto = { git = "https://github.com/paritytech/substrate", branc pallet-randomness-collective-flip = { git = "https://github.com/paritytech/substrate", branch = "master" } pallet-staking-reward-curve = { git = "https://github.com/paritytech/substrate", branch = "master" } pallet-treasury = { git = "https://github.com/paritytech/substrate", branch = "master" } -trie-db = "0.22.2" +trie-db = "0.22.3" serde_json = "1.0.61" libsecp256k1 = "0.3.5" diff --git a/runtime/polkadot/Cargo.toml b/runtime/polkadot/Cargo.toml index e95c94b3f443..2ed0d7c6e0a1 100644 --- a/runtime/polkadot/Cargo.toml +++ b/runtime/polkadot/Cargo.toml @@ -79,7 +79,6 @@ libsecp256k1 = "0.3.5" tiny-keccak = "2.0.2" keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate", branch = "master" } sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" } -trie-db = "0.22.2" serde_json = "1.0.61" [build-dependencies]