Add support for generating metadata from runtime wasm files (#1720)

closes #1660

Author: pkhry

Cargo.lock

@@ -116,6 +116,8 @@ which = "5.0.0"
 strip-ansi-escapes = "0.2.0"
 proptest = "1.5.0"
 hex-literal = "0.4.1"
+sc-executor = "0.40.0"
+sc-executor-common = "0.35.0"
 
 # Light client support:
 smoldot = { version = "0.16.0", default-features = false }
@@ -140,6 +142,9 @@ sp-core = { version = "31.0.0", default-features = false }
 sp-crypto-hashing = { version = "0.1.0", default-features = false }
 sp-runtime = "34.0.0"
 sp-keyring = "34.0.0"
+sp-maybe-compressed-blob = "11.0.0"
+sp-state-machine = "0.43.0"
+sp-io = "38.0.0"
 
 # Subxt workspace crates:
 subxt = { version = "0.37.0", path = "subxt", default-features = false }
@@ -167,6 +172,7 @@ base64 = { version = "0.22.1", default-features = false }
 scrypt = { version = "0.11.0", default-features = false }
 crypto_secretbox = { version = "0.1.1", default-features = false }
 
+
 [profile.dev.package.smoldot-light]
 opt-level = 2
 [profile.test.package.smoldot-light]

Cargo.toml

@@ -57,6 +57,9 @@ pub enum CodegenError {
     /// Cannot generate types.
     #[error("Type Generation failed: {0}")]
     TypeGeneration(#[from] TypegenError),
+    /// Error when generating metadata from Wasm-runtime
+    #[error("Failed to generate metadata from wasm file. reason: {0}")]
+    Wasm(String),
 }
 
 impl CodegenError {

artifacts/westend_runtime.compact.compressed.wasm

@@ -15,6 +15,7 @@ description = "Generate types and helpers for interacting with Substrate runtime
 
 [features]
 web = ["subxt-codegen/web"]
+runtime-path = ["sp-io", "sc-executor-common", "sp-state-machine", "sp-maybe-compressed-blob", "sc-executor"]
 
 [lib]
 proc-macro = true
@@ -27,6 +28,11 @@ syn = { workspace = true }
 quote = { workspace = true }
 subxt-codegen = { workspace = true, features = ["fetch-metadata"] }
 scale-typegen = { workspace = true }
+sc-executor = { workspace = true, optional = true }
+sp-maybe-compressed-blob = { workspace = true, optional = true }
+sp-state-machine = { workspace = true, optional = true }
+sp-io = { workspace = true, optional = true }
+sc-executor-common = { workspace = true, optional = true }
 
 [lints]
 workspace = true

artifacts/westend_runtime.wasm

@@ -23,6 +23,9 @@ use subxt_codegen::{
 };
 use syn::{parse_macro_input, punctuated::Punctuated};
 
+#[cfg(feature = "runtime-path")]
+mod wasm_loader;
+
 #[derive(Clone, Debug)]
 struct OuterAttribute(syn::Attribute);
 
@@ -60,6 +63,9 @@ struct RuntimeMetadataArgs {
     no_default_substitutions: bool,
     #[darling(default)]
     unstable_metadata: darling::util::Flag,
+    #[cfg(feature = "runtime-path")]
+    #[darling(default)]
+    runtime_path: Option<String>,
 }
 
 #[derive(Debug, FromMeta)]
@@ -206,6 +212,22 @@ fn validate_type_path(path: &syn::Path, metadata: &Metadata) {
 fn fetch_metadata(args: &RuntimeMetadataArgs) -> Result<subxt_codegen::Metadata, TokenStream> {
     // Do we want to fetch unstable metadata? This only works if fetching from a URL.
     let unstable_metadata = args.unstable_metadata.is_present();
+
+    #[cfg(feature = "runtime-path")]
+    if let Some(path) = &args.runtime_path {
+        if args.runtime_metadata_insecure_url.is_some() || args.runtime_metadata_path.is_some() {
+            abort_call_site!(
+                "Only one of 'runtime_metadata_path', 'runtime_metadata_insecure_url' or `runtime_path` must be provided"
+            );
+        };
+        let root = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
+        let root_path = std::path::Path::new(&root);
+        let path = root_path.join(path);
+
+        let metadata = wasm_loader::from_wasm_file(&path).map_err(|e| e.into_compile_error())?;
+        return Ok(metadata);
+    };
+
     let metadata = match (
         &args.runtime_metadata_path,
         &args.runtime_metadata_insecure_url,
@@ -239,12 +261,26 @@ fn fetch_metadata(args: &RuntimeMetadataArgs) -> Result<subxt_codegen::Metadata,
                 .and_then(|b| subxt_codegen::Metadata::decode(&mut &*b).map_err(Into::into))
                 .map_err(|e| e.into_compile_error())?
         }
+        #[cfg(feature = "runtime-path")]
         (None, None) => {
             abort_call_site!(
-                "One of 'runtime_metadata_path' or 'runtime_metadata_insecure_url' must be provided"
+                    "At least one of 'runtime_metadata_path', 'runtime_metadata_insecure_url'  or  'runtime_path` can be provided"
+                )
+        }
+        #[cfg(not(feature = "runtime-path"))]
+        (None, None) => {
+            abort_call_site!(
+                "At least one of 'runtime_metadata_path', 'runtime_metadata_insecure_url' can be provided"
             )
         }
-        (Some(_), Some(_)) => {
+        #[cfg(feature = "runtime-path")]
+        _ => {
+            abort_call_site!(
+                    "Only one of 'runtime_metadata_path', 'runtime_metadata_insecure_url'  or  'runtime_path` can be provided"
+                )
+        }
+        #[cfg(not(feature = "runtime-path"))]
+        _ => {
             abort_call_site!(
                 "Only one of 'runtime_metadata_path' or 'runtime_metadata_insecure_url' can be provided"
             )

codegen/src/error.rs

@@ -0,0 +1,55 @@
+// Copyright 2024 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use std::{borrow::Cow, path::Path};
+
+use codec::Decode;
+use sc_executor::{WasmExecutionMethod, WasmExecutor};
+use sc_executor_common::runtime_blob::RuntimeBlob;
+use sp_maybe_compressed_blob::CODE_BLOB_BOMB_LIMIT;
+use subxt_codegen::{fetch_metadata::fetch_metadata_from_file_blocking, CodegenError, Metadata};
+
+/// Result type shorthand
+pub type WasmMetadataResult<A> = Result<A, CodegenError>;
+
+/// Uses wasm artifact produced by compiling the runtime to generate metadata
+pub fn from_wasm_file(wasm_file_path: &Path) -> WasmMetadataResult<Metadata> {
+    let wasm_file = fetch_metadata_from_file_blocking(wasm_file_path)
+        .map_err(Into::<CodegenError>::into)
+        .and_then(maybe_decompress)?;
+    call_and_decode(wasm_file)
+}
+
+fn call_and_decode(wasm_file: Vec<u8>) -> WasmMetadataResult<Metadata> {
+    let mut ext: sp_state_machine::BasicExternalities = Default::default();
+
+    let executor: WasmExecutor<sp_io::SubstrateHostFunctions> = WasmExecutor::builder()
+        .with_execution_method(WasmExecutionMethod::default())
+        .with_offchain_heap_alloc_strategy(sc_executor::HeapAllocStrategy::Dynamic {
+            maximum_pages: Some(64),
+        })
+        .with_max_runtime_instances(1)
+        .with_runtime_cache_size(1)
+        .build();
+
+    let runtime_blob =
+        RuntimeBlob::new(&wasm_file).map_err(|e| CodegenError::Wasm(e.to_string()))?;
+    let metadata_encoded = executor
+        .uncached_call(runtime_blob, &mut ext, true, "Metadata_metadata", &[])
+        .map_err(|_| CodegenError::Wasm("method \"Metadata_metadata\" doesnt exist".to_owned()))?;
+
+    let metadata = <Vec<u8>>::decode(&mut &metadata_encoded[..]).map_err(CodegenError::Decode)?;
+
+    decode(metadata)
+}
+
+fn decode(encoded_metadata: Vec<u8>) -> WasmMetadataResult<Metadata> {
+    Metadata::decode(&mut encoded_metadata.as_ref()).map_err(Into::into)
+}
+
+fn maybe_decompress(file_contents: Vec<u8>) -> WasmMetadataResult<Vec<u8>> {
+    sp_maybe_compressed_blob::decompress(file_contents.as_ref(), CODE_BLOB_BOMB_LIMIT)
+        .map_err(|e| CodegenError::Wasm(e.to_string()))
+        .map(Cow::into_owned)
+}

macro/Cargo.toml

@@ -70,6 +70,9 @@ unstable-metadata = []
 # Note that this feature is experimental and things may break or not work as expected.
 unstable-light-client = ["subxt-lightclient"]
 
+# Activate this to expose the ability to generate metadata from Wasm runtime files.
+runtime-path = ["subxt-macro/runtime-path"]
+
 [dependencies]
 async-trait = { workspace = true }
 codec = { package = "parity-scale-codec", workspace = true, features = ["derive"] }

macro/src/lib.rs

@@ -14,6 +14,6 @@ hex = { workspace = true }
 scale-info = { workspace = true, features = ["bit-vec"] }
 frame-metadata = { workspace = true }
 codec = { package = "parity-scale-codec", workspace = true, features = ["derive", "bit-vec"] }
-subxt = { workspace = true, features = ["native", "jsonrpsee"] }
+subxt = { workspace = true, features = ["native", "jsonrpsee", "runtime-path"] }
 subxt-metadata = { workspace = true }
 generate-custom-metadata = { path = "../generate-custom-metadata" }

macro/src/wasm_loader.rs

@@ -0,0 +1,13 @@
+#[subxt::subxt(runtime_path = "../../../../artifacts/westend_runtime.wasm")]
+mod runtime {}
+
+#[subxt::subxt(runtime_path = "../../../../artifacts/westend_runtime.compact.compressed.wasm")]
+mod runtime_compressed {}
+
+fn main() {
+    use runtime;
+    use runtime_compressed;
+
+    let _ = runtime::system::events::CodeUpdated;
+    let _ = runtime_compressed::system::events::CodeUpdated;
+}

subxt/Cargo.toml

@@ -1,4 +1,4 @@
-error: One of 'runtime_metadata_path' or 'runtime_metadata_insecure_url' must be provided
+error: At least one of 'runtime_metadata_path', 'runtime_metadata_insecure_url'  or  'runtime_path` can be provided
  --> src/incorrect/need_url_or_path.rs:1:1
   |
 1 | #[subxt::subxt()]

testing/ui-tests/Cargo.toml

@@ -4,4 +4,11 @@
 )]
 pub mod node_runtime {}
 
+#[subxt::subxt(
+    runtime_metadata_path = "../../../../artifacts/polkadot_metadata_tiny.scale",
+    runtime_metadata_insecure_url = "wss://rpc.polkadot.io:443",
+    runtime_path = "../../../../artifacts/westend_runtime.wasm"
+)]
+pub mod node_runtime2 {}
+
 fn main() {}

testing/ui-tests/src/correct/wasm_runtime_metadata.rs

@@ -1,4 +1,4 @@
-error: Only one of 'runtime_metadata_path' or 'runtime_metadata_insecure_url' can be provided
+error: Only one of 'runtime_metadata_path', 'runtime_metadata_insecure_url'  or  'runtime_path` can be provided
  --> src/incorrect/url_and_path_provided.rs:1:1
   |
 1 | / #[subxt::subxt(
@@ -8,3 +8,15 @@ error: Only one of 'runtime_metadata_path' or 'runtime_metadata_insecure_url' ca
   | |__^
   |
   = note: this error originates in the attribute macro `subxt::subxt` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: Only one of 'runtime_metadata_path', 'runtime_metadata_insecure_url' or `runtime_path` must be provided
+  --> src/incorrect/url_and_path_provided.rs:7:1
+   |
+7  | / #[subxt::subxt(
+8  | |     runtime_metadata_path = "../../../../artifacts/polkadot_metadata_tiny.scale",
+9  | |     runtime_metadata_insecure_url = "wss://rpc.polkadot.io:443",
+10 | |     runtime_path = "../../../../artifacts/westend_runtime.wasm"
+11 | | )]
+   | |__^
+   |
+   = note: this error originates in the attribute macro `subxt::subxt` (in Nightly builds, run with -Z macro-backtrace for more info)