From 3d6f81f974b6bf6a89b8be3c21ca86a53fa29764 Mon Sep 17 00:00:00 2001 From: higherordertech Date: Tue, 28 Jan 2025 11:44:03 +1100 Subject: [PATCH 1/4] P-970: add pallet-state-trie-migration and update state_version to 1 --- parachain/Cargo.lock | 2 ++ parachain/Cargo.toml | 1 + parachain/runtime/litentry/Cargo.toml | 4 +++ parachain/runtime/litentry/src/lib.rs | 35 ++++++++++++++++++++++++++- parachain/runtime/paseo/Cargo.toml | 4 +++ parachain/runtime/paseo/src/lib.rs | 35 ++++++++++++++++++++++++++- 6 files changed, 79 insertions(+), 2 deletions(-) diff --git a/parachain/Cargo.lock b/parachain/Cargo.lock index 90032fc91d..cdd75da2ce 100644 --- a/parachain/Cargo.lock +++ b/parachain/Cargo.lock @@ -6022,6 +6022,7 @@ dependencies = [ "pallet-scheduler", "pallet-score-staking", "pallet-session", + "pallet-state-trie-migration", "pallet-teebag", "pallet-timestamp", "pallet-transaction-payment", @@ -9526,6 +9527,7 @@ dependencies = [ "pallet-scheduler", "pallet-score-staking", "pallet-session", + "pallet-state-trie-migration", "pallet-sudo", "pallet-teebag", "pallet-timestamp", diff --git a/parachain/Cargo.toml b/parachain/Cargo.toml index e50c1edd3a..1ff000ba2f 100644 --- a/parachain/Cargo.toml +++ b/parachain/Cargo.toml @@ -226,6 +226,7 @@ pallet-vesting = { git = "https://github.com/paritytech/polkadot-sdk", branch = pallet-utility = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false } pallet-whitelist = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false } pallet-session = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false } +pallet-state-trie-migration = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false } pallet-sudo = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false } pallet-tips = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false } pallet-timestamp = { git = "https://github.com/paritytech/polkadot-sdk", branch = "stable2407", default-features = false } diff --git a/parachain/runtime/litentry/Cargo.toml b/parachain/runtime/litentry/Cargo.toml index 42fd59fcbb..b89cda98b2 100644 --- a/parachain/runtime/litentry/Cargo.toml +++ b/parachain/runtime/litentry/Cargo.toml @@ -49,6 +49,7 @@ pallet-preimage = { workspace = true } pallet-proxy = { workspace = true } pallet-scheduler = { workspace = true } pallet-session = { workspace = true } +pallet-state-trie-migration = { workspace = true } pallet-timestamp = { workspace = true } pallet-transaction-payment = { workspace = true } pallet-transaction-payment-rpc-runtime-api = { workspace = true } @@ -174,6 +175,7 @@ runtime-benchmarks = [ "pallet-proxy/runtime-benchmarks", "pallet-scheduler/runtime-benchmarks", "pallet-score-staking/runtime-benchmarks", + "pallet-state-trie-migration/runtime-benchmarks", "pallet-teebag/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", "pallet-treasury/runtime-benchmarks", @@ -262,6 +264,7 @@ std = [ "pallet-scheduler/std", "pallet-score-staking/std", "pallet-session/std", + "pallet-state-trie-migration/std", "pallet-teebag/std", "pallet-timestamp/std", "pallet-transaction-payment-rpc-runtime-api/std", @@ -341,6 +344,7 @@ try-runtime = [ "pallet-scheduler/try-runtime", "pallet-score-staking/try-runtime", "pallet-session/try-runtime", + "pallet-state-trie-migration/try-runtime", "pallet-teebag/try-runtime", "pallet-timestamp/try-runtime", "pallet-transaction-payment/try-runtime", diff --git a/parachain/runtime/litentry/src/lib.rs b/parachain/runtime/litentry/src/lib.rs index a632477e17..62bd7bf1c7 100644 --- a/parachain/runtime/litentry/src/lib.rs +++ b/parachain/runtime/litentry/src/lib.rs @@ -234,7 +234,15 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 1, - state_version: 0, + // https://hackmd.io/JagpUd8tTjuKf9HQtpvHIQ + // The trie is an abstraction that sits between the Runtime (and its Overlays) and the actual database, providing an important abstraction to the blockchain, namely storage proofs and state roots. + // The trie format has changed since this pull request(#9732) in substrate. The main new difference is, that nodes that contain values larger than 256 bits will not storage the value itself, but rather store the hash of that value. The value itself, is consequently stored in the node that lives in the path traversed by this new hash. + // The main benefit of this optimization is better PoV (proof of validity) size for parachains, since large values are moved out of the common trie paths. + // The new trie has been included in Polkadot client since release v0.9.16. Although, new new trie format is not yet enabled. This is only done once state_version in RuntimeVersion is set to 1. Once set to 1, the trie works in a hybrid format, meaning that no migration is needed. Instead, migration is done lazily on the fly. Any storage key that's written to will be migrated, if needed. This means that a part of all chain's state is will migrated to the new format pretty soon after setting state_version to 1. + // Nonetheless, it might take a long time for all chain's entire state to be migrated to the new format. The sooner this happens, the better, since the lazy migration is a small overhead. Moreover, this hybrid/lazy state mode does not support warp-sync and state import/export. + // To do this faster, we have developed pallet-state-trie-migration. This pallet is a configurable background task that starts reading and writing all keys in the storage based on some given schedule, until they are all read, ergo migrated. This pallet can be deployed to a runtime to make sure all keys are read/written once, to ensure that all trie nodes are migrated to the new format. + // All substrate-based chains are advised to switch their state_version to 1, and use this pallet to migrate to the new trie format as soon as they can. Switching the state_version will enable the hybrid, lazy migration mode, and this pallet will speed up the migration process. + state_version: 1, }; /// The version information used to identify this runtime when compiled natively. @@ -1354,6 +1362,9 @@ construct_runtime! { EVM: pallet_evm = 120, Ethereum: pallet_ethereum = 121, + // State Trie Migration + StateTrieMigration: pallet_state_trie_migration = 130, + // TMP AccountFix: pallet_account_fix = 254, } @@ -2153,3 +2164,25 @@ cumulus_pallet_parachain_system::register_validate_block! { Runtime = Runtime, BlockExecutor = cumulus_pallet_aura_ext::BlockExecutor::, } + +parameter_types! { + // The deposit configuration for the singed migration. Specially if you want to allow any signed account to do the migration (see `SignedFilter`, these deposits should be high) + pub MigrationSignedDepositPerItem: Balance = 1 * CENTS; + pub MigrationSignedDepositBase: Balance = 20 * DOLLARS; + pub const MigrationMaxKeyLen: u32 = 512; +} + +impl pallet_state_trie_migration::Config for Runtime { + // An origin that can control the whole pallet: should be Root, or a part of your council. + type ControlOrigin = EnsureRootOrTwoThirdsTechnicalCommittee; + // specific account for the migration, can trigger the signed migrations. + type SignedFilter = frame_support::traits::NeverEnsureOrigin; + type RuntimeEvent = RuntimeEvent; + type Currency = Balances; + type RuntimeHoldReason = RuntimeHoldReason; + type MaxKeyLen = MigrationMaxKeyLen; + type SignedDepositPerItem = MigrationSignedDepositPerItem; + type SignedDepositBase = MigrationSignedDepositBase; + // Replace this with weight based on your runtime. + type WeightInfo = pallet_state_trie_migration::weights::SubstrateWeight; +} diff --git a/parachain/runtime/paseo/Cargo.toml b/parachain/runtime/paseo/Cargo.toml index 79300fef4f..8ad239d3c2 100644 --- a/parachain/runtime/paseo/Cargo.toml +++ b/parachain/runtime/paseo/Cargo.toml @@ -47,6 +47,7 @@ pallet-proxy = { workspace = true } pallet-referenda = { workspace = true } pallet-scheduler = { workspace = true } pallet-session = { workspace = true } +pallet-state-trie-migration = { workspace = true } pallet-sudo = { workspace = true } pallet-timestamp = { workspace = true } pallet-tips = { workspace = true } @@ -193,6 +194,7 @@ runtime-benchmarks = [ "pallet-referenda/runtime-benchmarks", "pallet-scheduler/runtime-benchmarks", "pallet-score-staking/runtime-benchmarks", + "pallet-state-trie-migration/runtime-benchmarks", "pallet-sudo/runtime-benchmarks", "pallet-teebag/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", @@ -293,6 +295,7 @@ std = [ "pallet-scheduler/std", "pallet-score-staking/std", "pallet-session/std", + "pallet-state-trie-migration/std", "pallet-sudo/std", "pallet-teebag/std", "pallet-timestamp/std", @@ -388,6 +391,7 @@ try-runtime = [ "pallet-scheduler/try-runtime", "pallet-score-staking/try-runtime", "pallet-session/try-runtime", + "pallet-state-trie-migration/try-runtime", "pallet-sudo/try-runtime", "pallet-teebag/try-runtime", "pallet-timestamp/try-runtime", diff --git a/parachain/runtime/paseo/src/lib.rs b/parachain/runtime/paseo/src/lib.rs index 65f76889a0..752da12b71 100644 --- a/parachain/runtime/paseo/src/lib.rs +++ b/parachain/runtime/paseo/src/lib.rs @@ -237,7 +237,15 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 1, - state_version: 0, + // https://hackmd.io/JagpUd8tTjuKf9HQtpvHIQ + // The trie is an abstraction that sits between the Runtime (and its Overlays) and the actual database, providing an important abstraction to the blockchain, namely storage proofs and state roots. + // The trie format has changed since this pull request(#9732) in substrate. The main new difference is, that nodes that contain values larger than 256 bits will not storage the value itself, but rather store the hash of that value. The value itself, is consequently stored in the node that lives in the path traversed by this new hash. + // The main benefit of this optimization is better PoV (proof of validity) size for parachains, since large values are moved out of the common trie paths. + // The new trie has been included in Polkadot client since release v0.9.16. Although, new new trie format is not yet enabled. This is only done once state_version in RuntimeVersion is set to 1. Once set to 1, the trie works in a hybrid format, meaning that no migration is needed. Instead, migration is done lazily on the fly. Any storage key that's written to will be migrated, if needed. This means that a part of all chain's state is will migrated to the new format pretty soon after setting state_version to 1. + // Nonetheless, it might take a long time for all chain's entire state to be migrated to the new format. The sooner this happens, the better, since the lazy migration is a small overhead. Moreover, this hybrid/lazy state mode does not support warp-sync and state import/export. + // To do this faster, we have developed pallet-state-trie-migration. This pallet is a configurable background task that starts reading and writing all keys in the storage based on some given schedule, until they are all read, ergo migrated. This pallet can be deployed to a runtime to make sure all keys are read/written once, to ensure that all trie nodes are migrated to the new format. + // All substrate-based chains are advised to switch their state_version to 1, and use this pallet to migrate to the new trie format as soon as they can. Switching the state_version will enable the hybrid, lazy migration mode, and this pallet will speed up the migration process. + state_version: 1, }; /// A timestamp: milliseconds since the unix epoch. @@ -1491,6 +1499,9 @@ construct_runtime! { EVM: pallet_evm = 120, Ethereum: pallet_ethereum = 121, + // State Trie Migration + StateTrieMigration: pallet_state_trie_migration = 130, + // CollabAI Curator: pallet_curator = 150, Guardian: pallet_guardian = 151, @@ -2324,3 +2335,25 @@ cumulus_pallet_parachain_system::register_validate_block! { Runtime = Runtime, BlockExecutor = cumulus_pallet_aura_ext::BlockExecutor::, } + +parameter_types! { + // The deposit configuration for the singed migration. Specially if you want to allow any signed account to do the migration (see `SignedFilter`, these deposits should be high) + pub MigrationSignedDepositPerItem: Balance = 1 * CENTS; + pub MigrationSignedDepositBase: Balance = 20 * DOLLARS; + pub const MigrationMaxKeyLen: u32 = 512; +} + +impl pallet_state_trie_migration::Config for Runtime { + // An origin that can control the whole pallet: should be Root, or a part of your council. + type ControlOrigin = EnsureRootOrTwoThirdsTechnicalCommittee; + // specific account for the migration, can trigger the signed migrations. + type SignedFilter = frame_support::traits::NeverEnsureOrigin; + type RuntimeEvent = RuntimeEvent; + type Currency = Balances; + type RuntimeHoldReason = RuntimeHoldReason; + type MaxKeyLen = MigrationMaxKeyLen; + type SignedDepositPerItem = MigrationSignedDepositPerItem; + type SignedDepositBase = MigrationSignedDepositBase; + // Replace this with weight based on your runtime. + type WeightInfo = pallet_state_trie_migration::weights::SubstrateWeight; +} From fbf3667cca53bf3f1a90fe08fd2a48339c798685 Mon Sep 17 00:00:00 2001 From: higherordertech Date: Tue, 28 Jan 2025 17:07:13 +1100 Subject: [PATCH 2/4] fix: typo --- parachain/runtime/litentry/src/lib.rs | 2 +- parachain/runtime/paseo/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parachain/runtime/litentry/src/lib.rs b/parachain/runtime/litentry/src/lib.rs index 62bd7bf1c7..c7c0a6a0f6 100644 --- a/parachain/runtime/litentry/src/lib.rs +++ b/parachain/runtime/litentry/src/lib.rs @@ -2166,7 +2166,7 @@ cumulus_pallet_parachain_system::register_validate_block! { } parameter_types! { - // The deposit configuration for the singed migration. Specially if you want to allow any signed account to do the migration (see `SignedFilter`, these deposits should be high) + // The deposit configuration for the signed migration. Specially if you want to allow any signed account to do the migration (see `SignedFilter`, these deposits should be high) pub MigrationSignedDepositPerItem: Balance = 1 * CENTS; pub MigrationSignedDepositBase: Balance = 20 * DOLLARS; pub const MigrationMaxKeyLen: u32 = 512; diff --git a/parachain/runtime/paseo/src/lib.rs b/parachain/runtime/paseo/src/lib.rs index 752da12b71..4a795e2601 100644 --- a/parachain/runtime/paseo/src/lib.rs +++ b/parachain/runtime/paseo/src/lib.rs @@ -2337,7 +2337,7 @@ cumulus_pallet_parachain_system::register_validate_block! { } parameter_types! { - // The deposit configuration for the singed migration. Specially if you want to allow any signed account to do the migration (see `SignedFilter`, these deposits should be high) + // The deposit configuration for the signed migration. Specially if you want to allow any signed account to do the migration (see `SignedFilter`, these deposits should be high) pub MigrationSignedDepositPerItem: Balance = 1 * CENTS; pub MigrationSignedDepositBase: Balance = 20 * DOLLARS; pub const MigrationMaxKeyLen: u32 = 512; From d7dbf99304ea4e0a4b78a3805663f12d37f75888 Mon Sep 17 00:00:00 2001 From: higherordertech Date: Wed, 29 Jan 2025 14:56:44 +1100 Subject: [PATCH 3/4] fix: update the state_trie_migration pallet index to 250+ --- parachain/runtime/litentry/src/lib.rs | 5 ++--- parachain/runtime/paseo/src/lib.rs | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/parachain/runtime/litentry/src/lib.rs b/parachain/runtime/litentry/src/lib.rs index c7c0a6a0f6..a02a9242e3 100644 --- a/parachain/runtime/litentry/src/lib.rs +++ b/parachain/runtime/litentry/src/lib.rs @@ -1362,11 +1362,10 @@ construct_runtime! { EVM: pallet_evm = 120, Ethereum: pallet_ethereum = 121, - // State Trie Migration - StateTrieMigration: pallet_state_trie_migration = 130, - // TMP AccountFix: pallet_account_fix = 254, + // State Trie Migration + StateTrieMigration: pallet_state_trie_migration = 255, } } diff --git a/parachain/runtime/paseo/src/lib.rs b/parachain/runtime/paseo/src/lib.rs index 4a795e2601..bd2548bac2 100644 --- a/parachain/runtime/paseo/src/lib.rs +++ b/parachain/runtime/paseo/src/lib.rs @@ -1499,9 +1499,6 @@ construct_runtime! { EVM: pallet_evm = 120, Ethereum: pallet_ethereum = 121, - // State Trie Migration - StateTrieMigration: pallet_state_trie_migration = 130, - // CollabAI Curator: pallet_curator = 150, Guardian: pallet_guardian = 151, @@ -1518,6 +1515,8 @@ construct_runtime! { // TMP AccountFix: pallet_account_fix = 254, Sudo: pallet_sudo = 255, + // State Trie Migration + StateTrieMigration: pallet_state_trie_migration = 256, } } From eae74149bd27c3bf92f8568bf648ef94ee4b0218 Mon Sep 17 00:00:00 2001 From: higherordertech Date: Wed, 29 Jan 2025 19:35:55 +1100 Subject: [PATCH 4/4] fix: update the state_trie_migration pallet index to 251 --- parachain/runtime/litentry/src/lib.rs | 4 ++-- parachain/runtime/paseo/src/lib.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parachain/runtime/litentry/src/lib.rs b/parachain/runtime/litentry/src/lib.rs index a02a9242e3..6710a1f524 100644 --- a/parachain/runtime/litentry/src/lib.rs +++ b/parachain/runtime/litentry/src/lib.rs @@ -1363,9 +1363,9 @@ construct_runtime! { Ethereum: pallet_ethereum = 121, // TMP - AccountFix: pallet_account_fix = 254, // State Trie Migration - StateTrieMigration: pallet_state_trie_migration = 255, + StateTrieMigration: pallet_state_trie_migration = 251, + AccountFix: pallet_account_fix = 254, } } diff --git a/parachain/runtime/paseo/src/lib.rs b/parachain/runtime/paseo/src/lib.rs index bd2548bac2..1b12726c55 100644 --- a/parachain/runtime/paseo/src/lib.rs +++ b/parachain/runtime/paseo/src/lib.rs @@ -1513,10 +1513,10 @@ construct_runtime! { Whitelist: pallet_whitelist::{Pallet, Call, Storage, Event} = 173, // TMP + // State Trie Migration + StateTrieMigration: pallet_state_trie_migration = 251, AccountFix: pallet_account_fix = 254, Sudo: pallet_sudo = 255, - // State Trie Migration - StateTrieMigration: pallet_state_trie_migration = 256, } }