[AHM] Multi-block staking election pallet (#7282)

## Multi Block Election Pallet

This PR adds the first iteration of the multi-block staking pallet. 

From this point onwards, the staking and its election provider pallets
are being customized to work in AssetHub. While usage in solo-chains is
still possible, it is not longer the main focus of this pallet. For a
safer usage, please fork and user an older version of this pallet.

---

## Replaces

- [x] #6034 
- [x] #5272

## Related PRs: 

- [x] #7483
- [ ] #7357
- [ ] #7424
- [ ] paritytech/polkadot-staking-miner#955

This branch can be periodically merged into
#7358 ->
#6996

## TODOs: 

- [x] rebase to master 
- Benchmarking for staking critical path
  - [x] snapshot
  - [x] election result
- Benchmarking for EPMB critical path
  - [x] snapshot
  - [x] verification
  - [x] submission
  - [x] unsigned submission
  - [ ] election results fetching
- [ ] Fix deletion weights. Either of
  - [ ] Garbage collector + lazy removal of all paged storage items
  - [ ] Confirm that deletion is small PoV footprint.
- [ ] Move election prediction to be push based. @tdimitrov 
- [ ] integrity checks for bounds 
- [ ] Properly benchmark this as a part of CI -- for now I will remove
them as they are too slow
- [x] add try-state to all pallets
- [x] Staking to allow genesis dev accounts to be created internally
- [x] Decouple miner config so @niklasad1 can work on the miner
72841b7
- [x] duplicate snapshot page reported by @niklasad1 
- [ ] #6520 or equivalent
-- during snapshot, `VoterList` must be locked
- [ ] Move target snapshot to a separate block

---------

Co-authored-by: Gonçalo Pestana <[email protected]>
Co-authored-by: Ankan <[email protected]>
Co-authored-by: command-bot <>
Co-authored-by: Guillaume Thiolliere <[email protected]>
Co-authored-by: Giuseppe Re <[email protected]>
Co-authored-by: cmd[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: 6 people

.github/workflows/runtimes-matrix.json

@@ -6,7 +6,7 @@
     "header": "substrate/HEADER-APACHE2",
     "template": "substrate/.maintain/frame-weight-template.hbs",
     "bench_features": "runtime-benchmarks",
-    "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_xcm_benchmarks::fungible,pallet_xcm_benchmarks::generic,pallet_nomination_pools,pallet_remark,pallet_transaction_storage",
+    "bench_flags": "--genesis-builder-policy=none --exclude-pallets=pallet_xcm,pallet_xcm_benchmarks::fungible,pallet_xcm_benchmarks::generic,pallet_nomination_pools,pallet_remark,pallet_transaction_storage,pallet_election_provider_multi_block,pallet_election_provider_multi_block::signed,pallet_election_provider_multi_block::unsigned,pallet_election_provider_multi_block::verifier",
     "uri": null,
     "is_relay": false
   },

.github/workflows/tests.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: script
-        run: forklift cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks --quiet -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet
+        run: forklift cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks --quiet -- benchmark pallet --chain dev --pallet "*" --exclude-pallets=pallet_election_provider_multi_block,pallet_election_provider_multi_block::signed,pallet_election_provider_multi_block::unsigned,pallet_election_provider_multi_block::verifier --extrinsic "*" --steps 2 --repeat 1 --quiet
 
   # cf https://github.com/paritytech/polkadot-sdk/issues/1652
   test-syscalls:

Cargo.lock

@@ -344,6 +344,7 @@ members = [
 	"substrate/frame/core-fellowship",
 	"substrate/frame/delegated-staking",
 	"substrate/frame/democracy",
+	"substrate/frame/election-provider-multi-block",
 	"substrate/frame/election-provider-multi-phase",
 	"substrate/frame/election-provider-multi-phase/test-staking-e2e",
 	"substrate/frame/election-provider-support",

Cargo.toml

@@ -19,7 +19,7 @@ use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId;
 use sp_consensus_babe::AuthorityId as BabeId;
 use sp_consensus_beefy::ecdsa_crypto::AuthorityId as BeefyId;
 use sp_core::storage::Storage;
-use sp_runtime::Perbill;
+use sp_runtime::{BoundedVec, Perbill};
 
 // Polkadot
 use polkadot_primitives::{AssignmentId, ValidatorId};
@@ -87,7 +87,13 @@ pub fn genesis() -> Storage {
 				.iter()
 				.map(|x| (x.0.clone(), x.1.clone(), STASH, pallet_staking::StakerStatus::Validator))
 				.collect(),
-			invulnerables: validators::initial_authorities().iter().map(|x| x.0.clone()).collect(),
+			invulnerables: BoundedVec::try_from(
+				validators::initial_authorities()
+					.iter()
+					.map(|x| x.0.clone())
+					.collect::<Vec<_>>(),
+			)
+			.expect("Limit for staking invulnerables must be less than initial authorities."),
 			force_era: pallet_staking::Forcing::ForceNone,
 			slash_reward_fraction: Perbill::from_percent(10),
 			..Default::default()

cumulus/parachains/integration-tests/emulated/chains/relays/westend/src/genesis.rs

@@ -36,7 +36,7 @@ where
 
 	let all_stakers = Ledger::<T>::iter().map(|(ctrl, l)| (ctrl, l.stash)).collect::<BTreeSet<_>>();
 	let mut all_exposed = BTreeSet::new();
-	ErasStakers::<T>::iter().for_each(|(_, val, expo)| {
+	ErasStakersPaged::<T>::iter().for_each(|((_era, val, _page), expo)| {
 		all_exposed.insert(val);
 		all_exposed.extend(expo.others.iter().map(|ie| ie.who.clone()))
 	});

polkadot/runtime/common/src/try_runtime.rs

@@ -82,8 +82,12 @@ where
 
 	pallet_session::Pallet::<T>::on_initialize(BlockNumberFor::<T>::one());
 	initializer::Pallet::<T>::on_initialize(BlockNumberFor::<T>::one());
+
 	// skip sessions until the new validator set is enacted
 	while pallet_session::Pallet::<T>::validators().len() < n as usize {
+		// initialize stakers in pallet_staking. This is suboptimal, but an easy way to avoid this
+		// being an infinite loop.
+		pallet_staking::Pallet::<T>::populate_staking_election_testing_benchmarking_only().unwrap();
 		pallet_session::Pallet::<T>::rotate_session();
 	}
 	initializer::Pallet::<T>::on_finalize(BlockNumberFor::<T>::one());

polkadot/runtime/parachains/src/disputes/slashing/benchmarking.rs

@@ -79,7 +79,7 @@ use polkadot_runtime_common::{
 use polkadot_runtime_parachains::reward_points::RewardValidatorsWithEraPoints;
 use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId;
 use sp_consensus_beefy::ecdsa_crypto::{AuthorityId as BeefyId, Signature as BeefySignature};
-use sp_core::{ConstU32, OpaqueMetadata};
+use sp_core::{ConstBool, ConstU32, OpaqueMetadata};
 use sp_mmr_primitives as mmr;
 use sp_runtime::{
 	curve::PiecewiseLinear,
@@ -348,7 +348,7 @@ parameter_types! {
 	pub const MaxExposurePageSize: u32 = 64;
 	pub const MaxNominators: u32 = 256;
 	pub const MaxAuthorities: u32 = 100_000;
-	pub const OnChainMaxWinners: u32 = u32::MAX;
+	pub const OnChainMaxWinners: u32 = MaxAuthorities::get();
 	// Unbounded number of election targets and voters.
 	pub ElectionBoundsOnChain: ElectionBounds = ElectionBoundsBuilder::default().build();
 }
@@ -361,7 +361,9 @@ impl onchain::Config for OnChainSeqPhragmen {
 	type DataProvider = Staking;
 	type WeightInfo = ();
 	type Bounds = ElectionBoundsOnChain;
-	type MaxWinners = OnChainMaxWinners;
+	type MaxWinnersPerPage = OnChainMaxWinners;
+	type MaxBackersPerWinner = ConstU32<{ u32::MAX }>;
+	type Sort = ConstBool<true>;
 }
 
 /// Upper limit on the number of NPOS nominations.
@@ -400,6 +402,9 @@ impl pallet_staking::Config for Runtime {
 	type EventListeners = ();
 	type WeightInfo = ();
 	type DisablingStrategy = pallet_staking::UpToLimitWithReEnablingDisablingStrategy;
+	type MaxValidatorSet = MaxAuthorities;
+	type MaxInvulnerables = ConstU32<20>;
+	type MaxDisabledValidators = ConstU32<100>;
 }
 
 parameter_types! {

polkadot/runtime/test-runtime/src/lib.rs

@@ -33,7 +33,7 @@ use sp_consensus_grandpa::AuthorityId as GrandpaId;
 use sp_core::{crypto::get_public_from_string_or_panic, sr25519};
 use sp_genesis_builder::PresetId;
 use sp_keyring::Sr25519Keyring;
-use sp_runtime::Perbill;
+use sp_runtime::{BoundedVec, Perbill};
 use westend_runtime_constants::currency::UNITS as WND;
 
 /// Helper function to generate stash, controller and session key from seed
@@ -202,7 +202,10 @@ fn westend_testnet_genesis(
 				.iter()
 				.map(|x| (x.0.clone(), x.0.clone(), STASH, StakerStatus::<AccountId>::Validator))
 				.collect::<Vec<_>>(),
-			invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect::<Vec<_>>(),
+			invulnerables: BoundedVec::try_from(
+					initial_authorities.iter().map(|x| x.0.clone()).collect::<Vec<_>>()
+				)
+				.expect("Too many invulnerable validators: upper limit is MaxInvulnerables from pallet staking config"),
 			force_era: Forcing::NotForcing,
 			slash_reward_fraction: Perbill::from_percent(10),
 		},
@@ -373,7 +376,10 @@ fn westend_staging_testnet_config_genesis() -> serde_json::Value {
 				.iter()
 				.map(|x| (x.0.clone(), x.0.clone(), STASH, StakerStatus::<AccountId>::Validator))
 				.collect::<Vec<_>>(),
-			invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect::<Vec<_>>(),
+			invulnerables: BoundedVec::try_from(
+					initial_authorities.iter().map(|x| x.0.clone()).collect::<Vec<_>>()
+				)
+				.expect("Too many invulnerable validators: upper limit is MaxInvulnerables from pallet staking config"),
 			force_era: Forcing::ForceNone,
 			slash_reward_fraction: Perbill::from_percent(10),
 		},

polkadot/runtime/westend/src/genesis_config_presets.rs

@@ -96,7 +96,7 @@ use sp_consensus_beefy::{
 	ecdsa_crypto::{AuthorityId as BeefyId, Signature as BeefySignature},
 	mmr::{BeefyDataProvider, MmrLeafVersion},
 };
-use sp_core::{ConstU8, OpaqueMetadata, RuntimeDebug, H256};
+use sp_core::{ConstBool, ConstU8, OpaqueMetadata, RuntimeDebug, H256};
 use sp_runtime::{
 	generic, impl_opaque_keys,
 	traits::{
@@ -585,7 +585,10 @@ parameter_types! {
 		ElectionBoundsBuilder::default().voters_count(MaxElectingVoters::get().into()).build();
 	// Maximum winners that can be chosen as active validators
 	pub const MaxActiveValidators: u32 = 1000;
-
+	// One page only, fill the whole page with the `MaxActiveValidators`.
+	pub const MaxWinnersPerPage: u32 = MaxActiveValidators::get();
+	// Unbonded, thus the max backers per winner maps to the max electing voters limit.
+	pub const MaxBackersPerWinner: u32 = MaxElectingVoters::get();
 }
 
 frame_election_provider_support::generate_solution_type!(
@@ -600,12 +603,14 @@ frame_election_provider_support::generate_solution_type!(
 
 pub struct OnChainSeqPhragmen;
 impl onchain::Config for OnChainSeqPhragmen {
+	type Sort = ConstBool<true>;
 	type System = Runtime;
 	type Solver = SequentialPhragmen<AccountId, OnChainAccuracy>;
 	type DataProvider = Staking;
 	type WeightInfo = weights::frame_election_provider_support::WeightInfo<Runtime>;
-	type MaxWinners = MaxActiveValidators;
 	type Bounds = ElectionBounds;
+	type MaxBackersPerWinner = MaxBackersPerWinner;
+	type MaxWinnersPerPage = MaxWinnersPerPage;
 }
 
 impl pallet_election_provider_multi_phase::MinerConfig for Runtime {
@@ -618,7 +623,8 @@ impl pallet_election_provider_multi_phase::MinerConfig for Runtime {
     as
     frame_election_provider_support::ElectionDataProvider
     >::MaxVotesPerVoter;
-	type MaxWinners = MaxActiveValidators;
+	type MaxBackersPerWinner = MaxBackersPerWinner;
+	type MaxWinners = MaxWinnersPerPage;
 
 	// The unsigned submissions have to respect the weight of the submit_unsigned call, thus their
 	// weight estimate function is wired to this call's weight.
@@ -652,6 +658,8 @@ impl pallet_election_provider_multi_phase::Config for Runtime {
 	type BetterSignedThreshold = ();
 	type OffchainRepeat = OffchainRepeat;
 	type MinerTxPriority = NposSolutionPriority;
+	type MaxWinners = MaxWinnersPerPage;
+	type MaxBackersPerWinner = MaxBackersPerWinner;
 	type DataProvider = Staking;
 	#[cfg(any(feature = "fast-runtime", feature = "runtime-benchmarks"))]
 	type Fallback = onchain::OnChainExecution<OnChainSeqPhragmen>;
@@ -660,7 +668,8 @@ impl pallet_election_provider_multi_phase::Config for Runtime {
 		AccountId,
 		BlockNumber,
 		Staking,
-		MaxActiveValidators,
+		MaxWinnersPerPage,
+		MaxBackersPerWinner,
 	)>;
 	type GovernanceFallback = onchain::OnChainExecution<OnChainSeqPhragmen>;
 	type Solver = SequentialPhragmen<
@@ -671,7 +680,6 @@ impl pallet_election_provider_multi_phase::Config for Runtime {
 	type BenchmarkingConfig = polkadot_runtime_common::elections::BenchmarkConfig;
 	type ForceOrigin = EnsureRoot<AccountId>;
 	type WeightInfo = weights::pallet_election_provider_multi_phase::WeightInfo<Self>;
-	type MaxWinners = MaxActiveValidators;
 	type ElectionBounds = ElectionBounds;
 }
 
@@ -753,6 +761,7 @@ impl pallet_staking::Config for Runtime {
 	type GenesisElectionProvider = onchain::OnChainExecution<OnChainSeqPhragmen>;
 	type VoterList = VoterList;
 	type TargetList = UseValidatorsMap<Self>;
+	type MaxValidatorSet = MaxActiveValidators;
 	type NominationsQuota = pallet_staking::FixedNominationsQuota<{ MaxNominations::get() }>;
 	type MaxUnlockingChunks = frame_support::traits::ConstU32<32>;
 	type HistoryDepth = frame_support::traits::ConstU32<84>;
@@ -761,6 +770,8 @@ impl pallet_staking::Config for Runtime {
 	type EventListeners = (NominationPools, DelegatedStaking);
 	type WeightInfo = weights::pallet_staking::WeightInfo<Runtime>;
 	type DisablingStrategy = pallet_staking::UpToLimitWithReEnablingDisablingStrategy;
+	type MaxInvulnerables = frame_support::traits::ConstU32<20>;
+	type MaxDisabledValidators = ConstU32<100>;
 }
 
 impl pallet_fast_unstake::Config for Runtime {