Allow for a dynamic number of nominators

bin/node/runtime/src/lib.rs

@@ -502,14 +502,14 @@ pallet_staking_reward_curve::build! {
 parameter_types! {
 	pub const SessionsPerEra: sp_staking::SessionIndex = 6;
 	pub const BondingDuration: pallet_staking::EraIndex = 24 * 28;
-	pub const SlashDeferDuration: pallet_staking::EraIndex = 24 * 7; // 1/4 the bonding duration.
+	pub const SlashDeferDuration: pallet_staking::EraIndex = 24 * 7;
 	pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE;
 	pub const MaxNominatorRewardedPerValidator: u32 = 256;
 	pub const OffendingValidatorsThreshold: Perbill = Perbill::from_percent(17);
 	pub OffchainRepeat: BlockNumber = 5;
 }
 
-use frame_election_provider_support::onchain;
+use frame_election_provider_support::{onchain, SnapshotBoundsBuilder};
 impl onchain::Config for Runtime {
 	type Accuracy = Perbill;
 	type DataProvider = Staking;
@@ -522,7 +522,6 @@ impl pallet_staking::BenchmarkingConfig for StakingBenchmarkingConfig {
 }
 
 impl pallet_staking::Config for Runtime {
-	const MAX_NOMINATIONS: u32 = MAX_NOMINATIONS;
 	type Currency = Balances;
 	type UnixTime = Timestamp;
 	type CurrencyToVote = U128CurrencyToVote;
@@ -548,10 +547,13 @@ impl pallet_staking::Config for Runtime {
 	// Alternatively, use pallet_staking::UseNominatorsMap<Runtime> to just use the nominators map.
 	// Note that the aforementioned does not scale to a very large number of nominators.
 	type SortedListProvider = BagsList;
+	// each nominator is allowed a fix number of nomination targets.
+	type NominationQuota = pallet_staking::FixedNominationQuota<MAX_NOMINATIONS>;
 	type WeightInfo = pallet_staking::weights::SubstrateWeight<Runtime>;
 	type BenchmarkingConfig = StakingBenchmarkingConfig;
 }
 
+use frame_election_provider_support::SnapshotBounds;
 parameter_types! {
 	// phase durations. 1/4 of the last session for each.
 	pub const SignedPhase: u32 = EPOCH_DURATION_IN_BLOCKS / 4;
@@ -577,10 +579,11 @@ parameter_types! {
 		.max
 		.get(DispatchClass::Normal);
 
-	// BagsList allows a practically unbounded count of nominators to participate in NPoS elections.
-	// To ensure we respect memory limits when using the BagsList this must be set to a number of
-	// voters we know can fit into a single vec allocation.
-	pub const VoterSnapshotPerBlock: u32 = 10_000;
+	/// maximum of 25k nominators, or 4MB.
+	pub VoterSnapshotBounds: SnapshotBounds = SnapshotBoundsBuilder::default().size(4 * 1024 * 1024).count(25_000).build();
+	/// maximum of 1k validator candidates, with no size limit.
+	pub TargetSnapshotBounds: SnapshotBounds = SnapshotBounds::new_count(1000);
+
 }
 
 sp_npos_elections::generate_solution_type!(
@@ -661,10 +664,11 @@ impl pallet_election_provider_multi_phase::Config for Runtime {
 		pallet_election_provider_multi_phase::SolutionAccuracyOf<Self>,
 		OffchainRandomBalancing,
 	>;
-	type WeightInfo = pallet_election_provider_multi_phase::weights::SubstrateWeight<Self>;
 	type ForceOrigin = EnsureRootOrHalfCouncil;
+	type VoterSnapshotBounds = VoterSnapshotBounds;
+	type TargetSnapshotBounds = TargetSnapshotBounds;
+	type WeightInfo = pallet_election_provider_multi_phase::weights::SubstrateWeight<Self>;
 	type BenchmarkingConfig = ElectionProviderBenchmarkConfig;
-	type VoterSnapshotPerBlock = VoterSnapshotPerBlock;
 }
 
 parameter_types! {
@@ -1788,4 +1792,16 @@ mod tests {
 			If the limit is too strong, maybe consider increase the limit to 300.",
 		);
 	}
+
+	#[test]
+	fn snapshot_details() {
+		let (_, _, max) =
+			pallet_staking::display_bounds_limits::<Runtime>(VoterSnapshotBounds::get());
+		// example of an assertion that a runtime could have to ensure no mis-configuration happens.
+		assert!(
+			max <= 25_000,
+			"under any configuration, the maximum number of voters should be less than 25_000"
+		);
+		let _ = pallet_staking::display_bounds_limits::<Runtime>(TargetSnapshotBounds::get());
+	}
 }

frame/babe/src/mock.rs

@@ -196,7 +196,6 @@ impl onchain::Config for Test {
 }
 
 impl pallet_staking::Config for Test {
-	const MAX_NOMINATIONS: u32 = 16;
 	type RewardRemainder = ();
 	type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote;
 	type Event = Event;
@@ -215,6 +214,7 @@ impl pallet_staking::Config for Test {
 	type NextNewSession = Session;
 	type ElectionProvider = onchain::OnChainSequentialPhragmen<Self>;
 	type GenesisElectionProvider = Self::ElectionProvider;
+	type NominationQuota = pallet_staking::FixedNominationQuota<16>;
 	type SortedListProvider = pallet_staking::UseNominatorsMap<Self>;
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;
 	type WeightInfo = ();

frame/bags-list/remote-tests/src/snapshot.rs

@@ -16,13 +16,14 @@
 
 //! Test to execute the snapshot using the voter bag.
 
+use frame_election_provider_support::SnapshotBounds;
 use frame_support::traits::PalletInfoAccess;
 use remote_externalities::{Builder, Mode, OnlineConfig};
 use sp_runtime::{traits::Block as BlockT, DeserializeOwned};
 
 /// Execute create a snapshot from pallet-staking.
 pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>(
-	voter_limit: Option<usize>,
+	voter_bounds: SnapshotBounds,
 	currency_unit: u64,
 	ws_url: String,
 ) {
@@ -57,7 +58,7 @@ pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>
 		let voters = <pallet_staking::Pallet<Runtime> as ElectionDataProvider<
 			Runtime::AccountId,
 			Runtime::BlockNumber,
-		>>::voters(voter_limit)
+		>>::voters(voter_bounds)
 		.unwrap();
 
 		let mut voters_nominator_only = voters
@@ -77,7 +78,7 @@ pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>
 		log::info!(
 			target: crate::LOG_TARGET,
 			"a snapshot with limit {:?} has been created, {} voters are taken. min nominator: {:?}, max: {:?}",
-			voter_limit,
+			voter_bounds,
 			voters.len(),
 			min_voter,
 			max_voter

frame/election-provider-multi-phase/src/benchmarking.rs

@@ -246,8 +246,8 @@ frame_benchmarking::benchmarks! {
 
 		// we don't directly need the data-provider to be populated, but it is just easy to use it.
 		set_up_data_provider::<T>(v, t);
-		let targets = T::DataProvider::targets(None)?;
-		let voters = T::DataProvider::voters(None)?;
+		let targets = T::DataProvider::targets(SnapshotBounds::new_unbounded())?;
+		let voters = T::DataProvider::voters(SnapshotBounds::new_unbounded())?;
 		let desired_targets = T::DataProvider::desired_targets()?;
 		assert!(<MultiPhase<T>>::snapshot().is_none());
 	}: {

frame/election-provider-multi-phase/src/lib.rs

@@ -230,7 +230,7 @@
 #![cfg_attr(not(feature = "std"), no_std)]
 
 use codec::{Decode, Encode};
-use frame_election_provider_support::{ElectionDataProvider, ElectionProvider};
+use frame_election_provider_support::{ElectionDataProvider, ElectionProvider, SnapshotBounds};
 use frame_support::{
 	dispatch::DispatchResultWithPostInfo,
 	ensure,
@@ -248,7 +248,6 @@ use sp_npos_elections::{
 	VoteWeight,
 };
 use sp_runtime::{
-	traits::Bounded,
 	transaction_validity::{
 		InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity,
 		TransactionValidityError, ValidTransaction,
@@ -309,6 +308,23 @@ pub trait BenchmarkingConfig {
 	const MAXIMUM_TARGETS: u32;
 }
 
+/// A benchmarking config, to be used only for testing.
+#[cfg(feature = "std")]
+pub struct TestBenchmarkingConfig;
+
+#[cfg(feature = "std")]
+impl BenchmarkingConfig for TestBenchmarkingConfig {
+	const VOTERS: [u32; 2] = [400, 600];
+	const ACTIVE_VOTERS: [u32; 2] = [100, 300];
+	const TARGETS: [u32; 2] = [200, 400];
+	const DESIRED_TARGETS: [u32; 2] = [100, 180];
+
+	const SNAPSHOT_MAXIMUM_VOTERS: u32 = 1000;
+	const MINER_MAXIMUM_VOTERS: u32 = 1000;
+
+	const MAXIMUM_TARGETS: u32 = 200;
+}
+
 /// A fallback implementation that transitions the pallet to the emergency phase.
 pub struct NoFallback<T>(sp_std::marker::PhantomData<T>);
 
@@ -631,14 +647,16 @@ pub mod pallet {
 		#[pallet::constant]
 		type SignedDepositWeight: Get<BalanceOf<Self>>;
 
-		/// The maximum number of voters to put in the snapshot. At the moment, snapshots are only
-		/// over a single block, but once multi-block elections are introduced they will take place
-		/// over multiple blocks.
+		/// The bound on the amount of voters to put in the snapshot, per block.
+		#[pallet::constant]
+		type VoterSnapshotBounds: Get<SnapshotBounds>;
+
+		/// The bound on the amount of targets to put in the snapshot, per block.
 		///
-		/// Also, note the data type: If the voters are represented by a `u32` in `type
-		/// CompactSolution`, the same `u32` is used here to ensure bounds are respected.
+		/// Note that the target snapshot happens next to voter snapshot. In essence, in a single
+		/// block, `TargetSnapshotSize + VoterSnapshotBounds` must not exhaust.
 		#[pallet::constant]
-		type VoterSnapshotPerBlock: Get<SolutionVoterIndexOf<Self>>;
+		type TargetSnapshotBounds: Get<SnapshotBounds>;
 
 		/// Handler for the slashed deposits.
 		type SlashHandler: OnUnbalanced<NegativeImbalanceOf<Self>>;
@@ -821,6 +839,14 @@ pub mod pallet {
 				<T::DataProvider as ElectionDataProvider<T::AccountId, T::BlockNumber>>::MAXIMUM_VOTES_PER_VOTER,
 				<SolutionOf<T> as NposSolution>::LIMIT as u32,
 			);
+
+			// ----------------------------
+			// maximum size of a snapshot should not exceed half the size of our allocator limit.
+			assert!(
+				T::VoterSnapshotBounds::get().size_bound().unwrap_or_default().saturating_add(
+					T::TargetSnapshotBounds::get().size_bound().unwrap_or_default()
+				) < sp_core::MAX_POSSIBLE_ALLOCATION as usize / 2 - 1
+			);
 		}
 	}
 
@@ -1286,22 +1312,22 @@ impl<T: Config> Pallet<T> {
 	/// Extracted for easier weight calculation.
 	fn create_snapshot_external(
 	) -> Result<(Vec<T::AccountId>, Vec<crate::unsigned::Voter<T>>, u32), ElectionError<T>> {
-		let target_limit = <SolutionTargetIndexOf<T>>::max_value().saturated_into::<usize>();
-		// for now we have just a single block snapshot.
-		let voter_limit = T::VoterSnapshotPerBlock::get().saturated_into::<usize>();
+		let target_bound = T::TargetSnapshotBounds::get();
+		let voter_bound = T::VoterSnapshotBounds::get();
 
 		let targets =
-			T::DataProvider::targets(Some(target_limit)).map_err(ElectionError::DataProvider)?;
-		let voters =
-			T::DataProvider::voters(Some(voter_limit)).map_err(ElectionError::DataProvider)?;
+			T::DataProvider::targets(target_bound).map_err(ElectionError::DataProvider)?;
+		let voters = T::DataProvider::voters(voter_bound).map_err(ElectionError::DataProvider)?;
 		let desired_targets =
 			T::DataProvider::desired_targets().map_err(ElectionError::DataProvider)?;
 
 		// Defensive-only.
-		if targets.len() > target_limit || voters.len() > voter_limit {
-			debug_assert!(false, "Snapshot limit has not been respected.");
-			return Err(ElectionError::DataProvider("Snapshot too big for submission."))
-		}
+		debug_assert!(!voter_bound
+			.exhausts_size_count_non_zero(|| voters.encoded_size() as u32, || voters.len() as u32));
+		debug_assert!(!target_bound.exhausts_size_count_non_zero(
+			|| targets.encoded_size() as u32,
+			|| targets.len() as u32
+		));
 
 		Ok((targets, voters, desired_targets))
 	}
@@ -1703,8 +1729,8 @@ mod tests {
 	use super::*;
 	use crate::{
 		mock::{
-			multi_phase_events, roll_to, AccountId, ExtBuilder, MockWeightInfo, MultiPhase,
-			Runtime, SignedMaxSubmissions, System, TargetIndex, Targets,
+			multi_phase_events, roll_to, ExtBuilder, MockWeightInfo, MultiPhase, Runtime,
+			SignedMaxSubmissions, System,
 		},
 		Phase,
 	};
@@ -1951,70 +1977,6 @@ mod tests {
 		})
 	}
 
-	#[test]
-	fn snapshot_too_big_failure_onchain_fallback() {
-		// the `MockStaking` is designed such that if it has too many targets, it simply fails.
-		ExtBuilder::default().build_and_execute(|| {
-			Targets::set((0..(TargetIndex::max_value() as AccountId) + 1).collect::<Vec<_>>());
-
-			// Signed phase failed to open.
-			roll_to(15);
-			assert_eq!(MultiPhase::current_phase(), Phase::Off);
-
-			// Unsigned phase failed to open.
-			roll_to(25);
-			assert_eq!(MultiPhase::current_phase(), Phase::Off);
-
-			// On-chain backup works though.
-			roll_to(29);
-			let supports = MultiPhase::elect().unwrap();
-			assert!(supports.len() > 0);
-		});
-	}
-
-	#[test]
-	fn snapshot_too_big_failure_no_fallback() {
-		// and if the backup mode is nothing, we go into the emergency mode..
-		ExtBuilder::default().onchain_fallback(false).build_and_execute(|| {
-			crate::mock::Targets::set(
-				(0..(TargetIndex::max_value() as AccountId) + 1).collect::<Vec<_>>(),
-			);
-
-			// Signed phase failed to open.
-			roll_to(15);
-			assert_eq!(MultiPhase::current_phase(), Phase::Off);
-
-			// Unsigned phase failed to open.
-			roll_to(25);
-			assert_eq!(MultiPhase::current_phase(), Phase::Off);
-
-			roll_to(29);
-			let err = MultiPhase::elect().unwrap_err();
-			assert_eq!(err, ElectionError::Fallback("NoFallback."));
-			assert_eq!(MultiPhase::current_phase(), Phase::Emergency);
-		});
-	}
-
-	#[test]
-	fn snapshot_too_big_truncate() {
-		// but if there are too many voters, we simply truncate them.
-		ExtBuilder::default().build_and_execute(|| {
-			// we have 8 voters in total.
-			assert_eq!(crate::mock::Voters::get().len(), 8);
-			// but we want to take 2.
-			crate::mock::VoterSnapshotPerBlock::set(2);
-
-			// Signed phase opens just fine.
-			roll_to(15);
-			assert_eq!(MultiPhase::current_phase(), Phase::Signed);
-
-			assert_eq!(
-				MultiPhase::snapshot_metadata().unwrap(),
-				SolutionOrSnapshotSize { voters: 2, targets: 4 }
-			);
-		})
-	}
-
 	#[test]
 	fn untrusted_score_verification_is_respected() {
 		ExtBuilder::default().build_and_execute(|| {