From 29fa3f47fde2f2b2dd74d1015de8c9185b4d9832 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Thu, 6 Apr 2023 09:51:22 +0400 Subject: [PATCH 1/5] remove pallet utility struct --- frame/support/src/lib.rs | 1 + frame/support/src/remove_pallet.rs | 137 +++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 frame/support/src/remove_pallet.rs diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index e0ebeb68a3eaf..fe6ac1eb20433 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -82,6 +82,7 @@ pub mod dispatch_context; pub mod instances; pub mod metadata_ir; pub mod migrations; +pub mod remove_pallet; pub mod traits; pub mod weights; #[doc(hidden)] diff --git a/frame/support/src/remove_pallet.rs b/frame/support/src/remove_pallet.rs new file mode 100644 index 0000000000000..72e11423f0fc3 --- /dev/null +++ b/frame/support/src/remove_pallet.rs @@ -0,0 +1,137 @@ +// Copyright 2017-2023 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Contains `RemovePallet`, a tool for deleting all storage items of a given pallet. +//! +//! This can be useful for cleaning up the storage of a pallet which has been removed from the +//! runtime. +#[cfg(feature = "try-runtime")] +use frame_support::storage::unhashed::contains_prefixed_key; +use frame_support::weights::RuntimeDbWeight; +use sp_core::Get; +use sp_io::{hashing::twox_128, storage::clear_prefix, KillStorageResult}; +use sp_std::marker::PhantomData; +#[cfg(feature = "try-runtime")] +use sp_std::vec::Vec; + +/// `RemovePallet` is a utility struct used to remove all storage items associated with a specific +/// pallet. +/// +/// This struct is generic over two parameters: +/// - `P` is a type that implements the `Get` trait for a static string, representing the pallet's +/// name. +/// - `DbWeight` is a type that implements the `Get` trait for `RuntimeDbWeight`, providing the +/// weight for database operations. +/// +/// On runtime upgrade, the `on_runtime_upgrade` function will clear all storage items associated +/// with the specified pallet, logging the number of keys removed. If the `try-runtime` feature is +/// enabled, the `pre_upgrade` and `post_upgrade` functions can be used to verify the storage +/// removal before and after the upgrade. +/// +/// # Examples: +/// ```ignore +/// construct_runtime! { +/// pub enum Runtime where +/// Block = Block, +/// NodeBlock = primitives::Block, +/// UncheckedExtrinsic = UncheckedExtrinsic +/// { +/// System: frame_system::{Pallet, Call, Storage, Config, Event} = 0, +/// +/// SomePalletToRemove: pallet_something::{Pallet, Call, Storage, Event} = 1, +/// AnotherPalletToRemove: pallet_something_else::{Pallet, Call, Storage, Event} = 2, +/// +/// YourOtherPallets... +/// } +/// }; +/// +/// parameter_types! { +/// pub const SomePalletToRemoveStr: &'static str = "SomePalletToRemove"; +/// pub const AnotherPalletToRemoveStr: &'static str = "AnotherPalletToRemove"; +/// } +/// +/// pub type Migrations = ( +/// RemovePallet, +/// RemovePallet, +/// AnyOtherMigrations... +/// ); +/// +/// pub type Executive = frame_executive::Executive< +/// Runtime, +/// Block, +/// frame_system::ChainContext, +/// Runtime, +/// Migrations +/// >; +/// ``` +/// +/// WARNING: `RemovePallet` has no guard rails preventing it from bricking the chain if the +/// operation of removing storage for the given pallet would exceed the block weight limit. +/// +/// If your pallet has too many keys to be removed in a single block, it is advised to wait for +/// a multi-block scheduler currently under development which will allow for removal of storage +/// items (and performing other heavy migrations) over multiple blocks. +/// (https://github.com/paritytech/substrate/issues/13690) +pub struct RemovePallet, DbWeight: Get>( + PhantomData<(P, DbWeight)>, +); +impl, DbWeight: Get> frame_support::traits::OnRuntimeUpgrade + for RemovePallet +{ + fn on_runtime_upgrade() -> frame_support::weights::Weight { + let hashed_prefix = twox_128(P::get().as_bytes()); + let keys_removed = match clear_prefix(&hashed_prefix, None) { + KillStorageResult::AllRemoved(value) => value, + KillStorageResult::SomeRemaining(value) => { + log::error!( + "`clear_prefix` failed to remove all keys for {}. THIS SHOULD NEVER HAPPEN! 🚨", + P::get() + ); + value + }, + } as u64; + + log::info!("Removed {} {} keys 🧹", keys_removed, P::get()); + + DbWeight::get().reads_writes(keys_removed + 1, keys_removed) + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, &'static str> { + let hashed_prefix = twox_128(P::get().as_bytes()); + match contains_prefixed_key(&hashed_prefix) { + true => log::info!("Found {} keys pre-removal πŸ‘€", P::get()), + false => log::warn!( + "Migration RemovePallet<{}> can be removed (no keys found pre-removal).", + P::get() + ), + }; + Ok(Vec::new()) + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(_state: Vec) -> Result<(), &'static str> { + let hashed_prefix = twox_128(P::get().as_bytes()); + match contains_prefixed_key(&hashed_prefix) { + true => { + log::error!("{} has keys remaining post-removal ❗", P::get()); + return Err("Keys remaining post-removal, this should never happen 🚨") + }, + false => log::info!("No {} keys found post-removal πŸŽ‰", P::get()), + }; + Ok(()) + } +} From 1389b7c0b7b98201c30ddf56e1fdd67e14a3469b Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Thu, 6 Apr 2023 19:11:09 +1000 Subject: [PATCH 2/5] Update frame/support/src/remove_pallet.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian KΓΆcher --- frame/support/src/remove_pallet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/support/src/remove_pallet.rs b/frame/support/src/remove_pallet.rs index 72e11423f0fc3..d4b1e18d03354 100644 --- a/frame/support/src/remove_pallet.rs +++ b/frame/support/src/remove_pallet.rs @@ -1,5 +1,5 @@ // Copyright 2017-2023 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. +// This file is part of Substrate. // Polkadot is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by From 17849938f040218ca3afb54b729663a0824775af Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Thu, 6 Apr 2023 19:11:19 +1000 Subject: [PATCH 3/5] Update frame/support/src/remove_pallet.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian KΓΆcher --- frame/support/src/remove_pallet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/support/src/remove_pallet.rs b/frame/support/src/remove_pallet.rs index d4b1e18d03354..8bf8f4070b5c4 100644 --- a/frame/support/src/remove_pallet.rs +++ b/frame/support/src/remove_pallet.rs @@ -1,4 +1,4 @@ -// Copyright 2017-2023 Parity Technologies (UK) Ltd. +// Copyright Parity Technologies (UK) Ltd. // This file is part of Substrate. // Polkadot is free software: you can redistribute it and/or modify From 15c408d44be8db4fd3433fd7a3d409da1e3d3764 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Thu, 6 Apr 2023 13:21:48 +0400 Subject: [PATCH 4/5] make removepallet a module of migrations --- frame/support/src/lib.rs | 1 - frame/support/src/migrations.rs | 116 ++++++++++++++++++++++++ frame/support/src/remove_pallet.rs | 137 ----------------------------- 3 files changed, 116 insertions(+), 138 deletions(-) delete mode 100644 frame/support/src/remove_pallet.rs diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index fe6ac1eb20433..e0ebeb68a3eaf 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -82,7 +82,6 @@ pub mod dispatch_context; pub mod instances; pub mod metadata_ir; pub mod migrations; -pub mod remove_pallet; pub mod traits; pub mod weights; #[doc(hidden)] diff --git a/frame/support/src/migrations.rs b/frame/support/src/migrations.rs index 645316042b38c..f50cae7f04eaa 100644 --- a/frame/support/src/migrations.rs +++ b/frame/support/src/migrations.rs @@ -15,11 +15,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(feature = "try-runtime")] +use crate::storage::unhashed::contains_prefixed_key; use crate::{ traits::{GetStorageVersion, PalletInfoAccess}, weights::{RuntimeDbWeight, Weight}, }; use impl_trait_for_tuples::impl_for_tuples; +use sp_core::Get; +use sp_io::{hashing::twox_128, storage::clear_prefix, KillStorageResult}; +use sp_std::marker::PhantomData; +#[cfg(feature = "try-runtime")] +use sp_std::vec::Vec; /// Trait used by [`migrate_from_pallet_version_to_storage_version`] to do the actual migration. pub trait PalletVersionToStorageVersionHelper { @@ -67,3 +74,112 @@ pub fn migrate_from_pallet_version_to_storage_version< ) -> Weight { Pallets::migrate(db_weight) } + +/// `RemovePallet` is a utility struct used to remove all storage items associated with a specific +/// pallet. +/// +/// This struct is generic over two parameters: +/// - `P` is a type that implements the `Get` trait for a static string, representing the pallet's +/// name. +/// - `DbWeight` is a type that implements the `Get` trait for `RuntimeDbWeight`, providing the +/// weight for database operations. +/// +/// On runtime upgrade, the `on_runtime_upgrade` function will clear all storage items associated +/// with the specified pallet, logging the number of keys removed. If the `try-runtime` feature is +/// enabled, the `pre_upgrade` and `post_upgrade` functions can be used to verify the storage +/// removal before and after the upgrade. +/// +/// # Examples: +/// ```ignore +/// construct_runtime! { +/// pub enum Runtime where +/// Block = Block, +/// NodeBlock = primitives::Block, +/// UncheckedExtrinsic = UncheckedExtrinsic +/// { +/// System: frame_system::{Pallet, Call, Storage, Config, Event} = 0, +/// +/// SomePalletToRemove: pallet_something::{Pallet, Call, Storage, Event} = 1, +/// AnotherPalletToRemove: pallet_something_else::{Pallet, Call, Storage, Event} = 2, +/// +/// YourOtherPallets... +/// } +/// }; +/// +/// parameter_types! { +/// pub const SomePalletToRemoveStr: &'static str = "SomePalletToRemove"; +/// pub const AnotherPalletToRemoveStr: &'static str = "AnotherPalletToRemove"; +/// } +/// +/// pub type Migrations = ( +/// RemovePallet, +/// RemovePallet, +/// AnyOtherMigrations... +/// ); +/// +/// pub type Executive = frame_executive::Executive< +/// Runtime, +/// Block, +/// frame_system::ChainContext, +/// Runtime, +/// Migrations +/// >; +/// ``` +/// +/// WARNING: `RemovePallet` has no guard rails preventing it from bricking the chain if the +/// operation of removing storage for the given pallet would exceed the block weight limit. +/// +/// If your pallet has too many keys to be removed in a single block, it is advised to wait for +/// a multi-block scheduler currently under development which will allow for removal of storage +/// items (and performing other heavy migrations) over multiple blocks. +/// (https://github.com/paritytech/substrate/issues/13690) +pub struct RemovePallet, DbWeight: Get>( + PhantomData<(P, DbWeight)>, +); +impl, DbWeight: Get> frame_support::traits::OnRuntimeUpgrade + for RemovePallet +{ + fn on_runtime_upgrade() -> frame_support::weights::Weight { + let hashed_prefix = twox_128(P::get().as_bytes()); + let keys_removed = match clear_prefix(&hashed_prefix, None) { + KillStorageResult::AllRemoved(value) => value, + KillStorageResult::SomeRemaining(value) => { + log::error!( + "`clear_prefix` failed to remove all keys for {}. THIS SHOULD NEVER HAPPEN! 🚨", + P::get() + ); + value + }, + } as u64; + + log::info!("Removed {} {} keys 🧹", keys_removed, P::get()); + + DbWeight::get().reads_writes(keys_removed + 1, keys_removed) + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, &'static str> { + let hashed_prefix = twox_128(P::get().as_bytes()); + match contains_prefixed_key(&hashed_prefix) { + true => log::info!("Found {} keys pre-removal πŸ‘€", P::get()), + false => log::warn!( + "Migration RemovePallet<{}> can be removed (no keys found pre-removal).", + P::get() + ), + }; + Ok(Vec::new()) + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(_state: Vec) -> Result<(), &'static str> { + let hashed_prefix = twox_128(P::get().as_bytes()); + match contains_prefixed_key(&hashed_prefix) { + true => { + log::error!("{} has keys remaining post-removal ❗", P::get()); + return Err("Keys remaining post-removal, this should never happen 🚨") + }, + false => log::info!("No {} keys found post-removal πŸŽ‰", P::get()), + }; + Ok(()) + } +} diff --git a/frame/support/src/remove_pallet.rs b/frame/support/src/remove_pallet.rs deleted file mode 100644 index 8bf8f4070b5c4..0000000000000 --- a/frame/support/src/remove_pallet.rs +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Contains `RemovePallet`, a tool for deleting all storage items of a given pallet. -//! -//! This can be useful for cleaning up the storage of a pallet which has been removed from the -//! runtime. -#[cfg(feature = "try-runtime")] -use frame_support::storage::unhashed::contains_prefixed_key; -use frame_support::weights::RuntimeDbWeight; -use sp_core::Get; -use sp_io::{hashing::twox_128, storage::clear_prefix, KillStorageResult}; -use sp_std::marker::PhantomData; -#[cfg(feature = "try-runtime")] -use sp_std::vec::Vec; - -/// `RemovePallet` is a utility struct used to remove all storage items associated with a specific -/// pallet. -/// -/// This struct is generic over two parameters: -/// - `P` is a type that implements the `Get` trait for a static string, representing the pallet's -/// name. -/// - `DbWeight` is a type that implements the `Get` trait for `RuntimeDbWeight`, providing the -/// weight for database operations. -/// -/// On runtime upgrade, the `on_runtime_upgrade` function will clear all storage items associated -/// with the specified pallet, logging the number of keys removed. If the `try-runtime` feature is -/// enabled, the `pre_upgrade` and `post_upgrade` functions can be used to verify the storage -/// removal before and after the upgrade. -/// -/// # Examples: -/// ```ignore -/// construct_runtime! { -/// pub enum Runtime where -/// Block = Block, -/// NodeBlock = primitives::Block, -/// UncheckedExtrinsic = UncheckedExtrinsic -/// { -/// System: frame_system::{Pallet, Call, Storage, Config, Event} = 0, -/// -/// SomePalletToRemove: pallet_something::{Pallet, Call, Storage, Event} = 1, -/// AnotherPalletToRemove: pallet_something_else::{Pallet, Call, Storage, Event} = 2, -/// -/// YourOtherPallets... -/// } -/// }; -/// -/// parameter_types! { -/// pub const SomePalletToRemoveStr: &'static str = "SomePalletToRemove"; -/// pub const AnotherPalletToRemoveStr: &'static str = "AnotherPalletToRemove"; -/// } -/// -/// pub type Migrations = ( -/// RemovePallet, -/// RemovePallet, -/// AnyOtherMigrations... -/// ); -/// -/// pub type Executive = frame_executive::Executive< -/// Runtime, -/// Block, -/// frame_system::ChainContext, -/// Runtime, -/// Migrations -/// >; -/// ``` -/// -/// WARNING: `RemovePallet` has no guard rails preventing it from bricking the chain if the -/// operation of removing storage for the given pallet would exceed the block weight limit. -/// -/// If your pallet has too many keys to be removed in a single block, it is advised to wait for -/// a multi-block scheduler currently under development which will allow for removal of storage -/// items (and performing other heavy migrations) over multiple blocks. -/// (https://github.com/paritytech/substrate/issues/13690) -pub struct RemovePallet, DbWeight: Get>( - PhantomData<(P, DbWeight)>, -); -impl, DbWeight: Get> frame_support::traits::OnRuntimeUpgrade - for RemovePallet -{ - fn on_runtime_upgrade() -> frame_support::weights::Weight { - let hashed_prefix = twox_128(P::get().as_bytes()); - let keys_removed = match clear_prefix(&hashed_prefix, None) { - KillStorageResult::AllRemoved(value) => value, - KillStorageResult::SomeRemaining(value) => { - log::error!( - "`clear_prefix` failed to remove all keys for {}. THIS SHOULD NEVER HAPPEN! 🚨", - P::get() - ); - value - }, - } as u64; - - log::info!("Removed {} {} keys 🧹", keys_removed, P::get()); - - DbWeight::get().reads_writes(keys_removed + 1, keys_removed) - } - - #[cfg(feature = "try-runtime")] - fn pre_upgrade() -> Result, &'static str> { - let hashed_prefix = twox_128(P::get().as_bytes()); - match contains_prefixed_key(&hashed_prefix) { - true => log::info!("Found {} keys pre-removal πŸ‘€", P::get()), - false => log::warn!( - "Migration RemovePallet<{}> can be removed (no keys found pre-removal).", - P::get() - ), - }; - Ok(Vec::new()) - } - - #[cfg(feature = "try-runtime")] - fn post_upgrade(_state: Vec) -> Result<(), &'static str> { - let hashed_prefix = twox_128(P::get().as_bytes()); - match contains_prefixed_key(&hashed_prefix) { - true => { - log::error!("{} has keys remaining post-removal ❗", P::get()); - return Err("Keys remaining post-removal, this should never happen 🚨") - }, - false => log::info!("No {} keys found post-removal πŸŽ‰", P::get()), - }; - Ok(()) - } -} From b564c9b6cf2d82150a2f9c455c6abc6b5538d024 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Thu, 6 Apr 2023 13:29:00 +0400 Subject: [PATCH 5/5] fix rust doc lint --- frame/support/src/migrations.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/support/src/migrations.rs b/frame/support/src/migrations.rs index f50cae7f04eaa..f54b73aad8889 100644 --- a/frame/support/src/migrations.rs +++ b/frame/support/src/migrations.rs @@ -131,8 +131,8 @@ pub fn migrate_from_pallet_version_to_storage_version< /// /// If your pallet has too many keys to be removed in a single block, it is advised to wait for /// a multi-block scheduler currently under development which will allow for removal of storage -/// items (and performing other heavy migrations) over multiple blocks. -/// (https://github.com/paritytech/substrate/issues/13690) +/// items (and performing other heavy migrations) over multiple blocks +/// (see ). pub struct RemovePallet, DbWeight: Get>( PhantomData<(P, DbWeight)>, );