feat: configurable fastnode (backport cosmos#13321) (cosmos#13338)

* feat: configurable fastnode (cosmos#13321)

(cherry picked from commit 412e2fc)

# Conflicts:
#	CHANGELOG.md
#	fuzz/tests/store_internal_proofs_createnonmembershipproof_test.go
#	go.mod
#	go.sum
#	server/config/config.go
#	server/config/toml.go
#	simapp/go.mod
#	simapp/go.sum
#	store/rootmulti/store.go
#	tests/go.mod
#	tests/go.sum

* fix conflicts

Co-authored-by: Marko <[email protected]>

CHANGELOG.md

@@ -52,6 +52,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * [#12693](https://github.com/cosmos/cosmos-sdk/pull/12693) Make sure the order of each node is consistent when emitting proto events.
 * (simapp) [#13107](https://github.com/cosmos/cosmos-sdk/pull/13107) Call `SetIAVLCacheSize` with the configured value in simapp.
 * (cli) [#12742](https://github.com/cosmos/cosmos-sdk/pull/12742) Add the `prune` CLI cmd to manually prune app store history versions based on the pruning options.
+* [#13321](https://github.com/cosmos/cosmos-sdk/pull/13321) Add flag to disable fast node migration and usage. 
 
 ### Bug Fixes
 

baseapp/options.go

@@ -72,6 +72,11 @@ func SetIAVLCacheSize(size int) func(*BaseApp) {
 	return func(bapp *BaseApp) { bapp.cms.SetIAVLCacheSize(size) }
 }
 
+// SetIAVLDisableFastNode enables(false)/disables(true) fast node usage from the IAVL store.
+func SetIAVLDisableFastNode(disable bool) func(*BaseApp) {
+	return func(bapp *BaseApp) { bapp.cms.SetIAVLDisableFastNode(disable) }
+}
+
 // SetInterBlockCache provides a BaseApp option function that sets the
 // inter-block cache.
 func SetInterBlockCache(cache storetypes.MultiStorePersistentCache) func(*BaseApp) {

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/confio/ics23/go v0.7.0
 	github.com/cosmos/btcutil v1.0.4
 	github.com/cosmos/go-bip39 v1.0.0
-	github.com/cosmos/iavl v0.19.1
+	github.com/cosmos/iavl v0.19.2-0.20220916140702-9b6be3095313
 	github.com/cosmos/ledger-cosmos-go v0.11.1
 	github.com/gogo/gateway v1.1.0
 	github.com/gogo/protobuf v1.3.3

go.sum

@@ -160,8 +160,8 @@ github.com/cosmos/btcutil v1.0.4/go.mod h1:Ffqc8Hn6TJUdDgHBwIZLtrLQC1KdJ9jGJl/Tv
 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
 github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY=
 github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw=
-github.com/cosmos/iavl v0.19.1 h1:3gaq9b6SjiB0KBTygRnAvEGml2pQlu1TH8uma5g63Ys=
-github.com/cosmos/iavl v0.19.1/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw=
+github.com/cosmos/iavl v0.19.2-0.20220916140702-9b6be3095313 h1:R7CnaI/0OLwOusy7n9750n8fqQ3yCQ8OJQI2L3ws9RA=
+github.com/cosmos/iavl v0.19.2-0.20220916140702-9b6be3095313/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw=
 github.com/cosmos/keyring v1.1.7-0.20210622111912-ef00f8ac3d76 h1:DdzS1m6o/pCqeZ8VOAit/gyATedRgjvkVI+UCrLpyuU=
 github.com/cosmos/keyring v1.1.7-0.20210622111912-ef00f8ac3d76/go.mod h1:0mkLWIoZuQ7uBoospo5Q9zIpqq6rYCPJDSUdeCJvPM8=
 github.com/cosmos/ledger-cosmos-go v0.11.1 h1:9JIYsGnXP613pb2vPjFeMMjBI5lEDsEaF6oYorTy6J4=

server/config/config.go

@@ -81,8 +81,12 @@ type BaseConfig struct {
 	// IndexEvents defines the set of events in the form {eventType}.{attributeKey},
 	// which informs CometBFT what to index. If empty, all events will be indexed.
 	IndexEvents []string `mapstructure:"index-events"`
+
 	// IavlCacheSize set the size of the iavl tree cache.
 	IAVLCacheSize uint64 `mapstructure:"iavl-cache-size"`
+
+	// IAVLDisableFastnNode enables or disables the fast sync node.
+	IAVLDisableFastnNode bool `mapstructure:"iavl-disable-fastnode"`
 }
 
 // APIConfig defines the API listener configuration.
@@ -204,15 +208,16 @@ func (c *Config) GetMinGasPrices() sdk.DecCoins {
 func DefaultConfig() *Config {
 	return &Config{
 		BaseConfig: BaseConfig{
-			MinGasPrices:      defaultMinGasPrices,
-			InterBlockCache:   true,
-			Pruning:           storetypes.PruningOptionDefault,
-			PruningKeepRecent: "0",
-			PruningKeepEvery:  "0",
-			PruningInterval:   "0",
-			MinRetainBlocks:   0,
-			IndexEvents:       make([]string, 0),
-			IAVLCacheSize:     781250, // 50 MB
+			MinGasPrices:         defaultMinGasPrices,
+			InterBlockCache:      true,
+			Pruning:              storetypes.PruningOptionDefault,
+			PruningKeepRecent:    "0",
+			PruningKeepEvery:     "0",
+			PruningInterval:      "0",
+			MinRetainBlocks:      0,
+			IndexEvents:          make([]string, 0),
+			IAVLCacheSize:        781250, // 50 MB
+			IAVLDisableFastnNode: false,
 		},
 		Telemetry: telemetry.Config{
 			Enabled:      false,
@@ -268,17 +273,17 @@ func GetConfig(v *viper.Viper) (Config, error) {
 
 	return Config{
 		BaseConfig: BaseConfig{
-			MinGasPrices:      v.GetString("minimum-gas-prices"),
-			InterBlockCache:   v.GetBool("inter-block-cache"),
-			Pruning:           v.GetString("pruning"),
-			PruningKeepRecent: v.GetString("pruning-keep-recent"),
-			PruningKeepEvery:  v.GetString("pruning-keep-every"),
-			PruningInterval:   v.GetString("pruning-interval"),
-			HaltHeight:        v.GetUint64("halt-height"),
-			HaltTime:          v.GetUint64("halt-time"),
-			IndexEvents:       v.GetStringSlice("index-events"),
-			MinRetainBlocks:   v.GetUint64("min-retain-blocks"),
-			IAVLCacheSize:     v.GetUint64("iavl-cache-size"),
+			MinGasPrices:         v.GetString("minimum-gas-prices"),
+			InterBlockCache:      v.GetBool("inter-block-cache"),
+			Pruning:              v.GetString("pruning"),
+			PruningKeepRecent:    v.GetString("pruning-keep-recent"),
+			PruningInterval:      v.GetString("pruning-interval"),
+			HaltHeight:           v.GetUint64("halt-height"),
+			HaltTime:             v.GetUint64("halt-time"),
+			IndexEvents:          v.GetStringSlice("index-events"),
+			MinRetainBlocks:      v.GetUint64("min-retain-blocks"),
+			IAVLCacheSize:        v.GetUint64("iavl-cache-size"),
+			IAVLDisableFastnNode: v.GetBool("iavl-disable-fastnode"),
 		},
 		Telemetry: telemetry.Config{
 			ServiceName:             v.GetString("telemetry.service-name"),

server/config/toml.go

@@ -76,6 +76,10 @@ index-events = {{ .BaseConfig.IndexEvents }}
 # Default cache size is 50mb.
 iavl-cache-size = {{ .BaseConfig.IAVLCacheSize }}
 
+# IavlDisableFastnNode enables or disables the fast node feature of IAVL. 
+# Default is false.
+iavl-disable-fastnode = {{ .BaseConfig.IAVLDisableFastnNode }}
+
 ###############################################################################
 ###                         Telemetry Configuration                         ###
 ###############################################################################

server/mock/store.go

@@ -145,6 +145,10 @@ func (ms multiStore) SetIAVLCacheSize(size int) {
 	panic("not implemented")
 }
 
+func (ms multiStore) SetIAVLDisableFastNode(disable bool) {
+	panic("not implemented")
+}
+
 func (ms multiStore) SetInitialVersion(version int64) error {
 	panic("not implemented")
 }

server/start.go

@@ -63,6 +63,7 @@ const (
 	FlagIndexEvents       = "index-events"
 	FlagMinRetainBlocks   = "min-retain-blocks"
 	FlagIAVLCacheSize     = "iavl-cache-size"
+	FlagIAVLFastNode      = "iavl-disable-fastnode"
 
 	// state sync-related flags
 	FlagStateSyncSnapshotInterval   = "state-sync.snapshot-interval"

simapp/simd/cmd/root.go

@@ -107,7 +107,58 @@ func NewRootCmd() *cobra.Command {
 			customAppTemplate, customAppConfig := initAppConfig()
 			customCMTConfig := initCometBFTConfig()
 
-			return server.InterceptConfigsPreRunHandler(cmd, customAppTemplate, customAppConfig, customCMTConfig)
+			return server.InterceptConfigsPreRunHandler(cmd, customAppTemplate, customAppConfig)
+		},
+	}
+
+	initRootCmd(rootCmd, encodingConfig)
+
+	return rootCmd, encodingConfig
+}
+
+// initAppConfig helps to override default appConfig template and configs.
+// return "", nil if no custom configuration is required for the application.
+func initAppConfig() (string, interface{}) {
+	// The following code snippet is just for reference.
+
+	// WASMConfig defines configuration for the wasm module.
+	type WASMConfig struct {
+		// This is the maximum sdk gas (wasm and storage) that we allow for any x/wasm "smart" queries
+		QueryGasLimit uint64 `mapstructure:"query_gas_limit"`
+
+		// Address defines the gRPC-web server to listen on
+		LruSize uint64 `mapstructure:"lru_size"`
+	}
+
+	type CustomAppConfig struct {
+		serverconfig.Config
+
+		WASM WASMConfig `mapstructure:"wasm"`
+	}
+
+	// Optionally allow the chain developer to overwrite the SDK's default
+	// server config.
+	srvCfg := serverconfig.DefaultConfig()
+	// The SDK's default minimum gas price is set to "" (empty value) inside
+	// app.toml. If left empty by validators, the node will halt on startup.
+	// However, the chain developer can set a default app.toml value for their
+	// validators here.
+	//
+	// In summary:
+	// - if you leave srvCfg.MinGasPrices = "", all validators MUST tweak their
+	//   own app.toml config,
+	// - if you set srvCfg.MinGasPrices non-empty, validators CAN tweak their
+	//   own app.toml to override, or use this default value.
+	//
+	// In simapp, we set the min gas prices to 0.
+	srvCfg.MinGasPrices = "0stake"
+	// srvCfg.BaseConfig.IAVLDisableFastnNode = true // disable fastnode by default
+
+	customAppConfig := CustomAppConfig{
+		Config: *srvCfg,
+		WASM: WASMConfig{
+			LruSize:       1,
+			QueryGasLimit: 300000,
 		},
 	}
 
@@ -252,6 +303,7 @@ func (a appCreator) newApp(logger log.Logger, db dbm.DB, traceStore io.Writer, a
 		baseapp.SetSnapshotInterval(cast.ToUint64(appOpts.Get(server.FlagStateSyncSnapshotInterval))),
 		baseapp.SetSnapshotKeepRecent(cast.ToUint32(appOpts.Get(server.FlagStateSyncSnapshotKeepRecent))),
 		baseapp.SetIAVLCacheSize(cast.ToInt(appOpts.Get(server.FlagIAVLCacheSize))),
+		baseapp.SetIAVLDisableFastNode(cast.ToBool(appOpts.Get(server.FlagIAVLFastNode))),
 	)
 }
 

store/cache/cache_test.go

@@ -20,7 +20,8 @@ func TestGetOrSetStoreCache(t *testing.T) {
 	mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize)
 
 	sKey := types.NewKVStoreKey("test")
-	tree := iavl.NewMutableTree(db, 100, false, log.NewNopLogger())
+	tree, err := iavl.NewMutableTree(db, 100, false)
+	require.NoError(t, err)
 	store := iavlstore.UnsafeNewStore(tree)
 	store2 := mngr.GetStoreCache(sKey, store)
 
@@ -33,7 +34,8 @@ func TestUnwrap(t *testing.T) {
 	mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize)
 
 	sKey := types.NewKVStoreKey("test")
-	tree := iavl.NewMutableTree(db, 100, false, log.NewNopLogger())
+	tree, err := iavl.NewMutableTree(db, 100, false)
+	require.NoError(t, err)
 	store := iavlstore.UnsafeNewStore(tree)
 	_ = mngr.GetStoreCache(sKey, store)
 
@@ -46,7 +48,8 @@ func TestStoreCache(t *testing.T) {
 	mngr := cache.NewCommitKVStoreCacheManager(cache.DefaultCommitKVStoreCacheSize)
 
 	sKey := types.NewKVStoreKey("test")
-	tree := iavl.NewMutableTree(db, 100, false, log.NewNopLogger())
+	tree, err := iavl.NewMutableTree(db, 100, false)
+	require.NoError(t, err)
 	store := iavlstore.UnsafeNewStore(tree)
 	kvStore := mngr.GetStoreCache(sKey, store)
 

store/iavl/store.go

@@ -45,16 +45,16 @@ type Store struct {
 // LoadStore returns an IAVL Store as a CommitKVStore. Internally, it will load the
 // store's version (id) from the provided DB. An error is returned if the version
 // fails to load, or if called with a positive version on an empty tree.
-func LoadStore(db dbm.DB, logger log.Logger, key types.StoreKey, id types.CommitID, lazyLoading bool, cacheSize int) (types.CommitKVStore, error) {
-	return LoadStoreWithInitialVersion(db, logger, key, id, lazyLoading, 0, cacheSize)
+func LoadStore(db dbm.DB, logger log.Logger, key types.StoreKey, id types.CommitID, lazyLoading bool, cacheSize int, disableFastNode bool) (types.CommitKVStore, error) {
+	return LoadStoreWithInitialVersion(db, logger, key, id, lazyLoading, 0, cacheSize, disableFastNode)
 }
 
 // LoadStoreWithInitialVersion returns an IAVL Store as a CommitKVStore setting its initialVersion
 // to the one given. Internally, it will load the store's version (id) from the
 // provided DB. An error is returned if the version fails to load, or if called with a positive
 // version on an empty tree.
-func LoadStoreWithInitialVersion(db dbm.DB, logger log.Logger, key types.StoreKey, id types.CommitID, lazyLoading bool, initialVersion uint64, cacheSize int) (types.CommitKVStore, error) {
-	tree, err := iavl.NewMutableTreeWithOpts(db, cacheSize, &iavl.Options{InitialVersion: initialVersion})
+func LoadStoreWithInitialVersion(db dbm.DB, logger log.Logger, key types.StoreKey, id types.CommitID, lazyLoading bool, initialVersion uint64, cacheSize int, disableFastNode bool) (types.CommitKVStore, error) {
+	tree, err := iavl.NewMutableTreeWithOpts(db, cacheSize, &iavl.Options{InitialVersion: initialVersion}, disableFastNode)
 	if err != nil {
 		return nil, err
 	}

store/iavl/store_test.go

@@ -39,9 +39,9 @@ func randBytes(numBytes int) []byte {
 }
 
 // make a tree with data from above and save it
-func newAlohaTree(t *testing.T, db corestore.KVStoreWithBatch) (*iavl.MutableTree, types.CommitID) {
-	t.Helper()
-	tree := iavl.NewMutableTree(db, cacheSize, false, log.NewNopLogger())
+func newAlohaTree(t *testing.T, db dbm.DB) (*iavl.MutableTree, types.CommitID) {
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
+	require.NoError(t, err)
 
 	for k, v := range treeData {
 		_, err := tree.Set([]byte(k), []byte(v))
@@ -108,17 +108,17 @@ func TestLoadStore(t *testing.T) {
 	require.Equal(t, string(hcStore.Get([]byte("hello"))), "ciao")
 
 	// Querying a new store at some previous non-pruned height H
-	newHStore, err := LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), cIDH, false, DefaultIAVLCacheSize)
+	newHStore, err := LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), cIDH, false, DefaultIAVLCacheSize, false)
 	require.NoError(t, err)
 	require.Equal(t, string(newHStore.Get([]byte("hello"))), "hallo")
 
 	// Querying a new store at some previous pruned height Hp
-	newHpStore, err := LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), cIDHp, false, DefaultIAVLCacheSize)
+	newHpStore, err := LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), cIDHp, false, DefaultIAVLCacheSize, false)
 	require.NoError(t, err)
 	require.Equal(t, string(newHpStore.Get([]byte("hello"))), "hola")
 
 	// Querying a new store at current height H
-	newHcStore, err := LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), cIDHc, false, DefaultIAVLCacheSize)
+	newHcStore, err := LoadStore(db, log.NewNopLogger(), types.NewKVStoreKey("test"), cIDHc, false, DefaultIAVLCacheSize, false)
 	require.NoError(t, err)
 	require.Equal(t, string(newHcStore.Get([]byte("hello"))), "ciao")
 }
@@ -290,7 +290,8 @@ func TestIAVLIterator(t *testing.T) {
 func TestIAVLReverseIterator(t *testing.T) {
 	db := coretesting.NewMemDB()
 
-	tree := iavl.NewMutableTree(db, cacheSize, false, log.NewNopLogger())
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
+	require.NoError(t, err)
 
 	iavlStore := UnsafeNewStore(tree)
 
@@ -321,8 +322,9 @@ func TestIAVLReverseIterator(t *testing.T) {
 }
 
 func TestIAVLPrefixIterator(t *testing.T) {
-	db := coretesting.NewMemDB()
-	tree := iavl.NewMutableTree(db, cacheSize, false, log.NewNopLogger())
+	db := dbm.NewMemDB()
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
+	require.NoError(t, err)
 
 	iavlStore := UnsafeNewStore(tree)
 
@@ -384,8 +386,9 @@ func TestIAVLPrefixIterator(t *testing.T) {
 }
 
 func TestIAVLReversePrefixIterator(t *testing.T) {
-	db := coretesting.NewMemDB()
-	tree := iavl.NewMutableTree(db, cacheSize, false, log.NewNopLogger())
+	db := dbm.NewMemDB()
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
+	require.NoError(t, err)
 
 	iavlStore := UnsafeNewStore(tree)
 
@@ -451,8 +454,9 @@ func nextVersion(iavl *Store) {
 }
 
 func TestIAVLNoPrune(t *testing.T) {
-	db := coretesting.NewMemDB()
-	tree := iavl.NewMutableTree(db, cacheSize, false, log.NewNopLogger())
+	db := dbm.NewMemDB()
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
+	require.NoError(t, err)
 
 	iavlStore := UnsafeNewStore(tree)
 	nextVersion(iavlStore)
@@ -469,8 +473,9 @@ func TestIAVLNoPrune(t *testing.T) {
 }
 
 func TestIAVLStoreQuery(t *testing.T) {
-	db := coretesting.NewMemDB()
-	tree := iavl.NewMutableTree(db, cacheSize, false, log.NewNopLogger())
+	db := dbm.NewMemDB()
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
+	require.NoError(t, err)
 
 	iavlStore := UnsafeNewStore(tree)
 
@@ -583,7 +588,8 @@ func BenchmarkIAVLIteratorNext(b *testing.B) {
 	b.ReportAllocs()
 	db := coretesting.NewMemDB()
 	treeSize := 1000
-	tree := iavl.NewMutableTree(db, cacheSize, false, log.NewNopLogger())
+	tree, err := iavl.NewMutableTree(db, cacheSize, false)
+	require.NoError(b, err)
 
 	for i := 0; i < treeSize; i++ {
 		key := randBytes(4)
@@ -616,17 +622,19 @@ func TestSetInitialVersion(t *testing.T) {
 	}{
 		{
 			"works with a mutable tree",
-			func(db corestore.KVStoreWithBatch) *Store {
-				tree := iavl.NewMutableTree(db, cacheSize, false, log.NewNopLogger())
+			func(db *dbm.MemDB) *Store {
+				tree, err := iavl.NewMutableTree(db, cacheSize, false)
+				require.NoError(t, err)
 				store := UnsafeNewStore(tree)
 
 				return store
 			}, false,
 		},
 		{
 			"throws error on immutable tree",
-			func(db corestore.KVStoreWithBatch) *Store {
-				tree := iavl.NewMutableTree(db, cacheSize, false, log.NewNopLogger())
+			func(db *dbm.MemDB) *Store {
+				tree, err := iavl.NewMutableTree(db, cacheSize, false)
+				require.NoError(t, err)
 				store := UnsafeNewStore(tree)
 				_, version, err := store.tree.SaveVersion()
 				require.NoError(t, err)

store/iavl/tree_test.go

@@ -12,7 +12,7 @@ import (
 
 func TestImmutableTreePanics(t *testing.T) {
 	t.Parallel()
-	immTree := iavl.NewImmutableTree(coretesting.NewMemDB(), 100, false, log.NewNopLogger())
+	immTree := iavl.NewImmutableTree(dbm.NewMemDB(), 100, false)
 	it := &immutableTree{immTree}
 	require.Panics(t, func() { it.Set([]byte{}, []byte{}) })
 	require.Panics(t, func() { it.Remove([]byte{}) })