Refactor store keys for variable-length addresses

CHANGELOG.md

@@ -36,8 +36,13 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ## [Unreleased]
 
+### Client Breaking Changes
+
+* [\#8363](https://github.com/cosmos/cosmos-sdk/issues/8363) Addresses no longer have a fixed 20-byte length. From the SDK modules' point of view, any 1-255 bytes-long byte array is a valid address.
+
 ### State Machine Breaking
 
+* (x/{bank,distrib,gov,slashing,staking}) [\#8363](https://github.com/cosmos/cosmos-sdk/issues/8363) Store keys have been modified to allow for variable-length addresses.
 * (x/ibc) [\#8266](https://github.com/cosmos/cosmos-sdk/issues/8266) Add amino JSON for IBC messages in order to support Ledger text signing.
 
 ### Improvements

simapp/export.go

@@ -157,7 +157,7 @@ func (app *SimApp) prepForZeroHeightGenesis(ctx sdk.Context, jailAllowedAddrs []
 	counter := int16(0)
 
 	for ; iter.Valid(); iter.Next() {
-		addr := sdk.ValAddress(iter.Key()[1:])
+		addr := sdk.ValAddress(stakingtypes.AddressFromValidatorsKey(iter.Key()))
 		validator, found := app.StakingKeeper.GetValidator(ctx, addr)
 		if !found {
 			panic("expected validator, not found")

types/address.go

@@ -13,6 +13,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/codec/legacy"
 	cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types"
 	"github.com/cosmos/cosmos-sdk/types/bech32"
+	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
 )
 
 const (
@@ -28,8 +29,9 @@ const (
 	//	config.SetFullFundraiserPath(yourFullFundraiserPath)
 	//	config.Seal()
 
-	// AddrLen defines a valid address length
-	AddrLen = 20
+	// Maximum allowed length (in bytes) for an address
+	maxAddrLen = 255
+
 	// Bech32MainPrefix defines the main SDK Bech32 prefix of an account's address
 	Bech32MainPrefix = "cosmos"
 
@@ -110,9 +112,15 @@ func VerifyAddressFormat(bz []byte) error {
 	if verifier != nil {
 		return verifier(bz)
 	}
-	if len(bz) != AddrLen {
-		return fmt.Errorf("incorrect address length (expected: %d, actual: %d)", AddrLen, len(bz))
+
+	if len(bz) == 0 {
+		return sdkerrors.Wrap(sdkerrors.ErrUnknownAddress, "addresses cannot be empty")
 	}
+
+	if len(bz) > maxAddrLen {
+		return sdkerrors.Wrapf(sdkerrors.ErrUnknownAddress, "address max length is %d, got %d", maxAddrLen, len(bz))
+	}
+
 	return nil
 }
 
@@ -256,6 +264,31 @@ func (aa AccAddress) Format(s fmt.State, verb rune) {
 	}
 }
 
+// LengthPrefixedAddressStoreKey prefixes the address bytes with its length, this is used
+// for variable-length components in store keys.
+func LengthPrefixedAddressStoreKey(bz []byte) ([]byte, error) {
+	bzLen := len(bz)
+	if bzLen == 0 {
+		return bz, nil
+	}
+
+	if bzLen > maxAddrLen {
+		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnknownAddress, "address length should be max %d bytes, got %d", maxAddrLen, bzLen)
+	}
+
+	return append([]byte{byte(bzLen)}, bz...), nil
+}
+
+// MustLengthPrefixedAddressStoreKey is LengthPrefixedAddressStoreKey with a panic.
+func MustLengthPrefixedAddressStoreKey(bz []byte) []byte {
+	res, err := LengthPrefixedAddressStoreKey(bz)
+	if err != nil {
+		panic(err)
+	}
+
+	return res
+}
+
 // ----------------------------------------------------------------------------
 // validator operator
 // ----------------------------------------------------------------------------

types/address_test.go

@@ -347,15 +347,18 @@ func (s *addressTestSuite) TestVerifyAddressFormat() {
 	addr5 := make([]byte, 5)
 	addr20 := make([]byte, 20)
 	addr32 := make([]byte, 32)
+	addr256 := make([]byte, 256)
 
 	err := types.VerifyAddressFormat(addr0)
-	s.Require().EqualError(err, "incorrect address length 0")
+	s.Require().EqualError(err, "addresses cannot be empty: unknown address")
 	err = types.VerifyAddressFormat(addr5)
-	s.Require().EqualError(err, "incorrect address length 5")
+	s.Require().NoError(err)
 	err = types.VerifyAddressFormat(addr20)
-	s.Require().Nil(err)
+	s.Require().NoError(err)
 	err = types.VerifyAddressFormat(addr32)
-	s.Require().EqualError(err, "incorrect address length 32")
+	s.Require().NoError(err)
+	err = types.VerifyAddressFormat(addr256)
+	s.Require().EqualError(err, "address max length is 255, got 256: unknown address")
 }
 
 func (s *addressTestSuite) TestCustomAddressVerifier() {
@@ -364,34 +367,39 @@ func (s *addressTestSuite) TestCustomAddressVerifier() {
 	accBech := types.AccAddress(addr).String()
 	valBech := types.ValAddress(addr).String()
 	consBech := types.ConsAddress(addr).String()
-	// Verifiy that the default logic rejects this 10 byte address
+	// Verify that the default logic doesn't reject this 10 byte address
+	// The default verifier is nil, we're only checking address length is
+	// between 1-255 bytes.
 	err := types.VerifyAddressFormat(addr)
-	s.Require().NotNil(err)
+	s.Require().Nil(err)
 	_, err = types.AccAddressFromBech32(accBech)
-	s.Require().NotNil(err)
+	s.Require().Nil(err)
 	_, err = types.ValAddressFromBech32(valBech)
-	s.Require().NotNil(err)
+	s.Require().Nil(err)
 	_, err = types.ConsAddressFromBech32(consBech)
-	s.Require().NotNil(err)
+	s.Require().Nil(err)
 
-	// Set a custom address verifier that accepts 10 or 20 byte addresses
+	// Set a custom address verifier only accepts 20 byte addresses
 	types.GetConfig().SetAddressVerifier(func(bz []byte) error {
 		n := len(bz)
-		if n == 10 || n == types.AddrLen {
+		if n == 20 {
 			return nil
 		}
 		return fmt.Errorf("incorrect address length %d", n)
 	})
 
-	// Verifiy that the custom logic accepts this 10 byte address
+	// Verifiy that the custom logic rejects this 10 byte address
 	err = types.VerifyAddressFormat(addr)
-	s.Require().Nil(err)
+	s.Require().NotNil(err)
 	_, err = types.AccAddressFromBech32(accBech)
-	s.Require().Nil(err)
+	s.Require().NotNil(err)
 	_, err = types.ValAddressFromBech32(valBech)
-	s.Require().Nil(err)
+	s.Require().NotNil(err)
 	_, err = types.ConsAddressFromBech32(consBech)
-	s.Require().Nil(err)
+	s.Require().NotNil(err)
+
+	// Reinitialize the global config to default address verifier (nil)
+	types.GetConfig().SetAddressVerifier(nil)
 }
 
 func (s *addressTestSuite) TestBech32ifyAddressBytes() {
@@ -520,3 +528,32 @@ func (s *addressTestSuite) TestGetFromBech32() {
 	s.Require().Error(err)
 	s.Require().Equal("invalid Bech32 prefix; expected x, got cosmos", err.Error())
 }
+
+func (s *addressTestSuite) TestLengthPrefixedAddressStoreKey() {
+	addr10byte := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+	addr20byte := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}
+	addr256byte := make([]byte, 256)
+
+	tests := []struct {
+		name        string
+		addr        []byte
+		expStoreKey []byte
+		expErr      bool
+	}{
+		{"10-byte address", addr10byte, append([]byte{byte(10)}, addr10byte...), false},
+		{"20-byte address", addr20byte, append([]byte{byte(20)}, addr20byte...), false},
+		{"256-byte address (too long)", addr256byte, nil, true},
+	}
+	for _, tt := range tests {
+		tt := tt
+		s.T().Run(tt.name, func(t *testing.T) {
+			storeKey, err := types.LengthPrefixedAddressStoreKey(tt.addr)
+			if tt.expErr {
+				s.Require().Error(err)
+			} else {
+				s.Require().NoError(err)
+				s.Require().Equal(tt.expStoreKey, storeKey)
+			}
+		})
+	}
+}

types/query/filtered_pagination_test.go

@@ -111,7 +111,7 @@ func ExampleFilteredPaginate() {
 	pageReq := &query.PageRequest{Key: nil, Limit: 1, CountTotal: true}
 	store := ctx.KVStore(app.GetKey(authtypes.StoreKey))
 	balancesStore := prefix.NewStore(store, types.BalancesPrefix)
-	accountStore := prefix.NewStore(balancesStore, addr1.Bytes())
+	accountStore := prefix.NewStore(balancesStore, sdk.MustLengthPrefixedAddressStoreKey(addr1))
 
 	var balResult sdk.Coins
 	pageRes, err := query.FilteredPaginate(accountStore, pageReq, func(key []byte, value []byte, accumulate bool) (bool, error) {
@@ -143,7 +143,7 @@ func ExampleFilteredPaginate() {
 
 func execFilterPaginate(store sdk.KVStore, pageReq *query.PageRequest, appCodec codec.Marshaler) (balances sdk.Coins, res *query.PageResponse, err error) {
 	balancesStore := prefix.NewStore(store, types.BalancesPrefix)
-	accountStore := prefix.NewStore(balancesStore, addr1.Bytes())
+	accountStore := prefix.NewStore(balancesStore, sdk.MustLengthPrefixedAddressStoreKey(addr1))
 
 	var balResult sdk.Coins
 	res, err = query.FilteredPaginate(accountStore, pageReq, func(key []byte, value []byte, accumulate bool) (bool, error) {

types/query/pagination_test.go

@@ -193,7 +193,7 @@ func ExamplePaginate() {
 	balResult := sdk.NewCoins()
 	authStore := ctx.KVStore(app.GetKey(authtypes.StoreKey))
 	balancesStore := prefix.NewStore(authStore, types.BalancesPrefix)
-	accountStore := prefix.NewStore(balancesStore, addr1.Bytes())
+	accountStore := prefix.NewStore(balancesStore, sdk.MustLengthPrefixedAddressStoreKey(addr1))
 	pageRes, err := query.Paginate(accountStore, request.Pagination, func(key []byte, value []byte) error {
 		var tempRes sdk.Coin
 		err := app.AppCodec().UnmarshalBinaryBare(value, &tempRes)

x/authz/types/keys.go

@@ -21,7 +21,7 @@ const (
 // Keys for authz store
 // Items are stored with the following key: values
 //
-// - 0x01<granterAddress_Bytes><granteeAddress_Bytes><msgType_Bytes>: Grant
+// - 0x01<granterAddressLen (1 Byte)><granterAddress_Bytes><granteeAddressLen (1 Byte)><granteeAddress_Bytes><msgType_Bytes>: Grant
 
 var (
 	// Keys for store prefixes
@@ -30,12 +30,22 @@ var (
 
 // GetAuthorizationStoreKey - return authorization store key
 func GetAuthorizationStoreKey(grantee sdk.AccAddress, granter sdk.AccAddress, msgType string) []byte {
-	return append(append(append(GrantKey, granter.Bytes()...), grantee.Bytes()...), []byte(msgType)...)
+	return append(append(append(
+		GrantKey,
+		sdk.MustLengthPrefixedAddressStoreKey(granter)...),
+		sdk.MustLengthPrefixedAddressStoreKey(grantee)...),
+		[]byte(msgType)...,
+	)
 }
 
 // ExtractAddressesFromGrantKey - split granter & grantee address from the authorization key
 func ExtractAddressesFromGrantKey(key []byte) (granterAddr, granteeAddr sdk.AccAddress) {
-	granterAddr = sdk.AccAddress(key[1 : sdk.AddrLen+1])
-	granteeAddr = sdk.AccAddress(key[sdk.AddrLen+1 : sdk.AddrLen*2+1])
+	// key if of format:
+	// 0x01<granterAddressLen (1 Byte)><granterAddress_Bytes><granteeAddressLen (1 Byte)><granteeAddress_Bytes><msgType_Bytes>
+	granterAddrLen := key[1] // remove prefix key
+	granterAddr = sdk.AccAddress(key[2 : 2+granterAddrLen])
+	granteeAddrLen := int(key[2+granterAddrLen])
+	granteeAddr = sdk.AccAddress(key[3+granterAddrLen : 3+granterAddrLen+byte(granteeAddrLen)])
+
 	return granterAddr, granteeAddr
 }

x/bank/keeper/grpc_query.go

@@ -57,9 +57,7 @@ func (k BaseKeeper) AllBalances(ctx context.Context, req *types.QueryAllBalances
 	sdkCtx := sdk.UnwrapSDKContext(ctx)
 
 	balances := sdk.NewCoins()
-	store := sdkCtx.KVStore(k.storeKey)
-	balancesStore := prefix.NewStore(store, types.BalancesPrefix)
-	accountStore := prefix.NewStore(balancesStore, addr.Bytes())
+	accountStore := k.getAccountStore(sdkCtx, addr)
 
 	pageRes, err := query.Paginate(accountStore, req.Pagination, func(_, value []byte) error {
 		var result sdk.Coin

x/bank/keeper/send.go

@@ -2,7 +2,6 @@ package keeper
 
 import (
 	"github.com/cosmos/cosmos-sdk/codec"
-	"github.com/cosmos/cosmos-sdk/store/prefix"
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
@@ -233,9 +232,7 @@ func (k BaseSendKeeper) ClearBalances(ctx sdk.Context, addr sdk.AccAddress) {
 		return false
 	})
 
-	store := ctx.KVStore(k.storeKey)
-	balancesStore := prefix.NewStore(store, types.BalancesPrefix)
-	accountStore := prefix.NewStore(balancesStore, addr.Bytes())
+	accountStore := k.getAccountStore(ctx, addr)
 
 	for _, key := range keys {
 		accountStore.Delete(key)
@@ -264,9 +261,7 @@ func (k BaseSendKeeper) SetBalance(ctx sdk.Context, addr sdk.AccAddress, balance
 		return sdkerrors.Wrap(sdkerrors.ErrInvalidCoins, balance.String())
 	}
 
-	store := ctx.KVStore(k.storeKey)
-	balancesStore := prefix.NewStore(store, types.BalancesPrefix)
-	accountStore := prefix.NewStore(balancesStore, addr.Bytes())
+	accountStore := k.getAccountStore(ctx, addr)
 
 	bz := k.cdc.MustMarshalBinaryBare(&balance)
 	accountStore.Set([]byte(balance.Denom), bz)

x/bank/keeper/view.go

@@ -97,9 +97,7 @@ func (k BaseViewKeeper) GetAccountsBalances(ctx sdk.Context) []types.Balance {
 // GetBalance returns the balance of a specific denomination for a given account
 // by address.
 func (k BaseViewKeeper) GetBalance(ctx sdk.Context, addr sdk.AccAddress, denom string) sdk.Coin {
-	store := ctx.KVStore(k.storeKey)
-	balancesStore := prefix.NewStore(store, types.BalancesPrefix)
-	accountStore := prefix.NewStore(balancesStore, addr.Bytes())
+	accountStore := k.getAccountStore(ctx, addr)
 
 	bz := accountStore.Get([]byte(denom))
 	if bz == nil {
@@ -116,9 +114,7 @@ func (k BaseViewKeeper) GetBalance(ctx sdk.Context, addr sdk.AccAddress, denom s
 // provides the token balance to a callback. If true is returned from the
 // callback, iteration is halted.
 func (k BaseViewKeeper) IterateAccountBalances(ctx sdk.Context, addr sdk.AccAddress, cb func(sdk.Coin) bool) {
-	store := ctx.KVStore(k.storeKey)
-	balancesStore := prefix.NewStore(store, types.BalancesPrefix)
-	accountStore := prefix.NewStore(balancesStore, addr.Bytes())
+	accountStore := k.getAccountStore(ctx, addr)
 
 	iterator := accountStore.Iterator(nil, nil)
 	defer iterator.Close()
@@ -214,3 +210,11 @@ func (k BaseViewKeeper) ValidateBalance(ctx sdk.Context, addr sdk.AccAddress) er
 
 	return nil
 }
+
+// getAccountStore gets the account store of the given address.
+func (k BaseViewKeeper) getAccountStore(ctx sdk.Context, addr sdk.AccAddress) prefix.Store {
+	store := ctx.KVStore(k.storeKey)
+	balancesStore := prefix.NewStore(store, types.BalancesPrefix)
+
+	return prefix.NewStore(balancesStore, sdk.MustLengthPrefixedAddressStoreKey(addr))
+}

x/bank/types/key.go

@@ -1,8 +1,6 @@
 package types
 
 import (
-	"fmt"
-
 	sdk "github.com/cosmos/cosmos-sdk/types"
 )
 
@@ -22,7 +20,9 @@ const (
 
 // KVStore keys
 var (
-	BalancesPrefix      = []byte("balances")
+	// BalancesPrefix is the for the account balances store. We use a byte
+	// (instead of say `[]]byte("balances")` to save some disk space).
+	BalancesPrefix      = []byte{0x02}
 	SupplyKey           = []byte{0x00}
 	DenomMetadataPrefix = []byte{0x1}
 )
@@ -37,10 +37,8 @@ func DenomMetadataKey(denom string) []byte {
 // store. The key must not contain the perfix BalancesPrefix as the prefix store
 // iterator discards the actual prefix.
 func AddressFromBalancesStore(key []byte) sdk.AccAddress {
-	addr := key[:sdk.AddrLen]
-	if len(addr) != sdk.AddrLen {
-		panic(fmt.Sprintf("unexpected account address key length; got: %d, expected: %d", len(addr), sdk.AddrLen))
-	}
+	addrLen := key[0]
+	addr := key[1 : addrLen+1]
 
 	return sdk.AccAddress(addr)
 }

x/bank/types/key_test.go

@@ -19,8 +19,10 @@ func cloneAppend(bz []byte, tail []byte) (res []byte) {
 func TestAddressFromBalancesStore(t *testing.T) {
 	addr, err := sdk.AccAddressFromBech32("cosmos1n88uc38xhjgxzw9nwre4ep2c8ga4fjxcar6mn7")
 	require.NoError(t, err)
+	addrLen := len(addr)
+	require.Equal(t, 20, addrLen)
 
-	key := cloneAppend(addr.Bytes(), []byte("stake"))
+	key := cloneAppend(sdk.MustLengthPrefixedAddressStoreKey(addr), []byte("stake"))
 	res := types.AddressFromBalancesStore(key)
 	require.Equal(t, res, addr)
 }