[dot/rpc] implement RPC state_getKeysPaged

dot/rpc/modules/api.go

@@ -19,6 +19,7 @@ type StorageAPI interface {
 	RegisterStorageChangeChannel(sub state.StorageSubscription) (byte, error)
 	UnregisterStorageChangeChannel(id byte)
 	GetStateRootFromBlock(bhash *common.Hash) (*common.Hash, error)
+	Keys(root *common.Hash) ([]string, error)
 }
 
 // BlockAPI is the interface for the block state

dot/rpc/modules/state.go

@@ -18,7 +18,9 @@ package modules
 
 import (
 	"encoding/hex"
+	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/lib/runtime"
@@ -41,8 +43,10 @@ type StateChildStorageRequest struct {
 
 // StateStorageKeyRequest holds json fields
 type StateStorageKeyRequest struct {
-	Key   []byte       `json:"key"`
-	Block *common.Hash `json:"block"`
+	Prefix   string       `json:"prefix"`
+	Qty      uint32       `json:"qty"`
+	AfterKey string       `json:"afterKey"`
+	Block    *common.Hash `json:"block"`
 }
 
 // StateRuntimeMetadataQuery is a hash value
@@ -117,7 +121,7 @@ type StateStorageResponse string
 type StatePairResponse []interface{}
 
 // StateStorageKeysResponse field for storage keys
-type StateStorageKeysResponse [][]byte
+type StateStorageKeysResponse []string
 
 // StateMetadataResponse holds the metadata
 //TODO: Determine actual type
@@ -233,10 +237,24 @@ func (sm *StateModule) GetChildStorageSize(r *http.Request, req *StateChildStora
 	return nil
 }
 
-// GetKeys isn't implemented properly yet.
-func (sm *StateModule) GetKeys(r *http.Request, req *StateStorageKeyRequest, res *StateStorageKeysResponse) error {
-	// TODO implement change storage trie so that block hash parameter works (See issue #834)
-	return nil
+// GetKeysPaged Returns the keys with prefix with pagination support.
+func (sm *StateModule) GetKeysPaged(r *http.Request, req *StateStorageKeyRequest, res *StateStorageKeysResponse) error {
+	keys, err := sm.storageAPI.Keys(req.Block)
+	resCount := uint32(0)
+	for _, k := range keys {
+		fKey := fmt.Sprintf("0x%x", k)
+		if strings.HasPrefix(fKey, req.Prefix) &&
+			strings.Compare(fKey, req.AfterKey) == 1 {
+			// sm.storageAPI.Keys sorts keys in lexicographical order, so we know that keys where strings.Compare = 1
+			//  are after the requested after key.
+			if resCount >= req.Qty {
+				break
+			}
+			*res = append(*res, fKey)
+			resCount++
+		}
+	}
+	return err
 }
 
 // GetMetadata calls runtime Metadata_metadata function

dot/rpc/modules/state_test.go

@@ -57,7 +57,7 @@ func TestStateModule_GetRuntimeVersion(t *testing.T) {
 		},
 	}
 
-	sm, hash := setupStateModule(t)
+	sm, hash, _ := setupStateModule(t)
 	randomHash, err := common.HexToHash(RandomHash)
 	require.NoError(t, err)
 
@@ -98,7 +98,7 @@ func TestStateModule_GetRuntimeVersion(t *testing.T) {
 }
 
 func TestStateModule_GetPairs(t *testing.T) {
-	sm, hash := setupStateModule(t)
+	sm, hash, _ := setupStateModule(t)
 
 	randomHash, err := common.HexToHash(RandomHash)
 	require.NoError(t, err)
@@ -166,7 +166,7 @@ func TestStateModule_GetPairs(t *testing.T) {
 }
 
 func TestStateModule_GetStorage(t *testing.T) {
-	sm, hash := setupStateModule(t)
+	sm, hash, _ := setupStateModule(t)
 	randomHash, err := common.HexToHash(RandomHash)
 	require.NoError(t, err)
 
@@ -216,7 +216,7 @@ func TestStateModule_GetStorage(t *testing.T) {
 }
 
 func TestStateModule_GetStorageHash(t *testing.T) {
-	sm, hash := setupStateModule(t)
+	sm, hash, _ := setupStateModule(t)
 	randomHash, err := common.HexToHash(RandomHash)
 	require.NoError(t, err)
 
@@ -268,7 +268,7 @@ func TestStateModule_GetStorageHash(t *testing.T) {
 }
 
 func TestStateModule_GetStorageSize(t *testing.T) {
-	sm, hash := setupStateModule(t)
+	sm, hash, _ := setupStateModule(t)
 	randomHash, err := common.HexToHash(RandomHash)
 	require.NoError(t, err)
 
@@ -313,7 +313,7 @@ func TestStateModule_GetStorageSize(t *testing.T) {
 }
 
 func TestStateModule_GetMetadata(t *testing.T) {
-	sm, hash := setupStateModule(t)
+	sm, hash, _ := setupStateModule(t)
 	randomHash, err := common.HexToHash(RandomHash)
 	require.NoError(t, err)
 
@@ -353,18 +353,79 @@ func TestStateModule_GetMetadata(t *testing.T) {
 	}
 }
 
-func setupStateModule(t *testing.T) (*StateModule, *common.Hash) {
+func TestStateModule_GetKeysPaged(t *testing.T) {
+	sm, _, stateRootHash := setupStateModule(t)
+
+	testCases := []struct {
+		name     string
+		params   StateStorageKeyRequest
+		expected []string
+	}{
+		{name: "allKeysNilBlockHash",
+			params: StateStorageKeyRequest{
+				Qty:   10,
+				Block: nil,
+			}, expected: []string{"0x3a6b657931", "0x3a6b657932"}},
+		{name: "allKeysTestBlockHash",
+			params: StateStorageKeyRequest{
+				Qty:   10,
+				Block: stateRootHash,
+			}, expected: []string{"0x3a6b657931", "0x3a6b657932"}},
+		{name: "prefixMatchAll",
+			params: StateStorageKeyRequest{
+				Prefix: "0x3a6b6579",
+				Qty:    10,
+			}, expected: []string{"0x3a6b657931", "0x3a6b657932"}},
+		{name: "prefixMatchOne",
+			params: StateStorageKeyRequest{
+				Prefix: "0x3a6b657931",
+				Qty:    10,
+			}, expected: []string{"0x3a6b657931"}},
+		{name: "prefixMatchNone",
+			params: StateStorageKeyRequest{
+				Prefix: "0x00",
+				Qty:    10,
+			}, expected: nil},
+		{name: "qtyOne",
+			params: StateStorageKeyRequest{
+				Qty: 1,
+			}, expected: []string{"0x3a6b657931"}},
+		{name: "afterKey",
+			params: StateStorageKeyRequest{
+				Qty:      10,
+				AfterKey: "0x3a6b657931",
+			}, expected: []string{"0x3a6b657932"}},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			var res StateStorageKeysResponse
+
+			err := sm.GetKeysPaged(nil, &test.params, &res)
+			require.NoError(t, err)
+
+			if test.expected == nil {
+				require.Empty(t, res)
+				return
+			}
+
+			require.Equal(t, StateStorageKeysResponse(test.expected), res)
+		})
+	}
+}
+
+func setupStateModule(t *testing.T) (*StateModule, *common.Hash, *common.Hash) {
 	// setup service
 	net := newNetworkService(t)
 	chain := newTestStateService(t)
 	// init storage with test data
 	ts, err := chain.Storage.TrieState(nil)
 	require.NoError(t, err)
 
-	err = ts.Set([]byte(`:key1`), []byte(`value1`))
-	require.NoError(t, err)
 	err = ts.Set([]byte(`:key2`), []byte(`value2`))
 	require.NoError(t, err)
+	err = ts.Set([]byte(`:key1`), []byte(`value1`))
+	require.NoError(t, err)
 
 	sr1, err := ts.Root()
 	require.NoError(t, err)
@@ -383,5 +444,5 @@ func setupStateModule(t *testing.T) (*StateModule, *common.Hash) {
 
 	hash, _ := chain.Block.GetBlockHash(big.NewInt(2))
 	core := newCoreService(t, chain)
-	return NewStateModule(net, chain.Storage, core), hash
+	return NewStateModule(net, chain.Storage, core), hash, &sr1
 }

dot/rpc/websocket_test.go

@@ -140,3 +140,6 @@ func (m *MockStorageAPI) UnregisterStorageChangeChannel(id byte) {
 func (m *MockStorageAPI) GetStateRootFromBlock(bhash *common.Hash) (*common.Hash, error) {
 	return nil, nil
 }
+func (m *MockStorageAPI) Keys(root *common.Hash) ([]string, error) {
+	return nil, nil
+}

dot/state/storage.go

@@ -20,6 +20,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"sort"
 	"sync"
 
 	"github.com/ChainSafe/gossamer/dot/types"
@@ -314,6 +315,22 @@ func (s *StorageState) Entries(hash *common.Hash) (map[string][]byte, error) {
 	return s.tries[*hash].Entries(), nil
 }
 
+// Keys returns all keys for given hash (or best block state root if hash is nil) in lexicographic order
+func (s *StorageState) Keys(hash *common.Hash) ([]string, error) {
+	entries, err := s.Entries(hash)
+	if err != nil {
+		return nil, err
+	}
+	keys := make([]string, 0, len(entries))
+
+	for k := range entries {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	return keys, nil
+}
+
 // GetStorageChild return GetChild from the trie
 func (s *StorageState) GetStorageChild(hash *common.Hash, keyToChild []byte) (*trie.Trie, error) {
 	if hash == nil {

go.mod

@@ -1,7 +1,7 @@
 module github.com/ChainSafe/gossamer
 
 require (
-	github.com/ChainSafe/chaindb v0.1.5-0.20210113205339-507b3a78c512
+	github.com/ChainSafe/chaindb v0.1.5-0.20210113220219-4e61b23c66d7
 	github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d
 	github.com/ChainSafe/log15 v1.0.0
 	github.com/OneOfOne/xxhash v1.2.5

go.sum

@@ -11,8 +11,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOv
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/ChainSafe/chaindb v0.1.5-0.20210113205339-507b3a78c512 h1:Qos8z4ReLzdaAoQTWm3rC3gapGHfPd8UlVPwpURl+xo=
-github.com/ChainSafe/chaindb v0.1.5-0.20210113205339-507b3a78c512/go.mod h1:WBsCSLGM7+DvSYU6cFVUltahwU7Sw4cN3e8kiLdNFJM=
+github.com/ChainSafe/chaindb v0.1.5-0.20210113220219-4e61b23c66d7 h1:xgn/gSkFpSfLBqWyU20t3WTUG+nJyN3vuCc+CcTlbAA=
+github.com/ChainSafe/chaindb v0.1.5-0.20210113220219-4e61b23c66d7/go.mod h1:WBsCSLGM7+DvSYU6cFVUltahwU7Sw4cN3e8kiLdNFJM=
 github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d h1:nalkkPQcITbvhmL4+C4cKA87NW0tfm3Kl9VXRoPywFg=
 github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4=
 github.com/ChainSafe/log15 v1.0.0 h1:vRDVtWtVwIH5uSCBvgTTZh6FA58UBJ6+QiiypaZfBf8=