From 0e5afc8e37678f5190d8de8d7418042690d61fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Thu, 19 Aug 2021 17:27:11 -0400 Subject: [PATCH 01/38] feat: implement chain subscribe all heads --- dot/rpc/subscription/listeners.go | 76 ++++++++++++++++++++++++++++ dot/rpc/subscription/subscription.go | 3 ++ dot/rpc/subscription/websocket.go | 34 +++++++++++++ 3 files changed, 113 insertions(+) diff --git a/dot/rpc/subscription/listeners.go b/dot/rpc/subscription/listeners.go index 01a4ad896d..e2570dd4d9 100644 --- a/dot/rpc/subscription/listeners.go +++ b/dot/rpc/subscription/listeners.go @@ -35,6 +35,7 @@ const ( authorExtrinsicUpdatesMethod = "author_extrinsicUpdate" chainFinalizedHeadMethod = "chain_finalizedHead" chainNewHeadMethod = "chain_newHead" + chainAllHeadMethod = "chain_allHead" stateStorageMethod = "state_storage" ) @@ -213,6 +214,81 @@ func (l *BlockFinalizedListener) Stop() error { return cancelWithTimeout(l.cancel, l.done, l.cancelTimeout) } +type AllBlocksListener struct { + finalizedChan chan *types.FinalisationInfo + importedChan chan *types.Block + + wsconn *WSConn + finalizedChanID byte + importedChanID byte + subID uint32 + done chan struct{} + cancel chan struct{} + cancelTimeout time.Duration +} + +func (l *AllBlocksListener) Listen() { + go func() { + defer func() { + l.wsconn.BlockAPI.UnregisterImportedChannel(l.importedChanID) + l.wsconn.BlockAPI.UnregisterFinalisedChannel(l.finalizedChanID) + close(l.done) + }() + + for { + select { + case <-l.cancel: + return + case fin, ok := <-l.finalizedChan: + if !ok { + return + } + + if fin == nil || fin.Header == nil { + continue + } + + finHead, err := modules.HeaderToJSON(*fin.Header) + if err != nil { + logger.Error("failed to convert finalized block header to JSON", "error", err) + continue + } + + res := newSubcriptionBaseResponseJSON() + res.Method = chainAllHeadMethod + res.Params.Result = finHead + res.Params.SubscriptionID = l.subID + l.wsconn.safeSend(res) + + case imp, ok := <-l.importedChan: + if !ok { + return + } + + if imp == nil || imp.Header == nil { + continue + } + + impHead, err := modules.HeaderToJSON(*imp.Header) + if err != nil { + logger.Error("failed to convert imported block header to JSON", "error", err) + continue + } + + res := newSubcriptionBaseResponseJSON() + res.Method = chainNewHeadMethod + res.Params.Result = impHead + res.Params.SubscriptionID = l.subID + l.wsconn.safeSend(res) + } + } + }() +} + +func (l *AllBlocksListener) Stop() error { + return cancelWithTimeout(l.cancel, l.done, l.cancelTimeout) +} + // ExtrinsicSubmitListener to handle listening for extrinsic events type ExtrinsicSubmitListener struct { wsconn *WSConn diff --git a/dot/rpc/subscription/subscription.go b/dot/rpc/subscription/subscription.go index cbf7648c78..870faf5410 100644 --- a/dot/rpc/subscription/subscription.go +++ b/dot/rpc/subscription/subscription.go @@ -11,6 +11,7 @@ const ( chainSubscribeNewHeads string = "chain_subscribeNewHeads" chainSubscribeNewHead string = "chain_subscribeNewHead" chainSubscribeFinalizedHeads string = "chain_subscribeFinalizedHeads" + chainSubscribeAllHeads string = "chain_subscribeAllHeads" stateSubscribeStorage string = "state_subscribeStorage" stateSubscribeRuntimeVersion string = "state_subscribeRuntimeVersion" grandpaSubscribeJustifications string = "grandpa_subscribeJustifications" @@ -35,6 +36,8 @@ func (c *WSConn) getSetupListener(method string) setupListener { return c.initStorageChangeListener case chainSubscribeFinalizedHeads: return c.initBlockFinalizedListener + case chainSubscribeAllHeads: + return c.initAllBlocksListerner case stateSubscribeRuntimeVersion: return c.initRuntimeVersionListener case grandpaSubscribeJustifications: diff --git a/dot/rpc/subscription/websocket.go b/dot/rpc/subscription/websocket.go index f3ede55f39..9c57c9556a 100644 --- a/dot/rpc/subscription/websocket.go +++ b/dot/rpc/subscription/websocket.go @@ -278,6 +278,40 @@ func (c *WSConn) initBlockFinalizedListener(reqID float64, _ interface{}) (Liste return bfl, nil } +func (c *WSConn) initAllBlocksListerner(reqID float64, _ interface{}) (Listener, error) { + listener := new(AllBlocksListener) + listener.cancel = make(chan struct{}) + listener.done = make(chan struct{}) + listener.cancelTimeout = defaultCancelTimeout + listener.wsconn = c + + if c.BlockAPI == nil { + c.safeSendError(reqID, nil, "error BlockAPI not set") + return nil, fmt.Errorf("error BlockAPI not set") + } + + var err error + listener.importedChanID, err = c.BlockAPI.RegisterImportedChannel(listener.importedChan) + if err != nil { + c.safeSendError(reqID, nil, "could not register imported channel") + return nil, fmt.Errorf("could not register imported channel") + } + + listener.finalizedChanID, err = c.BlockAPI.RegisterFinalizedChannel(listener.finalizedChan) + if err != nil { + c.safeSendError(reqID, nil, "could not register finalized channel") + return nil, fmt.Errorf("could not register finalized channel") + } + + c.mu.Lock() + listener.subID = atomic.AddUint32(&c.qtyListeners, 1) + c.Subscriptions[listener.subID] = listener + c.mu.Unlock() + + c.safeSend(NewSubscriptionResponseJSON(listener.subID, reqID)) + return listener, nil +} + func (c *WSConn) initExtrinsicWatch(reqID float64, params interface{}) (Listener, error) { pA := params.([]interface{}) extBytes, err := common.HexToBytes(pA[0].(string)) From 4586709f56b0b9ded8e761c0d3340462e848a1bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Thu, 19 Aug 2021 21:04:05 -0400 Subject: [PATCH 02/38] chore: add unit tests to subscribe all heads --- dot/rpc/subscription/listeners.go | 2 +- dot/rpc/subscription/websocket.go | 2 + dot/rpc/subscription/websocket_test.go | 113 +++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/dot/rpc/subscription/listeners.go b/dot/rpc/subscription/listeners.go index e2570dd4d9..dd6243c436 100644 --- a/dot/rpc/subscription/listeners.go +++ b/dot/rpc/subscription/listeners.go @@ -276,7 +276,7 @@ func (l *AllBlocksListener) Listen() { } res := newSubcriptionBaseResponseJSON() - res.Method = chainNewHeadMethod + res.Method = chainAllHeadMethod res.Params.Result = impHead res.Params.SubscriptionID = l.subID l.wsconn.safeSend(res) diff --git a/dot/rpc/subscription/websocket.go b/dot/rpc/subscription/websocket.go index 9c57c9556a..d07f706b07 100644 --- a/dot/rpc/subscription/websocket.go +++ b/dot/rpc/subscription/websocket.go @@ -284,6 +284,8 @@ func (c *WSConn) initAllBlocksListerner(reqID float64, _ interface{}) (Listener, listener.done = make(chan struct{}) listener.cancelTimeout = defaultCancelTimeout listener.wsconn = c + listener.finalizedChan = make(chan *types.FinalisationInfo, DEFAULT_BUFFER_SIZE) + listener.importedChan = make(chan *types.Block, DEFAULT_BUFFER_SIZE) if c.BlockAPI == nil { c.safeSendError(reqID, nil, "error BlockAPI not set") diff --git a/dot/rpc/subscription/websocket_test.go b/dot/rpc/subscription/websocket_test.go index 74d3c88796..7a35aa744f 100644 --- a/dot/rpc/subscription/websocket_test.go +++ b/dot/rpc/subscription/websocket_test.go @@ -1,11 +1,13 @@ package subscription import ( + "errors" "fmt" "math/big" "testing" "time" + "github.com/ChainSafe/gossamer/dot/rpc/modules/mocks" modulesmocks "github.com/ChainSafe/gossamer/dot/rpc/modules/mocks" "github.com/ChainSafe/gossamer/dot/rpc/modules" @@ -270,3 +272,114 @@ func TestWSConn_HandleComm(t *testing.T) { err = listener.Stop() require.NoError(t, err) } + +func TestSubscribeAllHeads(t *testing.T) { + wsconn, c, cancel := setupWSConn(t) + wsconn.Subscriptions = make(map[uint32]Listener) + defer cancel() + + go wsconn.HandleComm() + time.Sleep(time.Second * 2) + + _, err := wsconn.initAllBlocksListerner(1, nil) + require.EqualError(t, err, "error BlockAPI not set") + _, msg, err := c.ReadMessage() + require.NoError(t, err) + require.Equal(t, []byte(`{"jsonrpc":"2.0","error":{"code":null,"message":"error BlockAPI not set"},"id":1}`+"\n"), msg) + + mockBlockAPI := new(mocks.BlockAPI) + mockBlockAPI.On("RegisterImportedChannel", mock.AnythingOfType("chan<- *types.Block")). + Return(uint8(0), errors.New("some mocked error")).Once() + + wsconn.BlockAPI = mockBlockAPI + _, err = wsconn.initAllBlocksListerner(1, nil) + require.Error(t, err, "could not register imported channel") + + _, msg, err = c.ReadMessage() + require.NoError(t, err) + require.Equal(t, []byte(`{"jsonrpc":"2.0","error":{"code":null,"message":"could not register imported channel"},"id":1}`+"\n"), msg) + + mockBlockAPI.On("RegisterImportedChannel", mock.AnythingOfType("chan<- *types.Block")). + Return(uint8(10), nil).Once() + mockBlockAPI.On("RegisterFinalizedChannel", mock.AnythingOfType("chan<- *types.FinalisationInfo")). + Return(uint8(0), errors.New("failed")).Once() + + _, err = wsconn.initAllBlocksListerner(1, nil) + require.Error(t, err, "could not register finalized channel") + c.ReadMessage() + + importedChanID := uint8(10) + finalizedChanID := uint8(11) + + var fCh chan<- *types.FinalisationInfo + var iCh chan<- *types.Block + + mockBlockAPI.On("RegisterImportedChannel", mock.AnythingOfType("chan<- *types.Block")). + Run(func(args mock.Arguments) { + ch := args.Get(0).(chan<- *types.Block) + iCh = ch + }).Return(importedChanID, nil).Once() + + mockBlockAPI.On("RegisterFinalizedChannel", mock.AnythingOfType("chan<- *types.FinalisationInfo")). + Run(func(args mock.Arguments) { + ch := args.Get(0).(chan<- *types.FinalisationInfo) + fCh = ch + }). + Return(finalizedChanID, nil).Once() + + l, err := wsconn.initAllBlocksListerner(1, nil) + require.NoError(t, err) + require.NotNil(t, l) + require.IsType(t, &AllBlocksListener{}, l) + require.Len(t, wsconn.Subscriptions, 1) + + _, msg, err = c.ReadMessage() + require.NoError(t, err) + require.Equal(t, []byte(`{"jsonrpc":"2.0","result":1,"id":1}`+"\n"), msg) + + l.Listen() + time.Sleep(time.Millisecond * 500) + + expected := fmt.Sprintf( + `{"jsonrpc":"2.0","method":"chain_allHead","params":{"result":{"parentHash":"%s","number":"0x00","stateRoot":"%s","extrinsicsRoot":"%s","digest":{"logs":["0x064241424504ff"]}},"subscription":1}}`, + common.EmptyHash, + common.EmptyHash, + common.EmptyHash, + ) + + fCh <- &types.FinalisationInfo{ + Header: &types.Header{ + ParentHash: common.EmptyHash, + Number: big.NewInt(0), + StateRoot: common.EmptyHash, + ExtrinsicsRoot: common.EmptyHash, + Digest: types.NewDigest(types.NewBABEPreRuntimeDigest([]byte{0xff})), + }, + } + + time.Sleep(time.Millisecond * 500) + _, msg, err = c.ReadMessage() + require.NoError(t, err) + require.Equal(t, expected+"\n", string(msg)) + + iCh <- &types.Block{ + Header: &types.Header{ + ParentHash: common.EmptyHash, + Number: big.NewInt(0), + StateRoot: common.EmptyHash, + ExtrinsicsRoot: common.EmptyHash, + Digest: types.NewDigest(types.NewBABEPreRuntimeDigest([]byte{0xff})), + }, + } + time.Sleep(time.Millisecond * 500) + _, msg, err = c.ReadMessage() + require.NoError(t, err) + require.Equal(t, []byte(expected+"\n"), msg) + + mockBlockAPI.On("UnregisterImportedChannel", importedChanID) + mockBlockAPI.On("UnregisterFinalisedChannel", finalizedChanID) + + require.NoError(t, l.Stop()) + mockBlockAPI.AssertCalled(t, "UnregisterImportedChannel", importedChanID) + mockBlockAPI.AssertCalled(t, "UnregisterFinalisedChannel", finalizedChanID) +} From 91c0756ee781a7b3b5cb6787a88920faf40de2e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 23 Aug 2021 14:52:05 -0400 Subject: [PATCH 03/38] chore: fix imports --- dot/rpc/subscription/websocket_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dot/rpc/subscription/websocket_test.go b/dot/rpc/subscription/websocket_test.go index 7a35aa744f..65384cc478 100644 --- a/dot/rpc/subscription/websocket_test.go +++ b/dot/rpc/subscription/websocket_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/ChainSafe/gossamer/dot/rpc/modules/mocks" - modulesmocks "github.com/ChainSafe/gossamer/dot/rpc/modules/mocks" "github.com/ChainSafe/gossamer/dot/rpc/modules" "github.com/ChainSafe/gossamer/dot/types" @@ -231,7 +230,7 @@ func TestWSConn_HandleComm(t *testing.T) { mockedJustBytes, err := mockedJust.Encode() require.NoError(t, err) - BlockAPI := new(modulesmocks.BlockAPI) + BlockAPI := new(mocks.BlockAPI) BlockAPI.On("RegisterFinalizedChannel", mock.AnythingOfType("chan<- *types.FinalisationInfo")). Run(func(args mock.Arguments) { ch := args.Get(0).(chan<- *types.FinalisationInfo) From 659c503cefcda17366818f77ba79cf6a91f7b55d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 24 Aug 2021 20:26:35 -0400 Subject: [PATCH 04/38] feat: implement state_getReadProof rpc call --- dot/core/interface.go | 1 + dot/core/mocks/block_state.go | 37 ++++--- dot/core/mocks/storage_state.go | 147 +++++++++++++++++++++++++++ dot/core/service.go | 47 +++++++++ dot/core/service_test.go | 114 +++++++++++++++++++++ dot/rpc/modules/api.go | 1 + dot/rpc/modules/mocks/core_api.go | 32 ++++++ dot/rpc/modules/state.go | 27 +++++ dot/rpc/modules/state_test.go | 41 ++++++++ dot/state/block.go | 10 ++ lib/trie/proof.go | 80 +++++++-------- lib/trie/proof_test.go | 161 +++++++----------------------- lib/trie/test_utils.go | 27 +++++ 13 files changed, 546 insertions(+), 179 deletions(-) create mode 100644 dot/core/mocks/storage_state.go diff --git a/dot/core/interface.go b/dot/core/interface.go index 83fe0ab21f..67b1dc6a14 100644 --- a/dot/core/interface.go +++ b/dot/core/interface.go @@ -37,6 +37,7 @@ type BlockState interface { AddBlock(*types.Block) error GetAllBlocksAtDepth(hash common.Hash) []common.Hash GetBlockByHash(common.Hash) (*types.Block, error) + GetBlockStateRoot(bhash common.Hash) (common.Hash, error) GenesisHash() common.Hash GetSlotForBlock(common.Hash) (uint64, error) GetFinalisedHeader(uint64, uint64) (*types.Header, error) diff --git a/dot/core/mocks/block_state.go b/dot/core/mocks/block_state.go index 7686c23dab..a1bb6139c4 100644 --- a/dot/core/mocks/block_state.go +++ b/dot/core/mocks/block_state.go @@ -221,6 +221,29 @@ func (_m *MockBlockState) GetBlockByHash(_a0 common.Hash) (*types.Block, error) return r0, r1 } +// GetBlockStateRoot provides a mock function with given fields: bhash +func (_m *MockBlockState) GetBlockStateRoot(bhash common.Hash) (common.Hash, error) { + ret := _m.Called(bhash) + + var r0 common.Hash + if rf, ok := ret.Get(0).(func(common.Hash) common.Hash); ok { + r0 = rf(bhash) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(common.Hash) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(common.Hash) error); ok { + r1 = rf(bhash) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetFinalisedHash provides a mock function with given fields: _a0, _a1 func (_m *MockBlockState) GetFinalisedHash(_a0 uint64, _a1 uint64) (common.Hash, error) { ret := _m.Called(_a0, _a1) @@ -390,20 +413,6 @@ func (_m *MockBlockState) RegisterImportedChannel(ch chan<- *types.Block) (byte, return r0, r1 } -// SetFinalisedHash provides a mock function with given fields: _a0, _a1, _a2 -func (_m *MockBlockState) SetFinalisedHash(_a0 common.Hash, _a1 uint64, _a2 uint64) error { - ret := _m.Called(_a0, _a1, _a2) - - var r0 error - if rf, ok := ret.Get(0).(func(common.Hash, uint64, uint64) error); ok { - r0 = rf(_a0, _a1, _a2) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // StoreRuntime provides a mock function with given fields: _a0, _a1 func (_m *MockBlockState) StoreRuntime(_a0 common.Hash, _a1 runtime.Instance) { _m.Called(_a0, _a1) diff --git a/dot/core/mocks/storage_state.go b/dot/core/mocks/storage_state.go new file mode 100644 index 0000000000..8c7077ce75 --- /dev/null +++ b/dot/core/mocks/storage_state.go @@ -0,0 +1,147 @@ +// Code generated by mockery v2.8.0. DO NOT EDIT. + +package mocks + +import ( + common "github.com/ChainSafe/gossamer/lib/common" + + mock "github.com/stretchr/testify/mock" + + storage "github.com/ChainSafe/gossamer/lib/runtime/storage" + + types "github.com/ChainSafe/gossamer/dot/types" +) + +// MockStorageState is an autogenerated mock type for the StorageState type +type MockStorageState struct { + mock.Mock +} + +// GetStateRootFromBlock provides a mock function with given fields: bhash +func (_m *MockStorageState) GetStateRootFromBlock(bhash *common.Hash) (*common.Hash, error) { + ret := _m.Called(bhash) + + var r0 *common.Hash + if rf, ok := ret.Get(0).(func(*common.Hash) *common.Hash); ok { + r0 = rf(bhash) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*common.Hash) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*common.Hash) error); ok { + r1 = rf(bhash) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetStorage provides a mock function with given fields: root, key +func (_m *MockStorageState) GetStorage(root *common.Hash, key []byte) ([]byte, error) { + ret := _m.Called(root, key) + + var r0 []byte + if rf, ok := ret.Get(0).(func(*common.Hash, []byte) []byte); ok { + r0 = rf(root, key) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*common.Hash, []byte) error); ok { + r1 = rf(root, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// LoadCode provides a mock function with given fields: root +func (_m *MockStorageState) LoadCode(root *common.Hash) ([]byte, error) { + ret := _m.Called(root) + + var r0 []byte + if rf, ok := ret.Get(0).(func(*common.Hash) []byte); ok { + r0 = rf(root) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*common.Hash) error); ok { + r1 = rf(root) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// LoadCodeHash provides a mock function with given fields: root +func (_m *MockStorageState) LoadCodeHash(root *common.Hash) (common.Hash, error) { + ret := _m.Called(root) + + var r0 common.Hash + if rf, ok := ret.Get(0).(func(*common.Hash) common.Hash); ok { + r0 = rf(root) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(common.Hash) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*common.Hash) error); ok { + r1 = rf(root) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// StoreTrie provides a mock function with given fields: _a0, _a1 +func (_m *MockStorageState) StoreTrie(_a0 *storage.TrieState, _a1 *types.Header) error { + ret := _m.Called(_a0, _a1) + + var r0 error + if rf, ok := ret.Get(0).(func(*storage.TrieState, *types.Header) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// TrieState provides a mock function with given fields: root +func (_m *MockStorageState) TrieState(root *common.Hash) (*storage.TrieState, error) { + ret := _m.Called(root) + + var r0 *storage.TrieState + if rf, ok := ret.Get(0).(func(*common.Hash) *storage.TrieState); ok { + r0 = rf(root) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*storage.TrieState) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*common.Hash) error); ok { + r1 = rf(root) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/dot/core/service.go b/dot/core/service.go index 1bac4a4f26..451f92eb21 100644 --- a/dot/core/service.go +++ b/dot/core/service.go @@ -31,6 +31,7 @@ import ( rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/services" "github.com/ChainSafe/gossamer/lib/transaction" + "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" log "github.com/ChainSafe/log15" ) @@ -618,3 +619,49 @@ func (s *Service) tryQueryStorage(block common.Hash, keys ...string) (QueryKeyVa return changes, nil } + +// GetReadProofAt will return an array with the proofs for the keys passed as params +// based on the block hash passed as param as well, if block hash is nil then the current state will take place +func (s *Service) GetReadProofAt(block common.Hash, keys []common.Hash) (common.Hash, []string, error) { + if block == common.EmptyHash { + block = s.blockState.BestBlockHash() + } + + stateRoot, err := s.blockState.GetBlockStateRoot(block) + if err != nil { + return common.EmptyHash, nil, err + } + + ts, err := s.storageState.TrieState(&stateRoot) + if err != nil { + return common.EmptyHash, nil, err + } + + proofForKeys, err := readProofForKeys(ts.Trie(), keys) + if err != nil { + return common.EmptyHash, nil, err + } + + return block, proofForKeys, nil +} + +// readProofForKeys will go through the keys and generate the proof for each of them +// and merge the result into a string array containing the hashes in the hexadecimal format +func readProofForKeys(t *trie.Trie, keys []common.Hash) ([]string, error) { + storageKeys := make([][]byte, len(keys)) + for i, k := range keys { + storageKeys[i] = k.ToBytes() + } + + proof, err := t.GenerateProof(storageKeys) + if err != nil { + return nil, err + } + + proofSlice := make([]string, 0, len(proof)) + for k := range proof { + proofSlice = append(proofSlice, k) + } + + return proofSlice, nil +} diff --git a/dot/core/service_test.go b/dot/core/service_test.go index e5ef9188d8..d121d7c049 100644 --- a/dot/core/service_test.go +++ b/dot/core/service_test.go @@ -819,3 +819,117 @@ func TestDecodeSessionKeys_WhenGetRuntimeReturnError(t *testing.T) { require.Error(t, err, "problems") require.Nil(t, b) } + +func TestGetReadProofAt_WhenReceivedBlockIsEmpty(t *testing.T) { + tr, entries := trie.RandomTrieTest(t, 50) + require.Len(t, entries, 50) + + i := 0 + var keysToProof []common.Hash + var keys [][]byte + + // get the last 2 entries + for _, entry := range entries { + if i < len(entries)-2 { + i++ + continue + } + + keysToProof = append(keysToProof, common.BytesToHash(entry.K)) + keys = append(keys, entry.K) + } + require.Len(t, keysToProof, 2) + + expectedProof, err := tr.GenerateProof(keys) + require.NoError(t, err) + + ts, err := storage.NewTrieState(tr) + require.NoError(t, err) + root := ts.MustRoot() + + header, err := types.NewHeader(common.EmptyHash, root, common.EmptyHash, big.NewInt(1), nil) + require.NoError(t, err) + bestBlock := types.NewBlock(header, nil) + + blockStateMock := new(mocks.MockBlockState) + blockStateMock.On("BestBlockHash").Return(bestBlock.Header.Hash()) + blockStateMock.On("GetBlockStateRoot", bestBlock.Header.Hash()).Return(root, nil) + + storageMock := new(mocks.MockStorageState) + storageMock.On("TrieState", &root).Return(ts, nil) + + svc := new(Service) + svc.storageState = storageMock + svc.blockState = blockStateMock + + block, proof, err := svc.GetReadProofAt(common.EmptyHash, keysToProof) + require.NoError(t, err) + + blockStateMock.AssertCalled(t, "BestBlockHash") + blockStateMock.AssertCalled(t, "GetBlockStateRoot", bestBlock.Header.Hash()) + storageMock.AssertCalled(t, "TrieState", &root) + require.Equal(t, bestBlock.Header.Hash(), block) + + // all the proof must exists in the expected + require.Equal(t, len(expectedProof), len(proof)) + for _, p := range proof { + _, has := expectedProof[p] + require.True(t, has) + } +} + +func TestGetReadProofAt_OnSpecificBlock(t *testing.T) { + tr, entries := trie.RandomTrieTest(t, 50) + require.Len(t, entries, 50) + + i := 0 + var keysToProof []common.Hash + var keys [][]byte + + // get the last 2 entries + for _, entry := range entries { + if i < len(entries)-2 { + i++ + continue + } + + keysToProof = append(keysToProof, common.BytesToHash(entry.K)) + keys = append(keys, entry.K) + } + require.Len(t, keysToProof, 2) + + expectedProof, err := tr.GenerateProof(keys) + require.NoError(t, err) + + ts, err := storage.NewTrieState(tr) + require.NoError(t, err) + root := ts.MustRoot() + + header, err := types.NewHeader(common.EmptyHash, root, common.EmptyHash, big.NewInt(1), nil) + require.NoError(t, err) + specifcBlock := types.NewBlock(header, nil) + + blockStateMock := new(mocks.MockBlockState) + blockStateMock.On("GetBlockStateRoot", specifcBlock.Header.Hash()).Return(root, nil) + + storageMock := new(mocks.MockStorageState) + storageMock.On("TrieState", &root).Return(ts, nil) + + svc := new(Service) + svc.storageState = storageMock + svc.blockState = blockStateMock + + block, proof, err := svc.GetReadProofAt(specifcBlock.Header.Hash(), keysToProof) + require.NoError(t, err) + + blockStateMock.AssertCalled(t, "GetBlockStateRoot", specifcBlock.Header.Hash()) + storageMock.AssertCalled(t, "TrieState", &root) + require.Equal(t, specifcBlock.Header.Hash(), block) + + // all the proof must exists in the expected + require.Equal(t, len(expectedProof), len(proof)) + for _, p := range proof { + _, has := expectedProof[p] + require.True(t, has) + } +} diff --git a/dot/rpc/modules/api.go b/dot/rpc/modules/api.go index 1184b13251..d26d6e5719 100644 --- a/dot/rpc/modules/api.go +++ b/dot/rpc/modules/api.go @@ -84,6 +84,7 @@ type CoreAPI interface { GetMetadata(bhash *common.Hash) ([]byte, error) QueryStorage(from, to common.Hash, keys ...string) (map[common.Hash]core.QueryKeyValueChanges, error) DecodeSessionKeys(enc []byte) ([]byte, error) + GetReadProofAt(block common.Hash, keys []common.Hash) (common.Hash, []string, error) } // RPCAPI is the interface for methods related to RPC service diff --git a/dot/rpc/modules/mocks/core_api.go b/dot/rpc/modules/mocks/core_api.go index da382dbda6..df1d45ef4e 100644 --- a/dot/rpc/modules/mocks/core_api.go +++ b/dot/rpc/modules/mocks/core_api.go @@ -66,6 +66,38 @@ func (_m *MockCoreAPI) GetMetadata(bhash *common.Hash) ([]byte, error) { return r0, r1 } +// GetReadProofAt provides a mock function with given fields: block, keys +func (_m *MockCoreAPI) GetReadProofAt(block common.Hash, keys []common.Hash) (common.Hash, []string, error) { + ret := _m.Called(block, keys) + + var r0 common.Hash + if rf, ok := ret.Get(0).(func(common.Hash, []common.Hash) common.Hash); ok { + r0 = rf(block, keys) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(common.Hash) + } + } + + var r1 []string + if rf, ok := ret.Get(1).(func(common.Hash, []common.Hash) []string); ok { + r1 = rf(block, keys) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).([]string) + } + } + + var r2 error + if rf, ok := ret.Get(2).(func(common.Hash, []common.Hash) error); ok { + r2 = rf(block, keys) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + // GetRuntimeVersion provides a mock function with given fields: bhash func (_m *MockCoreAPI) GetRuntimeVersion(bhash *common.Hash) (runtime.Version, error) { ret := _m.Called(bhash) diff --git a/dot/rpc/modules/state.go b/dot/rpc/modules/state.go index f301b4e8ac..44962d1c2b 100644 --- a/dot/rpc/modules/state.go +++ b/dot/rpc/modules/state.go @@ -28,6 +28,12 @@ import ( "github.com/ChainSafe/gossamer/pkg/scale" ) +//StateGetReadProofRequest json fields +type StateGetReadProofRequest struct { + Keys []common.Hash + Hash common.Hash +} + // StateCallRequest holds json fields type StateCallRequest struct { Method string `json:"method"` @@ -128,6 +134,12 @@ type StateStorageKeysResponse []string //TODO: Determine actual type type StateMetadataResponse string +//StateGetReadProofResponse holds the response format +type StateGetReadProofResponse struct { + At common.Hash `json:"at"` + Proof []string `json:"proof"` +} + // StorageChangeSetResponse is the struct that holds the block and changes type StorageChangeSetResponse struct { Block *common.Hash `json:"block"` @@ -279,6 +291,21 @@ func (sm *StateModule) GetMetadata(r *http.Request, req *StateRuntimeMetadataQue return err } +// GetReadProof returns the proof to the received storage keys +func (sm *StateModule) GetReadProof(r *http.Request, req *StateGetReadProofRequest, res *StateGetReadProofResponse) error { + block, proofs, err := sm.coreAPI.GetReadProofAt(req.Hash, req.Keys) + if err != nil { + return err + } + + *res = StateGetReadProofResponse{ + At: block, + Proof: proofs, + } + + return nil +} + // GetRuntimeVersion Get the runtime version at a given block. // If no block hash is provided, the latest version gets returned. // TODO currently only returns latest version, add functionality to lookup runtime by block hash (see issue #834) diff --git a/dot/rpc/modules/state_test.go b/dot/rpc/modules/state_test.go index 19a844d80e..a4fa8e4e23 100644 --- a/dot/rpc/modules/state_test.go +++ b/dot/rpc/modules/state_test.go @@ -476,6 +476,47 @@ func TestStateModule_GetKeysPaged(t *testing.T) { } } +func TestGetReadProof_WhenCoreAPIReturnsError(t *testing.T) { + coreAPIMock := new(mocks.MockCoreAPI) + coreAPIMock. + On("GetReadProofAt", mock.AnythingOfType("common.Hash"), mock.AnythingOfType("[]common.Hash")). + Return(common.EmptyHash, nil, errors.New("mocked error")) + + sm := new(StateModule) + sm.coreAPI = coreAPIMock + + req := &StateGetReadProofRequest{ + Keys: []common.Hash{}, + Hash: common.EmptyHash, + } + err := sm.GetReadProof(nil, req, nil) + require.Error(t, err, "mocked error") +} + +func TestGetReadProof_WhenReturnsProof(t *testing.T) { + expectedBlock := common.BytesToHash([]byte("random hash")) + expectedProof := []string{"proof-1", "proof-2"} + + coreAPIMock := new(mocks.MockCoreAPI) + coreAPIMock. + On("GetReadProofAt", mock.AnythingOfType("common.Hash"), mock.AnythingOfType("[]common.Hash")). + Return(expectedBlock, expectedProof, nil) + + sm := new(StateModule) + sm.coreAPI = coreAPIMock + + req := &StateGetReadProofRequest{ + Keys: []common.Hash{}, + Hash: common.EmptyHash, + } + + res := new(StateGetReadProofResponse) + err := sm.GetReadProof(nil, req, res) + require.NoError(t, err) + require.Equal(t, res.At, expectedBlock) + require.Equal(t, res.Proof, expectedProof) +} + func setupStateModule(t *testing.T) (*StateModule, *common.Hash, *common.Hash) { // setup service net := newNetworkService(t) diff --git a/dot/state/block.go b/dot/state/block.go index 26a5114d5c..9875ec08db 100644 --- a/dot/state/block.go +++ b/dot/state/block.go @@ -555,6 +555,16 @@ func (bs *BlockState) BestBlockStateRoot() (common.Hash, error) { return header.StateRoot, nil } +// GetBlockStateRoot returns the state root of the given block hash +func (bs *BlockState) GetBlockStateRoot(bhash common.Hash) (common.Hash, error) { + header, err := bs.GetHeader(bhash) + if err != nil { + return common.EmptyHash, err + } + + return header.StateRoot, nil +} + // BestBlockNumber returns the block number of the current head of the chain func (bs *BlockState) BestBlockNumber() (*big.Int, error) { header, err := bs.GetHeader(bs.BestBlockHash()) diff --git a/lib/trie/proof.go b/lib/trie/proof.go index 537b4e7229..3d5a9beed6 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" - "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/lib/common" ) @@ -35,42 +34,46 @@ var ( // GenerateProof constructs the merkle-proof for key. The result contains all encoded nodes // on the path to the key. Returns the amount of nodes of the path and error if could not found the key -func (t *Trie) GenerateProof(key []byte, db chaindb.Writer) (int, error) { - key = keyToNibbles(key) - if len(key) == 0 { - return 0, ErrEmptyNibbles - } - +func (t *Trie) GenerateProof(keys [][]byte) (map[string][]byte, error) { var nodes []node - currNode := t.root -proveLoop: - for { - switch n := currNode.(type) { - case nil: - return 0, errors.New("no more paths to follow") + for _, k := range keys { + currNode := t.root - case *leaf: - nodes = append(nodes, n) + nk := keyToNibbles(k) + if len(nk) == 0 { + return nil, ErrEmptyNibbles + } - if bytes.Equal(n.key, key) { - break proveLoop - } + proveLoop: + for { + switch n := currNode.(type) { + case nil: + return nil, errors.New("no more paths to follow") - return 0, errors.New("leaf node doest not match the key") + case *leaf: + nodes = append(nodes, n) - case *branch: - nodes = append(nodes, n) - if bytes.Equal(n.key, key) || len(key) == 0 { - break proveLoop - } + if bytes.Equal(n.key, nk) { + break proveLoop + } - length := lenCommonPrefix(n.key, key) - currNode = n.children[key[length]] - key = key[length+1:] + return nil, errors.New("leaf node doest not match the key") + + case *branch: + nodes = append(nodes, n) + if bytes.Equal(n.key, nk) || len(nk) == 0 { + break proveLoop + } + + length := lenCommonPrefix(n.key, nk) + currNode = n.children[nk[length]] + nk = nk[length+1:] + } } } + proof := make(map[string][]byte) for _, n := range nodes { var ( hashNode []byte @@ -79,32 +82,29 @@ proveLoop: ) if encHashNode, hashNode, err = n.encodeAndHash(); err != nil { - return 0, fmt.Errorf("problems while encoding and hashing the node: %w", err) + return nil, fmt.Errorf("problems while encoding and hashing the node: %w", err) } - if err = db.Put(hashNode, encHashNode); err != nil { - return len(nodes), err - } + // avoid duplicate hashes + proof[common.BytesToHex(hashNode)] = encHashNode } - return len(nodes), nil + return proof, nil } // VerifyProof checks merkle proofs given an proof -func VerifyProof(rootHash common.Hash, key []byte, db chaindb.Reader) (bool, error) { +func VerifyProof(rootHash common.Hash, key []byte, proof map[string][]byte) (bool, error) { key = keyToNibbles(key) if len(key) == 0 { return false, ErrEmptyNibbles } - var wantedHash []byte - wantedHash = rootHash.ToBytes() + var wantedHash string + wantedHash = common.BytesToHex(rootHash.ToBytes()) for { - enc, err := db.Get(wantedHash) - if errors.Is(err, chaindb.ErrKeyNotFound) { - return false, nil - } else if err != nil { + enc, ok := proof[wantedHash] + if !ok { return false, nil } @@ -138,7 +138,7 @@ func VerifyProof(rootHash common.Hash, key []byte, db chaindb.Reader) (bool, err } key = key[length+1:] - wantedHash = next.getHash() + wantedHash = common.BytesToHex(next.getHash()) } } } diff --git a/lib/trie/proof_test.go b/lib/trie/proof_test.go index 8909246b72..047e228f15 100644 --- a/lib/trie/proof_test.go +++ b/lib/trie/proof_test.go @@ -17,72 +17,27 @@ package trie import ( - crand "crypto/rand" - "io/ioutil" + "fmt" "math/rand" - "os" - "sync" "testing" - "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/lib/common" "github.com/stretchr/testify/require" ) -func inMemoryChainDB(t *testing.T) (*chaindb.BadgerDB, func()) { - t.Helper() - - tmpdir, err := ioutil.TempDir("", "trie-chaindb-*") - require.NoError(t, err) - - db, err := chaindb.NewBadgerDB(&chaindb.Config{ - InMemory: true, - DataDir: tmpdir, - }) - require.NoError(t, err) - - clear := func() { - err = db.Close() - require.NoError(t, err) - - err = os.RemoveAll(tmpdir) - require.NoError(t, err) - } - - return db, clear -} - func TestVerifyProof(t *testing.T) { - trie, entries := randomTrie(t, 200) + trie, entries := RandomTrieTest(t, 200) root, err := trie.Hash() require.NoError(t, err) - amount := make(chan struct{}, 15) - wg := new(sync.WaitGroup) - for _, entry := range entries { - wg.Add(1) - go func(kv *kv) { - defer func() { - wg.Done() - <-amount - }() - - amount <- struct{}{} - - proof, clear := inMemoryChainDB(t) - defer clear() - - _, err := trie.GenerateProof(kv.k, proof) - require.NoError(t, err) - v, err := VerifyProof(root, kv.k, proof) + proof, err := trie.GenerateProof([][]byte{entry.K}) + require.NoError(t, err) - require.NoError(t, err) - require.True(t, v) - }(entry) + v, err := VerifyProof(root, entry.K, proof) + require.NoError(t, err) + require.True(t, v) } - - wg.Wait() } func TestVerifyProofOneElement(t *testing.T) { @@ -93,10 +48,8 @@ func TestVerifyProofOneElement(t *testing.T) { rootHash, err := trie.Hash() require.NoError(t, err) - proof, clear := inMemoryChainDB(t) - defer clear() - - _, err = trie.GenerateProof(key, proof) + proof, err := trie.GenerateProof([][]byte{key}) + fmt.Println(proof) require.NoError(t, err) val, err := VerifyProof(rootHash, key, proof) @@ -106,50 +59,40 @@ func TestVerifyProofOneElement(t *testing.T) { } func TestVerifyProof_BadProof(t *testing.T) { - trie, entries := randomTrie(t, 200) + trie, entries := RandomTrieTest(t, 200) rootHash, err := trie.Hash() require.NoError(t, err) - amount := make(chan struct{}, 15) - wg := new(sync.WaitGroup) - for _, entry := range entries { - wg.Add(1) + proof, err := trie.GenerateProof([][]byte{entry.K}) + require.Greater(t, len(proof), 0) + require.NoError(t, err) - go func(kv *kv) { - defer func() { - wg.Done() - <-amount - }() + i := 0 + d := rand.Intn(len(proof)) - amount <- struct{}{} - proof, clear := inMemoryChainDB(t) - defer clear() + var toTamper string + for k := range proof { + if i < d { + i++ + continue + } - nLen, err := trie.GenerateProof(kv.k, proof) - require.Greater(t, nLen, 0) - require.NoError(t, err) + toTamper = k + break + } - it := proof.NewIterator() - for i, d := 0, rand.Intn(nLen); i <= d; i++ { - it.Next() - } - key := it.Key() - val, _ := proof.Get(key) - proof.Del(key) - it.Release() - - newhash, err := common.Keccak256(val) - require.NoError(t, err) - proof.Put(newhash.ToBytes(), val) - - v, err := VerifyProof(rootHash, kv.k, proof) - require.NoError(t, err) - require.False(t, v) - }(entry) - } + val := proof[toTamper] + delete(proof, toTamper) + + newhash, err := common.Keccak256(val) + require.NoError(t, err) + proof[common.BytesToHex(newhash.ToBytes())] = val - wg.Wait() + v, err := VerifyProof(rootHash, entry.K, proof) + require.NoError(t, err) + require.False(t, v) + } } func TestGenerateProofMissingKey(t *testing.T) { @@ -163,16 +106,13 @@ func TestGenerateProofMissingKey(t *testing.T) { trie.Put(chieldKey, chieldValue) trie.Put(gransonKey, gransonValue) - proof, clear := inMemoryChainDB(t) - defer clear() - searchfor := make([]byte, len(gransonKey)) copy(searchfor[:], gransonKey[:]) // keep the path til the key but modify the last element searchfor[len(searchfor)-1] = searchfor[len(searchfor)-1] + byte(0xff) - _, err := trie.GenerateProof(searchfor, proof) + _, err := trie.GenerateProof([][]byte{searchfor}) require.Error(t, err, "leaf node doest not match the key") } @@ -187,9 +127,6 @@ func TestGenerateProofNoMorePathToFollow(t *testing.T) { trie.Put(chieldKey, chieldValue) trie.Put(gransonKey, gransonValue) - proof, clear := inMemoryChainDB(t) - defer clear() - searchfor := make([]byte, len(parentKey)) copy(searchfor[:], parentKey[:]) @@ -197,36 +134,10 @@ func TestGenerateProofNoMorePathToFollow(t *testing.T) { // value and the branch node will no be able to found the right slot searchfor[20] = searchfor[20] + byte(0xff) - _, err := trie.GenerateProof(searchfor, proof) + _, err := trie.GenerateProof([][]byte{searchfor}) require.Error(t, err, "no more paths to follow") } -type kv struct { - k []byte - v []byte -} - -func randomTrie(t *testing.T, n int) (*Trie, map[string]*kv) { - t.Helper() - - trie := NewEmptyTrie() - vals := make(map[string]*kv) - - for i := 0; i < n; i++ { - v := &kv{randBytes(32), randBytes(20)} - trie.Put(v.k, v.v) - vals[string(v.k)] = v - } - - return trie, vals -} - -func randBytes(n int) []byte { - r := make([]byte, n) - crand.Read(r) - return r -} - func modifyLastBytes(b []byte) []byte { newB := make([]byte, len(b)) copy(newB[:], b) diff --git a/lib/trie/test_utils.go b/lib/trie/test_utils.go index 80e262244f..6cabd20483 100644 --- a/lib/trie/test_utils.go +++ b/lib/trie/test_utils.go @@ -1,6 +1,7 @@ package trie import ( + crand "crypto/rand" "encoding/binary" "math/rand" "testing" @@ -67,3 +68,29 @@ func generateRandomTest(t testing.TB, kv map[string][]byte) Test { } } } + +type KV struct { + K []byte + V []byte +} + +func RandomTrieTest(t *testing.T, n int) (*Trie, map[string]*KV) { + t.Helper() + + trie := NewEmptyTrie() + vals := make(map[string]*KV) + + for i := 0; i < n; i++ { + v := &KV{randBytes(32), randBytes(20)} + trie.Put(v.K, v.V) + vals[string(v.K)] = v + } + + return trie, vals +} + +func randBytes(n int) []byte { + r := make([]byte, n) + crand.Read(r) + return r +} From 81605927ffd32520b20a71641fa19c6e196b8c47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 24 Aug 2021 20:27:13 -0400 Subject: [PATCH 05/38] chore: fix unused import --- dot/core/service_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dot/core/service_test.go b/dot/core/service_test.go index d121d7c049..62d2786cd6 100644 --- a/dot/core/service_test.go +++ b/dot/core/service_test.go @@ -26,7 +26,6 @@ import ( "time" "github.com/ChainSafe/gossamer/dot/core/mocks" - coremocks "github.com/ChainSafe/gossamer/dot/core/mocks" "github.com/ChainSafe/gossamer/dot/network" "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/sync" @@ -108,7 +107,7 @@ func TestStartService(t *testing.T) { } func TestAnnounceBlock(t *testing.T) { - net := new(coremocks.MockNetwork) + net := new(mocks.MockNetwork) cfg := &Config{ Network: net, } From 8d1b73befacad1ace32251d38fa54538bbf0eb88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 24 Aug 2021 20:59:11 -0400 Subject: [PATCH 06/38] chore: fix lint warnings --- lib/trie/test_utils.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/trie/test_utils.go b/lib/trie/test_utils.go index 78d16a2346..d60ad6577f 100644 --- a/lib/trie/test_utils.go +++ b/lib/trie/test_utils.go @@ -72,11 +72,13 @@ func generateRandomTest(t testing.TB, kv map[string][]byte) Test { } } +// KV helps to export the entries of the generated keys type KV struct { K []byte V []byte } +// RandomTrieTest generate a trie with random entries func RandomTrieTest(t *testing.T, n int) (*Trie, map[string]*KV) { t.Helper() @@ -94,6 +96,6 @@ func RandomTrieTest(t *testing.T, n int) (*Trie, map[string]*KV) { func randBytes(n int) []byte { r := make([]byte, n) - rand.Read(r) + rand.Read(r) //nolint return r } From 30c45811df22f421c98637a0e953e341756d00b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 7 Sep 2021 15:55:25 -0400 Subject: [PATCH 07/38] wip: proof generation algotithm --- lib/trie/lookup.go | 89 ++++++++++++++++++++++++++++++++++++++++++ lib/trie/node.go | 59 ++++++++++++++++++++++++++++ lib/trie/proof.go | 17 ++++++++ lib/trie/proof_test.go | 6 +++ lib/trie/recorder.go | 40 +++++++++++++++++++ 5 files changed, 211 insertions(+) create mode 100644 lib/trie/lookup.go create mode 100644 lib/trie/recorder.go diff --git a/lib/trie/lookup.go b/lib/trie/lookup.go new file mode 100644 index 0000000000..36bd7aa2f4 --- /dev/null +++ b/lib/trie/lookup.go @@ -0,0 +1,89 @@ +package trie + +import ( + "bytes" + "errors" + + "github.com/ChainSafe/chaindb" + "github.com/ChainSafe/gossamer/lib/common" +) + +var ( + ErrInvalidStateRoot = errors.New("cannot found the state root on storage") + ErrIncompleteDatabase = errors.New("cannot found the node hash on storage") +) + +type Lookup struct { + recorder *NodeRecorder + // root to start the lookup + hash common.Hash + db chaindb.Database +} + +// NewLookup returns a Lookup to helps the proof generator +func NewLookup(h common.Hash, db chaindb.Database, r *NodeRecorder) *Lookup { + return &Lookup{ + db: db, + hash: h, + recorder: r, + } +} + +// Find will return the desired value or nil if key cannot be found and will record visited nodes +func (l *Lookup) Find(nKeys []byte) ([]byte, error) { + partial := nKeys + hash := l.hash + + var depth uint32 + + for { + nodeData, err := l.db.Get(hash[:]) + if err != nil && depth == 0 { + return nil, ErrInvalidStateRoot + } else if err != nil { + return nil, ErrIncompleteDatabase + } + + l.recorder.Record(hash, nodeData, depth) + + decoded, err := decodeBytes(nodeData) + if err != nil { + return nil, err + } + + switch currNode := decoded.(type) { + case nil: + return nil, nil + + case *leaf: + if bytes.Equal(currNode.key, partial) { + return currNode.value, nil + } + return nil, nil + + case *branch: + switch len(partial) { + case 0: + return currNode.value, nil + default: + if !bytes.HasPrefix(partial, currNode.key) { + return nil, nil + } + + if bytes.Equal(partial, currNode.key) { + return currNode.value, nil + } + + switch child := currNode.children[partial[0]].(type) { + case nil: + return nil, nil + default: + partial = partial[1:] + copy(hash[:], child.getHash()) + } + } + } + + depth++ + } +} diff --git a/lib/trie/node.go b/lib/trie/node.go index 3e1f091f02..dffd9a5ca2 100644 --- a/lib/trie/node.go +++ b/lib/trie/node.go @@ -288,6 +288,65 @@ func decodeBytes(in []byte) (node, error) { return decode(r) } +type nodeHeaderType string + +const ( + LeafPlan nodeHeaderType = "leaf" + BranchPlan nodeHeaderType = "branch" + + LeafPrefix = 0b01 << 6 + BranchPrefxi = 0b11 << 6 +) + +var ( + ErrNilDecodedHeader = errors.New("decoded header with empty trie") +) + +func decodeNodeHeader(r io.Reader) (nodeHeaderType, uint32, error) { + b, err := readByte(r) + if err != nil { + return "", 0, err + } + + if b == 0 { + return "", 0, ErrNilDecodedHeader + } + + switch b & (0b11 << 6) { + case LeafPrefix: + s, err := decodeSize(b, r) + return LeafPlan, s, err + case BranchPrefxi: + s, err := decodeSize(b, r) + return BranchPlan, s, err + default: + return "", 0, errors.New("unknown type of node") + } +} + +func decodeSize(f byte, r io.Reader) (uint32, error) { + res := uint32(f & 0xff >> 2) + if res < 63 { + return res, nil + } + + res -= 1 + for res <= uint32((^uint16(0))) { + n, err := readByte(r) + if err != nil { + return 0, err + } + + if n < 255 { + return res + uint32(n+1), nil + } + + res += 255 + } + + return 0, errors.New("size limit reached for a nibble slice") +} + // Decode wraps the decoding of different node types back into a node func decode(r io.Reader) (node, error) { header, err := readByte(r) diff --git a/lib/trie/proof.go b/lib/trie/proof.go index 3d5a9beed6..e684802d5e 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" + "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/lib/common" ) @@ -32,6 +33,22 @@ var ( ErrEmptyNibbles = errors.New("empty nibbles provided from key") ) +func GenerateProofWithRecorder(root common.Hash, keys [][]byte, db chaindb.Database) ([]byte, error) { + for _, k := range keys { + rec := NewRecoder() + lookup := NewLookup(root, db, rec) + + v, err := lookup.Find(k) + if err != nil { + return nil, err + } + + println(v) + } + + return nil, nil +} + // GenerateProof constructs the merkle-proof for key. The result contains all encoded nodes // on the path to the key. Returns the amount of nodes of the path and error if could not found the key func (t *Trie) GenerateProof(keys [][]byte) (map[string][]byte, error) { diff --git a/lib/trie/proof_test.go b/lib/trie/proof_test.go index 047e228f15..7bf13b6126 100644 --- a/lib/trie/proof_test.go +++ b/lib/trie/proof_test.go @@ -21,10 +21,16 @@ import ( "math/rand" "testing" + "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/lib/common" "github.com/stretchr/testify/require" ) +func TestProofGenerationWithRecorder(t *testing.T) { + db, err := chaindb.NewBadgerDB(&chaindb.Config{InMemory: true}) + require.NoError(t, err) +} + func TestVerifyProof(t *testing.T) { trie, entries := RandomTrieTest(t, 200) root, err := trie.Hash() diff --git a/lib/trie/recorder.go b/lib/trie/recorder.go new file mode 100644 index 0000000000..c9dc232aeb --- /dev/null +++ b/lib/trie/recorder.go @@ -0,0 +1,40 @@ +package trie + +import "github.com/ChainSafe/gossamer/lib/common" + +// NodeRecord represets a record of a visited node +type NodeRecord struct { + Depth uint32 + RawData []byte + Hash common.Hash +} + +// NodeRecorder records trie nodes as they pass it +type NodeRecorder struct { + Nodes []NodeRecord + MinDepth uint32 +} + +// Record a visited node +func (r *NodeRecorder) Record(h common.Hash, rd []byte, depth uint32) { + if depth >= r.MinDepth { + r.Nodes = append(r.Nodes, NodeRecord{ + Depth: depth, + RawData: rd, + Hash: h, + }) + } +} + +// RecorderWithDepth create a NodeRecorder which only records nodes beyond a given depth +func NewRecorderWithDepth(d uint32) *NodeRecorder { + return &NodeRecorder{ + MinDepth: d, + Nodes: []NodeRecord{}, + } +} + +// NewRecoder create a NodeRecorder which records all given nodes +func NewRecoder() *NodeRecorder { + return NewRecorderWithDepth(0) +} From 58c73bb4f7315f7f8a62ba60037115486e37a796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 14 Sep 2021 10:47:22 -0400 Subject: [PATCH 08/38] wip: append not working --- dot/core/interface.go | 2 + dot/core/mocks/storage_state.go | 17 ++ dot/core/service.go | 28 +- dot/core/service_test.go | 114 -------- dot/rpc/modules/api.go | 2 +- dot/rpc/modules/mocks/core_api.go | 12 +- dot/rpc/modules/state.go | 21 +- dot/rpc/modules/state_test.go | 4 +- dot/state/storage.go | 4 + go.mod | 3 +- go.sum | 13 +- lib/.DS_Store | Bin 0 -> 6148 bytes lib/trie/lookup.go | 54 ++-- lib/trie/node.go | 7 + lib/trie/proof.go | 438 +++++++++++++++++++++++++++++- lib/trie/proof_test.go | 32 ++- lib/trie/recorder.go | 48 ++-- 17 files changed, 582 insertions(+), 217 deletions(-) create mode 100644 lib/.DS_Store diff --git a/dot/core/interface.go b/dot/core/interface.go index 67b1dc6a14..a71b198ae8 100644 --- a/dot/core/interface.go +++ b/dot/core/interface.go @@ -19,6 +19,7 @@ package core import ( "math/big" + "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/dot/network" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" @@ -62,6 +63,7 @@ type StorageState interface { StoreTrie(*rtstorage.TrieState, *types.Header) error GetStateRootFromBlock(bhash *common.Hash) (*common.Hash, error) GetStorage(root *common.Hash, key []byte) ([]byte, error) + ExposeDB() chaindb.Database } // TransactionState is the interface for transaction state methods diff --git a/dot/core/mocks/storage_state.go b/dot/core/mocks/storage_state.go index 8c7077ce75..4497b2b784 100644 --- a/dot/core/mocks/storage_state.go +++ b/dot/core/mocks/storage_state.go @@ -3,6 +3,7 @@ package mocks import ( + chaindb "github.com/ChainSafe/chaindb" common "github.com/ChainSafe/gossamer/lib/common" mock "github.com/stretchr/testify/mock" @@ -17,6 +18,22 @@ type MockStorageState struct { mock.Mock } +// ExposeDB provides a mock function with given fields: +func (_m *MockStorageState) ExposeDB() chaindb.Database { + ret := _m.Called() + + var r0 chaindb.Database + if rf, ok := ret.Get(0).(func() chaindb.Database); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(chaindb.Database) + } + } + + return r0 +} + // GetStateRootFromBlock provides a mock function with given fields: bhash func (_m *MockStorageState) GetStateRootFromBlock(bhash *common.Hash) (*common.Hash, error) { ret := _m.Called(bhash) diff --git a/dot/core/service.go b/dot/core/service.go index 451f92eb21..9a10d9d960 100644 --- a/dot/core/service.go +++ b/dot/core/service.go @@ -21,6 +21,7 @@ import ( "os" "sync" + "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/dot/network" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/blocktree" @@ -622,8 +623,8 @@ func (s *Service) tryQueryStorage(block common.Hash, keys ...string) (QueryKeyVa // GetReadProofAt will return an array with the proofs for the keys passed as params // based on the block hash passed as param as well, if block hash is nil then the current state will take place -func (s *Service) GetReadProofAt(block common.Hash, keys []common.Hash) (common.Hash, []string, error) { - if block == common.EmptyHash { +func (s *Service) GetReadProofAt(block common.Hash, keys [][]byte) (common.Hash, [][]byte, error) { + if len(block) == 0 { block = s.blockState.BestBlockHash() } @@ -632,12 +633,7 @@ func (s *Service) GetReadProofAt(block common.Hash, keys []common.Hash) (common. return common.EmptyHash, nil, err } - ts, err := s.storageState.TrieState(&stateRoot) - if err != nil { - return common.EmptyHash, nil, err - } - - proofForKeys, err := readProofForKeys(ts.Trie(), keys) + proofForKeys, err := readProofForKeys(stateRoot[:], keys, s.storageState.ExposeDB()) if err != nil { return common.EmptyHash, nil, err } @@ -647,21 +643,11 @@ func (s *Service) GetReadProofAt(block common.Hash, keys []common.Hash) (common. // readProofForKeys will go through the keys and generate the proof for each of them // and merge the result into a string array containing the hashes in the hexadecimal format -func readProofForKeys(t *trie.Trie, keys []common.Hash) ([]string, error) { - storageKeys := make([][]byte, len(keys)) - for i, k := range keys { - storageKeys[i] = k.ToBytes() - } - - proof, err := t.GenerateProof(storageKeys) +func readProofForKeys(root []byte, keys [][]byte, db chaindb.Database) ([][]byte, error) { + proof, err := trie.GenerateProofWithRecorder(root, keys, db) if err != nil { return nil, err } - proofSlice := make([]string, 0, len(proof)) - for k := range proof { - proofSlice = append(proofSlice, k) - } - - return proofSlice, nil + return proof, nil } diff --git a/dot/core/service_test.go b/dot/core/service_test.go index 62d2786cd6..8fcd3d736e 100644 --- a/dot/core/service_test.go +++ b/dot/core/service_test.go @@ -818,117 +818,3 @@ func TestDecodeSessionKeys_WhenGetRuntimeReturnError(t *testing.T) { require.Error(t, err, "problems") require.Nil(t, b) } - -func TestGetReadProofAt_WhenReceivedBlockIsEmpty(t *testing.T) { - tr, entries := trie.RandomTrieTest(t, 50) - require.Len(t, entries, 50) - - i := 0 - var keysToProof []common.Hash - var keys [][]byte - - // get the last 2 entries - for _, entry := range entries { - if i < len(entries)-2 { - i++ - continue - } - - keysToProof = append(keysToProof, common.BytesToHash(entry.K)) - keys = append(keys, entry.K) - } - require.Len(t, keysToProof, 2) - - expectedProof, err := tr.GenerateProof(keys) - require.NoError(t, err) - - ts, err := storage.NewTrieState(tr) - require.NoError(t, err) - root := ts.MustRoot() - - header, err := types.NewHeader(common.EmptyHash, root, common.EmptyHash, big.NewInt(1), nil) - require.NoError(t, err) - bestBlock := types.NewBlock(header, nil) - - blockStateMock := new(mocks.MockBlockState) - blockStateMock.On("BestBlockHash").Return(bestBlock.Header.Hash()) - blockStateMock.On("GetBlockStateRoot", bestBlock.Header.Hash()).Return(root, nil) - - storageMock := new(mocks.MockStorageState) - storageMock.On("TrieState", &root).Return(ts, nil) - - svc := new(Service) - svc.storageState = storageMock - svc.blockState = blockStateMock - - block, proof, err := svc.GetReadProofAt(common.EmptyHash, keysToProof) - require.NoError(t, err) - - blockStateMock.AssertCalled(t, "BestBlockHash") - blockStateMock.AssertCalled(t, "GetBlockStateRoot", bestBlock.Header.Hash()) - storageMock.AssertCalled(t, "TrieState", &root) - require.Equal(t, bestBlock.Header.Hash(), block) - - // all the proof must exists in the expected - require.Equal(t, len(expectedProof), len(proof)) - for _, p := range proof { - _, has := expectedProof[p] - require.True(t, has) - } -} - -func TestGetReadProofAt_OnSpecificBlock(t *testing.T) { - tr, entries := trie.RandomTrieTest(t, 50) - require.Len(t, entries, 50) - - i := 0 - var keysToProof []common.Hash - var keys [][]byte - - // get the last 2 entries - for _, entry := range entries { - if i < len(entries)-2 { - i++ - continue - } - - keysToProof = append(keysToProof, common.BytesToHash(entry.K)) - keys = append(keys, entry.K) - } - require.Len(t, keysToProof, 2) - - expectedProof, err := tr.GenerateProof(keys) - require.NoError(t, err) - - ts, err := storage.NewTrieState(tr) - require.NoError(t, err) - root := ts.MustRoot() - - header, err := types.NewHeader(common.EmptyHash, root, common.EmptyHash, big.NewInt(1), nil) - require.NoError(t, err) - specifcBlock := types.NewBlock(header, nil) - - blockStateMock := new(mocks.MockBlockState) - blockStateMock.On("GetBlockStateRoot", specifcBlock.Header.Hash()).Return(root, nil) - - storageMock := new(mocks.MockStorageState) - storageMock.On("TrieState", &root).Return(ts, nil) - - svc := new(Service) - svc.storageState = storageMock - svc.blockState = blockStateMock - - block, proof, err := svc.GetReadProofAt(specifcBlock.Header.Hash(), keysToProof) - require.NoError(t, err) - - blockStateMock.AssertCalled(t, "GetBlockStateRoot", specifcBlock.Header.Hash()) - storageMock.AssertCalled(t, "TrieState", &root) - require.Equal(t, specifcBlock.Header.Hash(), block) - - // all the proof must exists in the expected - require.Equal(t, len(expectedProof), len(proof)) - for _, p := range proof { - _, has := expectedProof[p] - require.True(t, has) - } -} diff --git a/dot/rpc/modules/api.go b/dot/rpc/modules/api.go index d26d6e5719..e880970729 100644 --- a/dot/rpc/modules/api.go +++ b/dot/rpc/modules/api.go @@ -84,7 +84,7 @@ type CoreAPI interface { GetMetadata(bhash *common.Hash) ([]byte, error) QueryStorage(from, to common.Hash, keys ...string) (map[common.Hash]core.QueryKeyValueChanges, error) DecodeSessionKeys(enc []byte) ([]byte, error) - GetReadProofAt(block common.Hash, keys []common.Hash) (common.Hash, []string, error) + GetReadProofAt(block common.Hash, keys [][]byte) (common.Hash, [][]byte, error) } // RPCAPI is the interface for methods related to RPC service diff --git a/dot/rpc/modules/mocks/core_api.go b/dot/rpc/modules/mocks/core_api.go index df1d45ef4e..fe20861ab2 100644 --- a/dot/rpc/modules/mocks/core_api.go +++ b/dot/rpc/modules/mocks/core_api.go @@ -67,11 +67,11 @@ func (_m *MockCoreAPI) GetMetadata(bhash *common.Hash) ([]byte, error) { } // GetReadProofAt provides a mock function with given fields: block, keys -func (_m *MockCoreAPI) GetReadProofAt(block common.Hash, keys []common.Hash) (common.Hash, []string, error) { +func (_m *MockCoreAPI) GetReadProofAt(block common.Hash, keys [][]byte) (common.Hash, [][]byte, error) { ret := _m.Called(block, keys) var r0 common.Hash - if rf, ok := ret.Get(0).(func(common.Hash, []common.Hash) common.Hash); ok { + if rf, ok := ret.Get(0).(func(common.Hash, [][]byte) common.Hash); ok { r0 = rf(block, keys) } else { if ret.Get(0) != nil { @@ -79,17 +79,17 @@ func (_m *MockCoreAPI) GetReadProofAt(block common.Hash, keys []common.Hash) (co } } - var r1 []string - if rf, ok := ret.Get(1).(func(common.Hash, []common.Hash) []string); ok { + var r1 [][]byte + if rf, ok := ret.Get(1).(func(common.Hash, [][]byte) [][]byte); ok { r1 = rf(block, keys) } else { if ret.Get(1) != nil { - r1 = ret.Get(1).([]string) + r1 = ret.Get(1).([][]byte) } } var r2 error - if rf, ok := ret.Get(2).(func(common.Hash, []common.Hash) error); ok { + if rf, ok := ret.Get(2).(func(common.Hash, [][]byte) error); ok { r2 = rf(block, keys) } else { r2 = ret.Error(2) diff --git a/dot/rpc/modules/state.go b/dot/rpc/modules/state.go index 44962d1c2b..6110d5dc49 100644 --- a/dot/rpc/modules/state.go +++ b/dot/rpc/modules/state.go @@ -30,7 +30,7 @@ import ( //StateGetReadProofRequest json fields type StateGetReadProofRequest struct { - Keys []common.Hash + Keys []string Hash common.Hash } @@ -293,14 +293,29 @@ func (sm *StateModule) GetMetadata(r *http.Request, req *StateRuntimeMetadataQue // GetReadProof returns the proof to the received storage keys func (sm *StateModule) GetReadProof(r *http.Request, req *StateGetReadProofRequest, res *StateGetReadProofResponse) error { - block, proofs, err := sm.coreAPI.GetReadProofAt(req.Hash, req.Keys) + keys := make([][]byte, len(req.Keys)) + for i, hexKey := range req.Keys { + bKey, err := common.HexToBytes(hexKey) + if err != nil { + return err + } + + keys[i] = bKey + } + + block, proofs, err := sm.coreAPI.GetReadProofAt(req.Hash, keys) if err != nil { return err } + var decProof []string + for _, p := range proofs { + decProof = append(decProof, fmt.Sprintf("0x%x", p)) + } + *res = StateGetReadProofResponse{ At: block, - Proof: proofs, + Proof: decProof, } return nil diff --git a/dot/rpc/modules/state_test.go b/dot/rpc/modules/state_test.go index a4fa8e4e23..78c16e576d 100644 --- a/dot/rpc/modules/state_test.go +++ b/dot/rpc/modules/state_test.go @@ -486,7 +486,7 @@ func TestGetReadProof_WhenCoreAPIReturnsError(t *testing.T) { sm.coreAPI = coreAPIMock req := &StateGetReadProofRequest{ - Keys: []common.Hash{}, + Keys: []string{}, Hash: common.EmptyHash, } err := sm.GetReadProof(nil, req, nil) @@ -506,7 +506,7 @@ func TestGetReadProof_WhenReturnsProof(t *testing.T) { sm.coreAPI = coreAPIMock req := &StateGetReadProofRequest{ - Keys: []common.Hash{}, + Keys: []string{}, Hash: common.EmptyHash, } diff --git a/dot/state/storage.go b/dot/state/storage.go index fd26ac24c0..8da900b075 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -91,6 +91,10 @@ func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, }, nil } +func (s *StorageState) ExposeDB() chaindb.Database { + return s.db +} + // SetSyncing sets whether the node is currently syncing or not func (s *StorageState) SetSyncing(syncing bool) { s.syncing = syncing diff --git a/go.mod b/go.mod index 61d56ed401..9d22db6e33 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,8 @@ require ( golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 - golang.org/x/tools v0.1.5 // indirect + golang.org/x/text v0.3.7 // indirect + golang.org/x/tools v0.1.6-0.20210908145159-e5f719fbe6d5 // indirect google.golang.org/appengine v1.6.6 // indirect google.golang.org/protobuf v1.26.0-rc.1 honnef.co/go/tools v0.2.0 // indirect diff --git a/go.sum b/go.sum index 43d7cfe4cc..9d58898dbb 100644 --- a/go.sum +++ b/go.sum @@ -1089,7 +1089,7 @@ github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6Ut github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -1221,7 +1221,6 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1297,12 +1296,11 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1315,8 +1313,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1363,8 +1362,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210908145159-e5f719fbe6d5 h1:VsF0qTw/aGuPGfWp6z2XrkKtnBt6pKLSsCo2CWW1yL0= +golang.org/x/tools v0.1.6-0.20210908145159-e5f719fbe6d5/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/lib/.DS_Store b/lib/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..928bfd12fd6d1f50a5d054f2b6d98f7042bea4e7 GIT binary patch literal 6148 zcmeH~K?=e^3`G;|LU7Zi%h`AUZ!n0SpcinVh#(b;x}Kx^lL>;=wTS#c@+X-IrEk$` zL`2uO{Ys=2kr~`nmKH{)$Q!xJASXH8yP-c{`Xg#qpTt>P!&@2c$2NroNPq-LfCNb3 zhX~lc4V%rYGLirZkie6GeIF9sG>4X|{^>yQ5dhkt?1r_^63}D?Xbvq^QGsbp4;roN zV~Ewg9h%}?4lPw{yJ!p_8c$Z6VqjX^MH3R3W)}t$Ab}BqY0X=^|F`f@^Z%%YDG89k zpApbz({0vxskmF;UeD^gsM@-~p?)0U 0x%x\n", recNodes.Hash) + } + + recorder := Recorder(nodes) + + fmt.Printf("Found at database\n\tkey:0x%x\n\tvalue:0x%x\n", k, expectedValue) + fmt.Println("Recorded nodes", recorder.Len()) + + stackIter := stack.iter() + logger.Warn("generate proof", "stackIter.HasNext()", stackIter.HasNext()) + + // Skip over recorded nodes already on the stack + for stackIter.HasNext() { + nxtRec, nxtEntry := recorder.Peek(), stackIter.Peek() + if !bytes.Equal(nxtRec.Hash[:], nxtEntry.nodeHash) { + break + } + + stackIter.Next() + recorder.Next() + } + + for { + var step Step + + fmt.Println("stack len", len(stack)) + fmt.Println("proofs len", len(proofs)) + + if len(stack) == 0 { + // as the stack is empty then start from the root node + step = Step{ + Found: false, + NextNode: root, + KeyOffset: 0, + } + } else { + entryOnTop := stack.Last() + logger.Warn("generate proof", "has last on stack", entryOnTop != nil) + + step, err = matchKeyToNode(entryOnTop, nk) + if err != nil { + return nil, err + } + } + + if step.Found { + if len(step.Value) > 0 && bytes.Equal(step.Value, expectedValue) && recorder.Len() > 0 { + return nil, errors.New("value found is not expected or there is recNodes to traverse") + } + + break + } + + rec := recorder.Next() + if rec == nil { + return nil, errors.New("recorder must has nodes to traverse") + } + + logger.Warn("generate proof", "expected", fmt.Sprintf("Recorded node: 0x%x\n", rec.Hash)) + logger.Warn("generate proof", "got", fmt.Sprintf("Step node: 0x%x\n", step.NextNode)) + + if !bytes.Equal(rec.Hash, step.NextNode) { + return nil, errors.New("recorded node does not match expected node") + } + + n, err := decodeBytes(rec.RawData) + if err != nil { + return nil, err + } + logger.Warn("generate proof", "has decoded node", n != nil) + + outputIndex := len(proofs) + proofs = append(proofs, []byte{}) + + ne, err := NewStackEntry(n, rec.RawData, outputIndex, step.KeyOffset) + if err != nil { + return nil, err + } + + stack.Push(ne) + } + } + + unwindStack(&stack, proofs, nil) + return proofs, nil +} + +func matchKeyToNode(e *StackEntry, nibbleKey []byte) (Step, error) { + switch ntype := e.node.(type) { + case nil: + return Step{Found: true}, nil + case *leaf: + keyToCompare := nibbleKey[e.keyOffset:] + if bytes.Equal(keyToCompare, ntype.key) && len(nibbleKey) == len(ntype.key)+e.keyOffset { + return Step{ + Found: true, + Value: ntype.value, + }, nil + } + + return Step{Found: true}, nil + case *branch: + return matchKeyToBranchNode(ntype, e, nibbleKey) + default: + return Step{}, errors.New("could not be possible to define the node type") } +} - return nil, nil +func matchKeyToBranchNode(n *branch, e *StackEntry, nibbleKey []byte) (Step, error) { + keyToCompare := nibbleKey[e.keyOffset:] + + logger.Warn("matchKeyToBranchNode", "keyToCompare", fmt.Sprintf("0x%x\n", keyToCompare), "len nibbles", len(nibbleKey)) + logger.Warn("matchKeyToBranchNode", "node key", fmt.Sprintf("x%x\n", n.key), "key offset", e.keyOffset) + + if !bytes.HasPrefix(keyToCompare, n.key) { + return Step{Found: true}, nil + } + + if len(nibbleKey) == len(n.key)+e.keyOffset { + return Step{ + Found: true, + Value: n.value, + }, nil + } + + newIndex := nibbleKey[e.keyOffset+len(n.key)] + for e.childIndex < int(newIndex) { + nodeChild := n.children[e.childIndex] + if nodeChild != nil { + var err error + if _, e.children[e.childIndex], err = nodeChild.encodeAndHash(); err != nil { + return Step{}, err + } + } + e.childIndex++ + } + + child := n.children[e.childIndex] + logger.Warn("matchKeyToBranchNode", "has child", child != nil, "childIndex", e.childIndex) + + if child == nil { + return Step{Found: true}, nil + } + + _, hash, err := child.encodeAndHash() + if err != nil { + return Step{}, err + } + + return Step{ + Found: false, + KeyOffset: e.keyOffset + len(n.key) + 1, + NextNode: hash, + }, nil +} + +func unwindStack(stack *Stack, proof [][]byte, key []byte) error { + for { + entry := stack.Pop() + if entry == nil { + break + } + + if key != nil && bytes.HasPrefix(key, entry.path) { + stack.Push(entry) + break + } + + index := entry.outputIndex + + enc, err := entry.encodeNode() + if err != nil { + return err + } + + parent := stack.Last() + + if parent != nil { + parent.setChild(enc) + } + + proof[index] = enc + } + + return nil +} + +func branchEncode(size int, prefix byte, output []byte) { + l := make([]byte, 0) + l1, rem := 62, 0 + + if size < 62 { + l1 = size + l = append(l, prefix+byte(l1)) + } else { + l = append(l, prefix+byte(63)) + rem = size - l1 + } + + for { + if rem > 0 { + if rem < 256 { + result := rem - 1 + l = append(l, byte(result)) + break + } else { + op := rem - 255 + if op < 0 { + rem = 0 + } else { + rem = op + } + l = append(l, byte(255)) + } + } else { + break + } + } + + copy(output[:len(l)], l[:]) +} + +func prefixIterator(size int, prefix byte) []byte { + if NibbleSizeBound < uint16(size) { + size = NibblePerByte + } + + output := make([]byte, 0, 3+(size/NibblePerByte)) + branchEncode(size, prefix, output) + return output +} + +func branchNodeNibbled(path []byte, children [][]byte, value []byte) []byte { + var output []byte + if len(value) == 0 { + output = prefixIterator(len(path), BranchWithMask) + } else { + output = prefixIterator(len(path), BranchWithoutMask) + } + output = append(output, path...) + bitmapIndex := len(output) + bitmap := make([]byte, BitmapLength) + + for i := 0; i < BitmapLength; i++ { + output = append(output, 0) + } + + if len(value) > 0 { + output = append(output, value...) + } + + var ( + bitmapValue uint16 = 0 + bitmapCursor uint16 = 1 + ) + + for _, c := range children { + if len(c) > 0 { + output = append(output, c...) + bitmapValue |= bitmapCursor + } + + bitmapCursor <<= 1 + } + + bitmap[0] = byte(bitmapValue % 256) + bitmap[1] = byte(bitmapValue / 256) + copy(output[bitmapIndex:bitmapIndex+BitmapLength], bitmap[:]) + + return output } // GenerateProof constructs the merkle-proof for key. The result contains all encoded nodes diff --git a/lib/trie/proof_test.go b/lib/trie/proof_test.go index 7bf13b6126..89dd7c2815 100644 --- a/lib/trie/proof_test.go +++ b/lib/trie/proof_test.go @@ -18,6 +18,7 @@ package trie import ( "fmt" + "io/ioutil" "math/rand" "testing" @@ -26,13 +27,38 @@ import ( "github.com/stretchr/testify/require" ) -func TestProofGenerationWithRecorder(t *testing.T) { - db, err := chaindb.NewBadgerDB(&chaindb.Config{InMemory: true}) +func TestGenerateProofWithRecorder(t *testing.T) { + tmp, err := ioutil.TempDir("", "*-test-trie") require.NoError(t, err) + + memdb, err := chaindb.NewBadgerDB(&chaindb.Config{ + InMemory: true, + DataDir: tmp, + }) + require.NoError(t, err) + + trie, entries := RandomTrieTest(t, 20) + err = trie.Store(memdb) + require.NoError(t, err) + + var lastEntryKey *KV + for _, kv := range entries { + lastEntryKey = kv + } + + fmt.Printf("Test\n\tkey:0x%x\n\tvalue:0x%x\n", lastEntryKey.K, lastEntryKey.V) + + rootHash := trie.root.getHash() + proof, err := GenerateProofWithRecorder(rootHash, [][]byte{lastEntryKey.K}, memdb) + require.NoError(t, err) + + for _, p := range proof { + fmt.Printf("0x%x\n", p) + } } func TestVerifyProof(t *testing.T) { - trie, entries := RandomTrieTest(t, 200) + trie, entries := RandomTrieTest(t, 1000) root, err := trie.Hash() require.NoError(t, err) diff --git a/lib/trie/recorder.go b/lib/trie/recorder.go index c9dc232aeb..65a5d064be 100644 --- a/lib/trie/recorder.go +++ b/lib/trie/recorder.go @@ -1,40 +1,40 @@ package trie -import "github.com/ChainSafe/gossamer/lib/common" +import "fmt" // NodeRecord represets a record of a visited node type NodeRecord struct { - Depth uint32 RawData []byte - Hash common.Hash + Hash []byte } -// NodeRecorder records trie nodes as they pass it -type NodeRecorder struct { - Nodes []NodeRecord - MinDepth uint32 +type Recorder []NodeRecord + +func (r *Recorder) Record(h, rd []byte) { + fmt.Printf("received ==> 0x%x\n", h) + *r = append(*r, NodeRecord{ + RawData: rd, + Hash: h, + }) } -// Record a visited node -func (r *NodeRecorder) Record(h common.Hash, rd []byte, depth uint32) { - if depth >= r.MinDepth { - r.Nodes = append(r.Nodes, NodeRecord{ - Depth: depth, - RawData: rd, - Hash: h, - }) - } +func (r *Recorder) Len() int { + return len(*r) } -// RecorderWithDepth create a NodeRecorder which only records nodes beyond a given depth -func NewRecorderWithDepth(d uint32) *NodeRecorder { - return &NodeRecorder{ - MinDepth: d, - Nodes: []NodeRecord{}, +func (r *Recorder) Next() *NodeRecord { + if r.Len() > 0 { + n := (*r)[0] + *r = (*r)[1:] + return &n } + + return nil } -// NewRecoder create a NodeRecorder which records all given nodes -func NewRecoder() *NodeRecorder { - return NewRecorderWithDepth(0) +func (r *Recorder) Peek() *NodeRecord { + if r.Len() > 0 { + return &(*r)[0] + } + return nil } From 4c1e63e1b293b2a8d91e6bec4dca0f9007a4b522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 15 Sep 2021 14:37:02 -0400 Subject: [PATCH 09/38] wip: fixing functions --- lib/trie/lookup.go | 36 +++--- lib/trie/node.go | 68 +---------- lib/trie/proof.go | 270 +++++++++++++++++++++++++------------------ lib/trie/recorder.go | 12 +- 4 files changed, 182 insertions(+), 204 deletions(-) diff --git a/lib/trie/lookup.go b/lib/trie/lookup.go index d8e65b5ad6..dcf1aff1d8 100644 --- a/lib/trie/lookup.go +++ b/lib/trie/lookup.go @@ -3,7 +3,6 @@ package trie import ( "bytes" "errors" - "fmt" "github.com/ChainSafe/chaindb" ) @@ -20,61 +19,58 @@ type Lookup struct { // NewLookup returns a Lookup to helps the proof generator func NewLookup(h []byte, db chaindb.Database) *Lookup { - return &Lookup{ - db: db, - hash: h, - } + lk := &Lookup{db: db} + lk.hash = make([]byte, len(h)) + copy(lk.hash, h) + + return lk } // Find will return the desired value or nil if key cannot be found and will record visited nodes -func (l *Lookup) Find(key []byte) ([]byte, []NodeRecord, error) { +func (l *Lookup) Find(key []byte, recorder *Recorder) ([]byte, error) { partial := key hash := l.hash - nr := make([]NodeRecord, 0) - for { nodeData, err := l.db.Get(hash[:]) if err != nil { - return nil, nil, ErrInvalidStateRoot + return nil, ErrInvalidStateRoot } - nodeRecord := NodeRecord{Hash: hash, RawData: nodeData} - fmt.Println(len(nr)) - nr = append(nr, nodeRecord) + recorder.Record(hash, nodeData) decoded, err := decodeBytes(nodeData) if err != nil { - return nil, nr, err + return nil, err } switch currNode := decoded.(type) { case nil: - return nil, nr, nil + return nil, nil case *leaf: if bytes.Equal(currNode.key, partial) { - return currNode.value, nr, nil + return currNode.value, nil } - return nil, nr, nil + return nil, nil case *branch: switch len(partial) { case 0: - return currNode.value, nr, nil + return currNode.value, nil default: if !bytes.HasPrefix(partial, currNode.key) { - return nil, nr, nil + return nil, nil } if bytes.Equal(partial, currNode.key) { - return currNode.value, nr, nil + return currNode.value, nil } length := lenCommonPrefix(currNode.key, partial) switch child := currNode.children[partial[length]].(type) { case nil: - return nil, nr, nil + return nil, nil default: partial = partial[length+1:] copy(hash[:], child.getHash()) diff --git a/lib/trie/node.go b/lib/trie/node.go index aa3fef754d..1df5bc0260 100644 --- a/lib/trie/node.go +++ b/lib/trie/node.go @@ -50,13 +50,6 @@ import ( "github.com/ChainSafe/gossamer/lib/scale" ) -type nodeType string - -const ( - branchNode nodeType = "branch" - leafNode nodeType = "leaf" -) - // node is the interface for trie methods type node interface { encodeAndHash() ([]byte, []byte, error) @@ -90,6 +83,7 @@ type ( hash []byte encoding []byte generation uint64 + offset int sync.RWMutex } ) @@ -295,65 +289,6 @@ func decodeBytes(in []byte) (node, error) { return decode(r) } -type nodeHeaderType string - -const ( - LeafPlan nodeHeaderType = "leaf" - BranchPlan nodeHeaderType = "branch" - - LeafPrefix = 0b01 << 6 - BranchPrefxi = 0b11 << 6 -) - -var ( - ErrNilDecodedHeader = errors.New("decoded header with empty trie") -) - -func decodeNodeHeader(r io.Reader) (nodeHeaderType, uint32, error) { - b, err := readByte(r) - if err != nil { - return "", 0, err - } - - if b == 0 { - return "", 0, ErrNilDecodedHeader - } - - switch b & (0b11 << 6) { - case LeafPrefix: - s, err := decodeSize(b, r) - return LeafPlan, s, err - case BranchPrefxi: - s, err := decodeSize(b, r) - return BranchPlan, s, err - default: - return "", 0, errors.New("unknown type of node") - } -} - -func decodeSize(f byte, r io.Reader) (uint32, error) { - res := uint32(f & 0xff >> 2) - if res < 63 { - return res, nil - } - - res -= 1 - for res <= uint32((^uint16(0))) { - n, err := readByte(r) - if err != nil { - return 0, err - } - - if n < 255 { - return res + uint32(n+1), nil - } - - res += 255 - } - - return 0, errors.New("size limit reached for a nibble slice") -} - // Decode wraps the decoding of different node types back into a node func decode(r io.Reader) (node, error) { header, err := readByte(r) @@ -448,6 +383,7 @@ func (l *leaf) decode(r io.Reader, header byte) (err error) { keyLen := header & 0x3f l.key, err = decodeKey(r, keyLen) + l.offset = int(keyLen) % 2 if err != nil { return err } diff --git a/lib/trie/proof.go b/lib/trie/proof.go index 62f84bc812..81d42f41d5 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -28,18 +28,19 @@ import ( ) const ( - NibbleSizeBound = ^uint16(0) - NibblePerByte = 2 - BranchWithMask = byte(0b11 << 6) - BranchWithoutMask = byte(0b_10 << 6) - BitmapLength = 2 + BitmapLength = 2 + NibbleLenght = 16 + NibblePerByte = 2 + NibbleSizeBound = ^uint16(0) + BranchWithMask = byte(0b11 << 6) + BranchWithoutMask = byte(0b10 << 6) + LeafPrefixWithMask = byte(0b01 << 6) ) var ( // ErrEmptyTrieRoot occurs when trying to craft a prove with an empty trie root ErrEmptyTrieRoot = errors.New("provided trie must have a root") - // ErrEmptyNibbles occurs when trying to prove or valid a proof to an empty key ErrEmptyNibbles = errors.New("empty nibbles provided from key") logger = log.New("lib", "trie") @@ -54,15 +55,22 @@ type StackEntry struct { children [][]byte childIndex int outputIndex int + omitValue bool } func (s *StackEntry) encodeNode() ([]byte, error) { switch ntype := s.node.(type) { - case nil, *leaf: + case nil: + return s.nodeRawData, nil + case *leaf: + if s.omitValue { + return leafNodeOmitValue(ntype.key), nil + } + return s.nodeRawData, nil case *branch: // Populate the remaining references in `children` - for i := s.childIndex; i < 16; i++ { + for i := s.childIndex; i < NibbleLenght; i++ { nodeChild := ntype.children[i] if nodeChild != nil { var err error @@ -72,18 +80,105 @@ func (s *StackEntry) encodeNode() ([]byte, error) { } } - return branchNodeNibbled(ntype.key, s.children, ntype.value), nil + v := ntype.value + if s.omitValue { + v = []byte{} + } + + return branchNodeNibbled(ntype.key, s.children, v), nil } return nil, nil } +func leafNodeOmitValue(path []byte) []byte { + split := len(path) % NibblePerByte / NibblePerByte + nb := byte(len(path) % NibblePerByte) + + var ( + first byte + second byte + data []byte + ) + + if nb > 0 { + first, second, data = nb, path[split]&0x0f, path[split+1:] + } else { + first, second, data = 0, 0, path[split:] + } + + nibbleCount := len(data)*NibblePerByte + int(first) + if nibbleCount > int(NibbleSizeBound) { + nibbleCount = int(NibbleSizeBound) + } + + output := make([]byte, 0, 3+len(data)) + sizeIterator(len(path), LeafPrefixWithMask, &output) + + if first > 0 { + output = append(output, second&0x0f) + } + + output = append(output, data...) + return output +} + +func branchNodeNibbled(path []byte, children [][]byte, value []byte) []byte { + nibbleCount := len(path) + if nibbleCount > int(NibbleSizeBound) { + nibbleCount = int(NibbleSizeBound) + } + + output := make([]byte, 0, 3+(nibbleCount/NibblePerByte)) + + var prefix byte + if len(value) == 0 { + prefix = BranchWithoutMask + } else { + prefix = BranchWithMask + } + + sizeIterator(nibbleCount, prefix, &output) + + output = append(output, path...) + bitmapIndex := len(output) + bitmap := make([]byte, BitmapLength) + + for i := 0; i < BitmapLength; i++ { + output = append(output, 0) + } + + if len(value) > 0 { + output = append(output, value...) + } + + var ( + bitmapValue uint16 = 0 + bitmapCursor uint16 = 1 + ) + + for _, c := range children { + if len(c) > 0 { + output = append(output, c...) + bitmapValue |= bitmapCursor + } + + bitmapCursor <<= 1 + } + + bitmap[0] = byte(bitmapValue % 256) + bitmap[1] = byte(bitmapValue / 256) + copy(output[bitmapIndex:bitmapIndex+BitmapLength], bitmap[:BitmapLength]) + + return output +} + func (s *StackEntry) setChild(enc []byte) error { switch s.node.(type) { case *branch: //child := ntype.children[s.childIndex] s.children[s.childIndex] = enc - s.childIndex++ + s.childIndex += 1 return nil default: return errors.New("nil, leaf or other nodes does not has children set") @@ -91,13 +186,18 @@ func (s *StackEntry) setChild(enc []byte) error { } func NewStackEntry(n node, rd []byte, outputIndex, keyOffset int) (*StackEntry, error) { - var children [][]byte + var ( + children [][]byte + path []byte + ) switch nt := n.(type) { case *leaf: children = make([][]byte, 0) + path = nt.key case *branch: - children = make([][]byte, 16, 16) + children = make([][]byte, NibbleLenght) + path = nt.key default: return nil, fmt.Errorf("could not define a stack entry for node: %s", nt) } @@ -110,39 +210,64 @@ func NewStackEntry(n node, rd []byte, outputIndex, keyOffset int) (*StackEntry, return &StackEntry{ keyOffset: keyOffset, nodeHash: h, + path: path, node: n, children: children, childIndex: 0, outputIndex: outputIndex, nodeRawData: rd, + omitValue: false, }, nil } type Stack []*StackEntry + +// Push adds a new item to the top of the stack +func (s *Stack) Push(e *StackEntry) { + (*s) = append((*s), e) +} + +// Pop removes and returns the top of the stack if there is some element there otherwise return nil +func (s *Stack) Pop() *StackEntry { + if len(*s) < 1 { + return nil + } + + // gets the top of the stack + entry := (*s)[len(*s)-1] + // removes the top of the stack + *s = (*s)[:len(*s)-1] + return entry +} + +// Last returns the top of the stack without remove from the stack +func (s *Stack) Last() *StackEntry { + if len(*s) < 1 { + return nil + } + return (*s)[len(*s)-1] +} + type StackIterator struct { index int set []*StackEntry } func (i *StackIterator) Next() *StackEntry { - var t *StackEntry - if i.HasNext() { - t = i.set[i.index] + t := i.set[i.index] i.index++ + return t } - return t + return nil } func (i *StackIterator) Peek() *StackEntry { - var t *StackEntry - if i.HasNext() { - t = i.set[i.index] + return i.set[i.index] } - - return t + return nil } func (i *StackIterator) HasNext() bool { @@ -157,30 +282,6 @@ func (s *Stack) iter() *StackIterator { return iter } -func (s *Stack) Push(e *StackEntry) { - (*s) = append((*s), e) -} - -func (s *Stack) Pop() *StackEntry { - if len(*s) < 1 { - return nil - } - - // gets the top of the stack - entry := (*s)[len(*s)-1] - - // removes the top of the stack - (*s) = (*s)[:len(*s)-1] - return entry -} - -func (s *Stack) Last() *StackEntry { - if len(*s) < 1 { - return nil - } - return (*s)[len(*s)-1] -} - type Step struct { Found bool Value []byte @@ -197,23 +298,21 @@ func GenerateProofWithRecorder(root []byte, keys [][]byte, db chaindb.Database) unwindStack(&stack, proofs, nk) lookup := NewLookup(root, db) - expectedValue, nodes, err := lookup.Find(nk) + + recorder := new(Recorder) + expectedValue, err := lookup.Find(nk, recorder) if err != nil { return nil, err } - for _, recNodes := range nodes { + for _, recNodes := range *recorder { fmt.Printf("recorded node ==> 0x%x\n", recNodes.Hash) } - recorder := Recorder(nodes) - fmt.Printf("Found at database\n\tkey:0x%x\n\tvalue:0x%x\n", k, expectedValue) fmt.Println("Recorded nodes", recorder.Len()) stackIter := stack.iter() - logger.Warn("generate proof", "stackIter.HasNext()", stackIter.HasNext()) - // Skip over recorded nodes already on the stack for stackIter.HasNext() { nxtRec, nxtEntry := recorder.Peek(), stackIter.Peek() @@ -227,10 +326,6 @@ func GenerateProofWithRecorder(root []byte, keys [][]byte, db chaindb.Database) for { var step Step - - fmt.Println("stack len", len(stack)) - fmt.Println("proofs len", len(proofs)) - if len(stack) == 0 { // as the stack is empty then start from the root node step = Step{ @@ -297,6 +392,7 @@ func matchKeyToNode(e *StackEntry, nibbleKey []byte) (Step, error) { case *leaf: keyToCompare := nibbleKey[e.keyOffset:] if bytes.Equal(keyToCompare, ntype.key) && len(nibbleKey) == len(ntype.key)+e.keyOffset { + e.omitValue = true return Step{ Found: true, Value: ntype.value, @@ -314,14 +410,15 @@ func matchKeyToNode(e *StackEntry, nibbleKey []byte) (Step, error) { func matchKeyToBranchNode(n *branch, e *StackEntry, nibbleKey []byte) (Step, error) { keyToCompare := nibbleKey[e.keyOffset:] - logger.Warn("matchKeyToBranchNode", "keyToCompare", fmt.Sprintf("0x%x\n", keyToCompare), "len nibbles", len(nibbleKey)) - logger.Warn("matchKeyToBranchNode", "node key", fmt.Sprintf("x%x\n", n.key), "key offset", e.keyOffset) + logger.Warn("matchKeyToBranchNode", "keyToCompare", fmt.Sprintf("0x%x", keyToCompare), "len nibbles", len(nibbleKey)) + logger.Warn("matchKeyToBranchNode", "node key", fmt.Sprintf("0x%x", n.key), "key offset", e.keyOffset) if !bytes.HasPrefix(keyToCompare, n.key) { return Step{Found: true}, nil } if len(nibbleKey) == len(n.key)+e.keyOffset { + e.omitValue = true return Step{ Found: true, Value: n.value, @@ -379,7 +476,6 @@ func unwindStack(stack *Stack, proof [][]byte, key []byte) error { } parent := stack.Last() - if parent != nil { parent.setChild(enc) } @@ -390,7 +486,7 @@ func unwindStack(stack *Stack, proof [][]byte, key []byte) error { return nil } -func branchEncode(size int, prefix byte, output []byte) { +func sizeIterator(size int, prefix byte, output *[]byte) { l := make([]byte, 0) l1, rem := 62, 0 @@ -406,8 +502,8 @@ func branchEncode(size int, prefix byte, output []byte) { if rem > 0 { if rem < 256 { result := rem - 1 + rem = 0 l = append(l, byte(result)) - break } else { op := rem - 255 if op < 0 { @@ -415,6 +511,7 @@ func branchEncode(size int, prefix byte, output []byte) { } else { rem = op } + l = append(l, byte(255)) } } else { @@ -422,57 +519,8 @@ func branchEncode(size int, prefix byte, output []byte) { } } - copy(output[:len(l)], l[:]) -} - -func prefixIterator(size int, prefix byte) []byte { - if NibbleSizeBound < uint16(size) { - size = NibblePerByte - } - - output := make([]byte, 0, 3+(size/NibblePerByte)) - branchEncode(size, prefix, output) - return output -} - -func branchNodeNibbled(path []byte, children [][]byte, value []byte) []byte { - var output []byte - if len(value) == 0 { - output = prefixIterator(len(path), BranchWithMask) - } else { - output = prefixIterator(len(path), BranchWithoutMask) - } - output = append(output, path...) - bitmapIndex := len(output) - bitmap := make([]byte, BitmapLength) - - for i := 0; i < BitmapLength; i++ { - output = append(output, 0) - } - - if len(value) > 0 { - output = append(output, value...) - } - - var ( - bitmapValue uint16 = 0 - bitmapCursor uint16 = 1 - ) - - for _, c := range children { - if len(c) > 0 { - output = append(output, c...) - bitmapValue |= bitmapCursor - } - - bitmapCursor <<= 1 - } - - bitmap[0] = byte(bitmapValue % 256) - bitmap[1] = byte(bitmapValue / 256) - copy(output[bitmapIndex:bitmapIndex+BitmapLength], bitmap[:]) - - return output + *output = append(*output, l...) + fmt.Printf("\n\non encode\noutput == 0x%x\nl == 0x%x\n", output, l) } // GenerateProof constructs the merkle-proof for key. The result contains all encoded nodes diff --git a/lib/trie/recorder.go b/lib/trie/recorder.go index 65a5d064be..75b8af4483 100644 --- a/lib/trie/recorder.go +++ b/lib/trie/recorder.go @@ -1,7 +1,5 @@ package trie -import "fmt" - // NodeRecord represets a record of a visited node type NodeRecord struct { RawData []byte @@ -11,11 +9,11 @@ type NodeRecord struct { type Recorder []NodeRecord func (r *Recorder) Record(h, rd []byte) { - fmt.Printf("received ==> 0x%x\n", h) - *r = append(*r, NodeRecord{ - RawData: rd, - Hash: h, - }) + nr := NodeRecord{RawData: rd} + nr.Hash = make([]byte, len(h)) + copy(nr.Hash, h[:]) + + *r = append(*r, nr) } func (r *Recorder) Len() int { From 26e1ad458c8fff02d8bb5a9b7705fcd70df3d1f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 15 Sep 2021 14:38:42 -0400 Subject: [PATCH 10/38] wip: remove unused code --- lib/trie/proof.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/trie/proof.go b/lib/trie/proof.go index 81d42f41d5..82eb5e21f3 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -92,7 +92,6 @@ func (s *StackEntry) encodeNode() ([]byte, error) { } func leafNodeOmitValue(path []byte) []byte { - split := len(path) % NibblePerByte / NibblePerByte nb := byte(len(path) % NibblePerByte) var ( @@ -102,9 +101,9 @@ func leafNodeOmitValue(path []byte) []byte { ) if nb > 0 { - first, second, data = nb, path[split]&0x0f, path[split+1:] + first, second, data = nb, path[0]&0x0f, path[0+1:] } else { - first, second, data = 0, 0, path[split:] + first, second, data = 0, 0, path[0:] } nibbleCount := len(data)*NibblePerByte + int(first) From 1758518bb6b1279657e2ba3601180f6d44d28e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Thu, 16 Sep 2021 23:40:11 -0400 Subject: [PATCH 11/38] feat: proof generation working --- dot/core/service.go | 2 +- dot/rpc/modules/state.go | 1 + lib/trie/lookup.go | 5 +- lib/trie/node.go | 2 - lib/trie/proof.go | 394 +++------------------------------------ lib/trie/proof_test.go | 140 ++------------ lib/trie/recorder.go | 5 +- 7 files changed, 40 insertions(+), 509 deletions(-) diff --git a/dot/core/service.go b/dot/core/service.go index 9a10d9d960..848dd34d9d 100644 --- a/dot/core/service.go +++ b/dot/core/service.go @@ -644,7 +644,7 @@ func (s *Service) GetReadProofAt(block common.Hash, keys [][]byte) (common.Hash, // readProofForKeys will go through the keys and generate the proof for each of them // and merge the result into a string array containing the hashes in the hexadecimal format func readProofForKeys(root []byte, keys [][]byte, db chaindb.Database) ([][]byte, error) { - proof, err := trie.GenerateProofWithRecorder(root, keys, db) + proof, err := trie.GenerateProof(root, keys, db) if err != nil { return nil, err } diff --git a/dot/rpc/modules/state.go b/dot/rpc/modules/state.go index 6110d5dc49..00d822b884 100644 --- a/dot/rpc/modules/state.go +++ b/dot/rpc/modules/state.go @@ -303,6 +303,7 @@ func (sm *StateModule) GetReadProof(r *http.Request, req *StateGetReadProofReque keys[i] = bKey } + fmt.Println("GET READ PROOF RPC", req.Keys, len(keys)) block, proofs, err := sm.coreAPI.GetReadProofAt(req.Hash, keys) if err != nil { return err diff --git a/lib/trie/lookup.go b/lib/trie/lookup.go index dcf1aff1d8..cf22e47f78 100644 --- a/lib/trie/lookup.go +++ b/lib/trie/lookup.go @@ -37,7 +37,10 @@ func (l *Lookup) Find(key []byte, recorder *Recorder) ([]byte, error) { return nil, ErrInvalidStateRoot } - recorder.Record(hash, nodeData) + nodeHash := make([]byte, len(hash)) + copy(nodeHash, hash) + + recorder.Record(nodeHash, nodeData) decoded, err := decodeBytes(nodeData) if err != nil { diff --git a/lib/trie/node.go b/lib/trie/node.go index 1df5bc0260..3e1f091f02 100644 --- a/lib/trie/node.go +++ b/lib/trie/node.go @@ -83,7 +83,6 @@ type ( hash []byte encoding []byte generation uint64 - offset int sync.RWMutex } ) @@ -383,7 +382,6 @@ func (l *leaf) decode(r io.Reader, header byte) (err error) { keyLen := header & 0x3f l.key, err = decodeKey(r, keyLen) - l.offset = int(keyLen) % 2 if err != nil { return err } diff --git a/lib/trie/proof.go b/lib/trie/proof.go index 82eb5e21f3..ff4ac9d28f 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -19,22 +19,8 @@ package trie import ( "bytes" "errors" - "fmt" "github.com/ChainSafe/chaindb" - "github.com/ChainSafe/gossamer/lib/common" - - log "github.com/ChainSafe/log15" -) - -const ( - BitmapLength = 2 - NibbleLenght = 16 - NibblePerByte = 2 - NibbleSizeBound = ^uint16(0) - BranchWithMask = byte(0b11 << 6) - BranchWithoutMask = byte(0b10 << 6) - LeafPrefixWithMask = byte(0b01 << 6) ) var ( @@ -42,180 +28,28 @@ var ( ErrEmptyTrieRoot = errors.New("provided trie must have a root") ErrEmptyNibbles = errors.New("empty nibbles provided from key") - - logger = log.New("lib", "trie") ) +// StackEntry is a entry on the nodes that is prooved already +// it stores the necessary infos to keep going and prooving the children nodes type StackEntry struct { keyOffset int - path []byte + key []byte nodeHash []byte node node nodeRawData []byte - children [][]byte - childIndex int outputIndex int - omitValue bool -} - -func (s *StackEntry) encodeNode() ([]byte, error) { - switch ntype := s.node.(type) { - case nil: - return s.nodeRawData, nil - case *leaf: - if s.omitValue { - return leafNodeOmitValue(ntype.key), nil - } - - return s.nodeRawData, nil - case *branch: - // Populate the remaining references in `children` - for i := s.childIndex; i < NibbleLenght; i++ { - nodeChild := ntype.children[i] - if nodeChild != nil { - var err error - if _, s.children[i], err = nodeChild.encodeAndHash(); err != nil { - return nil, err - } - } - } - - v := ntype.value - if s.omitValue { - v = []byte{} - } - - return branchNodeNibbled(ntype.key, s.children, v), nil - } - - return nil, nil } -func leafNodeOmitValue(path []byte) []byte { - nb := byte(len(path) % NibblePerByte) - - var ( - first byte - second byte - data []byte - ) - - if nb > 0 { - first, second, data = nb, path[0]&0x0f, path[0+1:] - } else { - first, second, data = 0, 0, path[0:] - } - - nibbleCount := len(data)*NibblePerByte + int(first) - if nibbleCount > int(NibbleSizeBound) { - nibbleCount = int(NibbleSizeBound) - } - - output := make([]byte, 0, 3+len(data)) - sizeIterator(len(path), LeafPrefixWithMask, &output) - - if first > 0 { - output = append(output, second&0x0f) - } - - output = append(output, data...) - return output -} - -func branchNodeNibbled(path []byte, children [][]byte, value []byte) []byte { - nibbleCount := len(path) - if nibbleCount > int(NibbleSizeBound) { - nibbleCount = int(NibbleSizeBound) - } - - output := make([]byte, 0, 3+(nibbleCount/NibblePerByte)) - - var prefix byte - if len(value) == 0 { - prefix = BranchWithoutMask - } else { - prefix = BranchWithMask - } - - sizeIterator(nibbleCount, prefix, &output) - - output = append(output, path...) - bitmapIndex := len(output) - bitmap := make([]byte, BitmapLength) - - for i := 0; i < BitmapLength; i++ { - output = append(output, 0) - } - - if len(value) > 0 { - output = append(output, value...) - } - - var ( - bitmapValue uint16 = 0 - bitmapCursor uint16 = 1 - ) - - for _, c := range children { - if len(c) > 0 { - output = append(output, c...) - bitmapValue |= bitmapCursor - } - - bitmapCursor <<= 1 - } - - bitmap[0] = byte(bitmapValue % 256) - bitmap[1] = byte(bitmapValue / 256) - copy(output[bitmapIndex:bitmapIndex+BitmapLength], bitmap[:BitmapLength]) - - return output -} - -func (s *StackEntry) setChild(enc []byte) error { - switch s.node.(type) { - case *branch: - //child := ntype.children[s.childIndex] - s.children[s.childIndex] = enc - s.childIndex += 1 - return nil - default: - return errors.New("nil, leaf or other nodes does not has children set") - } -} - -func NewStackEntry(n node, rd []byte, outputIndex, keyOffset int) (*StackEntry, error) { - var ( - children [][]byte - path []byte - ) - - switch nt := n.(type) { - case *leaf: - children = make([][]byte, 0) - path = nt.key - case *branch: - children = make([][]byte, NibbleLenght) - path = nt.key - default: - return nil, fmt.Errorf("could not define a stack entry for node: %s", nt) - } - - _, h, err := n.encodeAndHash() - if err != nil { - return nil, err - } - +// +func NewStackEntry(n node, hash, rd, prefix []byte, outputIndex, keyOffset int) (*StackEntry, error) { return &StackEntry{ keyOffset: keyOffset, - nodeHash: h, - path: path, + nodeHash: hash, + key: prefix, node: n, - children: children, - childIndex: 0, outputIndex: outputIndex, nodeRawData: rd, - omitValue: false, }, nil } @@ -281,6 +115,7 @@ func (s *Stack) iter() *StackIterator { return iter } +// type Step struct { Found bool Value []byte @@ -288,34 +123,29 @@ type Step struct { KeyOffset int } -func GenerateProofWithRecorder(root []byte, keys [][]byte, db chaindb.Database) ([][]byte, error) { +// GenerateProof receive the keys to proof, the trie root and a reference to database +// will +func GenerateProof(root []byte, keys [][]byte, db chaindb.Database) ([][]byte, error) { stack := make(Stack, 0) proofs := make([][]byte, 0) for _, k := range keys { nk := keyToNibbles(k) + unwindStack(&stack, proofs, nk) lookup := NewLookup(root, db) - recorder := new(Recorder) expectedValue, err := lookup.Find(nk, recorder) if err != nil { return nil, err } - for _, recNodes := range *recorder { - fmt.Printf("recorded node ==> 0x%x\n", recNodes.Hash) - } - - fmt.Printf("Found at database\n\tkey:0x%x\n\tvalue:0x%x\n", k, expectedValue) - fmt.Println("Recorded nodes", recorder.Len()) - - stackIter := stack.iter() // Skip over recorded nodes already on the stack + stackIter := stack.iter() for stackIter.HasNext() { nxtRec, nxtEntry := recorder.Peek(), stackIter.Peek() - if !bytes.Equal(nxtRec.Hash[:], nxtEntry.nodeHash) { + if !bytes.Equal(nxtRec.Hash, nxtEntry.nodeHash) { break } @@ -334,8 +164,6 @@ func GenerateProofWithRecorder(root []byte, keys [][]byte, db chaindb.Database) } } else { entryOnTop := stack.Last() - logger.Warn("generate proof", "has last on stack", entryOnTop != nil) - step, err = matchKeyToNode(entryOnTop, nk) if err != nil { return nil, err @@ -355,9 +183,6 @@ func GenerateProofWithRecorder(root []byte, keys [][]byte, db chaindb.Database) return nil, errors.New("recorder must has nodes to traverse") } - logger.Warn("generate proof", "expected", fmt.Sprintf("Recorded node: 0x%x\n", rec.Hash)) - logger.Warn("generate proof", "got", fmt.Sprintf("Step node: 0x%x\n", step.NextNode)) - if !bytes.Equal(rec.Hash, step.NextNode) { return nil, errors.New("recorded node does not match expected node") } @@ -366,12 +191,14 @@ func GenerateProofWithRecorder(root []byte, keys [][]byte, db chaindb.Database) if err != nil { return nil, err } - logger.Warn("generate proof", "has decoded node", n != nil) outputIndex := len(proofs) proofs = append(proofs, []byte{}) - ne, err := NewStackEntry(n, rec.RawData, outputIndex, step.KeyOffset) + prefix := make([]byte, len(nk[:step.KeyOffset])) + copy(prefix, nk[:step.KeyOffset]) + + ne, err := NewStackEntry(n, rec.Hash, rec.RawData, prefix, outputIndex, step.KeyOffset) if err != nil { return nil, err } @@ -391,7 +218,6 @@ func matchKeyToNode(e *StackEntry, nibbleKey []byte) (Step, error) { case *leaf: keyToCompare := nibbleKey[e.keyOffset:] if bytes.Equal(keyToCompare, ntype.key) && len(nibbleKey) == len(ntype.key)+e.keyOffset { - e.omitValue = true return Step{ Found: true, Value: ntype.value, @@ -408,16 +234,11 @@ func matchKeyToNode(e *StackEntry, nibbleKey []byte) (Step, error) { func matchKeyToBranchNode(n *branch, e *StackEntry, nibbleKey []byte) (Step, error) { keyToCompare := nibbleKey[e.keyOffset:] - - logger.Warn("matchKeyToBranchNode", "keyToCompare", fmt.Sprintf("0x%x", keyToCompare), "len nibbles", len(nibbleKey)) - logger.Warn("matchKeyToBranchNode", "node key", fmt.Sprintf("0x%x", n.key), "key offset", e.keyOffset) - if !bytes.HasPrefix(keyToCompare, n.key) { return Step{Found: true}, nil } if len(nibbleKey) == len(n.key)+e.keyOffset { - e.omitValue = true return Step{ Found: true, Value: n.value, @@ -425,20 +246,7 @@ func matchKeyToBranchNode(n *branch, e *StackEntry, nibbleKey []byte) (Step, err } newIndex := nibbleKey[e.keyOffset+len(n.key)] - for e.childIndex < int(newIndex) { - nodeChild := n.children[e.childIndex] - if nodeChild != nil { - var err error - if _, e.children[e.childIndex], err = nodeChild.encodeAndHash(); err != nil { - return Step{}, err - } - } - e.childIndex++ - } - - child := n.children[e.childIndex] - logger.Warn("matchKeyToBranchNode", "has child", child != nil, "childIndex", e.childIndex) - + child := n.children[newIndex] if child == nil { return Step{Found: true}, nil } @@ -462,173 +270,13 @@ func unwindStack(stack *Stack, proof [][]byte, key []byte) error { break } - if key != nil && bytes.HasPrefix(key, entry.path) { + if key != nil && bytes.HasPrefix(key, entry.key) { stack.Push(entry) break } - index := entry.outputIndex - - enc, err := entry.encodeNode() - if err != nil { - return err - } - - parent := stack.Last() - if parent != nil { - parent.setChild(enc) - } - - proof[index] = enc + proof[entry.outputIndex] = entry.nodeRawData } return nil } - -func sizeIterator(size int, prefix byte, output *[]byte) { - l := make([]byte, 0) - l1, rem := 62, 0 - - if size < 62 { - l1 = size - l = append(l, prefix+byte(l1)) - } else { - l = append(l, prefix+byte(63)) - rem = size - l1 - } - - for { - if rem > 0 { - if rem < 256 { - result := rem - 1 - rem = 0 - l = append(l, byte(result)) - } else { - op := rem - 255 - if op < 0 { - rem = 0 - } else { - rem = op - } - - l = append(l, byte(255)) - } - } else { - break - } - } - - *output = append(*output, l...) - fmt.Printf("\n\non encode\noutput == 0x%x\nl == 0x%x\n", output, l) -} - -// GenerateProof constructs the merkle-proof for key. The result contains all encoded nodes -// on the path to the key. Returns the amount of nodes of the path and error if could not found the key -func (t *Trie) GenerateProof(keys [][]byte) (map[string][]byte, error) { - var nodes []node - - for _, k := range keys { - currNode := t.root - - nk := keyToNibbles(k) - if len(nk) == 0 { - return nil, ErrEmptyNibbles - } - - proveLoop: - for { - switch n := currNode.(type) { - case nil: - return nil, errors.New("no more paths to follow") - - case *leaf: - nodes = append(nodes, n) - - if bytes.Equal(n.key, nk) { - break proveLoop - } - - return nil, errors.New("leaf node doest not match the key") - - case *branch: - nodes = append(nodes, n) - if bytes.Equal(n.key, nk) || len(nk) == 0 { - break proveLoop - } - - length := lenCommonPrefix(n.key, nk) - currNode = n.children[nk[length]] - nk = nk[length+1:] - } - } - } - - proof := make(map[string][]byte) - for _, n := range nodes { - var ( - hashNode []byte - encHashNode []byte - err error - ) - - if encHashNode, hashNode, err = n.encodeAndHash(); err != nil { - return nil, fmt.Errorf("problems while encoding and hashing the node: %w", err) - } - - // avoid duplicate hashes - proof[common.BytesToHex(hashNode)] = encHashNode - } - - return proof, nil -} - -// VerifyProof checks merkle proofs given an proof -func VerifyProof(rootHash common.Hash, key []byte, proof map[string][]byte) (bool, error) { - key = keyToNibbles(key) - if len(key) == 0 { - return false, ErrEmptyNibbles - } - - var wantedHash string - wantedHash = common.BytesToHex(rootHash.ToBytes()) - - for { - enc, ok := proof[wantedHash] - if !ok { - return false, nil - } - - currNode, err := decodeBytes(enc) - if err != nil { - return false, fmt.Errorf("could not decode node bytes: %w", err) - } - - switch n := currNode.(type) { - case nil: - return false, nil - case *leaf: - if bytes.Equal(n.key, key) { - return true, nil - } - - return false, nil - case *branch: - if bytes.Equal(n.key, key) { - return true, nil - } - - if len(key) == 0 { - return false, nil - } - - length := lenCommonPrefix(n.key, key) - next := n.children[key[length]] - if next == nil { - return false, nil - } - - key = key[length+1:] - wantedHash = common.BytesToHex(next.getHash()) - } - } -} diff --git a/lib/trie/proof_test.go b/lib/trie/proof_test.go index 89dd7c2815..685be62045 100644 --- a/lib/trie/proof_test.go +++ b/lib/trie/proof_test.go @@ -19,11 +19,9 @@ package trie import ( "fmt" "io/ioutil" - "math/rand" "testing" "github.com/ChainSafe/chaindb" - "github.com/ChainSafe/gossamer/lib/common" "github.com/stretchr/testify/require" ) @@ -37,145 +35,31 @@ func TestGenerateProofWithRecorder(t *testing.T) { }) require.NoError(t, err) - trie, entries := RandomTrieTest(t, 20) + trie, entries := RandomTrieTest(t, 200) err = trie.Store(memdb) require.NoError(t, err) + var otherKey *KV var lastEntryKey *KV + + i := 0 for _, kv := range entries { + if len(entries)-2 == i { + otherKey = kv + } lastEntryKey = kv + i++ } fmt.Printf("Test\n\tkey:0x%x\n\tvalue:0x%x\n", lastEntryKey.K, lastEntryKey.V) + fmt.Printf("Test2\n\tkey:0x%x\n\tvalue2:0x%x\n", otherKey.K, otherKey.V) rootHash := trie.root.getHash() - proof, err := GenerateProofWithRecorder(rootHash, [][]byte{lastEntryKey.K}, memdb) + proof, err := GenerateProof(rootHash, [][]byte{lastEntryKey.K, otherKey.K}, memdb) require.NoError(t, err) + fmt.Printf("\n\n") for _, p := range proof { - fmt.Printf("0x%x\n", p) + fmt.Printf("generated -> 0x%x\n", p) } } - -func TestVerifyProof(t *testing.T) { - trie, entries := RandomTrieTest(t, 1000) - root, err := trie.Hash() - require.NoError(t, err) - - for _, entry := range entries { - proof, err := trie.GenerateProof([][]byte{entry.K}) - require.NoError(t, err) - - v, err := VerifyProof(root, entry.K, proof) - require.NoError(t, err) - require.True(t, v) - } -} - -func TestVerifyProofOneElement(t *testing.T) { - trie := NewEmptyTrie() - key := randBytes(32) - trie.Put(key, []byte("V")) - - rootHash, err := trie.Hash() - require.NoError(t, err) - - proof, err := trie.GenerateProof([][]byte{key}) - fmt.Println(proof) - require.NoError(t, err) - - val, err := VerifyProof(rootHash, key, proof) - require.NoError(t, err) - - require.True(t, val) -} - -func TestVerifyProof_BadProof(t *testing.T) { - trie, entries := RandomTrieTest(t, 200) - rootHash, err := trie.Hash() - require.NoError(t, err) - - for _, entry := range entries { - proof, err := trie.GenerateProof([][]byte{entry.K}) - require.Greater(t, len(proof), 0) - require.NoError(t, err) - - i := 0 - d := rand.Intn(len(proof)) - - var toTamper string - for k := range proof { - if i < d { - i++ - continue - } - - toTamper = k - break - } - - val := proof[toTamper] - delete(proof, toTamper) - - newhash, err := common.Keccak256(val) - require.NoError(t, err) - proof[common.BytesToHex(newhash.ToBytes())] = val - - v, err := VerifyProof(rootHash, entry.K, proof) - require.NoError(t, err) - require.False(t, v) - } -} - -func TestGenerateProofMissingKey(t *testing.T) { - trie := NewEmptyTrie() - - parentKey, parentVal := randBytes(32), randBytes(20) - chieldKey, chieldValue := modifyLastBytes(parentKey), modifyLastBytes(parentVal) - gransonKey, gransonValue := modifyLastBytes(chieldKey), modifyLastBytes(chieldValue) - - trie.Put(parentKey, parentVal) - trie.Put(chieldKey, chieldValue) - trie.Put(gransonKey, gransonValue) - - searchfor := make([]byte, len(gransonKey)) - copy(searchfor[:], gransonKey[:]) - - // keep the path til the key but modify the last element - searchfor[len(searchfor)-1] = searchfor[len(searchfor)-1] + byte(0xff) - - _, err := trie.GenerateProof([][]byte{searchfor}) - require.Error(t, err, "leaf node doest not match the key") -} - -func TestGenerateProofNoMorePathToFollow(t *testing.T) { - trie := NewEmptyTrie() - - parentKey, parentVal := randBytes(32), randBytes(20) - chieldKey, chieldValue := modifyLastBytes(parentKey), modifyLastBytes(parentVal) - gransonKey, gransonValue := modifyLastBytes(chieldKey), modifyLastBytes(chieldValue) - - trie.Put(parentKey, parentVal) - trie.Put(chieldKey, chieldValue) - trie.Put(gransonKey, gransonValue) - - searchfor := make([]byte, len(parentKey)) - copy(searchfor[:], parentKey[:]) - - // the keys are equals until the byte number 20 so we modify the byte number 20 to another - // value and the branch node will no be able to found the right slot - searchfor[20] = searchfor[20] + byte(0xff) - - _, err := trie.GenerateProof([][]byte{searchfor}) - require.Error(t, err, "no more paths to follow") -} - -func modifyLastBytes(b []byte) []byte { - newB := make([]byte, len(b)) - copy(newB[:], b) - - rb := randBytes(12) - copy(newB[20:], rb) - - return newB -} diff --git a/lib/trie/recorder.go b/lib/trie/recorder.go index 75b8af4483..93273bf52a 100644 --- a/lib/trie/recorder.go +++ b/lib/trie/recorder.go @@ -9,10 +9,7 @@ type NodeRecord struct { type Recorder []NodeRecord func (r *Recorder) Record(h, rd []byte) { - nr := NodeRecord{RawData: rd} - nr.Hash = make([]byte, len(h)) - copy(nr.Hash, h[:]) - + nr := NodeRecord{RawData: rd, Hash: h} *r = append(*r, nr) } From b52a767fc6ff001b000f0fa7e793cff972928ffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 17 Sep 2021 13:39:23 -0400 Subject: [PATCH 12/38] chore: add multiple keys proof generation --- lib/trie/proof.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/trie/proof.go b/lib/trie/proof.go index ff4ac9d28f..82dea302fc 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -43,6 +43,11 @@ type StackEntry struct { // func NewStackEntry(n node, hash, rd, prefix []byte, outputIndex, keyOffset int) (*StackEntry, error) { + _, _, err := n.encodeAndHash() + if err != nil { + return nil, err + } + return &StackEntry{ keyOffset: keyOffset, nodeHash: hash, From b7239231e14f48742b98a407682807c3c950f24a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 17 Sep 2021 19:11:14 -0400 Subject: [PATCH 13/38] chore: add unit tests --- lib/trie/proof.go | 1 + lib/trie/proof_test.go | 35 ++++++++++++----------------------- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/lib/trie/proof.go b/lib/trie/proof.go index 82dea302fc..88c554ae71 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -252,6 +252,7 @@ func matchKeyToBranchNode(n *branch, e *StackEntry, nibbleKey []byte) (Step, err newIndex := nibbleKey[e.keyOffset+len(n.key)] child := n.children[newIndex] + if child == nil { return Step{Found: true}, nil } diff --git a/lib/trie/proof_test.go b/lib/trie/proof_test.go index 685be62045..4b062dd19a 100644 --- a/lib/trie/proof_test.go +++ b/lib/trie/proof_test.go @@ -17,7 +17,6 @@ package trie import ( - "fmt" "io/ioutil" "testing" @@ -25,7 +24,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestGenerateProofWithRecorder(t *testing.T) { +func TestProofGeneration(t *testing.T) { tmp, err := ioutil.TempDir("", "*-test-trie") require.NoError(t, err) @@ -35,31 +34,21 @@ func TestGenerateProofWithRecorder(t *testing.T) { }) require.NoError(t, err) - trie, entries := RandomTrieTest(t, 200) + trie := NewEmptyTrie() + trie.Put([]byte("cat"), randBytes(32)) + trie.Put([]byte("catapulta"), randBytes(32)) + trie.Put([]byte("catapora"), randBytes(32)) + trie.Put([]byte("dog"), randBytes(32)) + trie.Put([]byte("doguinho"), randBytes(32)) + err = trie.Store(memdb) require.NoError(t, err) - var otherKey *KV - var lastEntryKey *KV - - i := 0 - for _, kv := range entries { - if len(entries)-2 == i { - otherKey = kv - } - lastEntryKey = kv - i++ - } - - fmt.Printf("Test\n\tkey:0x%x\n\tvalue:0x%x\n", lastEntryKey.K, lastEntryKey.V) - fmt.Printf("Test2\n\tkey:0x%x\n\tvalue2:0x%x\n", otherKey.K, otherKey.V) + hash, err := trie.Hash() + require.NoError(t, err) - rootHash := trie.root.getHash() - proof, err := GenerateProof(rootHash, [][]byte{lastEntryKey.K, otherKey.K}, memdb) + proof, err := GenerateProof(hash.ToBytes(), [][]byte{[]byte("catapulta"), []byte("catapora")}, memdb) require.NoError(t, err) - fmt.Printf("\n\n") - for _, p := range proof { - fmt.Printf("generated -> 0x%x\n", p) - } + require.Equal(t, len(proof), 5) } From ae9e8ecaebf19f7b1685eadba46c4a927eea7d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 20 Sep 2021 09:57:04 -0400 Subject: [PATCH 14/38] chore: add unit test --- lib/trie/proof_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/trie/proof_test.go b/lib/trie/proof_test.go index 4b062dd19a..b7326c2980 100644 --- a/lib/trie/proof_test.go +++ b/lib/trie/proof_test.go @@ -50,5 +50,6 @@ func TestProofGeneration(t *testing.T) { proof, err := GenerateProof(hash.ToBytes(), [][]byte{[]byte("catapulta"), []byte("catapora")}, memdb) require.NoError(t, err) + // TODO: use the verify_proof function to assert the tests require.Equal(t, len(proof), 5) } From 3079a97812b2c05dc789bc7afbe9e432112532f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 20 Sep 2021 16:55:11 -0400 Subject: [PATCH 15/38] chore: fix lint warnings --- dot/state/storage.go | 4 ---- lib/trie/lookup.go | 15 +++++++++------ lib/trie/proof.go | 20 ++++++++++++-------- lib/trie/recorder.go | 5 +++++ 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/dot/state/storage.go b/dot/state/storage.go index 8da900b075..fd26ac24c0 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -91,10 +91,6 @@ func NewStorageState(db chaindb.Database, blockState *BlockState, t *trie.Trie, }, nil } -func (s *StorageState) ExposeDB() chaindb.Database { - return s.db -} - // SetSyncing sets whether the node is currently syncing or not func (s *StorageState) SetSyncing(syncing bool) { s.syncing = syncing diff --git a/lib/trie/lookup.go b/lib/trie/lookup.go index cf22e47f78..b2a009e595 100644 --- a/lib/trie/lookup.go +++ b/lib/trie/lookup.go @@ -8,20 +8,23 @@ import ( ) var ( - ErrInvalidStateRoot = errors.New("cannot found the state root on storage") + // ErrProofNodeNotFound when a needed proof node is not in the database + ErrProofNodeNotFound = errors.New("cannot found the state root on storage") ) +// Lookup struct holds the state root and database reference +// used to retrieve trie information from database type Lookup struct { // root to start the lookup - hash []byte + root []byte db chaindb.Database } // NewLookup returns a Lookup to helps the proof generator func NewLookup(h []byte, db chaindb.Database) *Lookup { lk := &Lookup{db: db} - lk.hash = make([]byte, len(h)) - copy(lk.hash, h) + lk.root = make([]byte, len(h)) + copy(lk.root, h) return lk } @@ -29,12 +32,12 @@ func NewLookup(h []byte, db chaindb.Database) *Lookup { // Find will return the desired value or nil if key cannot be found and will record visited nodes func (l *Lookup) Find(key []byte, recorder *Recorder) ([]byte, error) { partial := key - hash := l.hash + hash := l.root for { nodeData, err := l.db.Get(hash[:]) if err != nil { - return nil, ErrInvalidStateRoot + return nil, ErrProofNodeNotFound } nodeHash := make([]byte, len(hash)) diff --git a/lib/trie/proof.go b/lib/trie/proof.go index 88c554ae71..3548852874 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -26,11 +26,9 @@ import ( var ( // ErrEmptyTrieRoot occurs when trying to craft a prove with an empty trie root ErrEmptyTrieRoot = errors.New("provided trie must have a root") - - ErrEmptyNibbles = errors.New("empty nibbles provided from key") ) -// StackEntry is a entry on the nodes that is prooved already +// StackEntry is a entry on the nodes that is proved already // it stores the necessary infos to keep going and prooving the children nodes type StackEntry struct { keyOffset int @@ -41,7 +39,8 @@ type StackEntry struct { outputIndex int } -// +// NewStackEntry puts a node into the stack with its hash, the node key (prefix), position into +// the proof list (outputIndex) and the offset from the nibbled key (keyOffset) func NewStackEntry(n node, hash, rd, prefix []byte, outputIndex, keyOffset int) (*StackEntry, error) { _, _, err := n.encodeAndHash() if err != nil { @@ -58,6 +57,7 @@ func NewStackEntry(n node, hash, rd, prefix []byte, outputIndex, keyOffset int) }, nil } +// Stack represents a list of entries type Stack []*StackEntry // Push adds a new item to the top of the stack @@ -86,11 +86,13 @@ func (s *Stack) Last() *StackEntry { return (*s)[len(*s)-1] } +// StackIterator is used to transform the stack into a iterator, the iterator starts the cursor (index) from 0 type StackIterator struct { index int set []*StackEntry } +// Next returns the current item the cursor is on and increment the cursor by 1 func (i *StackIterator) Next() *StackEntry { if i.HasNext() { t := i.set[i.index] @@ -101,6 +103,7 @@ func (i *StackIterator) Next() *StackEntry { return nil } +// Peek returns the current item the cursor is on but dont increment the cursor by 1 func (i *StackIterator) Peek() *StackEntry { if i.HasNext() { return i.set[i.index] @@ -108,6 +111,7 @@ func (i *StackIterator) Peek() *StackEntry { return nil } +// HasNext returns true if there is more itens into the iterator func (i *StackIterator) HasNext() bool { return i.index < len(i.set) } @@ -120,7 +124,9 @@ func (s *Stack) iter() *StackIterator { return iter } -// +// Step struct helps the proof generation with the next step that should be taken when +// the seek node is found or to keep searching from the next node +// using the offset of the nibbled key type Step struct { Found bool Value []byte @@ -269,7 +275,7 @@ func matchKeyToBranchNode(n *branch, e *StackEntry, nibbleKey []byte) (Step, err }, nil } -func unwindStack(stack *Stack, proof [][]byte, key []byte) error { +func unwindStack(stack *Stack, proof [][]byte, key []byte) { for { entry := stack.Pop() if entry == nil { @@ -283,6 +289,4 @@ func unwindStack(stack *Stack, proof [][]byte, key []byte) error { proof[entry.outputIndex] = entry.nodeRawData } - - return nil } diff --git a/lib/trie/recorder.go b/lib/trie/recorder.go index 93273bf52a..3f5d8eb19a 100644 --- a/lib/trie/recorder.go +++ b/lib/trie/recorder.go @@ -6,17 +6,21 @@ type NodeRecord struct { Hash []byte } +// Recorder keeps the list of nodes find by Lookup.Find type Recorder []NodeRecord +// Record insert a node insede the recorded list func (r *Recorder) Record(h, rd []byte) { nr := NodeRecord{RawData: rd, Hash: h} *r = append(*r, nr) } +// Len returns the current length of the recorded list func (r *Recorder) Len() int { return len(*r) } +// Next returns the current item the cursor is on and increment the cursor by 1 func (r *Recorder) Next() *NodeRecord { if r.Len() > 0 { n := (*r)[0] @@ -27,6 +31,7 @@ func (r *Recorder) Next() *NodeRecord { return nil } +// Peek returns the current item the cursor is on but dont increment the cursor by 1 func (r *Recorder) Peek() *NodeRecord { if r.Len() > 0 { return &(*r)[0] From 56a726311f70fbbe1f32c059ab52a958296bcaec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 21 Sep 2021 19:13:15 -0400 Subject: [PATCH 16/38] chore: update GenerateTrieProof from block to storage state --- dot/core/interface.go | 2 +- dot/core/mocks/storage_state.go | 36 ++++++++++++++++++++++++--------- dot/core/service.go | 2 +- dot/rpc/modules/state.go | 1 - dot/state/block.go | 6 ------ dot/state/storage.go | 5 +++++ lib/trie/lookup.go | 6 +++--- 7 files changed, 36 insertions(+), 22 deletions(-) diff --git a/dot/core/interface.go b/dot/core/interface.go index c3a79d6d46..05071f7620 100644 --- a/dot/core/interface.go +++ b/dot/core/interface.go @@ -43,7 +43,6 @@ type BlockState interface { GetSlotForBlock(common.Hash) (uint64, error) GetFinalisedHeader(uint64, uint64) (*types.Header, error) GetFinalisedHash(uint64, uint64) (common.Hash, error) - GenerateTrieProof(stateRoot common.Hash, keys [][]byte) ([][]byte, error) RegisterImportedChannel(ch chan<- *types.Block) (byte, error) UnregisterImportedChannel(id byte) RegisterFinalizedChannel(ch chan<- *types.FinalisationInfo) (byte, error) @@ -64,6 +63,7 @@ type StorageState interface { StoreTrie(*rtstorage.TrieState, *types.Header) error GetStateRootFromBlock(bhash *common.Hash) (*common.Hash, error) GetStorage(root *common.Hash, key []byte) ([]byte, error) + GenerateTrieProof(stateRoot common.Hash, keys [][]byte) ([][]byte, error) sync.Locker } diff --git a/dot/core/mocks/storage_state.go b/dot/core/mocks/storage_state.go index 4497b2b784..654e1e8ec9 100644 --- a/dot/core/mocks/storage_state.go +++ b/dot/core/mocks/storage_state.go @@ -1,9 +1,8 @@ -// Code generated by mockery v2.8.0. DO NOT EDIT. +// Code generated by mockery v0.0.0-dev. DO NOT EDIT. package mocks import ( - chaindb "github.com/ChainSafe/chaindb" common "github.com/ChainSafe/gossamer/lib/common" mock "github.com/stretchr/testify/mock" @@ -18,20 +17,27 @@ type MockStorageState struct { mock.Mock } -// ExposeDB provides a mock function with given fields: -func (_m *MockStorageState) ExposeDB() chaindb.Database { - ret := _m.Called() +// GenerateTrieProof provides a mock function with given fields: stateRoot, keys +func (_m *MockStorageState) GenerateTrieProof(stateRoot common.Hash, keys [][]byte) ([][]byte, error) { + ret := _m.Called(stateRoot, keys) - var r0 chaindb.Database - if rf, ok := ret.Get(0).(func() chaindb.Database); ok { - r0 = rf() + var r0 [][]byte + if rf, ok := ret.Get(0).(func(common.Hash, [][]byte) [][]byte); ok { + r0 = rf(stateRoot, keys) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(chaindb.Database) + r0 = ret.Get(0).([][]byte) } } - return r0 + var r1 error + if rf, ok := ret.Get(1).(func(common.Hash, [][]byte) error); ok { + r1 = rf(stateRoot, keys) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } // GetStateRootFromBlock provides a mock function with given fields: bhash @@ -126,6 +132,11 @@ func (_m *MockStorageState) LoadCodeHash(root *common.Hash) (common.Hash, error) return r0, r1 } +// Lock provides a mock function with given fields: +func (_m *MockStorageState) Lock() { + _m.Called() +} + // StoreTrie provides a mock function with given fields: _a0, _a1 func (_m *MockStorageState) StoreTrie(_a0 *storage.TrieState, _a1 *types.Header) error { ret := _m.Called(_a0, _a1) @@ -162,3 +173,8 @@ func (_m *MockStorageState) TrieState(root *common.Hash) (*storage.TrieState, er return r0, r1 } + +// Unlock provides a mock function with given fields: +func (_m *MockStorageState) Unlock() { + _m.Called() +} diff --git a/dot/core/service.go b/dot/core/service.go index 4919e8757a..384584d180 100644 --- a/dot/core/service.go +++ b/dot/core/service.go @@ -640,7 +640,7 @@ func (s *Service) GetReadProofAt(block common.Hash, keys [][]byte) (common.Hash, return common.EmptyHash, nil, err } - proofForKeys, err := s.blockState.GenerateTrieProof(stateRoot, keys) + proofForKeys, err := s.storageState.GenerateTrieProof(stateRoot, keys) if err != nil { return common.EmptyHash, nil, err } diff --git a/dot/rpc/modules/state.go b/dot/rpc/modules/state.go index 00d822b884..6110d5dc49 100644 --- a/dot/rpc/modules/state.go +++ b/dot/rpc/modules/state.go @@ -303,7 +303,6 @@ func (sm *StateModule) GetReadProof(r *http.Request, req *StateGetReadProofReque keys[i] = bKey } - fmt.Println("GET READ PROOF RPC", req.Keys, len(keys)) block, proofs, err := sm.coreAPI.GetReadProofAt(req.Hash, keys) if err != nil { return err diff --git a/dot/state/block.go b/dot/state/block.go index 4f0fd4128a..44eed186a5 100644 --- a/dot/state/block.go +++ b/dot/state/block.go @@ -34,7 +34,6 @@ import ( rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/runtime/wasmer" - "github.com/ChainSafe/gossamer/lib/trie" ) var blockPrefix = "block" @@ -745,8 +744,3 @@ func (bs *BlockState) StoreRuntime(hash common.Hash, rt runtime.Instance) { func (bs *BlockState) GetNonFinalisedBlocks() []common.Hash { return bs.bt.GetAllBlocks() } - -// GenerateTrieProof returns the proofs related to the keys on the state root trie -func (bs *BlockState) GenerateTrieProof(stateRoot common.Hash, keys [][]byte) ([][]byte, error) { - return trie.GenerateProof(stateRoot[:], keys, bs.db) -} diff --git a/dot/state/storage.go b/dot/state/storage.go index fd26ac24c0..2291d8b880 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -325,6 +325,11 @@ func (s *StorageState) LoadCodeHash(hash *common.Hash) (common.Hash, error) { return common.Blake2bHash(code) } +// GenerateTrieProof returns the proofs related to the keys on the state root trie +func (s *StorageState) GenerateTrieProof(stateRoot common.Hash, keys [][]byte) ([][]byte, error) { + return trie.GenerateProof(stateRoot[:], keys, s.db) +} + // GetBalance gets the balance for an account with the given public key func (s *StorageState) GetBalance(hash *common.Hash, key [32]byte) (uint64, error) { skey, err := common.BalanceKey(key) diff --git a/lib/trie/lookup.go b/lib/trie/lookup.go index b2a009e595..94e1c98b02 100644 --- a/lib/trie/lookup.go +++ b/lib/trie/lookup.go @@ -21,10 +21,10 @@ type Lookup struct { } // NewLookup returns a Lookup to helps the proof generator -func NewLookup(h []byte, db chaindb.Database) *Lookup { +func NewLookup(rootHash []byte, db chaindb.Database) *Lookup { lk := &Lookup{db: db} - lk.root = make([]byte, len(h)) - copy(lk.root, h) + lk.root = make([]byte, len(rootHash)) + copy(lk.root, rootHash) return lk } From 449acb430994ee9c9be7813264633b1c6acd95e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 21 Sep 2021 20:02:52 -0400 Subject: [PATCH 17/38] chore: remove repeated code --- lib/trie/proof.go | 257 +++-------------------------------------- lib/trie/proof_test.go | 2 +- lib/trie/recorder.go | 13 +-- 3 files changed, 23 insertions(+), 249 deletions(-) diff --git a/lib/trie/proof.go b/lib/trie/proof.go index 3548852874..d264bd9773 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -17,10 +17,11 @@ package trie import ( - "bytes" "errors" + "fmt" "github.com/ChainSafe/chaindb" + "github.com/ChainSafe/gossamer/lib/common" ) var ( @@ -28,265 +29,39 @@ var ( ErrEmptyTrieRoot = errors.New("provided trie must have a root") ) -// StackEntry is a entry on the nodes that is proved already -// it stores the necessary infos to keep going and prooving the children nodes -type StackEntry struct { - keyOffset int - key []byte - nodeHash []byte - node node - nodeRawData []byte - outputIndex int -} - -// NewStackEntry puts a node into the stack with its hash, the node key (prefix), position into -// the proof list (outputIndex) and the offset from the nibbled key (keyOffset) -func NewStackEntry(n node, hash, rd, prefix []byte, outputIndex, keyOffset int) (*StackEntry, error) { - _, _, err := n.encodeAndHash() - if err != nil { - return nil, err - } - - return &StackEntry{ - keyOffset: keyOffset, - nodeHash: hash, - key: prefix, - node: n, - outputIndex: outputIndex, - nodeRawData: rd, - }, nil -} - -// Stack represents a list of entries -type Stack []*StackEntry - -// Push adds a new item to the top of the stack -func (s *Stack) Push(e *StackEntry) { - (*s) = append((*s), e) -} - -// Pop removes and returns the top of the stack if there is some element there otherwise return nil -func (s *Stack) Pop() *StackEntry { - if len(*s) < 1 { - return nil - } - - // gets the top of the stack - entry := (*s)[len(*s)-1] - // removes the top of the stack - *s = (*s)[:len(*s)-1] - return entry -} - -// Last returns the top of the stack without remove from the stack -func (s *Stack) Last() *StackEntry { - if len(*s) < 1 { - return nil - } - return (*s)[len(*s)-1] -} - -// StackIterator is used to transform the stack into a iterator, the iterator starts the cursor (index) from 0 -type StackIterator struct { - index int - set []*StackEntry -} - -// Next returns the current item the cursor is on and increment the cursor by 1 -func (i *StackIterator) Next() *StackEntry { - if i.HasNext() { - t := i.set[i.index] - i.index++ - return t - } - - return nil -} - -// Peek returns the current item the cursor is on but dont increment the cursor by 1 -func (i *StackIterator) Peek() *StackEntry { - if i.HasNext() { - return i.set[i.index] - } - return nil -} - -// HasNext returns true if there is more itens into the iterator -func (i *StackIterator) HasNext() bool { - return i.index < len(i.set) -} - -func (s *Stack) iter() *StackIterator { - iter := &StackIterator{index: 0} - iter.set = make([]*StackEntry, len(*s)) - copy(iter.set, *s) - - return iter -} - -// Step struct helps the proof generation with the next step that should be taken when -// the seek node is found or to keep searching from the next node -// using the offset of the nibbled key -type Step struct { - Found bool - Value []byte - NextNode []byte - KeyOffset int -} - // GenerateProof receive the keys to proof, the trie root and a reference to database // will func GenerateProof(root []byte, keys [][]byte, db chaindb.Database) ([][]byte, error) { - stack := make(Stack, 0) - proofs := make([][]byte, 0) + trackedProofs := make(map[string][]byte) for _, k := range keys { nk := keyToNibbles(k) - unwindStack(&stack, proofs, nk) - lookup := NewLookup(root, db) recorder := new(Recorder) - expectedValue, err := lookup.Find(nk, recorder) + + _, err := lookup.Find(nk, recorder) if err != nil { return nil, err } - // Skip over recorded nodes already on the stack - stackIter := stack.iter() - for stackIter.HasNext() { - nxtRec, nxtEntry := recorder.Peek(), stackIter.Peek() - if !bytes.Equal(nxtRec.Hash, nxtEntry.nodeHash) { - break - } - - stackIter.Next() - recorder.Next() - } + fmt.Println("Len of records ", len(*recorder)) - for { - var step Step - if len(stack) == 0 { - // as the stack is empty then start from the root node - step = Step{ - Found: false, - NextNode: root, - KeyOffset: 0, - } - } else { - entryOnTop := stack.Last() - step, err = matchKeyToNode(entryOnTop, nk) - if err != nil { - return nil, err - } + for recorder.HasNext() { + recNode := recorder.Next() + nodeHashHex := common.BytesToHex(recNode.Hash) + if _, ok := trackedProofs[nodeHashHex]; !ok { + trackedProofs[nodeHashHex] = recNode.RawData } - - if step.Found { - if len(step.Value) > 0 && bytes.Equal(step.Value, expectedValue) && recorder.Len() > 0 { - return nil, errors.New("value found is not expected or there is recNodes to traverse") - } - - break - } - - rec := recorder.Next() - if rec == nil { - return nil, errors.New("recorder must has nodes to traverse") - } - - if !bytes.Equal(rec.Hash, step.NextNode) { - return nil, errors.New("recorded node does not match expected node") - } - - n, err := decodeBytes(rec.RawData) - if err != nil { - return nil, err - } - - outputIndex := len(proofs) - proofs = append(proofs, []byte{}) - - prefix := make([]byte, len(nk[:step.KeyOffset])) - copy(prefix, nk[:step.KeyOffset]) - - ne, err := NewStackEntry(n, rec.Hash, rec.RawData, prefix, outputIndex, step.KeyOffset) - if err != nil { - return nil, err - } - - stack.Push(ne) } } - unwindStack(&stack, proofs, nil) - return proofs, nil -} - -func matchKeyToNode(e *StackEntry, nibbleKey []byte) (Step, error) { - switch ntype := e.node.(type) { - case nil: - return Step{Found: true}, nil - case *leaf: - keyToCompare := nibbleKey[e.keyOffset:] - if bytes.Equal(keyToCompare, ntype.key) && len(nibbleKey) == len(ntype.key)+e.keyOffset { - return Step{ - Found: true, - Value: ntype.value, - }, nil - } - - return Step{Found: true}, nil - case *branch: - return matchKeyToBranchNode(ntype, e, nibbleKey) - default: - return Step{}, errors.New("could not be possible to define the node type") - } -} - -func matchKeyToBranchNode(n *branch, e *StackEntry, nibbleKey []byte) (Step, error) { - keyToCompare := nibbleKey[e.keyOffset:] - if !bytes.HasPrefix(keyToCompare, n.key) { - return Step{Found: true}, nil - } - - if len(nibbleKey) == len(n.key)+e.keyOffset { - return Step{ - Found: true, - Value: n.value, - }, nil - } - - newIndex := nibbleKey[e.keyOffset+len(n.key)] - child := n.children[newIndex] - - if child == nil { - return Step{Found: true}, nil - } + proofs := make([][]byte, 0) - _, hash, err := child.encodeAndHash() - if err != nil { - return Step{}, err + for _, p := range trackedProofs { + fmt.Printf("tracked proofs: 0x%x\n", p) + proofs = append(proofs, p) } - return Step{ - Found: false, - KeyOffset: e.keyOffset + len(n.key) + 1, - NextNode: hash, - }, nil -} - -func unwindStack(stack *Stack, proof [][]byte, key []byte) { - for { - entry := stack.Pop() - if entry == nil { - break - } - - if key != nil && bytes.HasPrefix(key, entry.key) { - stack.Push(entry) - break - } - - proof[entry.outputIndex] = entry.nodeRawData - } + return proofs, nil } diff --git a/lib/trie/proof_test.go b/lib/trie/proof_test.go index b7326c2980..d54d57b414 100644 --- a/lib/trie/proof_test.go +++ b/lib/trie/proof_test.go @@ -51,5 +51,5 @@ func TestProofGeneration(t *testing.T) { require.NoError(t, err) // TODO: use the verify_proof function to assert the tests - require.Equal(t, len(proof), 5) + require.Equal(t, 5, len(proof)) } diff --git a/lib/trie/recorder.go b/lib/trie/recorder.go index 3f5d8eb19a..5691d2aa3c 100644 --- a/lib/trie/recorder.go +++ b/lib/trie/recorder.go @@ -15,14 +15,9 @@ func (r *Recorder) Record(h, rd []byte) { *r = append(*r, nr) } -// Len returns the current length of the recorded list -func (r *Recorder) Len() int { - return len(*r) -} - // Next returns the current item the cursor is on and increment the cursor by 1 func (r *Recorder) Next() *NodeRecord { - if r.Len() > 0 { + if r.HasNext() { n := (*r)[0] *r = (*r)[1:] return &n @@ -33,8 +28,12 @@ func (r *Recorder) Next() *NodeRecord { // Peek returns the current item the cursor is on but dont increment the cursor by 1 func (r *Recorder) Peek() *NodeRecord { - if r.Len() > 0 { + if r.HasNext() { return &(*r)[0] } return nil } + +func (r *Recorder) HasNext() bool { + return len(*r) > 0 +} From 6ce68841274a99146b3819972c044f8b471b3ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 21 Sep 2021 20:07:03 -0400 Subject: [PATCH 18/38] chore: resolve fix lint --- lib/trie/recorder.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/trie/recorder.go b/lib/trie/recorder.go index 5691d2aa3c..3617126067 100644 --- a/lib/trie/recorder.go +++ b/lib/trie/recorder.go @@ -34,6 +34,7 @@ func (r *Recorder) Peek() *NodeRecord { return nil } +// HasNext returns bool if there is data inside the slice func (r *Recorder) HasNext() bool { return len(*r) > 0 } From f6aafb050bd1393a229395cb47082f3056725200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 22 Sep 2021 14:16:08 -0400 Subject: [PATCH 19/38] chore: update tests --- dot/rpc/modules/state.go | 2 +- dot/rpc/modules/state_test.go | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/dot/rpc/modules/state.go b/dot/rpc/modules/state.go index 6110d5dc49..10d2f3a195 100644 --- a/dot/rpc/modules/state.go +++ b/dot/rpc/modules/state.go @@ -310,7 +310,7 @@ func (sm *StateModule) GetReadProof(r *http.Request, req *StateGetReadProofReque var decProof []string for _, p := range proofs { - decProof = append(decProof, fmt.Sprintf("0x%x", p)) + decProof = append(decProof, common.BytesToHex(p)) } *res = StateGetReadProofResponse{ diff --git a/dot/rpc/modules/state_test.go b/dot/rpc/modules/state_test.go index 715dfe797b..5bbbc40e1a 100644 --- a/dot/rpc/modules/state_test.go +++ b/dot/rpc/modules/state_test.go @@ -479,7 +479,7 @@ func TestStateModule_GetKeysPaged(t *testing.T) { func TestGetReadProof_WhenCoreAPIReturnsError(t *testing.T) { coreAPIMock := new(mocks.MockCoreAPI) coreAPIMock. - On("GetReadProofAt", mock.AnythingOfType("common.Hash"), mock.AnythingOfType("[]common.Hash")). + On("GetReadProofAt", mock.AnythingOfType("common.Hash"), mock.AnythingOfType("[][]uint8")). Return(common.EmptyHash, nil, errors.New("mocked error")) sm := new(StateModule) @@ -495,12 +495,12 @@ func TestGetReadProof_WhenCoreAPIReturnsError(t *testing.T) { func TestGetReadProof_WhenReturnsProof(t *testing.T) { expectedBlock := common.BytesToHash([]byte("random hash")) - expectedProof := []string{"proof-1", "proof-2"} + mockedProof := [][]byte{[]byte("proof-1"), []byte("proof-2")} coreAPIMock := new(mocks.MockCoreAPI) coreAPIMock. - On("GetReadProofAt", mock.AnythingOfType("common.Hash"), mock.AnythingOfType("[]common.Hash")). - Return(expectedBlock, expectedProof, nil) + On("GetReadProofAt", mock.AnythingOfType("common.Hash"), mock.AnythingOfType("[][]uint8")). + Return(expectedBlock, mockedProof, nil) sm := new(StateModule) sm.coreAPI = coreAPIMock @@ -514,6 +514,12 @@ func TestGetReadProof_WhenReturnsProof(t *testing.T) { err := sm.GetReadProof(nil, req, res) require.NoError(t, err) require.Equal(t, res.At, expectedBlock) + + expectedProof := []string{ + common.BytesToHex([]byte("proof-1")), + common.BytesToHex([]byte("proof-2")), + } + require.Equal(t, res.Proof, expectedProof) } From bdfbe804caf4cfa4a88da9979c7b2dbe1a0a41b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 22 Sep 2021 14:22:50 -0400 Subject: [PATCH 20/38] chore: simplify to sliced value itself --- lib/trie/lookup.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/trie/lookup.go b/lib/trie/lookup.go index 94e1c98b02..e98cce4f1a 100644 --- a/lib/trie/lookup.go +++ b/lib/trie/lookup.go @@ -35,7 +35,7 @@ func (l *Lookup) Find(key []byte, recorder *Recorder) ([]byte, error) { hash := l.root for { - nodeData, err := l.db.Get(hash[:]) + nodeData, err := l.db.Get(hash) if err != nil { return nil, ErrProofNodeNotFound } @@ -79,7 +79,7 @@ func (l *Lookup) Find(key []byte, recorder *Recorder) ([]byte, error) { return nil, nil default: partial = partial[length+1:] - copy(hash[:], child.getHash()) + copy(hash, child.getHash()) } } } From 7f130a88fc4c398d74f73caa1c15c2489917f87d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 22 Sep 2021 14:27:53 -0400 Subject: [PATCH 21/38] chore: deepsource ignore auto generated files --- .deepsource.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.deepsource.toml b/.deepsource.toml index 1c41320658..9053d32f39 100644 --- a/.deepsource.toml +++ b/.deepsource.toml @@ -11,7 +11,9 @@ exclude_patterns = [ "dot/config/**/*", "dot/rpc/modules/test_data", "lib/runtime/test_data", - "**/*_test.go" + "**/*_test.go", + "**/mocks/*", + "**/mock_*" ] [[analyzers]] From e800b0b07c89e662b24c2b37260849d983ba67bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 22 Sep 2021 14:32:47 -0400 Subject: [PATCH 22/38] chore: remove unused refs --- dot/rpc/modules/state.go | 10 +++++----- dot/state/storage.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dot/rpc/modules/state.go b/dot/rpc/modules/state.go index 10d2f3a195..4ec809a3f3 100644 --- a/dot/rpc/modules/state.go +++ b/dot/rpc/modules/state.go @@ -228,25 +228,25 @@ func (sm *StateModule) Call(r *http.Request, req *StateCallRequest, res *StateCa } // GetChildKeys isn't implemented properly yet. -func (sm *StateModule) GetChildKeys(r *http.Request, req *StateChildStorageRequest, res *StateKeysResponse) error { +func (*StateModule) GetChildKeys(r *http.Request, req *StateChildStorageRequest, res *StateKeysResponse) error { // TODO implement change storage trie so that block hash parameter works (See issue #834) return nil } // GetChildStorage isn't implemented properly yet. -func (sm *StateModule) GetChildStorage(r *http.Request, req *StateChildStorageRequest, res *StateStorageDataResponse) error { +func (*StateModule) GetChildStorage(r *http.Request, req *StateChildStorageRequest, res *StateStorageDataResponse) error { // TODO implement change storage trie so that block hash parameter works (See issue #834) return nil } // GetChildStorageHash isn't implemented properly yet. -func (sm *StateModule) GetChildStorageHash(r *http.Request, req *StateChildStorageRequest, res *StateChildStorageResponse) error { +func (*StateModule) GetChildStorageHash(r *http.Request, req *StateChildStorageRequest, res *StateChildStorageResponse) error { // TODO implement change storage trie so that block hash parameter works (See issue #834) return nil } // GetChildStorageSize isn't implemented properly yet. -func (sm *StateModule) GetChildStorageSize(r *http.Request, req *StateChildStorageRequest, res *StateChildStorageSizeResponse) error { +func (*StateModule) GetChildStorageSize(r *http.Request, req *StateChildStorageRequest, res *StateChildStorageSizeResponse) error { // TODO implement change storage trie so that block hash parameter works (See issue #834) return nil } @@ -469,7 +469,7 @@ func (sm *StateModule) SubscribeRuntimeVersion(r *http.Request, req *StateStorag // SubscribeStorage Storage subscription. If storage keys are specified, it creates a message for each block which // changes the specified storage keys. If none are specified, then it creates a message for every block. // This endpoint communicates over the Websocket protocol, but this func should remain here so it's added to rpc_methods list -func (sm *StateModule) SubscribeStorage(r *http.Request, req *StateStorageQueryRangeRequest, res *StorageChangeSetResponse) error { +func (*StateModule) SubscribeStorage(r *http.Request, req *StateStorageQueryRangeRequest, res *StorageChangeSetResponse) error { return nil } diff --git a/dot/state/storage.go b/dot/state/storage.go index 2291d8b880..33e81e1600 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -265,7 +265,7 @@ func (s *StorageState) StorageRoot() (common.Hash, error) { } // EnumeratedTrieRoot not implemented -func (s *StorageState) EnumeratedTrieRoot(values [][]byte) { +func (*StorageState) EnumeratedTrieRoot(values [][]byte) { //TODO panic("not implemented") } From 76ff9c937aea27f996c6efeb15e83e039860fb24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 22 Sep 2021 14:38:54 -0400 Subject: [PATCH 23/38] chore: addressing deepsource warngins --- dot/rpc/modules/state.go | 30 +++++++++++++++--------------- dot/state/storage.go | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dot/rpc/modules/state.go b/dot/rpc/modules/state.go index 4ec809a3f3..bdb0064f7d 100644 --- a/dot/rpc/modules/state.go +++ b/dot/rpc/modules/state.go @@ -221,38 +221,38 @@ func (sm *StateModule) GetPairs(r *http.Request, req *StatePairRequest, res *Sta } // Call isn't implemented properly yet. -func (sm *StateModule) Call(r *http.Request, req *StateCallRequest, res *StateCallResponse) error { +func (sm *StateModule) Call(r *http.Request, _ *StateCallRequest, _ *StateCallResponse) error { _ = sm.networkAPI _ = sm.storageAPI return nil } // GetChildKeys isn't implemented properly yet. -func (*StateModule) GetChildKeys(r *http.Request, req *StateChildStorageRequest, res *StateKeysResponse) error { +func (*StateModule) GetChildKeys(_ *http.Request, _ *StateChildStorageRequest, _ *StateKeysResponse) error { // TODO implement change storage trie so that block hash parameter works (See issue #834) return nil } // GetChildStorage isn't implemented properly yet. -func (*StateModule) GetChildStorage(r *http.Request, req *StateChildStorageRequest, res *StateStorageDataResponse) error { +func (*StateModule) GetChildStorage(_ *http.Request, _ *StateChildStorageRequest, _ *StateStorageDataResponse) error { // TODO implement change storage trie so that block hash parameter works (See issue #834) return nil } // GetChildStorageHash isn't implemented properly yet. -func (*StateModule) GetChildStorageHash(r *http.Request, req *StateChildStorageRequest, res *StateChildStorageResponse) error { +func (*StateModule) GetChildStorageHash(_ *http.Request, _ *StateChildStorageRequest, _ *StateChildStorageResponse) error { // TODO implement change storage trie so that block hash parameter works (See issue #834) return nil } // GetChildStorageSize isn't implemented properly yet. -func (*StateModule) GetChildStorageSize(r *http.Request, req *StateChildStorageRequest, res *StateChildStorageSizeResponse) error { +func (*StateModule) GetChildStorageSize(_ *http.Request, _ *StateChildStorageRequest, _ *StateChildStorageSizeResponse) error { // TODO implement change storage trie so that block hash parameter works (See issue #834) return nil } // GetKeysPaged Returns the keys with prefix with pagination support. -func (sm *StateModule) GetKeysPaged(r *http.Request, req *StateStorageKeyRequest, res *StateStorageKeysResponse) error { +func (sm *StateModule) GetKeysPaged(_ *http.Request, req *StateStorageKeyRequest, res *StateStorageKeysResponse) error { if req.Prefix == "" { req.Prefix = "0x" } @@ -278,7 +278,7 @@ func (sm *StateModule) GetKeysPaged(r *http.Request, req *StateStorageKeyRequest } // GetMetadata calls runtime Metadata_metadata function -func (sm *StateModule) GetMetadata(r *http.Request, req *StateRuntimeMetadataQuery, res *StateMetadataResponse) error { +func (sm *StateModule) GetMetadata(_ *http.Request, req *StateRuntimeMetadataQuery, res *StateMetadataResponse) error { // TODO implement change storage trie so that block hash parameter works (See issue #834) metadata, err := sm.coreAPI.GetMetadata(req.Bhash) if err != nil { @@ -292,7 +292,7 @@ func (sm *StateModule) GetMetadata(r *http.Request, req *StateRuntimeMetadataQue } // GetReadProof returns the proof to the received storage keys -func (sm *StateModule) GetReadProof(r *http.Request, req *StateGetReadProofRequest, res *StateGetReadProofResponse) error { +func (sm *StateModule) GetReadProof(_ *http.Request, req *StateGetReadProofRequest, res *StateGetReadProofResponse) error { keys := make([][]byte, len(req.Keys)) for i, hexKey := range req.Keys { bKey, err := common.HexToBytes(hexKey) @@ -324,7 +324,7 @@ func (sm *StateModule) GetReadProof(r *http.Request, req *StateGetReadProofReque // GetRuntimeVersion Get the runtime version at a given block. // If no block hash is provided, the latest version gets returned. // TODO currently only returns latest version, add functionality to lookup runtime by block hash (see issue #834) -func (sm *StateModule) GetRuntimeVersion(r *http.Request, req *StateRuntimeVersionRequest, res *StateRuntimeVersionResponse) error { +func (sm *StateModule) GetRuntimeVersion(_ *http.Request, req *StateRuntimeVersionRequest, res *StateRuntimeVersionResponse) error { rtVersion, err := sm.coreAPI.GetRuntimeVersion(req.Bhash) if err != nil { return err @@ -342,7 +342,7 @@ func (sm *StateModule) GetRuntimeVersion(r *http.Request, req *StateRuntimeVersi } // GetStorage Returns a storage entry at a specific block's state. If not block hash is provided, the latest value is returned. -func (sm *StateModule) GetStorage(r *http.Request, req *StateStorageRequest, res *StateStorageResponse) error { +func (sm *StateModule) GetStorage(_ *http.Request, req *StateStorageRequest, res *StateStorageResponse) error { var ( item []byte err error @@ -372,7 +372,7 @@ func (sm *StateModule) GetStorage(r *http.Request, req *StateStorageRequest, res // GetStorageHash returns the hash of a storage entry at a block's state. // If no block hash is provided, the latest value is returned. // TODO implement change storage trie so that block hash parameter works (See issue #834) -func (sm *StateModule) GetStorageHash(r *http.Request, req *StateStorageHashRequest, res *StateStorageHashResponse) error { +func (sm *StateModule) GetStorageHash(_ *http.Request, req *StateStorageHashRequest, res *StateStorageHashResponse) error { var ( item []byte err error @@ -402,7 +402,7 @@ func (sm *StateModule) GetStorageHash(r *http.Request, req *StateStorageHashRequ // GetStorageSize returns the size of a storage entry at a block's state. // If no block hash is provided, the latest value is used. // TODO implement change storage trie so that block hash parameter works (See issue #834) -func (sm *StateModule) GetStorageSize(r *http.Request, req *StateStorageSizeRequest, res *StateStorageSizeResponse) error { +func (sm *StateModule) GetStorageSize(_ *http.Request, req *StateStorageSizeRequest, res *StateStorageSizeResponse) error { var ( item []byte err error @@ -430,7 +430,7 @@ func (sm *StateModule) GetStorageSize(r *http.Request, req *StateStorageSizeRequ } // QueryStorage isn't implemented properly yet. -func (sm *StateModule) QueryStorage(r *http.Request, req *StateStorageQueryRangeRequest, res *[]StorageChangeSetResponse) error { +func (sm *StateModule) QueryStorage(_ *http.Request, req *StateStorageQueryRangeRequest, res *[]StorageChangeSetResponse) error { if req.StartBlock == common.EmptyHash { return errors.New("the start block hash cannot be an empty value") } @@ -461,7 +461,7 @@ func (sm *StateModule) QueryStorage(r *http.Request, req *StateStorageQueryRange // SubscribeRuntimeVersion isn't implemented properly yet. // TODO make this actually a subscription that pushes data -func (sm *StateModule) SubscribeRuntimeVersion(r *http.Request, req *StateStorageQueryRangeRequest, res *StateRuntimeVersionResponse) error { +func (sm *StateModule) SubscribeRuntimeVersion(r *http.Request, _ *StateStorageQueryRangeRequest, res *StateRuntimeVersionResponse) error { // TODO implement change storage trie so that block hash parameter works (See issue #834) return sm.GetRuntimeVersion(r, nil, res) } @@ -469,7 +469,7 @@ func (sm *StateModule) SubscribeRuntimeVersion(r *http.Request, req *StateStorag // SubscribeStorage Storage subscription. If storage keys are specified, it creates a message for each block which // changes the specified storage keys. If none are specified, then it creates a message for every block. // This endpoint communicates over the Websocket protocol, but this func should remain here so it's added to rpc_methods list -func (*StateModule) SubscribeStorage(r *http.Request, req *StateStorageQueryRangeRequest, res *StorageChangeSetResponse) error { +func (*StateModule) SubscribeStorage(_ *http.Request, _ *StateStorageQueryRangeRequest, _ *StorageChangeSetResponse) error { return nil } diff --git a/dot/state/storage.go b/dot/state/storage.go index 33e81e1600..96e9643ca7 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -265,7 +265,7 @@ func (s *StorageState) StorageRoot() (common.Hash, error) { } // EnumeratedTrieRoot not implemented -func (*StorageState) EnumeratedTrieRoot(values [][]byte) { +func (*StorageState) EnumeratedTrieRoot(_ [][]byte) { //TODO panic("not implemented") } From 1758e912f24a27365bebd1304a509eba6608ce60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Wed, 22 Sep 2021 14:45:22 -0400 Subject: [PATCH 24/38] chore: addressing deepsource wars --- dot/rpc/modules/state.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dot/rpc/modules/state.go b/dot/rpc/modules/state.go index bdb0064f7d..cb9c8dcba8 100644 --- a/dot/rpc/modules/state.go +++ b/dot/rpc/modules/state.go @@ -180,7 +180,7 @@ func NewStateModule(net NetworkAPI, storage StorageAPI, core CoreAPI) *StateModu } // GetPairs returns the keys with prefix, leave empty to get all the keys. -func (sm *StateModule) GetPairs(r *http.Request, req *StatePairRequest, res *StatePairResponse) error { +func (sm *StateModule) GetPairs(_ *http.Request, req *StatePairRequest, res *StatePairResponse) error { // TODO implement change storage trie so that block hash parameter works (See issue #834) var ( stateRootHash *common.Hash @@ -221,7 +221,7 @@ func (sm *StateModule) GetPairs(r *http.Request, req *StatePairRequest, res *Sta } // Call isn't implemented properly yet. -func (sm *StateModule) Call(r *http.Request, _ *StateCallRequest, _ *StateCallResponse) error { +func (sm *StateModule) Call(_ *http.Request, _ *StateCallRequest, _ *StateCallResponse) error { _ = sm.networkAPI _ = sm.storageAPI return nil From 6fd30fc353fd34ba462573e8af29c9901e97ff3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 24 Sep 2021 13:59:13 -0400 Subject: [PATCH 25/38] chore: remove DS_Store --- lib/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 lib/.DS_Store diff --git a/lib/.DS_Store b/lib/.DS_Store deleted file mode 100644 index 928bfd12fd6d1f50a5d054f2b6d98f7042bea4e7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~K?=e^3`G;|LU7Zi%h`AUZ!n0SpcinVh#(b;x}Kx^lL>;=wTS#c@+X-IrEk$` zL`2uO{Ys=2kr~`nmKH{)$Q!xJASXH8yP-c{`Xg#qpTt>P!&@2c$2NroNPq-LfCNb3 zhX~lc4V%rYGLirZkie6GeIF9sG>4X|{^>yQ5dhkt?1r_^63}D?Xbvq^QGsbp4;roN zV~Ewg9h%}?4lPw{yJ!p_8c$Z6VqjX^MH3R3W)}t$Ab}BqY0X=^|F`f@^Z%%YDG89k zpApbz({0vxskmF;UeD^gsM@-~p?)0U Date: Fri, 24 Sep 2021 14:00:44 -0400 Subject: [PATCH 26/38] chore: fix typo --- lib/trie/lookup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/trie/lookup.go b/lib/trie/lookup.go index e98cce4f1a..0c4ea991cc 100644 --- a/lib/trie/lookup.go +++ b/lib/trie/lookup.go @@ -9,7 +9,7 @@ import ( var ( // ErrProofNodeNotFound when a needed proof node is not in the database - ErrProofNodeNotFound = errors.New("cannot found the state root on storage") + ErrProofNodeNotFound = errors.New("cannot found the state root in storage") ) // Lookup struct holds the state root and database reference From ba421be8f0522efb7ddb3e7335daf7d19d465477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 24 Sep 2021 14:03:41 -0400 Subject: [PATCH 27/38] chore: use IsEmpty as record method --- lib/trie/proof.go | 6 +----- lib/trie/recorder.go | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/trie/proof.go b/lib/trie/proof.go index d264bd9773..0f2fb08c12 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -18,7 +18,6 @@ package trie import ( "errors" - "fmt" "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/lib/common" @@ -45,9 +44,7 @@ func GenerateProof(root []byte, keys [][]byte, db chaindb.Database) ([][]byte, e return nil, err } - fmt.Println("Len of records ", len(*recorder)) - - for recorder.HasNext() { + for !recorder.IsEmpty() { recNode := recorder.Next() nodeHashHex := common.BytesToHex(recNode.Hash) if _, ok := trackedProofs[nodeHashHex]; !ok { @@ -59,7 +56,6 @@ func GenerateProof(root []byte, keys [][]byte, db chaindb.Database) ([][]byte, e proofs := make([][]byte, 0) for _, p := range trackedProofs { - fmt.Printf("tracked proofs: 0x%x\n", p) proofs = append(proofs, p) } diff --git a/lib/trie/recorder.go b/lib/trie/recorder.go index 3617126067..9a5851330f 100644 --- a/lib/trie/recorder.go +++ b/lib/trie/recorder.go @@ -17,7 +17,7 @@ func (r *Recorder) Record(h, rd []byte) { // Next returns the current item the cursor is on and increment the cursor by 1 func (r *Recorder) Next() *NodeRecord { - if r.HasNext() { + if !r.IsEmpty() { n := (*r)[0] *r = (*r)[1:] return &n @@ -28,13 +28,13 @@ func (r *Recorder) Next() *NodeRecord { // Peek returns the current item the cursor is on but dont increment the cursor by 1 func (r *Recorder) Peek() *NodeRecord { - if r.HasNext() { + if !r.IsEmpty() { return &(*r)[0] } return nil } // HasNext returns bool if there is data inside the slice -func (r *Recorder) HasNext() bool { +func (r *Recorder) IsEmpty() bool { return len(*r) > 0 } From d65244488c19a768e46aadd1d66df2763c6706c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Fri, 24 Sep 2021 14:10:30 -0400 Subject: [PATCH 28/38] chore: update comment export --- lib/trie/recorder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/trie/recorder.go b/lib/trie/recorder.go index 9a5851330f..29f4cb7ab1 100644 --- a/lib/trie/recorder.go +++ b/lib/trie/recorder.go @@ -34,7 +34,7 @@ func (r *Recorder) Peek() *NodeRecord { return nil } -// HasNext returns bool if there is data inside the slice +// IsEmpty returns bool if there is data inside the slice func (r *Recorder) IsEmpty() bool { return len(*r) > 0 } From 9a06653c816efed5a4d47f55a15fae7c40858ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 27 Sep 2021 15:47:35 -0400 Subject: [PATCH 29/38] chore: resolve errors --- lib/trie/lookup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/trie/lookup.go b/lib/trie/lookup.go index 0c4ea991cc..1971728522 100644 --- a/lib/trie/lookup.go +++ b/lib/trie/lookup.go @@ -9,7 +9,7 @@ import ( var ( // ErrProofNodeNotFound when a needed proof node is not in the database - ErrProofNodeNotFound = errors.New("cannot found the state root in storage") + ErrProofNodeNotFound = errors.New("cannot find a trie node in the database") ) // Lookup struct holds the state root and database reference From b48fbce9b4ba7d837a14b2b5eaa0d2d85c6d31f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 27 Sep 2021 16:06:54 -0400 Subject: [PATCH 30/38] chore: resolve exported --- lib/trie/lookup.go | 14 +++++++------- lib/trie/proof.go | 8 ++++---- lib/trie/recorder.go | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/trie/lookup.go b/lib/trie/lookup.go index 1971728522..0725a818ef 100644 --- a/lib/trie/lookup.go +++ b/lib/trie/lookup.go @@ -12,25 +12,25 @@ var ( ErrProofNodeNotFound = errors.New("cannot find a trie node in the database") ) -// Lookup struct holds the state root and database reference +// lookup struct holds the state root and database reference // used to retrieve trie information from database -type Lookup struct { +type lookup struct { // root to start the lookup root []byte db chaindb.Database } // NewLookup returns a Lookup to helps the proof generator -func NewLookup(rootHash []byte, db chaindb.Database) *Lookup { - lk := &Lookup{db: db} +func NewLookup(rootHash []byte, db chaindb.Database) *lookup { + lk := &lookup{db: db} lk.root = make([]byte, len(rootHash)) copy(lk.root, rootHash) return lk } -// Find will return the desired value or nil if key cannot be found and will record visited nodes -func (l *Lookup) Find(key []byte, recorder *Recorder) ([]byte, error) { +// find will return the desired value or nil if key cannot be found and will record visited nodes +func (l *lookup) find(key []byte, recorder *recorder) ([]byte, error) { partial := key hash := l.root @@ -43,7 +43,7 @@ func (l *Lookup) Find(key []byte, recorder *Recorder) ([]byte, error) { nodeHash := make([]byte, len(hash)) copy(nodeHash, hash) - recorder.Record(nodeHash, nodeData) + recorder.record(nodeHash, nodeData) decoded, err := decodeBytes(nodeData) if err != nil { diff --git a/lib/trie/proof.go b/lib/trie/proof.go index 0f2fb08c12..1078fa5771 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -37,15 +37,15 @@ func GenerateProof(root []byte, keys [][]byte, db chaindb.Database) ([][]byte, e nk := keyToNibbles(k) lookup := NewLookup(root, db) - recorder := new(Recorder) + recorder := new(recorder) - _, err := lookup.Find(nk, recorder) + _, err := lookup.find(nk, recorder) if err != nil { return nil, err } - for !recorder.IsEmpty() { - recNode := recorder.Next() + for !recorder.isEmpty() { + recNode := recorder.next() nodeHashHex := common.BytesToHex(recNode.Hash) if _, ok := trackedProofs[nodeHashHex]; !ok { trackedProofs[nodeHashHex] = recNode.RawData diff --git a/lib/trie/recorder.go b/lib/trie/recorder.go index 29f4cb7ab1..ed865637fe 100644 --- a/lib/trie/recorder.go +++ b/lib/trie/recorder.go @@ -1,23 +1,23 @@ package trie -// NodeRecord represets a record of a visited node -type NodeRecord struct { +// nodeRecord represets a record of a visited node +type nodeRecord struct { RawData []byte Hash []byte } // Recorder keeps the list of nodes find by Lookup.Find -type Recorder []NodeRecord +type recorder []nodeRecord // Record insert a node insede the recorded list -func (r *Recorder) Record(h, rd []byte) { - nr := NodeRecord{RawData: rd, Hash: h} +func (r *recorder) record(h, rd []byte) { + nr := nodeRecord{RawData: rd, Hash: h} *r = append(*r, nr) } // Next returns the current item the cursor is on and increment the cursor by 1 -func (r *Recorder) Next() *NodeRecord { - if !r.IsEmpty() { +func (r *recorder) next() *nodeRecord { + if !r.isEmpty() { n := (*r)[0] *r = (*r)[1:] return &n @@ -27,14 +27,14 @@ func (r *Recorder) Next() *NodeRecord { } // Peek returns the current item the cursor is on but dont increment the cursor by 1 -func (r *Recorder) Peek() *NodeRecord { - if !r.IsEmpty() { +func (r *recorder) peek() *nodeRecord { + if !r.isEmpty() { return &(*r)[0] } return nil } // IsEmpty returns bool if there is data inside the slice -func (r *Recorder) IsEmpty() bool { +func (r *recorder) isEmpty() bool { return len(*r) > 0 } From 9fc6711be28dea9a6cde4f66d1f14bb3d5bc48cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 27 Sep 2021 16:19:22 -0400 Subject: [PATCH 31/38] chore: resolve test to dont exported --- lib/trie/test_utils.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/trie/test_utils.go b/lib/trie/test_utils.go index d60ad6577f..226718db94 100644 --- a/lib/trie/test_utils.go +++ b/lib/trie/test_utils.go @@ -72,21 +72,21 @@ func generateRandomTest(t testing.TB, kv map[string][]byte) Test { } } -// KV helps to export the entries of the generated keys -type KV struct { +// kv helps to export the entries of the generated keys +type kv struct { K []byte V []byte } // RandomTrieTest generate a trie with random entries -func RandomTrieTest(t *testing.T, n int) (*Trie, map[string]*KV) { +func randomTrieTest(t *testing.T, n int) (*Trie, map[string]*kv) { t.Helper() trie := NewEmptyTrie() - vals := make(map[string]*KV) + vals := make(map[string]*kv) for i := 0; i < n; i++ { - v := &KV{randBytes(32), randBytes(20)} + v := &kv{randBytes(32), randBytes(20)} trie.Put(v.K, v.V) vals[string(v.K)] = v } From 3406405152dd21bec7d25ebeb07ed480b2e89d05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 27 Sep 2021 16:29:51 -0400 Subject: [PATCH 32/38] chore: resolve interface mocks --- dot/core/mocks/block_state.go | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/dot/core/mocks/block_state.go b/dot/core/mocks/block_state.go index 55e9eab695..efb499f96f 100644 --- a/dot/core/mocks/block_state.go +++ b/dot/core/mocks/block_state.go @@ -143,27 +143,9 @@ func (_m *MockBlockState) BestBlockStateRoot() (common.Hash, error) { return r0, r1 } -// GenerateTrieProof provides a mock function with given fields: stateRoot, keys -func (_m *MockBlockState) GenerateTrieProof(stateRoot common.Hash, keys [][]byte) ([][]byte, error) { - ret := _m.Called(stateRoot, keys) - - var r0 [][]byte - if rf, ok := ret.Get(0).(func(common.Hash, [][]byte) [][]byte); ok { - r0 = rf(stateRoot, keys) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([][]byte) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(common.Hash, [][]byte) error); ok { - r1 = rf(stateRoot, keys) - } else { - r1 = ret.Error(1) - } - - return r0, r1 +// FreeImportedBlockNotifierChannel provides a mock function with given fields: ch +func (_m *MockBlockState) FreeImportedBlockNotifierChannel(ch chan *types.Block) { + _m.Called(ch) } // GenesisHash provides a mock function with given fields: From 17c430214c3025d3987728a433d3983454a05279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 27 Sep 2021 16:36:40 -0400 Subject: [PATCH 33/38] chore: remove unused code --- lib/trie/lookup.go | 4 ++-- lib/trie/proof.go | 2 +- lib/trie/test_utils.go | 22 ---------------------- 3 files changed, 3 insertions(+), 25 deletions(-) diff --git a/lib/trie/lookup.go b/lib/trie/lookup.go index 0725a818ef..dd8600963e 100644 --- a/lib/trie/lookup.go +++ b/lib/trie/lookup.go @@ -20,8 +20,8 @@ type lookup struct { db chaindb.Database } -// NewLookup returns a Lookup to helps the proof generator -func NewLookup(rootHash []byte, db chaindb.Database) *lookup { +// newLookup returns a Lookup to helps the proof generator +func newLookup(rootHash []byte, db chaindb.Database) *lookup { lk := &lookup{db: db} lk.root = make([]byte, len(rootHash)) copy(lk.root, rootHash) diff --git a/lib/trie/proof.go b/lib/trie/proof.go index 1078fa5771..d4b8dda636 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -36,7 +36,7 @@ func GenerateProof(root []byte, keys [][]byte, db chaindb.Database) ([][]byte, e for _, k := range keys { nk := keyToNibbles(k) - lookup := NewLookup(root, db) + lookup := newLookup(root, db) recorder := new(recorder) _, err := lookup.find(nk, recorder) diff --git a/lib/trie/test_utils.go b/lib/trie/test_utils.go index 226718db94..2636c6984c 100644 --- a/lib/trie/test_utils.go +++ b/lib/trie/test_utils.go @@ -72,28 +72,6 @@ func generateRandomTest(t testing.TB, kv map[string][]byte) Test { } } -// kv helps to export the entries of the generated keys -type kv struct { - K []byte - V []byte -} - -// RandomTrieTest generate a trie with random entries -func randomTrieTest(t *testing.T, n int) (*Trie, map[string]*kv) { - t.Helper() - - trie := NewEmptyTrie() - vals := make(map[string]*kv) - - for i := 0; i < n; i++ { - v := &kv{randBytes(32), randBytes(20)} - trie.Put(v.K, v.V) - vals[string(v.K)] = v - } - - return trie, vals -} - func randBytes(n int) []byte { r := make([]byte, n) rand.Read(r) //nolint From 391401e0dc11d7a999974ae236691bd6aa28491d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 27 Sep 2021 16:40:08 -0400 Subject: [PATCH 34/38] chore: resolve unues funcs --- lib/trie/proof_test.go | 10 +++++----- lib/trie/recorder.go | 8 -------- lib/trie/test_utils.go | 4 ++-- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/lib/trie/proof_test.go b/lib/trie/proof_test.go index d54d57b414..9129d503c2 100644 --- a/lib/trie/proof_test.go +++ b/lib/trie/proof_test.go @@ -35,11 +35,11 @@ func TestProofGeneration(t *testing.T) { require.NoError(t, err) trie := NewEmptyTrie() - trie.Put([]byte("cat"), randBytes(32)) - trie.Put([]byte("catapulta"), randBytes(32)) - trie.Put([]byte("catapora"), randBytes(32)) - trie.Put([]byte("dog"), randBytes(32)) - trie.Put([]byte("doguinho"), randBytes(32)) + trie.Put([]byte("cat"), rand32Bytes()) + trie.Put([]byte("catapulta"), rand32Bytes()) + trie.Put([]byte("catapora"), rand32Bytes()) + trie.Put([]byte("dog"), rand32Bytes()) + trie.Put([]byte("doguinho"), rand32Bytes()) err = trie.Store(memdb) require.NoError(t, err) diff --git a/lib/trie/recorder.go b/lib/trie/recorder.go index ed865637fe..cef66b18dd 100644 --- a/lib/trie/recorder.go +++ b/lib/trie/recorder.go @@ -26,14 +26,6 @@ func (r *recorder) next() *nodeRecord { return nil } -// Peek returns the current item the cursor is on but dont increment the cursor by 1 -func (r *recorder) peek() *nodeRecord { - if !r.isEmpty() { - return &(*r)[0] - } - return nil -} - // IsEmpty returns bool if there is data inside the slice func (r *recorder) isEmpty() bool { return len(*r) > 0 diff --git a/lib/trie/test_utils.go b/lib/trie/test_utils.go index 2636c6984c..907e17a0e8 100644 --- a/lib/trie/test_utils.go +++ b/lib/trie/test_utils.go @@ -72,8 +72,8 @@ func generateRandomTest(t testing.TB, kv map[string][]byte) Test { } } -func randBytes(n int) []byte { - r := make([]byte, n) +func rand32Bytes() []byte { + r := make([]byte, 32) rand.Read(r) //nolint return r } From 00d3f7cbdd8812612a66deac6287c6d07d2dda3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Mon, 27 Sep 2021 17:47:21 -0400 Subject: [PATCH 35/38] chore: resolve failing unit tests --- lib/trie/proof.go | 4 ++-- lib/trie/recorder.go | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/trie/proof.go b/lib/trie/proof.go index d4b8dda636..7668b69df8 100644 --- a/lib/trie/proof.go +++ b/lib/trie/proof.go @@ -46,9 +46,9 @@ func GenerateProof(root []byte, keys [][]byte, db chaindb.Database) ([][]byte, e for !recorder.isEmpty() { recNode := recorder.next() - nodeHashHex := common.BytesToHex(recNode.Hash) + nodeHashHex := common.BytesToHex(recNode.hash) if _, ok := trackedProofs[nodeHashHex]; !ok { - trackedProofs[nodeHashHex] = recNode.RawData + trackedProofs[nodeHashHex] = recNode.rawData } } } diff --git a/lib/trie/recorder.go b/lib/trie/recorder.go index cef66b18dd..7c2b9a40c9 100644 --- a/lib/trie/recorder.go +++ b/lib/trie/recorder.go @@ -2,8 +2,8 @@ package trie // nodeRecord represets a record of a visited node type nodeRecord struct { - RawData []byte - Hash []byte + rawData []byte + hash []byte } // Recorder keeps the list of nodes find by Lookup.Find @@ -11,8 +11,7 @@ type recorder []nodeRecord // Record insert a node insede the recorded list func (r *recorder) record(h, rd []byte) { - nr := nodeRecord{RawData: rd, Hash: h} - *r = append(*r, nr) + *r = append(*r, nodeRecord{rawData: rd, hash: h}) } // Next returns the current item the cursor is on and increment the cursor by 1 @@ -28,5 +27,5 @@ func (r *recorder) next() *nodeRecord { // IsEmpty returns bool if there is data inside the slice func (r *recorder) isEmpty() bool { - return len(*r) > 0 + return len(*r) <= 0 } From f5b151b9d69192c050a302b63e0f14636db13a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 28 Sep 2021 12:20:49 -0400 Subject: [PATCH 36/38] chore: update dot/core/services.go unit test coverage --- dot/core/service.go | 2 +- dot/core/service_test.go | 73 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/dot/core/service.go b/dot/core/service.go index 384584d180..b466aa3712 100644 --- a/dot/core/service.go +++ b/dot/core/service.go @@ -631,7 +631,7 @@ func (s *Service) tryQueryStorage(block common.Hash, keys ...string) (QueryKeyVa // GetReadProofAt will return an array with the proofs for the keys passed as params // based on the block hash passed as param as well, if block hash is nil then the current state will take place func (s *Service) GetReadProofAt(block common.Hash, keys [][]byte) (common.Hash, [][]byte, error) { - if len(block) == 0 { + if common.EmptyHash.Equal(block) { block = s.blockState.BestBlockHash() } diff --git a/dot/core/service_test.go b/dot/core/service_test.go index 002893e3f7..9774857e25 100644 --- a/dot/core/service_test.go +++ b/dot/core/service_test.go @@ -828,3 +828,76 @@ func TestDecodeSessionKeys_WhenGetRuntimeReturnError(t *testing.T) { require.Error(t, err, "problems") require.Nil(t, b) } + +func TestGetReadProofAt(t *testing.T) { + keysToProof := [][]byte{[]byte("first_key"), []byte("another_key")} + mockedProofs := [][]byte{[]byte("proof01"), []byte("proof02")} + + t.Run("When Has Block Is Empty", func(t *testing.T) { + mockedStateRootHash := common.NewHash([]byte("state root hash")) + expectedBlockHash := common.NewHash([]byte("expected block hash")) + + mockBlockState := new(mocks.MockBlockState) + mockBlockState.On("BestBlockHash").Return(expectedBlockHash) + mockBlockState.On("GetBlockStateRoot", expectedBlockHash). + Return(mockedStateRootHash, nil) + + mockStorageStage := new(mocks.MockStorageState) + mockStorageStage.On("GenerateTrieProof", mockedStateRootHash, keysToProof). + Return(mockedProofs, nil) + + s := &Service{ + blockState: mockBlockState, + storageState: mockStorageStage, + } + + b, p, err := s.GetReadProofAt(common.EmptyHash, keysToProof) + require.NoError(t, err) + require.Equal(t, p, mockedProofs) + require.Equal(t, expectedBlockHash, b) + + mockBlockState.AssertCalled(t, "BestBlockHash") + mockBlockState.AssertCalled(t, "GetBlockStateRoot", expectedBlockHash) + mockStorageStage.AssertCalled(t, "GenerateTrieProof", mockedStateRootHash, keysToProof) + }) + + t.Run("When GetStateRoot fails", func(t *testing.T) { + mockedBlockHash := common.NewHash([]byte("fake block hash")) + + mockBlockState := new(mocks.MockBlockState) + mockBlockState.On("GetBlockStateRoot", mockedBlockHash). + Return(common.EmptyHash, errors.New("problems while getting state root")) + + s := &Service{ + blockState: mockBlockState, + } + + b, p, err := s.GetReadProofAt(mockedBlockHash, keysToProof) + require.Equal(t, common.EmptyHash, b) + require.Nil(t, p) + require.Error(t, err) + }) + + t.Run("When GenerateTrieProof fails", func(t *testing.T) { + mockedBlockHash := common.NewHash([]byte("fake block hash")) + mockedStateRootHash := common.NewHash([]byte("state root hash")) + + mockBlockState := new(mocks.MockBlockState) + mockBlockState.On("GetBlockStateRoot", mockedBlockHash). + Return(mockedStateRootHash, nil) + + mockStorageStage := new(mocks.MockStorageState) + mockStorageStage.On("GenerateTrieProof", mockedStateRootHash, keysToProof). + Return(nil, errors.New("problems to generate trie proof")) + + s := &Service{ + blockState: mockBlockState, + storageState: mockStorageStage, + } + + b, p, err := s.GetReadProofAt(mockedBlockHash, keysToProof) + require.Equal(t, common.EmptyHash, b) + require.Nil(t, p) + require.Error(t, err) + }) +} From 73d69558fa8dcb079ccc5a399d8a007e58c4a026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 28 Sep 2021 14:45:02 -0400 Subject: [PATCH 37/38] chore: remove skip from test --- lib/babe/build_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/babe/build_test.go b/lib/babe/build_test.go index 7977ddc081..14cdc9553c 100644 --- a/lib/babe/build_test.go +++ b/lib/babe/build_test.go @@ -307,7 +307,7 @@ func TestApplyExtrinsic(t *testing.T) { func TestBuildAndApplyExtrinsic(t *testing.T) { // TODO (ed) currently skipping this because it's failing on github with error: // failed to sign with subkey: fork/exec /Users/runner/.local/bin/subkey: exec format error - t.Skip() + //t.Skip() cfg := &ServiceConfig{ TransactionState: state.NewTransactionState(), LogLvl: log.LvlInfo, @@ -331,7 +331,7 @@ func TestBuildAndApplyExtrinsic(t *testing.T) { rawMeta, err := rt.Metadata() require.NoError(t, err) var decoded []byte - err = scale.Unmarshal(rawMeta, []byte{}) + err = scale.Unmarshal(rawMeta, &decoded) require.NoError(t, err) meta := &ctypes.Metadata{} From d84077015d432f09aeeeac8a1bfc55375112e7af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20J=C3=BAnior?= Date: Tue, 28 Sep 2021 16:07:21 -0400 Subject: [PATCH 38/38] chore: remove comment --- lib/babe/build_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/babe/build_test.go b/lib/babe/build_test.go index 14cdc9553c..cb0dd91468 100644 --- a/lib/babe/build_test.go +++ b/lib/babe/build_test.go @@ -305,9 +305,6 @@ func TestApplyExtrinsic(t *testing.T) { } func TestBuildAndApplyExtrinsic(t *testing.T) { - // TODO (ed) currently skipping this because it's failing on github with error: - // failed to sign with subkey: fork/exec /Users/runner/.local/bin/subkey: exec format error - //t.Skip() cfg := &ServiceConfig{ TransactionState: state.NewTransactionState(), LogLvl: log.LvlInfo,