Skip to content

Commit

Permalink
core, trie: rework trie commiter (#560)
Browse files Browse the repository at this point in the history
* core, trie: rework trie commiter

changed the commit procedure, introduce new struct called nodeSet for
returning including all dirty nodes of a trie. Multiple nodeset will be
merged to MergedNodeSet struct. then be submitted to in-memory database
from block to block

* trie,core: fix comments
  • Loading branch information
huyngopt1994 committed Nov 21, 2024
1 parent 8a8f0e4 commit 06ad1b3
Show file tree
Hide file tree
Showing 23 changed files with 550 additions and 380 deletions.
8 changes: 5 additions & 3 deletions core/state/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,11 @@ type Trie interface {
// can be used even if the trie doesn't have one.
Hash() common.Hash

// Commit writes all nodes to the trie's memory database, tracking the internal
// and external (for account tries) references.
Commit(onleaf trie.LeafCallback) (common.Hash, int, error)
// Commit collects all dirty nodes in the trie and replace them with the
// corresponding node hash. All collected nodes(including dirty leaves if
// collectLeaf is true) will be encapsulated into a nodeset for return.
// The returned nodeset can be nil if the trie is clean(nothing to commit).
Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error)

// NodeIterator returns an iterator that returns nodes of the trie. Iteration
// starts at the key after the given start key.
Expand Down
12 changes: 6 additions & 6 deletions core/state/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ package state
import "github.com/ethereum/go-ethereum/metrics"

var (
accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil)
storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil)
accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil)
storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil)
accountCommittedMeter = metrics.NewRegisteredMeter("state/commit/account", nil)
storageCommittedMeter = metrics.NewRegisteredMeter("state/commit/storage", nil)
accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil)
storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil)
accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil)
storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil)
accountTrieCommittedMeter = metrics.NewRegisteredMeter("state/commit/accountnodes", nil)
storageTriesCommittedMeter = metrics.NewRegisteredMeter("state/commit/storagenodes", nil)
)
5 changes: 4 additions & 1 deletion core/state/snapshot/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,10 @@ func (dl *diskLayer) generateRange(owner common.Hash, root common.Hash, prefix [
for i, key := range result.keys {
snapTrie.Update(key, result.vals[i])
}
root, _, _ := snapTrie.Commit(nil)
root, nodes, _ := snapTrie.Commit(false)
if nodes != nil {
snapTrieDb.Update(trie.NewWithNodeSet(nodes))
}
snapTrieDb.Commit(root, false, nil)
}
tr := result.tr
Expand Down
46 changes: 15 additions & 31 deletions core/state/snapshot/generate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ type testHelper struct {
diskdb ethdb.Database
triedb *trie.Database
accTrie *trie.SecureTrie
nodes *trie.MergedNodeSet
}

func newHelper() *testHelper {
Expand All @@ -148,6 +149,7 @@ func newHelper() *testHelper {
diskdb: diskdb,
triedb: triedb,
accTrie: accTrie,
nodes: trie.NewMergedNodeSet(),
}
}

Expand Down Expand Up @@ -179,17 +181,22 @@ func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string
for i, k := range keys {
stTrie.Update([]byte(k), []byte(vals[i]))
}
var root common.Hash
if !commit {
root = stTrie.Hash()
} else {
root, _, _ = stTrie.Commit(nil)
return stTrie.Hash().Bytes()
}
root, nodes, _ := stTrie.Commit(false)
if nodes != nil {
t.nodes.Merge(nodes)
}
return root.Bytes()
}

func (t *testHelper) Commit() common.Hash {
root, _, _ := t.accTrie.Commit(nil)
root, nodes, _ := t.accTrie.Commit(true)
if nodes != nil {
t.nodes.Merge(nodes)
}
t.triedb.Update(t.nodes)
t.triedb.Commit(root, false, nil)
return root
}
Expand Down Expand Up @@ -375,7 +382,7 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4

root, _, _ := helper.accTrie.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978

// Delete an account trie leaf and ensure the generator chokes
helper.triedb.Commit(root, false, nil)
Expand Down Expand Up @@ -410,18 +417,7 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
root, _, _ := helper.accTrie.Commit(nil)

// We can only corrupt the disk database, so flush the tries out
helper.triedb.Reference(
common.BytesToHash(stRoot),
common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"),
)
helper.triedb.Reference(
common.BytesToHash(stRoot),
common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"),
)
helper.triedb.Commit(root, false, nil)
root := helper.Commit()

// Delete a storage trie root and ensure the generator chokes
helper.diskdb.Delete(stRoot)
Expand Down Expand Up @@ -455,19 +451,7 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2

root, _, _ := helper.accTrie.Commit(nil)

// We can only corrupt the disk database, so flush the tries out
helper.triedb.Reference(
common.BytesToHash(stRoot),
common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"),
)
helper.triedb.Reference(
common.BytesToHash(stRoot),
common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"),
)
helper.triedb.Commit(root, false, nil)

root := helper.Commit()
// Delete a storage trie leaf and ensure the generator chokes
helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())

Expand Down
11 changes: 6 additions & 5 deletions core/state/state_object.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)

var emptyCodeHash = crypto.Keccak256(nil)
Expand Down Expand Up @@ -393,23 +394,23 @@ func (s *stateObject) updateRoot(db Database) {

// CommitTrie the storage trie of the object to db.
// This updates the trie root.
func (s *stateObject) CommitTrie(db Database) (int, error) {
func (s *stateObject) CommitTrie(db Database) (*trie.NodeSet, error) {
// If nothing changed, don't bother with hashing anything
if s.updateTrie(db) == nil {
return 0, nil
return nil, nil
}
if s.dbErr != nil {
return 0, s.dbErr
return nil, s.dbErr
}
// Track the amount of time wasted on committing the storage trie
if metrics.EnabledExpensive {
defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now())
}
root, committed, err := s.trie.Commit(nil)
root, nodes, err := s.trie.Commit(false)
if err == nil {
s.data.Root = root
}
return committed, err
return nodes, err
}

// AddBalance adds amount to s's balance.
Expand Down
47 changes: 30 additions & 17 deletions core/state/statedb.go
Original file line number Diff line number Diff line change
Expand Up @@ -972,7 +972,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
s.IntermediateRoot(deleteEmptyObjects)

// Commit objects to the trie, measuring the elapsed time
var storageCommitted int
var (
accountTrieNodes int
storageTrieNodes int
nodes = trie.NewMergedNodeSet()
)
codeWriter := s.db.TrieDB().DiskDB().NewBatch()
for addr := range s.stateObjectsDirty {
if obj := s.stateObjects[addr]; !obj.deleted {
Expand All @@ -982,11 +986,18 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
obj.dirtyCode = false
}
// Write any storage changes in the state object to its storage trie
committed, err := obj.CommitTrie(s.db)
nodeSet, err := obj.CommitTrie(s.db)
if err != nil {
return common.Hash{}, err
}
storageCommitted += committed

// Merge the dirty nodes of storage trie into global set
if nodeSet != nil {
if err := nodes.Merge(nodeSet); err != nil {
return common.Hash{}, err
}
storageTrieNodes += nodeSet.Len()
}
}
}
if len(s.stateObjectsDirty) > 0 {
Expand All @@ -1002,30 +1013,27 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
if metrics.EnabledExpensive {
start = time.Now()
}
// The onleaf func is called _serially_, so we can reuse the same account
// for unmarshalling every time.
var account types.StateAccount
root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error {
if err := rlp.DecodeBytes(leaf, &account); err != nil {
return nil
}
if account.Root != emptyRoot {
s.db.TrieDB().Reference(account.Root, parent)
}
return nil
})
root, nodeSet, err := s.trie.Commit(true)
if err != nil {
return common.Hash{}, err
}

// Merge the dirty nodes of account trie into global set
if nodeSet != nil {
if err := nodes.Merge(nodeSet); err != nil {
return common.Hash{}, err
}
accountTrieNodes = nodeSet.Len()
}
if metrics.EnabledExpensive {
s.AccountCommits += time.Since(start)

accountUpdatedMeter.Mark(int64(s.AccountUpdated))
storageUpdatedMeter.Mark(int64(s.StorageUpdated))
accountDeletedMeter.Mark(int64(s.AccountDeleted))
storageDeletedMeter.Mark(int64(s.StorageDeleted))
accountCommittedMeter.Mark(int64(accountCommitted))
storageCommittedMeter.Mark(int64(storageCommitted))
accountTrieCommittedMeter.Mark(int64(accountTrieNodes))
storageTriesCommittedMeter.Mark(int64(storageTrieNodes))
s.AccountUpdated, s.AccountDeleted = 0, 0
s.StorageUpdated, s.StorageDeleted = 0, 0
}
Expand All @@ -1049,6 +1057,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
}
s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil
}

// Update Trie MergeNodeSets.
if err := s.db.TrieDB().Update(nodes); err != nil {
return common.Hash{}, err
}
s.originalRoot = root
return root, err
}
Expand Down
Loading

0 comments on commit 06ad1b3

Please sign in to comment.