Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Clean up merkleDB interface and duplicate code #2445

Merged
merged 36 commits into from
Dec 28, 2023
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
c417eec
Clean Up interfaces
dboehm-avalabs Nov 22, 2023
e54d60c
Merge branch 'dev' into CleanUpInterfaces
dboehm-avalabs Dec 7, 2023
0de83eb
rename
dboehm-avalabs Dec 7, 2023
e193cd7
Update db.go
dboehm-avalabs Dec 7, 2023
494de38
cleanup
dboehm-avalabs Dec 12, 2023
84df3c9
Update mock_db.go
dboehm-avalabs Dec 13, 2023
18ffba9
Merge branch 'dev' into CleanUpInterfaces
dboehm-avalabs Dec 13, 2023
8a56516
Update trieview.go
dboehm-avalabs Dec 13, 2023
7334c03
Merge branch 'dev' into CleanUpInterfaces
dboehm-avalabs Dec 14, 2023
127f367
Update db_test.go
dboehm-avalabs Dec 14, 2023
7a1029c
Update db.go
dboehm-avalabs Dec 14, 2023
a455451
rename trieview -> view
dboehm-avalabs Dec 14, 2023
b96ffe8
rename
dboehm-avalabs Dec 14, 2023
60a919d
gofumpt
dboehm-avalabs Dec 14, 2023
d5187f1
Update db.go
dboehm-avalabs Dec 14, 2023
6c5f4dc
Merge remote-tracking branch 'upstream/dev' into CleanUpInterfaces
Dec 19, 2023
22eec2d
newline nits
Dec 19, 2023
6cf97ec
nit if --> switch
Dec 19, 2023
0637ed9
prioritize returning error from getting proof, not validity
Dec 19, 2023
6438c68
Merge branch 'dev' into CleanUpInterfaces
Dec 19, 2023
07eaf12
Merge branch 'dev' into CleanUpInterfaces
dboehm-avalabs Dec 19, 2023
1a6dc5e
Merge branch 'CleanUpInterfaces' of https://github.com/ava-labs/avala…
dboehm-avalabs Dec 19, 2023
6428ae1
comments
dboehm-avalabs Dec 19, 2023
ad6bcf0
Update db.go
dboehm-avalabs Dec 19, 2023
95c7eca
Update trie_test.go
dboehm-avalabs Dec 19, 2023
a5c23d8
comments
dboehm-avalabs Dec 19, 2023
4cdb1aa
remove references to trieView
Dec 22, 2023
dc31fe1
fix test name
Dec 27, 2023
84c35a0
Merge branch 'dev' into CleanUpInterfaces
Dec 27, 2023
5cba51d
re-add database closed checks in GetProof
Dec 27, 2023
01bc75c
Merge branch 'CleanUpInterfaces' of github.com:ava-labs/avalanchego i…
Dec 27, 2023
ff774ad
nit
Dec 27, 2023
82019a0
nits
Dec 27, 2023
dc36c65
comments
Dec 27, 2023
2b5982e
comment
Dec 27, 2023
837a1b7
Merge branch 'dev' into CleanUpInterfaces
Dec 28, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 38 additions & 68 deletions x/merkledb/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ type Config struct {
Tracer trace.Tracer
}

// merkleDB can only be edited by committing changes from a trieView.
// merkleDB can only be edited by committing changes from a view.
type merkleDB struct {
// Must be held when reading/writing fields.
lock sync.RWMutex
Expand Down Expand Up @@ -216,7 +216,7 @@ type merkleDB struct {
rootID ids.ID

// Valid children of this trie.
childViews []*trieView
childViews []*view

// calculateNodeIDsSema controls the number of goroutines inside
// [calculateNodeIDsHelper] at any given time.
Expand Down Expand Up @@ -264,7 +264,7 @@ func newDatabase(
history: newTrieHistory(int(config.HistoryLength)),
debugTracer: getTracerIfEnabled(config.TraceLevel, DebugTrace, config.Tracer),
infoTracer: getTracerIfEnabled(config.TraceLevel, InfoTrace, config.Tracer),
childViews: make([]*trieView, 0, defaultPreallocationSize),
childViews: make([]*view, 0, defaultPreallocationSize),
calculateNodeIDsSema: semaphore.NewWeighted(int64(rootGenConcurrency)),
tokenSize: BranchFactorToTokenSize[config.BranchFactor],
}
Expand Down Expand Up @@ -462,13 +462,8 @@ func (db *merkleDB) PrefetchPaths(keys [][]byte) error {
return database.ErrClosed
}

// reuse the view so that it can keep repeated nodes in memory
tempView, err := newTrieView(db, db, ViewChanges{})
if err != nil {
return err
}
for _, key := range keys {
if err := db.prefetchPath(tempView, key); err != nil {
if err := db.prefetchPath(key); err != nil {
return err
}
}
Expand All @@ -483,16 +478,11 @@ func (db *merkleDB) PrefetchPath(key []byte) error {
if db.closed {
return database.ErrClosed
}
tempView, err := newTrieView(db, db, ViewChanges{})
if err != nil {
return err
}

return db.prefetchPath(tempView, key)
return db.prefetchPath(key)
}

func (db *merkleDB) prefetchPath(view *trieView, keyBytes []byte) error {
return view.visitPathToKey(ToKey(keyBytes), func(n *node) error {
func (db *merkleDB) prefetchPath(keyBytes []byte) error {
return visitPathToKey(db, ToKey(keyBytes), func(n *node) error {
if !n.hasValue() {
// this value is already in the cache, so skip writing
// to avoid grabbing the cache write lock
Expand Down Expand Up @@ -607,22 +597,9 @@ func (db *merkleDB) getMerkleRoot() ids.ID {
func (db *merkleDB) GetProof(ctx context.Context, key []byte) (*Proof, error) {
db.commitLock.RLock()
defer db.commitLock.RUnlock()

return db.getProof(ctx, key)
}

// Assumes [db.commitLock] is read locked.
func (db *merkleDB) getProof(ctx context.Context, key []byte) (*Proof, error) {
if db.closed {
return nil, database.ErrClosed
}

view, err := newTrieView(db, db, ViewChanges{})
if err != nil {
return nil, err
}
// Don't need to lock [view] because nobody else has a reference to it.
return view.getProof(ctx, key)
_, span := db.infoTracer.Start(ctx, "MerkleDB.GetProof")
defer span.End()
return getProof(db, key)
}

func (db *merkleDB) GetRangeProof(
Expand All @@ -631,10 +608,9 @@ func (db *merkleDB) GetRangeProof(
end maybe.Maybe[[]byte],
maxLength int,
) (*RangeProof, error) {
db.commitLock.RLock()
defer db.commitLock.RUnlock()

return db.getRangeProofAtRoot(ctx, db.getMerkleRoot(), start, end, maxLength)
_, span := db.infoTracer.Start(ctx, "MerkleDB.GetRangeProof")
defer span.End()
return getRangeProof(db, start, end, maxLength)
dboehm-avalabs marked this conversation as resolved.
Show resolved Hide resolved
}

func (db *merkleDB) GetRangeProofAtRoot(
Expand All @@ -647,18 +623,8 @@ func (db *merkleDB) GetRangeProofAtRoot(
db.commitLock.RLock()
defer db.commitLock.RUnlock()

return db.getRangeProofAtRoot(ctx, rootID, start, end, maxLength)
}

// Assumes [db.commitLock] is read locked.
// Assumes [db.lock] is not held
func (db *merkleDB) getRangeProofAtRoot(
ctx context.Context,
rootID ids.ID,
start maybe.Maybe[[]byte],
end maybe.Maybe[[]byte],
maxLength int,
) (*RangeProof, error) {
_, span := db.infoTracer.Start(ctx, "MerkleDB.GetRangeProofAtRoot")
defer span.End()
switch {
case db.closed:
return nil, database.ErrClosed
Expand All @@ -668,11 +634,11 @@ func (db *merkleDB) getRangeProofAtRoot(
return nil, ErrEmptyProof
}

historicalView, err := db.getHistoricalViewForRange(rootID, start, end)
historicalTrie, err := db.getTrieAtRootForRange(rootID, start, end)
if err != nil {
return nil, err
}
return historicalView.GetRangeProof(ctx, start, end, maxLength)
return getRangeProof(historicalTrie, start, end, maxLength)
}

func (db *merkleDB) GetChangeProof(
Expand All @@ -683,6 +649,8 @@ func (db *merkleDB) GetChangeProof(
end maybe.Maybe[[]byte],
maxLength int,
) (*ChangeProof, error) {
_, span := db.infoTracer.Start(ctx, "MerkleDB.GetChangeProof")
defer span.End()
switch {
case start.HasValue() && end.HasValue() && bytes.Compare(start.Value(), end.Value()) == 1:
return nil, ErrStartAfterEnd
Expand Down Expand Up @@ -731,21 +699,21 @@ func (db *merkleDB) GetChangeProof(

// Since we hold [db.commitlock] we must still have sufficient
// history to recreate the trie at [endRootID].
historicalView, err := db.getHistoricalViewForRange(endRootID, start, largestKey)
historicalTrie, err := db.getTrieAtRootForRange(endRootID, start, largestKey)
if err != nil {
return nil, err
}

if largestKey.HasValue() {
endProof, err := historicalView.getProof(ctx, largestKey.Value())
endProof, err := getProof(historicalTrie, largestKey.Value())
if err != nil {
return nil, err
}
result.EndProof = endProof.Path
}

if start.HasValue() {
startProof, err := historicalView.getProof(ctx, start.Value())
startProof, err := getProof(historicalTrie, start.Value())
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -782,7 +750,7 @@ func (db *merkleDB) GetChangeProof(
func (db *merkleDB) NewView(
_ context.Context,
changes ViewChanges,
) (TrieView, error) {
) (View, error) {
// ensure the db doesn't change while creating the new view
db.commitLock.RLock()
defer db.commitLock.RUnlock()
Expand Down Expand Up @@ -916,7 +884,7 @@ func (db *merkleDB) commitBatch(ops []database.BatchOp) error {

// commitChanges commits the changes in [trieToCommit] to [db].
// Assumes [trieToCommit]'s node IDs have been calculated.
func (db *merkleDB) commitChanges(ctx context.Context, trieToCommit *trieView) error {
func (db *merkleDB) commitChanges(ctx context.Context, trieToCommit *view) error {
db.lock.Lock()
defer db.lock.Unlock()

Expand Down Expand Up @@ -1002,15 +970,15 @@ func (db *merkleDB) commitChanges(ctx context.Context, trieToCommit *trieView) e

// moveChildViewsToDB removes any child views from the trieToCommit and moves them to the db
// assumes [db.lock] is held
func (db *merkleDB) moveChildViewsToDB(trieToCommit *trieView) {
func (db *merkleDB) moveChildViewsToDB(trieToCommit *view) {
trieToCommit.validityTrackingLock.Lock()
defer trieToCommit.validityTrackingLock.Unlock()

for _, childView := range trieToCommit.childViews {
childView.updateParent(db)
db.childViews = append(db.childViews, childView)
}
trieToCommit.childViews = make([]*trieView, 0, defaultPreallocationSize)
trieToCommit.childViews = make([]*view, 0, defaultPreallocationSize)
}

// CommitToDB is a no-op for db since it is already in sync with itself.
Expand Down Expand Up @@ -1159,7 +1127,7 @@ func (db *merkleDB) VerifyChangeProof(

// Invalidates and removes any child views that aren't [exception].
// Assumes [db.lock] is held.
func (db *merkleDB) invalidateChildrenExcept(exception *trieView) {
func (db *merkleDB) invalidateChildrenExcept(exception *view) {
isTrackedView := false

for _, childView := range db.childViews {
Expand All @@ -1169,7 +1137,7 @@ func (db *merkleDB) invalidateChildrenExcept(exception *trieView) {
isTrackedView = true
}
}
db.childViews = make([]*trieView, 0, defaultPreallocationSize)
db.childViews = make([]*view, 0, defaultPreallocationSize)
if isTrackedView {
db.childViews = append(db.childViews, exception)
}
Expand Down Expand Up @@ -1217,24 +1185,22 @@ func (db *merkleDB) initializeRoot() error {
// If [start] is Nothing, there's no lower bound on the range.
// If [end] is Nothing, there's no upper bound on the range.
// Assumes [db.commitLock] is read locked.
func (db *merkleDB) getHistoricalViewForRange(
// Assumes [db.lock] isn't held.
dboehm-avalabs marked this conversation as resolved.
Show resolved Hide resolved
func (db *merkleDB) getTrieAtRootForRange(
rootID ids.ID,
start maybe.Maybe[[]byte],
end maybe.Maybe[[]byte],
) (*trieView, error) {
currentRootID := db.getMerkleRoot()

) (Trie, error) {
// looking for the trie's current root id, so return the trie unmodified
if currentRootID == rootID {
// create an empty trie
return newTrieView(db, db, ViewChanges{})
if rootID == db.getMerkleRoot() {
return db, nil
}

changeHistory, err := db.history.getChangesToGetToRoot(rootID, start, end)
if err != nil {
return nil, err
}
return newHistoricalTrieView(db, changeHistory)
return newViewWithChanges(db, changeHistory)
}

// Returns all keys in range [start, end] that aren't in [keySet].
Expand Down Expand Up @@ -1327,6 +1293,10 @@ func (db *merkleDB) Clear() error {
return nil
}

func (db *merkleDB) getTokenSize() int {
return db.tokenSize
}

// Returns [key] prefixed by [prefix].
// The returned []byte is taken from [bufferPool] and
// should be returned to it when the caller is done with it.
Expand Down
38 changes: 19 additions & 19 deletions x/merkledb/db_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -297,15 +297,15 @@ func Test_MerkleDB_Invalidate_Siblings_On_Commit(t *testing.T) {
sibling2, err := dbTrie.NewView(context.Background(), ViewChanges{})
require.NoError(err)

require.False(sibling1.(*trieView).isInvalid())
require.False(sibling2.(*trieView).isInvalid())
require.False(sibling1.(*view).isInvalid())
require.False(sibling2.(*view).isInvalid())

// Committing viewToCommit should invalidate siblings
require.NoError(viewToCommit.CommitToDB(context.Background()))

require.True(sibling1.(*trieView).isInvalid())
require.True(sibling2.(*trieView).isInvalid())
require.False(viewToCommit.(*trieView).isInvalid())
require.True(sibling1.(*view).isInvalid())
require.True(sibling2.(*view).isInvalid())
require.False(viewToCommit.(*view).isInvalid())
}

func Test_MerkleDB_CommitRangeProof_DeletesValuesInRange(t *testing.T) {
Expand Down Expand Up @@ -561,7 +561,7 @@ func TestDatabaseCommitChanges(t *testing.T) {
// Committing an invalid view should fail.
invalidView, err := db.NewView(context.Background(), ViewChanges{})
require.NoError(err)
invalidView.(*trieView).invalidate()
invalidView.(*view).invalidate()
err = invalidView.CommitToDB(context.Background())
require.ErrorIs(err, ErrInvalid)

Expand All @@ -582,22 +582,22 @@ func TestDatabaseCommitChanges(t *testing.T) {
},
)
require.NoError(err)
require.IsType(&trieView{}, view1Intf)
view1 := view1Intf.(*trieView)
require.IsType(&view{}, view1Intf)
view1 := view1Intf.(*view)
view1Root, err := view1.GetMerkleRoot(context.Background())
require.NoError(err)

// Make a second view
view2Intf, err := db.NewView(context.Background(), ViewChanges{})
require.NoError(err)
require.IsType(&trieView{}, view2Intf)
view2 := view2Intf.(*trieView)
require.IsType(&view{}, view2Intf)
view2 := view2Intf.(*view)

// Make a view atop a view
view3Intf, err := view1.NewView(context.Background(), ViewChanges{})
require.NoError(err)
require.IsType(&trieView{}, view3Intf)
view3 := view3Intf.(*trieView)
require.IsType(&view{}, view3Intf)
view3 := view3Intf.(*view)

// view3
// |
Expand Down Expand Up @@ -646,18 +646,18 @@ func TestDatabaseInvalidateChildrenExcept(t *testing.T) {
// Create children
view1Intf, err := db.NewView(context.Background(), ViewChanges{})
require.NoError(err)
require.IsType(&trieView{}, view1Intf)
view1 := view1Intf.(*trieView)
require.IsType(&view{}, view1Intf)
view1 := view1Intf.(*view)

view2Intf, err := db.NewView(context.Background(), ViewChanges{})
require.NoError(err)
require.IsType(&trieView{}, view2Intf)
view2 := view2Intf.(*trieView)
require.IsType(&view{}, view2Intf)
view2 := view2Intf.(*view)

view3Intf, err := db.NewView(context.Background(), ViewChanges{})
require.NoError(err)
require.IsType(&trieView{}, view3Intf)
view3 := view3Intf.(*trieView)
require.IsType(&view{}, view3Intf)
view3 := view3Intf.(*view)

db.invalidateChildrenExcept(view1)

Expand Down Expand Up @@ -1255,7 +1255,7 @@ func TestGetRangeProofAtRootEmptyRootID(t *testing.T) {
db, err := getBasicDB()
require.NoError(err)

_, err = db.getRangeProofAtRoot(
_, err = db.GetRangeProofAtRoot(
context.Background(),
ids.Empty,
maybe.Nothing[[]byte](),
Expand Down
Loading
Loading