Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

eth/downloader: separate state sync from queue #14460

Merged
merged 19 commits into from
Jun 22, 2017
Merged

eth/downloader: separate state sync from queue #14460

Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
1b19b21
eth/downloader: separate state sync from queue
fjl May 29, 2017
c7e5270
eth/downloader: remove cancelTimeout channel
fjl May 30, 2017
4209354
eth/downloader: retry state requests on timeout
fjl May 30, 2017
ac085e0
eth/downloader: improve comment
fjl May 30, 2017
465acb2
eth/downloader: mark peers idle when state sync is done
fjl May 30, 2017
4c6040c
eth/downloader: move pivot block splitting to processContent
fjl May 30, 2017
44b8cb8
eth/downloader: limit state node retries
fjl May 30, 2017
5339c79
eth/downloader: improve state node error handling and retry check
fjl May 30, 2017
339bc10
eth/downloader: remove maxStateNodeRetries
fjl May 30, 2017
baa79e9
eth/downloader: remove last use of cancelCh in statesync.go
fjl May 31, 2017
6588e66
eth/downloader: fix leak in runStateSync
fjl May 31, 2017
5029223
eth/downloader: don't run processFullSyncContent in LightSync mode
fjl May 31, 2017
50d0c3f
eth/downloader: improve comments
fjl May 31, 2017
b48521a
eth/downloader: fix vet, megacheck
fjl May 31, 2017
829a042
eth/downloader: remove unrequested tasks anyway
fjl May 31, 2017
e3717b4
eth/downloader, trie: various polishes around duplicate items
karalabe Jun 16, 2017
115768a
eth/downloader: fix state request leak
karalabe Jun 16, 2017
2410187
core, eth, trie: support batched trie sync db writes
karalabe Jun 21, 2017
8a22650
trie: rename SyncMemCache to syncMemBatch
karalabe Jun 22, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions core/state/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,16 @@ func (s *StateSync) Missing(max int) []common.Hash {
}

// Process injects a batch of retrieved trie nodes data, returning if something
// was committed to the database and also the index of an entry if processing of
// was committed to the memcache and also the index of an entry if processing of
// it failed.
func (s *StateSync) Process(list []trie.SyncResult, dbw trie.DatabaseWriter) (bool, int, error) {
return (*trie.TrieSync)(s).Process(list, dbw)
func (s *StateSync) Process(list []trie.SyncResult) (bool, int, error) {
return (*trie.TrieSync)(s).Process(list)
}

// Commit flushes the data stored in the internal memcache out to persistent
// storage, returning th enumber of items written and any occurred error.
func (s *StateSync) Commit(dbw trie.DatabaseWriter) (int, error) {
return (*trie.TrieSync)(s).Commit(dbw)
}

// Pending returns the number of state entries currently pending for download.
Expand Down
25 changes: 20 additions & 5 deletions core/state/sync_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,9 +138,12 @@ func testIterativeStateSync(t *testing.T, batch int) {
}
results[i] = trie.SyncResult{Hash: hash, Data: data}
}
if _, index, err := sched.Process(results, dstDb); err != nil {
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
}
if index, err := sched.Commit(dstDb); err != nil {
t.Fatalf("failed to commit data #%d: %v", index, err)
}
queue = append(queue[:0], sched.Missing(batch)...)
}
// Cross check that the two states are in sync
Expand Down Expand Up @@ -168,9 +171,12 @@ func TestIterativeDelayedStateSync(t *testing.T) {
}
results[i] = trie.SyncResult{Hash: hash, Data: data}
}
if _, index, err := sched.Process(results, dstDb); err != nil {
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
}
if index, err := sched.Commit(dstDb); err != nil {
t.Fatalf("failed to commit data #%d: %v", index, err)
}
queue = append(queue[len(results):], sched.Missing(0)...)
}
// Cross check that the two states are in sync
Expand Down Expand Up @@ -206,9 +212,12 @@ func testIterativeRandomStateSync(t *testing.T, batch int) {
results = append(results, trie.SyncResult{Hash: hash, Data: data})
}
// Feed the retrieved results back and queue new tasks
if _, index, err := sched.Process(results, dstDb); err != nil {
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
}
if index, err := sched.Commit(dstDb); err != nil {
t.Fatalf("failed to commit data #%d: %v", index, err)
}
queue = make(map[common.Hash]struct{})
for _, hash := range sched.Missing(batch) {
queue[hash] = struct{}{}
Expand Down Expand Up @@ -249,9 +258,12 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
}
}
// Feed the retrieved results back and queue new tasks
if _, index, err := sched.Process(results, dstDb); err != nil {
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
}
if index, err := sched.Commit(dstDb); err != nil {
t.Fatalf("failed to commit data #%d: %v", index, err)
}
for _, hash := range sched.Missing(0) {
queue[hash] = struct{}{}
}
Expand Down Expand Up @@ -283,9 +295,12 @@ func TestIncompleteStateSync(t *testing.T) {
results[i] = trie.SyncResult{Hash: hash, Data: data}
}
// Process each of the state nodes
if _, index, err := sched.Process(results, dstDb); err != nil {
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
}
if index, err := sched.Commit(dstDb); err != nil {
t.Fatalf("failed to commit data #%d: %v", index, err)
}
for _, result := range results {
added = append(added, result.Hash)
}
Expand Down
Loading