perf(db): reduce batch size for chain history indexer (#105)

* perf(db): reduce batch size for chain history indexer Makes batch size configurable. The high batch size was causing enormous insert queries to be sent to postgres which were always taking > 10s to process, holding locks in the process. * Update commands/run.go Co-authored-by: Will <[email protected]> Co-authored-by: Will <[email protected]>
filecoin-project · Oct 14, 2020 · 60c41e5 · 60c41e5
1 parent 8c4303c
commit 60c41e5
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 3 deletions.
diff --git a/commands/run.go b/commands/run.go
@@ -55,6 +55,13 @@ var Run = &cli.Command{
 			Usage:   "Start indexing tipsets by walking the chain history",
 			EnvVars: []string{"VISOR_INDEXHISTORY"},
 		},
+		&cli.IntFlag{
+			Name:    "indexhistory-batch",
+			Aliases: []string{"ihb"},
+			Value:   25,
+			Usage:   "Batch size for the chain history indexer",
+			EnvVars: []string{"VISOR_INDEXHISTORY_BATCH"},
+		},
 
 		&cli.DurationFlag{
 			Name:    "statechange-lease",
@@ -246,7 +253,7 @@ var Run = &cli.Command{
 		if cctx.Bool("indexhistory") {
 			scheduler.Add(schedule.TaskConfig{
 				Name:                "ChainHistoryIndexer",
-				Task:                indexer.NewChainHistoryIndexer(rctx.db, rctx.api),
+				Task:                indexer.NewChainHistoryIndexer(rctx.db, rctx.api, cctx.Int("indexhistory-batch")),
 				Locker:              NewGlobalSingleton(ChainHistoryIndexerLockID, rctx.db), // only want one history indexer anywhere to be running
 				RestartOnFailure:    true,
 				RestartOnCompletion: true,

diff --git a/tasks/indexer/chainhistoryindexer.go b/tasks/indexer/chainhistoryindexer.go
@@ -15,12 +15,12 @@ import (
 	"github.com/filecoin-project/sentinel-visor/storage"
 )
 
-func NewChainHistoryIndexer(d *storage.Database, node lens.API) *ChainHistoryIndexer {
+func NewChainHistoryIndexer(d *storage.Database, node lens.API, batchSize int) *ChainHistoryIndexer {
 	return &ChainHistoryIndexer{
 		node:      node,
 		storage:   d,
 		finality:  900,
-		batchSize: 500,
+		batchSize: batchSize,
 	}
 }