Merge remote-tracking branch 'origin/master' into feat/minerposts

* origin/master:
  feat: Add historical indexer metrics (#92)
  fix(lens): Include dependencies needed for Repo Lens (#90)
  Move migration to schema version 10
  fix(migrations): migrations require version 0
  feat: add message gas economy processing
  perf(db): reduce batch size for chain history indexer (#105)
  fix: use hash index type for visor_processing_actors_code_idx (#106)
  feat: set application name in postgres connection (#104)
  feat: add visor processing stats table (#96)
  feat(task): add chain economics processing (#94)
  get actor name for both versions of specs-actors (#101)

.gitignore

@@ -15,6 +15,7 @@
 # vendor/
 .idea
 visor
+!model/visor
 sentinel-visor
 
 build/.*

commands/run.go

@@ -8,14 +8,17 @@ import (
 	"time"
 
 	"github.com/filecoin-project/specs-actors/actors/builtin"
+	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/sentinel-visor/schedule"
 	"github.com/filecoin-project/sentinel-visor/tasks/actorstate"
+	"github.com/filecoin-project/sentinel-visor/tasks/chain"
 	"github.com/filecoin-project/sentinel-visor/tasks/indexer"
 	"github.com/filecoin-project/sentinel-visor/tasks/message"
+	"github.com/filecoin-project/sentinel-visor/tasks/stats"
 	"github.com/filecoin-project/sentinel-visor/tasks/views"
 )
 
@@ -53,6 +56,13 @@ var Run = &cli.Command{
 			Usage:   "Start indexing tipsets by walking the chain history",
 			EnvVars: []string{"VISOR_INDEXHISTORY"},
 		},
+		&cli.IntFlag{
+			Name:    "indexhistory-batch",
+			Aliases: []string{"ihb"},
+			Value:   25,
+			Usage:   "Batch size for the chain history indexer",
+			EnvVars: []string{"VISOR_INDEXHISTORY_BATCH"},
+		},
 
 		&cli.DurationFlag{
 			Name:    "statechange-lease",
@@ -161,6 +171,36 @@ var Run = &cli.Command{
 			Usage:   "Refresh frequency for chain visualization views (0 = disables refresh)",
 			EnvVars: []string{"VISOR_CHAINVIS_REFRESH"},
 		},
+
+		&cli.DurationFlag{
+			Name:    "processingstats-refresh-rate",
+			Aliases: []string{"psr"},
+			Value:   0,
+			Usage:   "Refresh frequency for processing stats (0 = disables refresh)",
+			EnvVars: []string{"VISOR_PROCESSINGSTATS_REFRESH"},
+		},
+
+		&cli.IntFlag{
+			Name:    "chaineconomics-workers",
+			Aliases: []string{"cew"},
+			Value:   5,
+			Usage:   "Number of chain economics processors to start",
+			EnvVars: []string{"VISOR_CHAINECONOMICS_WORKERS"},
+		},
+		&cli.IntFlag{
+			Name:    "chaineconomics-batch",
+			Aliases: []string{"ceb"},
+			Value:   50, // chain economics processing is quite fast
+			Usage:   "Batch size for the chain economics processor",
+			EnvVars: []string{"VISOR_CHAINECONOMICS_BATCH"},
+		},
+		&cli.DurationFlag{
+			Name:    "chaineconomics-lease",
+			Aliases: []string{"cel"},
+			Value:   time.Minute * 15,
+			Usage:   "Lease time for the chain economics processor",
+			EnvVars: []string{"VISOR_CHAINECONOMICS_LEASE"},
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		// Validate flags
@@ -214,7 +254,7 @@ var Run = &cli.Command{
 		if cctx.Bool("indexhistory") {
 			scheduler.Add(schedule.TaskConfig{
 				Name:                "ChainHistoryIndexer",
-				Task:                indexer.NewChainHistoryIndexer(rctx.db, rctx.api),
+				Task:                indexer.NewChainHistoryIndexer(rctx.db, rctx.api, cctx.Int("indexhistory-batch")),
 				Locker:              NewGlobalSingleton(ChainHistoryIndexerLockID, rctx.db), // only want one history indexer anywhere to be running
 				RestartOnFailure:    true,
 				RestartOnCompletion: true,
@@ -266,6 +306,16 @@ var Run = &cli.Command{
 			})
 		}
 
+		// Add several chain economics tasks to read gas outputs from indexed messages
+		for i := 0; i < cctx.Int("chaineconomics-workers"); i++ {
+			scheduler.Add(schedule.TaskConfig{
+				Name:                fmt.Sprintf("ChainEconomicsProcessor%03d", i),
+				Task:                chain.NewChainEconomicsProcessor(rctx.db, rctx.api, cctx.Duration("chaineconomics-lease"), cctx.Int("chaineconomics-batch"), heightFrom, heightTo),
+				RestartOnFailure:    true,
+				RestartOnCompletion: true,
+			})
+		}
+
 		// Include optional refresher for Chain Visualization views
 		// Zero duration will cause ChainVisRefresher to exit and should not restart
 		scheduler.Add(schedule.TaskConfig{
@@ -276,6 +326,17 @@ var Run = &cli.Command{
 			RestartOnCompletion: false,
 		})
 
+		// Include optional refresher for processing stats
+		if cctx.Duration("processingstats-refresh-rate") != 0 {
+			scheduler.Add(schedule.TaskConfig{
+				Name:                "ProcessingStatsRefresher",
+				Locker:              NewGlobalSingleton(ProcessingStatsRefresherLockID, rctx.db),
+				Task:                stats.NewProcessingStatsRefresher(rctx.db, cctx.Duration("processingstats-refresh-rate")),
+				RestartOnFailure:    true,
+				RestartOnCompletion: true,
+			})
+		}
+
 		// Start the scheduler and wait for it to complete or to be cancelled.
 		err = scheduler.Run(ctx)
 		if !errors.Is(err, context.Canceled) {
@@ -351,15 +412,26 @@ func parseActorCodes(ss []string) ([]cid.Cid, error) {
 }
 
 var actorNamesToCodes = map[string]cid.Cid{
-	"fil/2/system":           builtin.SystemActorCodeID,
-	"fil/2/init":             builtin.InitActorCodeID,
-	"fil/2/cron":             builtin.CronActorCodeID,
-	"fil/2/storagepower":     builtin.StoragePowerActorCodeID,
-	"fil/2/storageminer":     builtin.StorageMinerActorCodeID,
-	"fil/2/storagemarket":    builtin.StorageMarketActorCodeID,
-	"fil/2/paymentchannel":   builtin.PaymentChannelActorCodeID,
-	"fil/2/reward":           builtin.RewardActorCodeID,
-	"fil/2/verifiedregistry": builtin.VerifiedRegistryActorCodeID,
-	"fil/2/account":          builtin.AccountActorCodeID,
-	"fil/2/multisig":         builtin.MultisigActorCodeID,
+	"fil/1/system":           builtin.SystemActorCodeID,
+	"fil/1/init":             builtin.InitActorCodeID,
+	"fil/1/cron":             builtin.CronActorCodeID,
+	"fil/1/storagepower":     builtin.StoragePowerActorCodeID,
+	"fil/1/storageminer":     builtin.StorageMinerActorCodeID,
+	"fil/1/storagemarket":    builtin.StorageMarketActorCodeID,
+	"fil/1/paymentchannel":   builtin.PaymentChannelActorCodeID,
+	"fil/1/reward":           builtin.RewardActorCodeID,
+	"fil/1/verifiedregistry": builtin.VerifiedRegistryActorCodeID,
+	"fil/1/account":          builtin.AccountActorCodeID,
+	"fil/1/multisig":         builtin.MultisigActorCodeID,
+	"fil/2/system":           builtin2.SystemActorCodeID,
+	"fil/2/init":             builtin2.InitActorCodeID,
+	"fil/2/cron":             builtin2.CronActorCodeID,
+	"fil/2/storagepower":     builtin2.StoragePowerActorCodeID,
+	"fil/2/storageminer":     builtin2.StorageMinerActorCodeID,
+	"fil/2/storagemarket":    builtin2.StorageMarketActorCodeID,
+	"fil/2/paymentchannel":   builtin2.PaymentChannelActorCodeID,
+	"fil/2/reward":           builtin2.RewardActorCodeID,
+	"fil/2/verifiedregistry": builtin2.VerifiedRegistryActorCodeID,
+	"fil/2/account":          builtin2.AccountActorCodeID,
+	"fil/2/multisig":         builtin2.MultisigActorCodeID,
 }

commands/setup.go

@@ -173,9 +173,10 @@ func setupLogging(cctx *cli.Context) error {
 }
 
 const (
-	ChainHeadIndexerLockID    = 98981111
-	ChainHistoryIndexerLockID = 98981112
-	ChainVisRefresherLockID   = 98981113
+	ChainHeadIndexerLockID         = 98981111
+	ChainHistoryIndexerLockID      = 98981112
+	ChainVisRefresherLockID        = 98981113
+	ProcessingStatsRefresherLockID = 98981114
 )
 
 func NewGlobalSingleton(id int64, d *storage.Database) *GlobalSingleton {

go.mod

@@ -5,10 +5,13 @@ go 1.14
 require (
 	contrib.go.opencensus.io/exporter/prometheus v0.1.0
 	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
+	github.com/cskr/pubsub v1.0.2
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/filecoin-project/go-address v0.0.4
 	github.com/filecoin-project/go-bitfield v0.2.1
+	github.com/filecoin-project/go-fil-markets v0.7.0
 	github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52
+	github.com/filecoin-project/go-multistore v0.0.3
 	github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab
 	github.com/filecoin-project/lotus v0.9.0
 	github.com/filecoin-project/specs-actors v0.9.12
@@ -18,9 +21,13 @@ require (
 	github.com/go-pg/pgext v0.1.4
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/ipfs/go-cid v0.0.7
+	github.com/ipfs/go-ipfs-blockstore v1.0.1
 	github.com/ipfs/go-ipld-cbor v0.0.5-0.20200428170625-a0bd04d3cbdf
 	github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4
 	github.com/lib/pq v1.8.0
+	github.com/libp2p/go-libp2p-core v0.6.1
+	github.com/libp2p/go-libp2p-host v0.0.3
+	github.com/libp2p/go-libp2p-peer v0.2.0
 	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/multiformats/go-multiaddr v0.3.1
 	github.com/multiformats/go-multihash v0.0.14