fix: run more backfill tipsetkey during import chain (#1222)

* backfill tipsetkey during import chain * Add option for init to control backfill range --------- Co-authored-by: Terry <[email protected]>
filecoin-project · Jun 6, 2023 · bdf1d1c · bdf1d1c
1 parent fd2168e
commit bdf1d1c
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 5 deletions.
diff --git a/commands/init.go b/commands/init.go
@@ -17,9 +17,10 @@ import (
 )
 
 var initFlags struct {
-	repo           string
-	config         string
-	importSnapshot string
+	repo                   string
+	config                 string
+	importSnapshot         string
+	backfillTipsetKeyRange int
 }
 
 var InitCmd = &cli.Command{
@@ -45,6 +46,13 @@ var InitCmd = &cli.Command{
 			EnvVars:     []string{"LILY_SNAPSHOT"},
 			Destination: &initFlags.importSnapshot,
 		},
+		&cli.IntFlag{
+			Name:        "backfill-tipsetkey-range",
+			Usage:       "Determine the extent of backfilling from the head.",
+			EnvVars:     []string{"LILY_BACKFILL_TIPSETKEY_RANGE"},
+			Value:       3600,
+			Destination: &initFlags.backfillTipsetKeyRange,
+		},
 	},
 	Action: func(c *cli.Context) error {
 		lotuslog.SetupLogLevels()
@@ -80,7 +88,7 @@ var InitCmd = &cli.Command{
 		}
 
 		if initFlags.importSnapshot != "" {
-			if err := util.ImportChain(ctx, r, initFlags.importSnapshot, true); err != nil {
+			if err := util.ImportChain(ctx, r, initFlags.importSnapshot, true, initFlags.backfillTipsetKeyRange); err != nil {
 				return err
 			}
 		}

diff --git a/commands/util/import.go b/commands/util/import.go
@@ -22,6 +22,7 @@ import (
 	"github.com/filecoin-project/lotus/storage/sealer/ffiwrapper"
 	"github.com/mitchellh/go-homedir"
 
+	"github.com/filecoin-project/lotus/chain/types"
 	"golang.org/x/xerrors"
 	"gopkg.in/cheggaaa/pb.v1"
 )
@@ -92,7 +93,7 @@ func ImportFromFsFile(ctx context.Context, r repo.Repo, fs fs.File, snapshot boo
 	return nil
 }
 
-func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool) (err error) {
+func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool, backfillTipsetkeyRange int) (err error) {
 	var rd io.Reader
 	var l int64
 	if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") {
@@ -189,6 +190,13 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool)
 		return fmt.Errorf("importing chain failed: %w", err)
 	}
 
+	// The cst.Import function will only backfill 1800 epochs of tipsetkey,
+	// Hence, the function is to backfill more epochs covered by the snapshot.
+	err = backfillTipsetKey(ctx, ts, cst, backfillTipsetkeyRange)
+	if err != nil {
+		log.Errorf("backfill tipsetkey failed: %w", err)
+	}
+
 	if err := cst.FlushValidationCache(ctx); err != nil {
 		return fmt.Errorf("flushing validation cache failed: %w", err)
 	}
@@ -222,3 +230,21 @@ func ImportChain(ctx context.Context, r repo.Repo, fname string, snapshot bool)
 
 	return nil
 }
+
+func backfillTipsetKey(ctx context.Context, root *types.TipSet, cs *store.ChainStore, backfillRange int) (err error) {
+	ts := root
+	log.Infof("backfilling the tipsetkey into chainstore, attempt to backfill the last %v epochs starting from the head.", backfillRange)
+	for i := 0; i < backfillRange; i++ {
+		err = cs.PersistTipset(ctx, ts)
+		if err != nil {
+			return
+		}
+		parentTsKey := ts.Parents()
+		ts, err = cs.LoadTipSet(ctx, parentTsKey)
+		if ts == nil || err != nil {
+			log.Infof("Only able to load the last %d tipsets", i)
+			break
+		}
+	}
+	return
+}