From e32de4a21dc7f2cfd613583b8ff8aa1be700d48d Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 23 Nov 2021 08:35:57 -0800 Subject: [PATCH] fix(find): find only consideres lack of OK's when finding gaps - closes #773 --- chain/find.go | 45 +++++++++++++++++++++++++-------------------- chain/find_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/chain/find.go b/chain/find.go index 531006295..cd574cd5a 100644 --- a/chain/find.go +++ b/chain/find.go @@ -45,40 +45,45 @@ func NewGapIndexer(node lens.API, db *storage.Database, name string, minHeight, } } -func (g *GapIndexer) Run(ctx context.Context) error { - startTime := time.Now() - - head, err := g.node.ChainHead(ctx) - if err != nil { - return err - } - if uint64(head.Height()) < g.maxHeight { - return xerrors.Errorf("cannot look for gaps beyond chain head height %d", head.Height()) - } - +func (g *GapIndexer) findGapsAndNullRounds(ctx context.Context) (visor.GapReportList, []abi.ChainEpoch, error) { findLog := log.With("type", "find") + var gaps []*visor.GapReport // looks for incomplete epochs. An incomplete epoch has some, but not all tasks in the processing report table. taskGaps, err := g.findTaskEpochGaps(ctx) if err != nil { - return xerrors.Errorf("finding task epoch gaps: %w", err) + return nil, nil, xerrors.Errorf("finding task epoch gaps: %w", err) } findLog.Infow("found gaps in tasks", "count", len(taskGaps)) + gaps = append(gaps, taskGaps...) // looks for missing epochs and null rounds. A missing epoch is a non-null-round height missing from the processing report table heightGaps, nulls, err := g.findEpochGapsAndNullRounds(ctx, g.node) if err != nil { - return xerrors.Errorf("finding epoch gaps: %w", err) + return nil, nil, xerrors.Errorf("finding epoch gaps: %w", err) } findLog.Infow("found gaps in epochs", "count", len(heightGaps)) findLog.Infow("found null rounds", "count", len(nulls)) + gaps = append(gaps, heightGaps...) + + return gaps, nulls, nil +} - // looks for entriest in the lily processing report table that have been skipped. - skipGaps, err := g.findEpochSkips(ctx) +func (g *GapIndexer) Run(ctx context.Context) error { + startTime := time.Now() + + head, err := g.node.ChainHead(ctx) + if err != nil { + return err + } + if uint64(head.Height()) < g.maxHeight { + return xerrors.Errorf("cannot look for gaps beyond chain head height %d", head.Height()) + } + + gaps, nulls, err := g.findGapsAndNullRounds(ctx) if err != nil { - return xerrors.Errorf("detecting skipped gaps: %w", err) + return err } - findLog.Infow("found skipped epochs", "count", len(skipGaps)) var nullRounds visor.ProcessingReportList for _, epoch := range nulls { @@ -99,7 +104,7 @@ func (g *GapIndexer) Run(ctx context.Context) error { }) } - return g.DB.PersistBatch(ctx, skipGaps, heightGaps, taskGaps, nullRounds) + return g.DB.PersistBatch(ctx, gaps, nullRounds) } type GapIndexerLens interface { @@ -139,7 +144,7 @@ func (g *GapIndexer) findEpochSkips(ctx context.Context) (visor.GapReportList, e return gapReport, nil } -func (g *GapIndexer) findEpochGapsAndNullRounds(ctx context.Context, node GapIndexerLens) (visor.GapReportList, []abi.ChainEpoch, error) { +func (g *GapIndexer) findEpochGapsAndNullRounds(ctx context.Context, node GapIndexerLens) ([]*visor.GapReport, []abi.ChainEpoch, error) { log.Debug("finding epoch gaps and null rounds") reportTime := time.Now() @@ -199,7 +204,7 @@ type TaskHeight struct { // TODO rather than use the length of `tasks` to determine where gaps are, use the contents to look for // gaps in specific task. Forrest' SQL-Foo isn't good enough for this yet. -func (g *GapIndexer) findTaskEpochGaps(ctx context.Context) (visor.GapReportList, error) { +func (g *GapIndexer) findTaskEpochGaps(ctx context.Context) ([]*visor.GapReport, error) { log.Debug("finding task epoch gaps") start := time.Now() var result []TaskHeight diff --git a/chain/find_test.go b/chain/find_test.go index 360d42fc9..17d5fd9c2 100644 --- a/chain/find_test.go +++ b/chain/find_test.go @@ -319,6 +319,23 @@ func TestFind(t *testing.T) { expected := makeGapReportList(tsh2, MessagesTask, ActorStatesInitTask) assertGapReportsEqual(t, expected, actual) }) + + t.Run("(#773) for each task at epoch 2 there exists a SKIP and an OK", func(t *testing.T) { + truncateVPR(t, db) + initializeVPR(t, db, maxHeight, t.Name(), AllTasks...) + skipEpochSkippedVRP(t, db, 2, AllTasks...) + appendOKAtEpochVPR(t, db, 2, AllTasks...) + + strg, err := storage.NewDatabaseFromDB(ctx, db, "public") + require.NoError(t, err, "NewDatabaseFromDB") + + actual, _, err := NewGapIndexer(nil, strg, t.Name(), minHeight, maxHeight, AllTasks). + findGapsAndNullRounds(ctx) + require.NoError(t, err) + + // no gaps should be found since the epoch has OK's for all tasks; the SKIPS are ignored. + require.Len(t, actual, 0) + }) } type assertFields struct { @@ -421,6 +438,17 @@ func skipEpochSkippedVRP(tb testing.TB, db *pg.DB, epoch int, tasks ...string) { } } +func appendOKAtEpochVPR(tb testing.TB, db *pg.DB, epoch int, tasks ...string) { + for _, task := range tasks { + qsrt := fmt.Sprintf(` + insert into public.visor_processing_reports(height, state_root, reporter, task, started_at, completed_at, status, status_information, errors_detected) + values(%d, concat(%d, '_state_root'), '%s_appendok', '%s', '2021-01-01 00:00:00.000000 +00:00', '2021-01-21 00:00:00.000000 +00:00', 'OK',null, null); + `, epoch, epoch, tb.Name(), task) + _, err := db.Exec(qsrt) + require.NoError(tb, err) + } +} + func truncateVPR(tb testing.TB, db *pg.DB) { _, err := db.Exec(`TRUNCATE TABLE visor_processing_reports`) require.NoError(tb, err, "visor_processing_report")