-
Notifications
You must be signed in to change notification settings - Fork 46
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(gas outputs): Add Height and ActorName #270
Changes from 9 commits
4e6a5ff
545c9d9
746d903
6e5079c
c9dbcdd
a361a33
f3161ac
4f8e2d3
943a68b
f83a401
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
package migrations | ||
|
||
import "github.com/go-pg/migrations/v8" | ||
|
||
// Schema version 22 adds Height and ActorName to gas outputs table | ||
|
||
func init() { | ||
up := batch(` | ||
ALTER TABLE public.derived_gas_outputs ADD COLUMN height bigint NOT NULL; | ||
ALTER TABLE public.derived_gas_outputs ADD COLUMN actor_name text NOT NULL; | ||
ALTER TABLE public.derived_gas_outputs DROP CONSTRAINT derived_gas_outputs_pkey; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I also see in other migrations we do There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. pkey is the convention used by postgres when you create a primary key |
||
ALTER TABLE public.derived_gas_outputs ADD PRIMARY KEY (cid, height); | ||
`) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @iand can shall we leave like this, or add state_root too? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. let's add state_root - I'll do it |
||
down := batch(` | ||
ALTER TABLE public.derived_gas_outputs DROP CONSTRAINT derived_gas_outputs_pkey; | ||
ALTER TABLE public.derived_gas_outputs ADD PRIMARY KEY (cid); | ||
ALTER TABLE public.derived_gas_outputs DROP COLUMN height; | ||
ALTER TABLE public.derived_gas_outputs DROP COLUMN actor_name; | ||
`) | ||
|
||
migrations.MustRegisterTx(up, down) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,10 +2,16 @@ package message | |
|
||
import ( | ||
"context" | ||
"errors" | ||
"time" | ||
|
||
"github.com/filecoin-project/go-address" | ||
"github.com/filecoin-project/go-state-types/abi" | ||
"github.com/filecoin-project/go-state-types/big" | ||
"github.com/filecoin-project/lotus/chain/state" | ||
"github.com/filecoin-project/lotus/chain/types" | ||
"github.com/go-pg/pg/v10" | ||
"github.com/ipfs/go-cid" | ||
"github.com/raulk/clock" | ||
"go.opencensus.io/tag" | ||
"go.opentelemetry.io/otel/api/global" | ||
|
@@ -15,6 +21,7 @@ import ( | |
"github.com/filecoin-project/sentinel-visor/metrics" | ||
"github.com/filecoin-project/sentinel-visor/model/derived" | ||
"github.com/filecoin-project/sentinel-visor/storage" | ||
"github.com/filecoin-project/sentinel-visor/tasks/actorstate" | ||
"github.com/filecoin-project/sentinel-visor/wait" | ||
) | ||
|
||
|
@@ -65,7 +72,7 @@ func (p *GasOutputsProcessor) processBatch(ctx context.Context, node lens.API) ( | |
|
||
claimUntil := p.clock.Now().Add(p.leaseLength) | ||
|
||
var batch []*derived.ProcessingGasOutputs | ||
var batch []*derived.GasOutputs | ||
var err error | ||
|
||
if p.useLeases { | ||
|
@@ -104,7 +111,7 @@ func (p *GasOutputsProcessor) processBatch(ctx context.Context, node lens.API) ( | |
|
||
errorLog := log.With("cid", item.Cid) | ||
|
||
if err := p.processItem(ctx, node, &item.GasOutputs); err != nil { | ||
if err := p.processItem(ctx, node, item); err != nil { | ||
// Any errors are likely to be problems using the lens, mark this tipset as failed and exit this batch | ||
errorLog.Errorw("failed to process message", "error", err.Error()) | ||
if err := p.storage.MarkGasOutputsMessagesComplete(ctx, item.Height, item.Cid, p.clock.Now(), err.Error()); err != nil { | ||
|
@@ -125,6 +132,35 @@ func (p *GasOutputsProcessor) processItem(ctx context.Context, node lens.API, it | |
stop := metrics.Timer(ctx, metrics.ProcessingDuration) | ||
defer stop() | ||
|
||
// Note: this item will only be processed if there are receipts for | ||
// it, which means there should be a tipset at height+1. This is only | ||
// used to get the destination actor code, so we don't care about side | ||
// chains. | ||
Comment on lines
+135
to
+138
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Perhaps its worth adding a check to ensure There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Given that I just want to extract the dstActorCode, is it a problem if child == parent? The actor code will be the same even if it comes from a slightly wrong state. But maybe I'm missing something? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I believe the destination actor may no exist in |
||
child, err := node.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(item.Height+1), types.NewTipSetKey()) | ||
if err != nil { | ||
return xerrors.Errorf("Failed to load child tipset: %w", err) | ||
} | ||
|
||
st, err := state.LoadStateTree(node.Store(), child.ParentState()) | ||
if err != nil { | ||
return xerrors.Errorf("load state tree when gas outputs for %s: %w", item.Cid, err) | ||
} | ||
|
||
dstAddr, err := address.NewFromString(item.To) | ||
if err != nil { | ||
return xerrors.Errorf("parse to address failed for gas outputs in %s: %w", item.Cid, err) | ||
} | ||
|
||
var dstActorCode cid.Cid | ||
dstActor, err := st.GetActor(dstAddr) | ||
if err != nil { | ||
if !errors.Is(err, types.ErrActorNotFound) { | ||
return xerrors.Errorf("get destination actor for gas outputs %s failed: %w", item.Cid, err) | ||
} | ||
} else { | ||
dstActorCode = dstActor.Code | ||
} | ||
|
||
Comment on lines
+139
to
+163
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wish there was a better way, but this should be ok if the state is cached at some layer. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. #249 is much more efficient but will be a few days before it's ready. Currently checking that the data it produces matches the existing data. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yeah, still need to support it in the "standard" way which I trust will become way faster with store improvement and block caching at some point. |
||
baseFee, err := big.FromString(item.ParentBaseFee) | ||
if err != nil { | ||
return xerrors.Errorf("parse fee cap: %w", err) | ||
|
@@ -144,6 +180,7 @@ func (p *GasOutputsProcessor) processItem(ctx context.Context, node lens.API, it | |
outputs := node.ComputeGasOutputs(item.GasUsed, item.GasLimit, baseFee, feeCap, gasPremium) | ||
cgoStop() | ||
|
||
item.ActorName = actorstate.ActorNameByCode(dstActorCode) | ||
item.BaseFeeBurn = outputs.BaseFeeBurn.String() | ||
item.OverEstimationBurn = outputs.OverEstimationBurn.String() | ||
item.MinerPenalty = outputs.MinerPenalty.String() | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We will need to backfill the value for this column.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The value corresponds to the height of the message I think (not that of the receipts).
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, it's the height of the message. Backfilling is a problem. With the current version it will take some time. With the new in-order processing it takes <1s per tipset, but that's still ~211000 seconds = ~2.4 days unless we shard the processing.
An alternative is to figure out a query that can update this table by joining with messages. That will still be slow to run, several hours at least.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Custom in-order processing that just does messages and just updates this should be faster (?). That said, 2.4 days should be ok.