diff --git a/.gitignore b/.gitignore index 3d23c963b..47d0b5378 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ visor sentinel-visor build/.* +vector/data/*.json diff --git a/Makefile b/Makefile index e7127e301..4aef60c36 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,7 @@ build/.update-modules: .PHONY: deps deps: build/.update-modules + cd ./vector; ./fetch_vectors.sh # test starts dependencies and runs all tests .PHONY: test @@ -77,6 +78,7 @@ docker-image: clean: rm -rf $(CLEAN) $(BINS) + rm ./vector/data/*json .PHONY: clean dist-clean: diff --git a/chain/actor.go b/chain/actor.go index 2e9fbfb73..a44881c7d 100644 --- a/chain/actor.go +++ b/chain/actor.go @@ -193,6 +193,10 @@ type ActorExtractorMap interface { GetExtractor(code cid.Cid) (actorstate.ActorStateExtractor, bool) } +type ActorExtractorFilter interface { + AllowAddress(addr string) bool +} + // A RawActorExtractorMap extracts all types of actors using basic actor extraction which only parses shallow state. type RawActorExtractorMap struct{} diff --git a/chain/filter.go b/chain/filter.go new file mode 100644 index 000000000..cceaf0464 --- /dev/null +++ b/chain/filter.go @@ -0,0 +1,13 @@ +package chain + +func NewAddressFilter(addr string) *AddressFilter { + return &AddressFilter{address: addr} +} + +type AddressFilter struct { + address string +} + +func (f *AddressFilter) Allow(addr string) bool { + return f.address == addr +} diff --git a/chain/indexer.go b/chain/indexer.go index 04677b0ea..b585dea91 100644 --- a/chain/indexer.go +++ b/chain/indexer.go @@ -53,13 +53,22 @@ type TipSetIndexer struct { node lens.API opener lens.APIOpener closer lens.APICloser + addressFilter *AddressFilter +} + +type TipSetIndexerOpt func(t *TipSetIndexer) + +func AddressFilterOpt(f *AddressFilter) TipSetIndexerOpt { + return func(t *TipSetIndexer) { + t.addressFilter = f + } } // A TipSetIndexer extracts block, message and actor state data from a tipset and persists it to storage. Extraction // and persistence are concurrent. Extraction of the a tipset can proceed while data from the previous extraction is // being persisted. The indexer may be given a time window in which to complete data extraction. The name of the // indexer is used as the reporter in the visor_processing_reports table. -func NewTipSetIndexer(o lens.APIOpener, d model.Storage, window time.Duration, name string, tasks []string) (*TipSetIndexer, error) { +func NewTipSetIndexer(o lens.APIOpener, d model.Storage, window time.Duration, name string, tasks []string, options ...TipSetIndexerOpt) (*TipSetIndexer, error) { tsi := &TipSetIndexer{ storage: d, window: window, @@ -123,6 +132,11 @@ func NewTipSetIndexer(o lens.APIOpener, d model.Storage, window time.Duration, n return nil, xerrors.Errorf("unknown task: %s", task) } } + + for _, opt := range options { + opt(tsi) + } + return tsi, nil } @@ -223,6 +237,13 @@ func (t *TipSetIndexer) TipSet(ctx context.Context, ts *types.TipSet) error { if len(t.actorProcessors) > 0 { changes, err := t.node.StateChangedActors(tctx, parent.ParentState(), child.ParentState()) if err == nil { + if t.addressFilter != nil { + for addr := range changes { + if !t.addressFilter.Allow(addr) { + delete(changes, addr) + } + } + } for name, p := range t.actorProcessors { inFlight++ go t.runActorProcessor(tctx, p, name, child, parent, changes, results) diff --git a/commands/vector.go b/commands/vector.go new file mode 100644 index 000000000..f2aa1ba55 --- /dev/null +++ b/commands/vector.go @@ -0,0 +1,140 @@ +package commands + +import ( + "context" + "os" + "os/signal" + "strings" + "syscall" + + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/sentinel-visor/chain" + "github.com/filecoin-project/sentinel-visor/vector" +) + +var Vector = &cli.Command{ + Name: "vector", + Usage: "Vector tooling for Visor.", + Subcommands: []*cli.Command{ + BuildVector, + ExecuteVector, + }, +} + +var BuildVector = &cli.Command{ + Name: "build", + Usage: "Create a vector.", + Flags: []cli.Flag{ + &cli.Int64Flag{ + Name: "from", + Usage: "Limit actor and message processing to tipsets at or above `HEIGHT`", + EnvVars: []string{"VISOR_HEIGHT_FROM"}, + }, + &cli.Int64Flag{ + Name: "to", + Usage: "Limit actor and message processing to tipsets at or below `HEIGHT`", + Value: estimateCurrentEpoch(), + DefaultText: "current epoch", + EnvVars: []string{"VISOR_HEIGHT_TO"}, + }, + &cli.StringFlag{ + Name: "tasks", + Usage: "Comma separated list of tasks to build. Each task is reported separately in the database.", + Value: strings.Join([]string{chain.BlocksTask}, ","), + EnvVars: []string{"VISOR_VECTOR_TASKS"}, + }, + &cli.StringFlag{ + Name: "actor-address", + Usage: "Address of an actor.", + }, + &cli.StringFlag{ + Name: "vector-file", + Usage: "Path of vector file.", + Required: true, + }, + &cli.StringFlag{ + Name: "vector-desc", + Usage: "Short description of the test vector.", + Required: true, + }, + }, + Action: build, +} + +func build(cctx *cli.Context) error { + // Set up a context that is canceled when the command is interrupted + ctx, cancel := context.WithCancel(cctx.Context) + + // Set up a signal handler to cancel the context + go func() { + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, syscall.SIGTERM, syscall.SIGINT) + select { + case <-interrupt: + cancel() + case <-ctx.Done(): + } + }() + + if err := setupLogging(cctx); err != nil { + return xerrors.Errorf("setup logging: %w", err) + } + + builder, err := vector.NewBuilder(cctx) + if err != nil { + return err + } + + schema, err := builder.Build(ctx) + if err != nil { + return err + } + + return schema.Persist(cctx.String("vector-file")) +} + +var ExecuteVector = &cli.Command{ + Name: "execute", + Usage: "execute a test vector", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "vector-file", + Usage: "Path to vector file.", + Required: true, + }, + }, + Action: execute, +} + +func execute(cctx *cli.Context) error { + // Set up a context that is canceled when the command is interrupted + ctx, cancel := context.WithCancel(cctx.Context) + + // Set up a signal handler to cancel the context + go func() { + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, syscall.SIGTERM, syscall.SIGINT) + select { + case <-interrupt: + cancel() + case <-ctx.Done(): + } + }() + + if err := setupLogging(cctx); err != nil { + return xerrors.Errorf("setup logging: %w", err) + } + runner, err := vector.NewRunner(ctx, cctx.String("vector-file"), 0) + if err != nil { + return err + } + + err = runner.Run(ctx) + if err != nil { + return err + } + + return runner.Validate(ctx) +} diff --git a/go.mod b/go.mod index 01335b952..bfd08090d 100644 --- a/go.mod +++ b/go.mod @@ -20,12 +20,18 @@ require ( github.com/go-pg/migrations/v8 v8.0.1 github.com/go-pg/pg/v10 v10.3.1 github.com/go-pg/pgext v0.1.4 + github.com/google/go-cmp v0.5.2 github.com/hashicorp/golang-lru v0.5.4 github.com/ipfs/go-block-format v0.0.2 + github.com/ipfs/go-blockservice v0.1.4 github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-datastore v0.4.5 + github.com/ipfs/go-ipfs-exchange-offline v0.0.1 github.com/ipfs/go-ipld-cbor v0.0.5 + github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4 + github.com/ipfs/go-merkledag v0.3.2 + github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018 github.com/jackc/pgx/v4 v4.9.0 github.com/lib/pq v1.8.0 diff --git a/go.sum b/go.sum index fea24a0af..d503692cb 100644 --- a/go.sum +++ b/go.sum @@ -1572,6 +1572,7 @@ github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/ github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= diff --git a/lens/carrepo/carrepo.go b/lens/carrepo/carrepo.go index 939713d23..2d93f85b4 100644 --- a/lens/carrepo/carrepo.go +++ b/lens/carrepo/carrepo.go @@ -33,5 +33,5 @@ func NewAPIOpener(c *cli.Context) (lens.APIOpener, lens.APICloser, error) { return &tsk, nil } - return util.NewAPIOpener(c, cacheDB, h) + return util.NewAPIOpener(c.Context, cacheDB, h, c.Int("lens-cache-hint")) } diff --git a/lens/interface.go b/lens/interface.go index 8b5b2d083..76f86abc1 100644 --- a/lens/interface.go +++ b/lens/interface.go @@ -2,6 +2,7 @@ package lens import ( "context" + "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" @@ -55,6 +56,7 @@ type StateAPI interface { StateReadState(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*api.ActorState, error) StateGetReceipt(ctx context.Context, bcid cid.Cid, tsk types.TipSetKey) (*types.MessageReceipt, error) StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error) + StateNetworkName(context.Context) (dtypes.NetworkName, error) } type APICloser func() diff --git a/lens/s3repo/repo.go b/lens/s3repo/repo.go index b2f218147..d8370eca9 100644 --- a/lens/s3repo/repo.go +++ b/lens/s3repo/repo.go @@ -12,5 +12,5 @@ func NewAPIOpener(c *cli.Context) (lens.APIOpener, lens.APICloser, error) { return nil, nil, err } - return util.NewAPIOpener(c, bs, bs.(*S3Blockstore).getMasterTsKey) + return util.NewAPIOpener(c.Context, bs, bs.(*S3Blockstore).getMasterTsKey, c.Int("lens-cache-hint")) } diff --git a/lens/sqlrepo/repo.go b/lens/sqlrepo/repo.go index 082edaecb..cbb52cd0a 100644 --- a/lens/sqlrepo/repo.go +++ b/lens/sqlrepo/repo.go @@ -13,5 +13,5 @@ func NewAPIOpener(c *cli.Context) (lens.APIOpener, lens.APICloser, error) { return nil, nil, err } - return util.NewAPIOpener(c, bs, bs.(*SqlBlockstore).getMasterTsKey) + return util.NewAPIOpener(c.Context, bs, bs.(*SqlBlockstore).getMasterTsKey, c.Int("lens-cache-hint")) } diff --git a/lens/util/repo.go b/lens/util/repo.go index a7d517c9b..0f952626b 100644 --- a/lens/util/repo.go +++ b/lens/util/repo.go @@ -30,7 +30,6 @@ import ( "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" peer "github.com/libp2p/go-libp2p-core/peer" - "github.com/urfave/cli/v2" "golang.org/x/xerrors" "github.com/filecoin-project/sentinel-visor/lens" @@ -43,7 +42,7 @@ type APIOpener struct { type HeadMthd func(ctx context.Context, lookback int) (*types.TipSetKey, error) -func NewAPIOpener(c *cli.Context, bs blockstore.Blockstore, head HeadMthd) (*APIOpener, lens.APICloser, error) { +func NewAPIOpener(ctx context.Context, bs blockstore.Blockstore, head HeadMthd, cacheHint int) (*APIOpener, lens.APICloser, error) { rapi := LensAPI{} if _, _, err := ulimit.ManageFdLimit(); err != nil { @@ -65,7 +64,7 @@ func NewAPIOpener(c *cli.Context, bs blockstore.Blockstore, head HeadMthd) (*API const safetyLookBack = 5 - headKey, err := head(c.Context, safetyLookBack) + headKey, err := head(ctx, safetyLookBack) if err != nil { return nil, nil, err } @@ -91,8 +90,8 @@ func NewAPIOpener(c *cli.Context, bs blockstore.Blockstore, head HeadMthd) (*API lr.Close() } - rapi.Context = c.Context - rapi.cacheSize = c.Int("lens-cache-hint") + rapi.Context = ctx + rapi.cacheSize = cacheHint return &APIOpener{rapi: &rapi}, sf, nil } diff --git a/lens/vector/repo.go b/lens/vector/repo.go new file mode 100644 index 000000000..ad23f4544 --- /dev/null +++ b/lens/vector/repo.go @@ -0,0 +1,179 @@ +package vector + +import ( + "context" + "fmt" + "io" + + "github.com/urfave/cli/v2" + + "github.com/ipfs/go-blockservice" + cid "github.com/ipfs/go-cid" + offline "github.com/ipfs/go-ipfs-exchange-offline" + cbor "github.com/ipfs/go-ipld-cbor" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-merkledag" + car "github.com/ipld/go-car" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/journal" + "github.com/filecoin-project/lotus/lib/bufbstore" + "github.com/filecoin-project/lotus/lib/ulimit" + "github.com/filecoin-project/lotus/node/impl" + "github.com/filecoin-project/lotus/node/impl/full" + "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/sentinel-visor/lens" + "github.com/filecoin-project/sentinel-visor/lens/util" + "github.com/filecoin-project/specs-actors/actors/runtime/proof" + "github.com/filecoin-project/specs-actors/actors/util/adt" +) + +func NewAPIOpener(c *cli.Context) (*APIOpener, lens.APICloser, error) { + capi := CaptureAPI{} + + if _, _, err := ulimit.ManageFdLimit(); err != nil { + return nil, nil, fmt.Errorf("setting file descriptor limit: %s", err) + } + + r, err := repo.NewFS(c.String("repo")) + if err != nil { + return nil, nil, err + } + + exists, err := r.Exists() + if err != nil { + return nil, nil, err + } + if !exists { + return nil, nil, fmt.Errorf("lotus repo doesn't exist") + } + + var lr repo.LockedRepo + if c.Bool("repo-read-only") { + lr, err = r.LockRO(repo.FullNode) + } else { + lr, err = r.Lock(repo.FullNode) + } + if err != nil { + return nil, nil, err + } + + sf := func() { + lr.Close() + } + + bs, err := lr.Blockstore(repo.BlockstoreChain) + if err != nil { + return nil, nil, err + } + + // wrap the repos blockstore with a tracing implementation capturing all cid's read. + capi.tbs = NewTracingBlockstore(bs) + + mds, err := lr.Datastore("/metadata") + if err != nil { + return nil, nil, err + } + + cs := store.NewChainStore(capi.tbs, capi.tbs, mds, vm.Syscalls(&fakeVerifier{}), journal.NilJournal()) + if err := cs.Load(); err != nil { + return nil, nil, err + } + + sm := stmgr.NewStateManager(cs) + + capi.FullNodeAPI.ChainAPI.Chain = cs + capi.FullNodeAPI.ChainAPI.ChainModuleAPI = &full.ChainModule{Chain: cs} + capi.FullNodeAPI.StateAPI.Chain = cs + capi.FullNodeAPI.StateAPI.StateManager = sm + capi.FullNodeAPI.StateAPI.StateModuleAPI = &full.StateModule{Chain: cs, StateManager: sm} + + capi.Context = c.Context + capi.cacheSize = c.Int("lens-cache-hint") + return &APIOpener{capi: &capi}, sf, nil +} + +type APIOpener struct { + capi *CaptureAPI +} + +func (o *APIOpener) Open(ctx context.Context) (lens.API, lens.APICloser, error) { + return o.capi, lens.APICloser(func() {}), nil +} +func (c *APIOpener) CaptureAsCAR(ctx context.Context, w io.Writer, roots ...cid.Cid) error { + carWalkFn := func(nd format.Node) (out []*format.Link, err error) { + for _, link := range nd.Links() { + if _, ok := c.capi.tbs.traced[link.Cid]; !ok { + continue + } + if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed { + continue + } + out = append(out, link) + } + return out, nil + } + + var ( + offl = offline.Exchange(c.capi.tbs) + blkserv = blockservice.New(c.capi.tbs, offl) + dserv = merkledag.NewDAGService(blkserv) + ) + + return car.WriteCarWithWalker(ctx, dserv, roots, w, carWalkFn) +} + +type CaptureAPI struct { + impl.FullNodeAPI + context.Context + cacheSize int + + tbs *TracingBlockstore +} + +func (c *CaptureAPI) Store() adt.Store { + cachedStore := bufbstore.NewBufferedBstore(c.tbs) + cs := cbor.NewCborStore(cachedStore) + adtStore := adt.WrapStore(c.Context, cs) + return adtStore +} + +func (c *CaptureAPI) GetExecutedMessagesForTipset(ctx context.Context, ts, pts *types.TipSet) ([]*lens.ExecutedMessage, error) { + return util.GetExecutedMessagesForTipset(ctx, c.FullNodeAPI.ChainAPI.Chain, ts, pts) +} + +func (c *CaptureAPI) StateGetActor(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) { + act, err := lens.OptimizedStateGetActorWithFallback(ctx, c.ChainAPI.Chain.Store(ctx), c.ChainAPI, c.StateAPI, addr, tsk) + if err != nil { + return nil, err + } + //c.tbs.Record(act.Head) + return act, nil +} + +// From https://github.com/ribasushi/ltsh/blob/5b0211033020570217b0ae37b50ee304566ac218/cmd/lotus-shed/deallifecycles.go#L41-L171 +type fakeVerifier struct{} + +var _ ffiwrapper.Verifier = (*fakeVerifier)(nil) + +func (m fakeVerifier) VerifySeal(svi proof.SealVerifyInfo) (bool, error) { + return true, nil +} + +func (m fakeVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) { + return true, nil +} + +func (m fakeVerifier) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) { + return true, nil +} + +func (m fakeVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proof abi.RegisteredPoStProof, id abi.ActorID, randomness abi.PoStRandomness, u uint64) ([]uint64, error) { + panic("GenerateWinningPoStSectorChallenge not supported") +} diff --git a/lens/vector/tracing_bs.go b/lens/vector/tracing_bs.go new file mode 100644 index 000000000..a8b7af789 --- /dev/null +++ b/lens/vector/tracing_bs.go @@ -0,0 +1,73 @@ +package vector + +import ( + "sync" + + "github.com/filecoin-project/lotus/lib/blockstore" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" +) + +func NewTracingBlockstore(bs blockstore.Blockstore) *TracingBlockstore { + return &TracingBlockstore{ + tracedMu: sync.Mutex{}, + traced: make(map[cid.Cid]struct{}), + Blockstore: bs, + } +} + +var ( + _ cbor.IpldBlockstore = (*TracingBlockstore)(nil) + _ blockstore.Viewer = (*TracingBlockstore)(nil) + _ blockstore.Blockstore = (*TracingBlockstore)(nil) +) + +type TracingBlockstore struct { + tracedMu sync.Mutex + traced map[cid.Cid]struct{} + + blockstore.Blockstore +} + +func (tb *TracingBlockstore) Traced() map[cid.Cid]struct{} { + // TODO maybe returning a copy? + return tb.traced +} + +// implements blockstore viewer interface. +func (tb *TracingBlockstore) View(k cid.Cid, callback func([]byte) error) error { + blk, err := tb.Get(k) + if err == nil && blk != nil { + return callback(blk.RawData()) + } + return err +} + +func (tb *TracingBlockstore) Get(cid cid.Cid) (blocks.Block, error) { + tb.tracedMu.Lock() + tb.traced[cid] = struct{}{} + tb.tracedMu.Unlock() + + block, err := tb.Blockstore.Get(cid) + if err != nil { + return nil, err + } + return block, err +} + +func (tb *TracingBlockstore) Put(block blocks.Block) error { + tb.tracedMu.Lock() + tb.traced[block.Cid()] = struct{}{} + tb.tracedMu.Unlock() + return tb.Blockstore.Put(block) +} + +func (tb *TracingBlockstore) PutMany(blocks []blocks.Block) error { + tb.tracedMu.Lock() + for _, b := range blocks { + tb.traced[b.Cid()] = struct{}{} + } + tb.tracedMu.Unlock() + return tb.Blockstore.PutMany(blocks) +} diff --git a/main.go b/main.go index 8c41d5e04..7094a9188 100644 --- a/main.go +++ b/main.go @@ -150,6 +150,7 @@ func main() { }, Commands: []*cli.Command{ commands.Migrate, + commands.Vector, commands.Walk, commands.Watch, }, diff --git a/storage/mem.go b/storage/mem.go new file mode 100644 index 000000000..7be2e7593 --- /dev/null +++ b/storage/mem.go @@ -0,0 +1,63 @@ +package storage + +import ( + "context" + "reflect" + "sync" + + "github.com/go-pg/pg/v10/orm" + + "github.com/filecoin-project/sentinel-visor/model" +) + +func NewMemStorage() *MemStorage { + return &MemStorage{Data: map[string][]interface{}{}} +} + +type MemStorage struct { + // TODO parallel map? + Data map[string][]interface{} + DataMu sync.Mutex +} + +func (j *MemStorage) PersistModel(ctx context.Context, m interface{}) error { + if len(models) == 0 { + return nil + } + + value := reflect.ValueOf(m) + if value.Kind() == reflect.Ptr { + value = value.Elem() + } + + switch value.Kind() { + case reflect.Slice, reflect.Array: + for i := 0; i < value.Len(); i++ { + if err := j.PersistModel(ctx, value.Index(i).Interface()); err != nil { + return err + } + } + return nil + case reflect.Struct: + q := orm.NewQuery(nil, m) + tm := q.TableModel() + n := tm.Table() + name := stripQuotes(n.SQLNameForSelects) + j.DataMu.Lock() + j.Data[name] = append(j.Data[name], m) + j.DataMu.Unlock() + return nil + default: + return ErrMarshalUnsupportedType + + } +} + +func (j *MemStorage) PersistBatch(ctx context.Context, ps ...model.Persistable) error { + for _, p := range ps { + if err := p.Persist(ctx, j); err != nil { + return err + } + } + return nil +} diff --git a/storage/sql.go b/storage/sql.go index f989617c9..8651e00d2 100644 --- a/storage/sql.go +++ b/storage/sql.go @@ -57,6 +57,7 @@ var models = []interface{}{ (*multisig.MultisigTransaction)(nil), (*power.ChainPower)(nil), + (*power.PowerActorClaim)(nil), (*reward.ChainReward)(nil), (*common.Actor)(nil), (*common.ActorState)(nil), diff --git a/vector/README.md b/vector/README.md new file mode 100644 index 000000000..d58ee93b5 --- /dev/null +++ b/vector/README.md @@ -0,0 +1,110 @@ +# Visor Vector How-To +You can use the `visor vector` command to create and execute test vectors with Visor. Vectors serve as a set of reproducible scenarios that allow developers to ensure their changes haven't broken Visor. All that's required is a Lotus repo. + +## How to build a vector +To build a test vector, use the `vector build` command. The example below creates a test vector file named "blocks_471000-471010.json". +``` +visor --repo-read-only=false --repo="~/.lotus" vector build --tasks=blocks --from=471000 --to=471010 --vector-file="./blocks_471000-471010.json" --vector-desc="block models from epoch 471000 to 471010" +``` +The file contains information about the version of Visor used to build it: +``` +$ cat blocks_471000-471010.json | jq ".metadata" +{ + "version": "v0.4.0+rc2-55-g17c8329-dirty", + "description": "block models from epoch 471000 to 471010", + "network": "mainnet", + "time": 1613515320 +} +``` +the commands to produce it: +``` +$ cat blocks_471000-471010.json | jq ".parameters" +{ + "from": 471000, + "to": 471010, + "tasks": [ + "blocks" + ], + "actor-address": "" +} +``` +the extracted chain state as a base64 encoded CAR file: +``` +$ cat blocks_471000-471010.json | jq ".car" + +``` +and the models extracted: +``` +$ cat blocks_471000-471010.json | jq ".expected[].block_headers" +[ + { + "Height": 471010, + "Cid": "bafy2bzacecq2fysiktcuc5r762hvuxstwv4qzx645ld4h66spenheckyuko2y", + "Miner": "f02770", + "ParentWeight": "10230517226", + "ParentBaseFee": "3275882399", + "ParentStateRoot": "bafy2bzacecsckqgj6ufbefj7bx6odzvvbftky77py5x2th3wrqx3yvpksyuc6", + "WinCount": 1, + "Timestamp": 1612436700, + "ForkSignaling": 0 + }, + { + "Height": 471010, + "Cid": "bafy2bzacecxmra4cdbqq7lxao7e2gmxvf7ef3o464et7ejykcn24itps7vnau", + "Miner": "f02775", + "ParentWeight": "10230517226", + "ParentBaseFee": "3275882399", + "ParentStateRoot": "bafy2bzacecsckqgj6ufbefj7bx6odzvvbftky77py5x2th3wrqx3yvpksyuc6", + "WinCount": 1, + "Timestamp": 1612436700, + "ForkSignaling": 0 + }, + ... +``` + +## How to execute a vector +To execute a vector, use the `vector execute` command. The example below executes a test vector file named "blocks_471000-471010.json". +``` +$ ./visor vector execute --vector-file=blocks_471000-471010.json +2021-02-16T16:32:29.225-0800 INFO vector vector/runner.go:146 Validate Model block_headers: Passed +2021-02-16T16:32:29.230-0800 INFO vector vector/runner.go:146 Validate Model block_parents: Passed +2021-02-16T16:32:29.230-0800 INFO vector vector/runner.go:146 Validate Model drand_block_entries: Passed +``` +When executing the vector file Visor uses the data in the `car` field as its data source, executes the commands in the `parameters` field, then validates the data returned from the execution matches the `expected` models in the vector file. + +## How to save a vector +Vector files are stored on the IPFS network, and a list of their hashes is kept in the `VECTOR_MANIFEST` file. Storing the hashes in the git repo and the files in IPFS helps keep our repo compact. The example below will demonstrate how to save a vector file to run as a part of CI. We will use the aforementioned `blocks_471000-471010.json` in this example. +``` +$ ipfs add blocks_471000-471010.json +added QmPTKSr1uwExGTVByDUoVm4CfrjU34oNV6gfvWvufaN9h1 blocks_471000-471010.json + 146.43 KiB / 146.43 KiB [==] 100.00% +``` +next, add the hash to the manifest: +``` +$ echo QmPTKSr1uwExGTVByDUoVm4CfrjU34oNV6gfvWvufaN9h1 >> ./vector/VECTOR_MANIFEST +``` +then make the vector deps: +``` +$ make deps +cd ./vector; ./fetch_vectors.sh +Fetching QmPTKSr1uwExGTVByDUoVm4CfrjU34oNV6gfvWvufaN9h1 + ``` +You can see the vector file is in `vectors/data/`: +``` +ls vector/data/ +QmPTKSr1uwExGTVByDUoVm4CfrjU34oNV6gfvWvufaN9h1_block_models_from_epoch_471000_to_471010.json +``` +Finally, execute the vector unit tests: +``` +$ go test -v ./vector/... +=== RUN TestExecuteVectors +=== RUN TestExecuteVectors/QmPTKSr1uwExGTVByDUoVm4CfrjU34oNV6gfvWvufaN9h1_block_models_from_epoch_471000_to_471010.json +--- PASS: TestExecuteVectors (65.08s) + --- PASS: TestExecuteVectors/QmPTKSr1uwExGTVByDUoVm4CfrjU34oNV6gfvWvufaN9h1_block_models_from_epoch_471000_to_471010.json (0.02s) +``` +Push your changes to `VECTOR_MANIFEST` up execution in CI. + + + + + diff --git a/vector/VECTOR_MANIFEST b/vector/VECTOR_MANIFEST new file mode 100644 index 000000000..16000e1a5 --- /dev/null +++ b/vector/VECTOR_MANIFEST @@ -0,0 +1,9 @@ +Qmdz1XTcneY8VE2s8jXnkyUqaY31ad6cEYMVtoFCLCZ2uc +QmUSity2Y24fDRcZkxwd8YmXan48jGZ1Udqrr8XqrJ5ysJ +QmNcur1q4dvjWU36f1ysR4eXZNkUi5XBZ6eiLPdbQnf7uw +QmagPzhnJzLSsNk53F28vXRVyH8DbNRPiooQeg8TqEDN7p +QmXtJaXsktYgGkZw9zGYNCEVFbLu6kwiKYcJ8rJTjwat81 +QmRpzYRnU6PfcVW5XmbB2dqymqGKCejz8wYZbhDBZQDpfj +QmWiKwqPYdou5sc6J78Ci1i3FNJwxA14qd4Rc5sQ89q1Ab +QmYhszKDYDFvXyFQF8KefF4M8gcNaoTC8FMYFcgKmfq6pi +QmZMXpmkNDMVmUammvgN5UbgUWXXBCyJL5wFn6YNo5JQVh diff --git a/vector/builder.go b/vector/builder.go new file mode 100644 index 000000000..8b5ec68f1 --- /dev/null +++ b/vector/builder.go @@ -0,0 +1,162 @@ +package vector + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "os" + "strings" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/sentinel-visor/chain" + "github.com/filecoin-project/sentinel-visor/lens" + "github.com/filecoin-project/sentinel-visor/lens/vector" + "github.com/filecoin-project/sentinel-visor/storage" + "github.com/filecoin-project/sentinel-visor/version" +) + +var log = logging.Logger("vector") + +type BuilderSchema struct { + Meta Metadata `json:"metadata"` + Params Parameters `json:"parameters"` + CAR Base64EncodedBytes `json:"car"` + Exp BuilderExpected `json:"expected"` +} + +func (bs *BuilderSchema) Persist(path string) error { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + return err + } + return json.NewEncoder(f).Encode(bs) +} + +type Builder struct { + From int64 + To int64 + Tasks []string + AddressFilter string + Description string + + storage *storage.MemStorage + + opener lens.APIOpener + closer lens.APICloser +} + +func NewBuilder(cctx *cli.Context) (*Builder, error) { + from := cctx.Int64("from") + to := cctx.Int64("to") + if from > to { + return nil, xerrors.Errorf("--from must not be greater than --to") + } + + lensOpener, lensCloser, err := vector.NewAPIOpener(cctx) + if err != nil { + return nil, xerrors.Errorf("setup lens: %w", err) + } + + return &Builder{ + From: from, + To: to, + Tasks: strings.Split(cctx.String("tasks"), ","), + AddressFilter: cctx.String("actor-address"), + Description: cctx.String("vector-desc"), + storage: storage.NewMemStorage(), + opener: lensOpener, + closer: lensCloser, + }, nil +} + +func (b *Builder) Build(ctx context.Context) (*BuilderSchema, error) { + // get root CIDs of vector (for the car file header). + node, closer, err := b.opener.Open(ctx) + if err != nil { + return nil, xerrors.Errorf("open lens: %w", err) + } + defer closer() + + network, err := node.StateNetworkName(ctx) + if err != nil { + return nil, err + } + + roots, err := node.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(b.To), types.EmptyTSK) + if err != nil { + return nil, err + } + + var opts []chain.TipSetIndexerOpt + if len(b.AddressFilter) > 0 { + opts = append(opts, chain.AddressFilterOpt(chain.NewAddressFilter(b.AddressFilter))) + } + + // perform a walk over the chain. + tsIndexer, err := chain.NewTipSetIndexer(b.opener, b.storage, 0, "build_vector", b.Tasks, opts...) + if err != nil { + return nil, xerrors.Errorf("setup indexer: %w", err) + } + defer func() { + if err := tsIndexer.Close(); err != nil { + log.Errorw("failed to close tipset indexer cleanly", "error", err) + } + }() + + if err := chain.NewWalker(tsIndexer, b.opener, b.From, b.To).Run(ctx); err != nil && !errors.Is(err, context.Canceled) { + return nil, err + } + + // persist the chain data read during the above walk to `out` + out := new(bytes.Buffer) + gw := gzip.NewWriter(out) + cameraOpener, ok := b.opener.(*vector.APIOpener) + if !ok { + panic("developer error") + } + if err := cameraOpener.CaptureAsCAR(ctx, gw, roots.Cids()...); err != nil { + return nil, err + } + + if err := gw.Flush(); err != nil { + return nil, err + } + if err := gw.Close(); err != nil { + return nil, err + } + + schema := &BuilderSchema{ + Meta: Metadata{ + Version: version.String(), + Description: b.Description, + Network: string(network), + Date: time.Now().UTC().Unix(), + }, + Params: Parameters{ + From: b.From, + To: b.To, + Tasks: b.Tasks, + AddressFilter: b.AddressFilter, + }, + CAR: out.Bytes(), + Exp: BuilderExpected{ + Models: b.storage.Data, + }, + } + + log.Infow("built Schema", + "version", schema.Meta.Version, + "network", schema.Meta.Network, + "date", schema.Meta.Date, + ) + + return schema, nil +} diff --git a/vector/data/.gitkeep b/vector/data/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/vector/fetch_vectors.sh b/vector/fetch_vectors.sh new file mode 100755 index 000000000..d331b038a --- /dev/null +++ b/vector/fetch_vectors.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +while read hash; do + lines=$(find ./data -name "${hash}*" | wc -l) + if [ $lines -eq 0 ]; then + echo "Fetching $hash" + curl https://ipfs.io/ipfs/"$hash" -o tmp + filename=$(cat ./tmp | jq ".metadata.description" | sed 's/ /_/g' | sed -e 's/^"//' -e 's/"$//') + mv ./tmp ./data/"$hash"_"$filename".json + fi +done <./VECTOR_MANIFEST diff --git a/vector/runner.go b/vector/runner.go new file mode 100644 index 000000000..40a4b0518 --- /dev/null +++ b/vector/runner.go @@ -0,0 +1,582 @@ +package vector + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "time" + + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/filecoin-project/sentinel-visor/model/actors/common" + init_ "github.com/filecoin-project/sentinel-visor/model/actors/init" + "github.com/filecoin-project/sentinel-visor/model/actors/market" + "github.com/filecoin-project/sentinel-visor/model/actors/multisig" + "github.com/filecoin-project/sentinel-visor/model/actors/power" + "github.com/filecoin-project/sentinel-visor/model/actors/reward" + modelchain "github.com/filecoin-project/sentinel-visor/model/chain" + "github.com/filecoin-project/sentinel-visor/model/derived" + "github.com/filecoin-project/sentinel-visor/model/messages" + + "github.com/google/go-cmp/cmp" + "github.com/ipld/go-car" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/blockstore" + + "github.com/filecoin-project/sentinel-visor/chain" + "github.com/filecoin-project/sentinel-visor/lens" + "github.com/filecoin-project/sentinel-visor/lens/util" + "github.com/filecoin-project/sentinel-visor/model/actors/miner" + "github.com/filecoin-project/sentinel-visor/model/blocks" + "github.com/filecoin-project/sentinel-visor/storage" +) + +type RunnerSchema struct { + Meta Metadata `json:"metadata"` + Params Parameters `json:"parameters"` + CAR Base64EncodedBytes `json:"car"` + Exp RunnerExpected `json:"expected"` +} + +type Runner struct { + schema RunnerSchema + + storage *storage.MemStorage + + opener lens.APIOpener + closer lens.APICloser +} + +func NewRunner(ctx context.Context, vectorPath string, cacheHint int) (*Runner, error) { + fecVile, err := os.OpenFile(vectorPath, os.O_RDONLY, 0o644) + if err != nil { + return nil, err + } + var vs RunnerSchema + if err := json.NewDecoder(fecVile).Decode(&vs); err != nil { + return nil, err + } + // need to go from bytes representing a car file to a blockstore, then to a Lotus API. + bs := blockstore.Blockstore(blockstore.NewTemporary()) + + // Read the base64-encoded CAR from the vector, and inflate the gzip. + buf := bytes.NewReader(vs.CAR) + r, err := gzip.NewReader(buf) + if err != nil { + return nil, fmt.Errorf("failed to inflate gzipped CAR: %s", err) + } + defer r.Close() + + // put the thing in the blockstore. + // TODO: the entire car file is now in memory, this is an unrealistic expectation, adjust at some point. + carHeader, err := car.LoadCar(bs, r) + if err != nil { + return nil, fmt.Errorf("failed to load state tree car from test vector: %s", err) + } + + cacheDB := util.NewCachingStore(bs) + + h := func(ctx context.Context, lookback int) (*types.TipSetKey, error) { + tsk := types.NewTipSetKey(carHeader.Roots...) + return &tsk, nil + } + + opener, closer, err := util.NewAPIOpener(ctx, cacheDB, h, cacheHint) + if err != nil { + return nil, err + } + + return &Runner{ + schema: vs, + storage: storage.NewMemStorage(), + opener: opener, + closer: closer, + }, nil +} + +func (r *Runner) Run(ctx context.Context) error { + var opt []chain.TipSetIndexerOpt + if len(r.schema.Params.AddressFilter) > 0 { + opt = append(opt, chain.AddressFilterOpt(chain.NewAddressFilter(r.schema.Params.AddressFilter))) + } + tsIndexer, err := chain.NewTipSetIndexer(r.opener, r.storage, 0, "run_vector", r.schema.Params.Tasks, opt...) + if err != nil { + return xerrors.Errorf("setup indexer: %w", err) + } + defer func() { + if err := tsIndexer.Close(); err != nil { + log.Errorw("failed to close tipset indexer cleanly", "error", err) + } + }() + + if err := chain.NewWalker(tsIndexer, r.opener, r.schema.Params.From, r.schema.Params.To).Run(ctx); err != nil && !errors.Is(err, context.Canceled) { + return err + } + // TODO remove when https://github.com/filecoin-project/sentinel-visor/issues/374 is fixed + time.Sleep(3 * time.Second) + return nil +} + +func (r *Runner) Validate(ctx context.Context) error { + actual := r.storage.Data + expected := r.schema.Exp.Models + + for expTable, expData := range expected { + if expTable == "visor_processing_reports" { + continue + } + + actData, ok := actual[expTable] + if !ok { + return xerrors.Errorf("Missing Table: %s", expTable) + } + + diff, err := modelTypeFromTable(expTable, expData, actData) + if err != nil { + return err + } + + if diff != "" { + log.Errorf("Validate Model %s: Failed\n", expTable) + fmt.Println(diff) + } else { + log.Infof("Validate Model %s: Passed\n", expTable) + } + } + return nil +} + +func modelTypeFromTable(tableName string, expected json.RawMessage, actual []interface{}) (string, error) { + // TODO: something with reflection someday + switch tableName { + default: + return "", xerrors.Errorf("validation no implemented for model table %s", tableName) + + case "block_headers": + var expType blocks.BlockHeaders + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType blocks.BlockHeaders + for _, raw := range actual { + act, ok := raw.(*blocks.BlockHeader) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "block_parents": + var expType blocks.BlockParents + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType blocks.BlockParents + for _, raw := range actual { + act, ok := raw.(*blocks.BlockParent) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "drand_block_entries": + var expType blocks.DrandBlockEntries + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType blocks.DrandBlockEntries + for _, raw := range actual { + act, ok := raw.(*blocks.DrandBlockEntrie) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "derived_gas_outputs": + var expType derived.GasOutputsList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType derived.GasOutputsList + for _, raw := range actual { + act, ok := raw.(*derived.GasOutputs) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType, cmpopts.IgnoreUnexported(derived.GasOutputs{})), nil + case "receipts": + var expType messages.Receipts + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType messages.Receipts + for _, raw := range actual { + act, ok := raw.(*messages.Receipt) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "parsed_messages": + var expType messages.ParsedMessages + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType messages.ParsedMessages + for _, raw := range actual { + act, ok := raw.(*messages.ParsedMessage) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "block_messages": + var expType messages.BlockMessages + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType messages.BlockMessages + for _, raw := range actual { + act, ok := raw.(*messages.BlockMessage) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "message_gas_economy": + var expType []*messages.MessageGasEconomy + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType []*messages.MessageGasEconomy + for _, raw := range actual { + act, ok := raw.(*messages.MessageGasEconomy) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType, cmpopts.IgnoreUnexported(messages.MessageGasEconomy{})), nil + case "messages": + var expType messages.Messages + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType messages.Messages + for _, raw := range actual { + act, ok := raw.(*messages.Message) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "miner_current_deadline_infos": + var expType miner.MinerCurrentDeadlineInfoList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType miner.MinerCurrentDeadlineInfoList + for _, raw := range actual { + act, ok := raw.(*miner.MinerCurrentDeadlineInfo) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "miner_fee_debts": + var expType miner.MinerFeeDebtList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType miner.MinerFeeDebtList + for _, raw := range actual { + act, ok := raw.(*miner.MinerFeeDebt) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "miner_locked_funds": + var expType miner.MinerLockedFundsList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType miner.MinerLockedFundsList + for _, raw := range actual { + act, ok := raw.(*miner.MinerLockedFund) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "miner_pre_commit_infos": + var expType miner.MinerPreCommitInfoList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType miner.MinerPreCommitInfoList + for _, raw := range actual { + act, ok := raw.(*miner.MinerPreCommitInfo) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "miner_sector_events": + var expType miner.MinerSectorEventList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType miner.MinerSectorEventList + for _, raw := range actual { + act, ok := raw.(*miner.MinerSectorEvent) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "miner_sector_infos": + var expType miner.MinerSectorInfoList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType miner.MinerSectorInfoList + for _, raw := range actual { + act, ok := raw.(*miner.MinerSectorInfo) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "miner_infos": + var expType miner.MinerInfoList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType miner.MinerInfoList + for _, raw := range actual { + act, ok := raw.(*miner.MinerInfo) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "miner_sector_posts": + var expType miner.MinerSectorPostList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType miner.MinerSectorPostList + for _, raw := range actual { + act, ok := raw.(*miner.MinerSectorPost) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "miner_sector_deals": + var expType miner.MinerSectorDealList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType miner.MinerSectorDealList + for _, raw := range actual { + act, ok := raw.(*miner.MinerSectorDeal) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "market_deal_proposals": + var expType market.MarketDealProposals + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType market.MarketDealProposals + for _, raw := range actual { + act, ok := raw.(*market.MarketDealProposal) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "market_deal_states": + var expType market.MarketDealStates + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType market.MarketDealStates + for _, raw := range actual { + act, ok := raw.(*market.MarketDealState) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "multisig_transactions": + var expType multisig.MultisigTransactionList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType multisig.MultisigTransactionList + for _, raw := range actual { + act, ok := raw.(*multisig.MultisigTransaction) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "chain_powers": + var expType power.ChainPowerList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType power.ChainPowerList + for _, raw := range actual { + act, ok := raw.(*power.ChainPower) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "power_actor_claims": + var expType power.PowerActorClaimList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType power.PowerActorClaimList + for _, raw := range actual { + act, ok := raw.(*power.PowerActorClaim) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "chain_rewards": + var expType []*reward.ChainReward + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType []*reward.ChainReward + for _, raw := range actual { + act, ok := raw.(*reward.ChainReward) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "actors": + var expType common.ActorList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType common.ActorList + for _, raw := range actual { + act, ok := raw.(*common.Actor) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "actor_states": + var expType common.ActorStateList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType common.ActorStateList + for _, raw := range actual { + act, ok := raw.(*common.ActorState) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "id_addresses": + var expType init_.IdAddressList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType init_.IdAddressList + for _, raw := range actual { + act, ok := raw.(*init_.IdAddress) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + case "chain_economics": + var expType modelchain.ChainEconomicsList + if err := json.Unmarshal(expected, &expType); err != nil { + return "", err + } + + var actType modelchain.ChainEconomicsList + for _, raw := range actual { + act, ok := raw.(*modelchain.ChainEconomics) + if !ok { + panic("developer error") + } + actType = append(actType, act) + } + return cmp.Diff(actType, expType), nil + } +} diff --git a/vector/schema.go b/vector/schema.go new file mode 100644 index 000000000..167f2f951 --- /dev/null +++ b/vector/schema.go @@ -0,0 +1,62 @@ +package vector + +import ( + "encoding/base64" + "encoding/json" +) + +type Options map[string]interface{} + +// Base64EncodedBytes is a base64-encoded binary value. +type Base64EncodedBytes []byte + +func (b Base64EncodedBytes) String() string { + return base64.StdEncoding.EncodeToString(b) +} + +// MarshalJSON implements json.Marshal for Base64EncodedBytes +func (b Base64EncodedBytes) MarshalJSON() ([]byte, error) { + return json.Marshal(b.String()) +} + +// UnmarshalJSON implements json.Unmarshal for Base64EncodedBytes +func (b *Base64EncodedBytes) UnmarshalJSON(v []byte) error { + var s string + if err := json.Unmarshal(v, &s); err != nil { + return err + } + + if len(s) == 0 { + *b = nil + return nil + } + + bytes, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return err + } + *b = bytes + return nil +} + +type Metadata struct { + Version string `json:"version"` + Description string `json:"description"` + Network string `json:"network"` + Date int64 `json:"time"` +} + +type Parameters struct { + From int64 `json:"from"` + To int64 `json:"to"` + Tasks []string `json:"tasks"` + AddressFilter string `json:"address-filter"` +} + +type BuilderExpected struct { + Models map[string][]interface{} +} + +type RunnerExpected struct { + Models map[string]json.RawMessage +} diff --git a/vector/vector_test.go b/vector/vector_test.go new file mode 100644 index 000000000..9d75411ae --- /dev/null +++ b/vector/vector_test.go @@ -0,0 +1,41 @@ +package vector + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestExecuteVectors(t *testing.T) { + ctx := context.Background() + var vectorPaths []string + if err := filepath.Walk("./data", func(path string, info os.FileInfo, _ error) error { + if filepath.Ext(path) != ".json" { + return nil + } + full, err := filepath.Abs(path) + if err != nil { + t.Fatal(err) + } + vectorPaths = append(vectorPaths, full) + return nil + }); err != nil { + t.Fatal(err) + } + + for _, vp := range vectorPaths { + t.Run(filepath.Base(vp), func(t *testing.T) { + runner, err := NewRunner(ctx, vp, 0) + if err != nil { + t.Fatal(err) + } + if err := runner.Run(ctx); err != nil { + t.Fatal(err) + } + if err := runner.Validate(ctx); err != nil { + t.Fatal(err) + } + }) + } +}