From 5a700d3168734effce48d384194b884607792e4f Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 24 Apr 2023 16:02:25 +0200 Subject: [PATCH 01/15] feat: yugabyte db impl --- extern/boostd-data/go.mod | 3 + extern/boostd-data/go.sum | 9 + extern/boostd-data/model/model.go | 7 +- extern/boostd-data/svc/svc.go | 5 + extern/boostd-data/svc/svc_test.go | 47 ++- extern/boostd-data/yugabyte/create.cql | 35 ++ extern/boostd-data/yugabyte/encode.go | 35 ++ extern/boostd-data/yugabyte/err.go | 35 ++ extern/boostd-data/yugabyte/service.go | 487 +++++++++++++++++++++++++ extern/boostd-data/yugabyte/setup.go | 41 +++ go.mod | 3 + go.sum | 9 + 12 files changed, 706 insertions(+), 10 deletions(-) create mode 100644 extern/boostd-data/yugabyte/create.cql create mode 100644 extern/boostd-data/yugabyte/encode.go create mode 100644 extern/boostd-data/yugabyte/err.go create mode 100644 extern/boostd-data/yugabyte/service.go create mode 100644 extern/boostd-data/yugabyte/setup.go diff --git a/extern/boostd-data/go.mod b/extern/boostd-data/go.mod index 9d1c890d0..35d9994de 100644 --- a/extern/boostd-data/go.mod +++ b/extern/boostd-data/go.mod @@ -24,6 +24,7 @@ require ( github.com/stretchr/testify v1.8.1 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/urfave/cli/v2 v2.24.4 + github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0 go.opentelemetry.io/otel v1.13.0 go.opentelemetry.io/otel/exporters/jaeger v1.13.0 go.opentelemetry.io/otel/sdk v1.13.0 @@ -51,6 +52,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-block-format v0.0.3 // indirect @@ -121,6 +123,7 @@ require ( google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.4.0 // indirect lukechampine.com/blake3 v1.1.7 // indirect diff --git a/extern/boostd-data/go.sum b/extern/boostd-data/go.sum index cf22ef94e..33f551568 100644 --- a/extern/boostd-data/go.sum +++ b/extern/boostd-data/go.sum @@ -128,9 +128,13 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/briandowns/spinner v1.11.1/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= @@ -617,6 +621,8 @@ github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3vH7VqgtMxQ= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE= github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I8h3MITA53gN9OnWGCgaMa0JWVRdXthWw4M3CPM54OY= @@ -1998,6 +2004,8 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/ybbus/jsonrpc/v2 v2.1.6/go.mod h1:rIuG1+ORoiqocf9xs/v+ecaAVeo3zcZHQgInyKFMeg0= +github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0 h1:68nrJsrWe0A6JiKnsjWChAaWhj20v+AwYJObtp86D1k= +github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0/go.mod h1:LAokR6+vevDCrTxk52U7p6ki+4qELu4XU7JUGYa2O2M= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -2659,6 +2667,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/extern/boostd-data/model/model.go b/extern/boostd-data/model/model.go index a994ef7c2..fa919b3a4 100644 --- a/extern/boostd-data/model/model.go +++ b/extern/boostd-data/model/model.go @@ -12,8 +12,8 @@ import ( ) // DealInfo is information about a single deal for a given piece -// PieceOffset -// v +// . PieceOffset +// . v // Sector [..........................] // Piece ......[ ]...... // CAR ......[ ]............ @@ -39,7 +39,8 @@ type Metadata struct { IndexedAt time.Time `json:"i"` // CompleteIndex indicates whether the index has all information or is // missing block size information. Note that indexes imported from the - // dagstore do not have block size information. + // dagstore do not have block size information (they only have block + // offsets). CompleteIndex bool `json:"c"` Deals []DealInfo `json:"d"` Error string `json:"e"` diff --git a/extern/boostd-data/svc/svc.go b/extern/boostd-data/svc/svc.go index 56ba36871..d08ae3ae2 100644 --- a/extern/boostd-data/svc/svc.go +++ b/extern/boostd-data/svc/svc.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/boostd-data/couchbase" "github.com/filecoin-project/boostd-data/ldb" "github.com/filecoin-project/boostd-data/svc/types" + "github.com/filecoin-project/boostd-data/yugabyte" "github.com/filecoin-project/go-jsonrpc" "github.com/gorilla/mux" logging "github.com/ipfs/go-log/v2" @@ -25,6 +26,10 @@ type Service struct { impl types.ServiceImpl } +func NewYugabyte(settings yugabyte.DBSettings) *Service { + return &Service{impl: yugabyte.NewStore(settings)} +} + func NewCouchbase(settings couchbase.DBSettings) *Service { return &Service{impl: couchbase.NewStore(settings)} } diff --git a/extern/boostd-data/svc/svc_test.go b/extern/boostd-data/svc/svc_test.go index 498829caa..d91ccf04f 100644 --- a/extern/boostd-data/svc/svc_test.go +++ b/extern/boostd-data/svc/svc_test.go @@ -11,12 +11,11 @@ import ( "testing" "time" - "golang.org/x/sync/errgroup" - "github.com/filecoin-project/boost/testutil" "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/couchbase" "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/yugabyte" "github.com/filecoin-project/go-state-types/abi" "github.com/google/uuid" "github.com/ipfs/go-cid" @@ -26,6 +25,7 @@ import ( "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) var testCouchSettings = couchbase.DBSettings{ @@ -46,6 +46,10 @@ var testCouchSettings = couchbase.DBSettings{ TestMode: true, } +var testYugaSettings = yugabyte.DBSettings{ + ConnectString: "127.0.0.1", +} + func TestService(t *testing.T) { _ = logging.SetLogLevel("*", "debug") @@ -54,8 +58,14 @@ func TestService(t *testing.T) { defer cancel() bdsvc, err := NewLevelDB("") require.NoError(t, err) + + addr := "localhost:8042" + err = bdsvc.Start(ctx, addr) + require.NoError(t, err) + testService(ctx, t, bdsvc, "localhost:8042") }) + t.Run("couchbase", func(t *testing.T) { // TODO: Unskip this test once the couchbase instance can be created // from a docker container in CI as part of the test @@ -66,16 +76,39 @@ func TestService(t *testing.T) { defer cancel() SetupCouchbase(t, testCouchSettings) bdsvc := NewCouchbase(testCouchSettings) - testService(ctx, t, bdsvc, "localhost:8043") + + addr := "localhost:8043" + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + + testService(ctx, t, bdsvc, addr) + }) + + t.Run("yugabyte", func(t *testing.T) { + // Running couchbase tests may require download the docker container + // so set a high timeout + //ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + bdsvc := NewYugabyte(testYugaSettings) + + addr := "localhost:8044" + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + + ybstore := bdsvc.impl.(*yugabyte.Store) + err = ybstore.Drop(ctx) + require.NoError(t, err) + err = ybstore.Create(ctx) + require.NoError(t, err) + + testService(ctx, t, bdsvc, addr) }) } func testService(ctx context.Context, t *testing.T, bdsvc *Service, addr string) { - err := bdsvc.Start(ctx, addr) - require.NoError(t, err) - cl := client.NewStore() - err = cl.Dial(context.Background(), fmt.Sprintf("http://%s", addr)) + err := cl.Dial(context.Background(), fmt.Sprintf("http://%s", addr)) require.NoError(t, err) defer cl.Close(ctx) diff --git a/extern/boostd-data/yugabyte/create.cql b/extern/boostd-data/yugabyte/create.cql new file mode 100644 index 000000000..038a0be78 --- /dev/null +++ b/extern/boostd-data/yugabyte/create.cql @@ -0,0 +1,35 @@ +create keyspace if not exists idx with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + +CREATE TABLE idx.PayloadToPieces ( + PayloadMultihash BLOB PRIMARY KEY, + PieceCids BLOB +); + +CREATE TABLE idx.PieceBlockOffsetSize ( + PieceCid BLOB, + PayloadMultihash BLOB, + BlockOffset BIGINT, + BlockSize BIGINT, + PRIMARY KEY (PieceCid, PayloadMultihash) +); + +CREATE TABLE idx.PieceMetadata ( + PieceCid BLOB PRIMARY KEY, + Version TEXT, + IndexedAt TIMESTAMP, + CompleteIndex BOOLEAN, + Error TEXT, + ErrorType Text +); + +CREATE TABLE idx.PieceDeal ( + DealUuid TEXT PRIMARY KEY, + PieceCid BLOB, + IsLegacy BOOLEAN, + ChainDealID BIGINT, + MinerAddr TEXT, + SectorID BIGINT, + PieceOffset BIGINT, + PieceLength BIGINT, + CarLength BIGINT +); diff --git a/extern/boostd-data/yugabyte/encode.go b/extern/boostd-data/yugabyte/encode.go new file mode 100644 index 000000000..71da9de7d --- /dev/null +++ b/extern/boostd-data/yugabyte/encode.go @@ -0,0 +1,35 @@ +package yugabyte + +import ( + "fmt" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" +) + +func bytesToCids(bz []byte) ([]cid.Cid, error) { + var bytesIdx int + var pieceCids []cid.Cid + for bytesIdx < len(bz) { + readCount, pcid, err := cid.CidFromBytes(bz[bytesIdx:]) + if err != nil { + return nil, fmt.Errorf("parsing bytes to cid: %w", err) + } + + bytesIdx += readCount + pieceCids = append(pieceCids, pcid) + } + return pieceCids, nil +} + +// Probability of a collision in two 24 byte hashes (birthday problem): +// 2^(24*8/2) = 8 x 10^28 +const multihashLimitBytes = 24 + +// trimMultihash trims the multihash to the last multihashLimitBytes bytes +func trimMultihash(mh multihash.Multihash) []byte { + var idx int + if len(mh) > multihashLimitBytes { + idx = len(mh) - multihashLimitBytes + } + return mh[idx:] +} diff --git a/extern/boostd-data/yugabyte/err.go b/extern/boostd-data/yugabyte/err.go new file mode 100644 index 000000000..acf4a7391 --- /dev/null +++ b/extern/boostd-data/yugabyte/err.go @@ -0,0 +1,35 @@ +package yugabyte + +import ( + "fmt" + "github.com/filecoin-project/boostd-data/svc/types" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + "strings" +) + +func normalizePieceCidError(pieceCid cid.Cid, err error) error { + if err == nil { + return nil + } + if isNotFoundErr(err) { + return fmt.Errorf("piece %s: %s", pieceCid, types.ErrNotFound) + } + return err +} + +func normalizeMultihashError(m mh.Multihash, err error) error { + if err == nil { + return nil + } + if isNotFoundErr(err) { + return fmt.Errorf("multihash %s: %s", m, types.ErrNotFound) + } + return err +} + +func isNotFoundErr(err error) bool { + // TODO: is there some other way to know if the error is a not found error? + // Check yugabyte cassandra driver docs + return strings.Contains(err.Error(), "not found") +} diff --git a/extern/boostd-data/yugabyte/service.go b/extern/boostd-data/yugabyte/service.go new file mode 100644 index 000000000..46dcdbdc3 --- /dev/null +++ b/extern/boostd-data/yugabyte/service.go @@ -0,0 +1,487 @@ +package yugabyte + +import ( + "context" + _ "embed" + "fmt" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/tracing" + "github.com/filecoin-project/boostd-data/svc/types" + "github.com/filecoin-project/go-address" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "github.com/multiformats/go-multihash" + mh "github.com/multiformats/go-multihash" + "github.com/yugabyte/gocql" + "go.opentelemetry.io/otel/attribute" + "golang.org/x/sync/errgroup" + "time" +) + +var log = logging.Logger("boostd-data-yb") + +// The current piece metadata version. This version will be used when doing +// data migrations (migrations are not yet implemented in version 1). +const pieceMetadataVersion = "1" + +type DBSettings struct { + ConnectString string + PayloadPiecesParallelism int +} + +type Store struct { + settings DBSettings + cluster *gocql.ClusterConfig + session *gocql.Session +} + +var _ types.ServiceImpl = (*Store)(nil) + +func NewStore(settings DBSettings) *Store { + if settings.PayloadPiecesParallelism == 0 { + settings.PayloadPiecesParallelism = 16 + } + + cluster := gocql.NewCluster(settings.ConnectString) + //cluster.Timeout = 30 * time.Second + //cluster.ConnectTimeout = 30 * time.Second + return &Store{ + settings: settings, + cluster: cluster, + } +} + +func (s *Store) Start(ctx context.Context) error { + session, err := s.cluster.CreateSession() + if err != nil { + return fmt.Errorf("creating yugabyte cluster: %w", err) + } + s.session = session + return nil +} + +func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { + ctx, span := tracing.Tracer.Start(ctx, "store.add_deal_for_piece") + defer span.End() + + err := s.createPieceMetadata(ctx, pieceCid) + if err != nil { + return err + } + + qry := `INSERT INTO idx.PieceDeal ` + + `(DealUuid, PieceCid, IsLegacy, ChainDealID, MinerAddr, SectorID, PieceOffset, PieceLength, CarLength) ` + + `VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ` + + `IF NOT EXISTS` + err = s.session.Query(qry, + dealInfo.DealUuid, pieceCid.Bytes(), dealInfo.IsLegacy, dealInfo.ChainDealID, dealInfo.MinerAddr.String(), + dealInfo.SectorID, dealInfo.PieceOffset, dealInfo.PieceLength, dealInfo.CarLength). + WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("inserting deal %s for piece %s", dealInfo.DealUuid, pieceCid) + } + + return nil +} + +func (s *Store) createPieceMetadata(ctx context.Context, pieceCid cid.Cid) error { + qry := `INSERT INTO idx.PieceMetadata (PieceCid, Version) VALUES (?, ?) IF NOT EXISTS` + err := s.session.Query(qry, pieceCid.String(), pieceMetadataVersion).WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("inserting piece metadata for piece %s: %w", pieceCid, err) + } + return nil +} + +// TODO: I don't think we're using this functionality anymore, we can probably +// just remove it from the interface and data model +func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { + ctx, span := tracing.Tracer.Start(ctx, "store.set-car-size") + defer span.End() + + //err := s.db.SetCarSize(ctx, pieceCid, size) + //return normalizePieceCidError(pieceCid, err) + return nil +} + +// TODO: Do we need this? +func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr string) error { + ctx, span := tracing.Tracer.Start(ctx, "store.mark-piece-index-errored") + defer span.End() + + //err := s.db.MarkIndexErrored(ctx, pieceCid, errors.New(idxErr)) + //if err != nil { + // return normalizePieceCidError(pieceCid, err) + //} + // + //return s.FlagPiece(ctx, pieceCid) + return nil +} + +func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.get_offset_size") + defer span.End() + + var offset, size uint64 + qry := `SELECT BlockOffset, BlockSize FROM idx.PieceBlockOffsetSize WHERE PieceCid = ? AND PayloadMultihash = ?` + err := s.session.Query(qry, pieceCid.Bytes(), hash).WithContext(ctx).Scan(&offset, &size) + if err != nil { + err = normalizePieceCidError(pieceCid, err) + return nil, fmt.Errorf("getting offset / size: %w", err) + } + + return &model.OffsetSize{Offset: offset, Size: size}, nil +} + +// Get piece metadata with deals +func (s *Store) GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) { + md, err := s.getPieceMetadata(ctx, pieceCid) + if err != nil { + return md, nil + } + + deals, err := s.GetPieceDeals(ctx, pieceCid) + if err != nil { + return md, fmt.Errorf("getting deals for piece %s: %w", pieceCid, err) + } + md.Deals = deals + + return md, nil +} + +// Get the piece metadata without deal information +func (s *Store) getPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.get_piece_metadata") + defer span.End() + + // Get piece metadata + var md model.Metadata + qry := `SELECT Version, IndexedAt, CompleteIndex, Error, ErrorType ` + + `FROM idx.PieceMetadata WHERE PieceCid = ?` + err := s.session.Query(qry, pieceCid.String()).WithContext(ctx). + Scan(&md.Version, &md.IndexedAt, &md.CompleteIndex, &md.Error, &md.ErrorType) + if err != nil { + err = normalizePieceCidError(pieceCid, err) + return md, fmt.Errorf("getting piece metadata: %w", err) + } + + return md, nil +} + +func (s *Store) GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.DealInfo, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.get_piece_deals") + defer span.End() + + // Get deals for piece + qry := `SELECT DealUuid, IsLegacy, ChainDealID, MinerAddr, ` + + `SectorID, PieceOffset, PieceLength, CarLength ` + + `FROM idx.PieceDeal WHERE PieceCid = ?` + iter := s.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Iter() + + var deals []model.DealInfo + var deal model.DealInfo + var minerAddr string + for iter.Scan(&deal.DealUuid, &deal.IsLegacy, &deal.ChainDealID, &minerAddr, + &deal.SectorID, &deal.PieceOffset, &deal.PieceLength, &deal.CarLength) { + + ma, err := address.NewFromString(minerAddr) + if err != nil { + return nil, fmt.Errorf("parsing miner address for piece %s / deal %s: %w", pieceCid, deal.DealUuid, err) + } + + deal.MinerAddr = ma + deals = append(deals, deal) + } + if err := iter.Close(); err != nil { + return nil, fmt.Errorf("getting piece deals: %w", err) + } + + return deals, nil +} + +// Get all pieces that contain a multihash (used when retrieving by payload CID) +func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.pieces_containing_multihash") + defer span.End() + + var bz []byte + qry := `SELECT PieceCids FROM idx.PayloadToPieces WHERE PayloadMultihash = ?` + err := s.session.Query(qry, trimMultihash(m)).WithContext(ctx).Scan(&bz) + if err != nil { + err = normalizeMultihashError(m, err) + return nil, fmt.Errorf("getting pieces containing multihash: %w", err) + } + + return bytesToCids(bz) +} + +func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) ([]model.Record, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.get_index") + defer span.End() + + qry := `SELECT PayloadMultihash, BlockOffset, BlockSize FROM idx.PieceBlockOffsetSize WHERE PieceCid = ?` + iter := s.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Iter() + + var records []model.Record + var payloadMHBz []byte + var offset, size uint64 + for iter.Scan(&payloadMHBz, &offset, &size) { + _, pmh, err := multihash.MHFromBytes(payloadMHBz) + if err != nil { + return nil, fmt.Errorf("scanning mulithash: %w", err) + } + + records = append(records, model.Record{ + Cid: cid.NewCidV1(cid.Raw, pmh), + OffsetSize: model.OffsetSize{ + Offset: offset, + Size: size, + }, + }) + } + if err := iter.Close(); err != nil { + return nil, fmt.Errorf("getting piece index for piece %s: %w", pieceCid, err) + } + + return records, nil +} + +func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, recs []model.Record, isCompleteIndex bool) error { + ctx, span := tracing.Tracer.Start(ctx, "store.add_index") + defer span.End() + + if len(recs) == 0 { + return nil + } + + // Add a mapping from multihash -> piece cid so that clients can look up + // which pieces contain a multihash + err := s.addMultihashesToPieces(ctx, pieceCid, recs) + if err != nil { + return err + } + + // Add a mapping from piece cid -> offset / size of each block so that + // clients can get the block info for all blocks in a piece + err = s.addPieceInfos(ctx, pieceCid, recs) + if err != nil { + return err + } + + // Ensure the piece metadata exists + err = s.createPieceMetadata(ctx, pieceCid) + if err != nil { + return err + } + + // Mark indexing as complete for the piece + qry := `UPDATE idx.PieceMetadata ` + + `SET IndexedAt = ?, CompleteIndex = ?, Error = '', ErrorType = '' ` + + `WHERE PieceCid = ?` + err = s.session.Query(qry, time.Now(), isCompleteIndex, pieceCid.String()).WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("marking indexing as complete for piece %s", pieceCid) + } + + return nil +} + +func (s *Store) addMultihashesToPieces(ctx context.Context, pieceCid cid.Cid, recs []model.Record) error { + ctx, span := tracing.Tracer.Start(ctx, "store.add_index.payloadpiece") + defer span.End() + + queue := make(chan []byte, len(recs)) + for _, rec := range recs { + queue <- rec.Cid.Hash() + } + close(queue) + + var eg errgroup.Group + for i := 0; i < s.settings.PayloadPiecesParallelism; i++ { + eg.Go(func() error { + for ctx.Err() == nil { + select { + case <-ctx.Done(): + return ctx.Err() + case multihashBytes, ok := <-queue: + if !ok { + // Finished adding all the queued items, exit the thread + return nil + } + + q := `INSERT INTO idx.PayloadToPieces (PayloadMultihash, PieceCids) VALUES (?, ?)` + err := s.session.Query(q, trimMultihash(multihashBytes), pieceCid.Bytes()).Exec() + if err != nil { + return fmt.Errorf("inserting into PayloadToPieces: %w", err) + } + } + } + + return ctx.Err() + }) + } + return eg.Wait() +} + +func (s *Store) addPieceInfos(ctx context.Context, pieceCid cid.Cid, recs []model.Record) error { + ctx, span := tracing.Tracer.Start(ctx, "store.add_index.pieceinfo") + defer span.End() + + batchEntries := make([]gocql.BatchEntry, 0, len(recs)) + insertPieceOffsetsQry := `INSERT INTO idx.PieceBlockOffsetSize (PieceCid, PayloadMultihash, BlockOffset, BlockSize) VALUES (?, ?, ?, ?)` + for _, rec := range recs { + batchEntries = append(batchEntries, gocql.BatchEntry{ + Stmt: insertPieceOffsetsQry, + Args: []interface{}{pieceCid.Bytes(), rec.Cid.Hash(), rec.Offset, rec.Size}, + Idempotent: true, + }) + } + + // The Cassandra driver has a 50k limit on batch statements. Keeping + // batch size small makes sure we're under the limit. + const batchSize = 49000 + var batch *gocql.Batch + for allIdx, entry := range batchEntries { + if batch == nil { + batch = s.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + } + + batch.Entries = append(batch.Entries, entry) + + if allIdx == len(batchEntries)-1 || len(batch.Entries) == batchSize { + err := s.session.ExecuteBatch(batch) + if err != nil { + return fmt.Errorf("executing offset / size batch insert for piece %s: %w", pieceCid, err) + } + batch = nil + continue + } + } + + return nil +} + +func (s *Store) IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.is_incomplete_index") + defer span.End() + + md, err := s.getPieceMetadata(ctx, pieceCid) + if err != nil { + return false, err + } + + return md.CompleteIndex, nil +} + +func (s *Store) IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) { + t, err := s.IndexedAt(ctx, pieceCid) + if err != nil { + return false, err + } + return !t.IsZero(), nil +} + +func (s *Store) IndexedAt(ctx context.Context, pieceCid cid.Cid) (time.Time, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.indexed_at") + defer span.End() + + md, err := s.getPieceMetadata(ctx, pieceCid) + if err != nil { + return time.Time{}, err + } + + return md.IndexedAt, nil +} + +func (s *Store) ListPieces(ctx context.Context) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.list_pieces") + defer span.End() + + iter := s.session.Query("SELECT PieceCid FROM idx.PieceMetadata").WithContext(ctx).Iter() + var pcids []cid.Cid + var cstr string + for iter.Scan(&cstr) { + c, err := cid.Parse(cstr) + if err != nil { + return nil, fmt.Errorf("parsing piece cid: %w", err) + } + + pcids = append(pcids, c) + } + if err := iter.Close(); err != nil { + return nil, fmt.Errorf("getting piece cids: %w", err) + } + + return pcids, nil +} + +func (s *Store) NextPiecesToCheck(ctx context.Context) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.next_pieces_to_check") + defer span.End() + + return nil, nil +} + +func (s *Store) FlagPiece(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "store.flag_piece") + span.SetAttributes(attribute.String("pieceCid", pieceCid.String())) + defer span.End() + + return nil +} + +func (s *Store) UnflagPiece(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "store.unflag_piece") + span.SetAttributes(attribute.String("pieceCid", pieceCid.String())) + defer span.End() + + return nil +} + +func (s *Store) FlaggedPiecesList(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.flagged_pieces") + var spanCursor int + if cursor != nil { + spanCursor = int(cursor.UnixMilli()) + } + span.SetAttributes(attribute.Int("cursor", spanCursor)) + span.SetAttributes(attribute.Int("offset", offset)) + span.SetAttributes(attribute.Int("limit", limit)) + defer span.End() + + return nil, nil +} + +func (s *Store) FlaggedPiecesCount(ctx context.Context) (int, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.flagged_pieces_count") + defer span.End() + + return 0, nil +} + +// RemoveDealForPiece removes Single deal for pieceCID. If []Deals is empty then Metadata is removed as well +func (s *Store) RemoveDealForPiece(ctx context.Context, pieceCid cid.Cid, dealId string) error { + ctx, span := tracing.Tracer.Start(ctx, "store.remove_deal_for_piece") + defer span.End() + + return nil +} + +// RemovePieceMetadata removes all Metadata for pieceCID. To be used manually in case of failure +// in RemoveDealForPiece +func (s *Store) RemovePieceMetadata(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "store.remove_piece_metadata") + defer span.End() + + return nil +} + +// RemoveIndexes removes all MultiHashes for pieceCID. To be used manually in case of failure +// in RemoveDealForPiece or RemovePieceMetadata. Metadata for the piece must be +// present in the database +func (s *Store) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "store.remove_indexes") + defer span.End() + + return nil +} diff --git a/extern/boostd-data/yugabyte/setup.go b/extern/boostd-data/yugabyte/setup.go new file mode 100644 index 000000000..9cb217ed3 --- /dev/null +++ b/extern/boostd-data/yugabyte/setup.go @@ -0,0 +1,41 @@ +package yugabyte + +import ( + "context" + _ "embed" + "fmt" + "strings" +) + +//go:embed create.cql +var createCQL string + +func (s *Store) Create(ctx context.Context) error { + createTablesLines := strings.Split(createCQL, ";") + for _, line := range createTablesLines { + line = strings.Trim(line, "\n \t") + if line == "" { + continue + } + log.Debug(line) + err := s.session.Query(line).WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("creating tables: executing\n%s\n%w", line, err) + } + } + + return nil +} + +func (s *Store) Drop(ctx context.Context) error { + tables := []string{`PayloadToPieces`, `PieceBlockOffsetSize`, `PieceMetadata`, `PieceDeal`} + for _, tbl := range tables { + qry := `drop table if exists idx.` + tbl + log.Debug(qry) + err := s.session.Query(qry).WithContext(ctx).Exec() + if err != nil { + return err + } + } + return nil +} diff --git a/go.mod b/go.mod index 574fa987d..32276bc5a 100644 --- a/go.mod +++ b/go.mod @@ -369,6 +369,7 @@ require ( require ( github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/onsi/ginkgo/v2 v2.5.1 // indirect @@ -379,6 +380,8 @@ require ( github.com/quic-go/webtransport-go v0.5.2 // indirect github.com/tidwall/gjson v1.14.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect + github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0 // indirect github.com/zyedidia/generic v1.2.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index c7d3cfb1c..e45216233 100644 --- a/go.sum +++ b/go.sum @@ -134,8 +134,12 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= @@ -664,6 +668,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 h1:BpJ2o0OR5FV7vrkDYfXYVJQeMNWa8RhklZOpW2ITAIQ= github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE= github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c h1:iiD+p+U0M6n/FsO6XIZuOgobnNa48FxtyYFfWwLttUQ= @@ -1872,6 +1878,8 @@ github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 h1:oWgZJmC1DorFZD github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0 h1:68nrJsrWe0A6JiKnsjWChAaWhj20v+AwYJObtp86D1k= +github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0/go.mod h1:LAokR6+vevDCrTxk52U7p6ki+4qELu4XU7JUGYa2O2M= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -2477,6 +2485,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= From b3317409bc552d31563c924c75696c4b27360d8e Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 25 Apr 2023 15:02:06 +0200 Subject: [PATCH 02/15] feat: run yugabyte tests against a dockerized yugabyte --- .../svc/setup_yugabyte_test_util.go | 87 +++++++++++++++++++ extern/boostd-data/svc/svc_test.go | 61 ++++++++++--- 2 files changed, 138 insertions(+), 10 deletions(-) create mode 100644 extern/boostd-data/svc/setup_yugabyte_test_util.go diff --git a/extern/boostd-data/svc/setup_yugabyte_test_util.go b/extern/boostd-data/svc/setup_yugabyte_test_util.go new file mode 100644 index 000000000..7ee2e2089 --- /dev/null +++ b/extern/boostd-data/svc/setup_yugabyte_test_util.go @@ -0,0 +1,87 @@ +package svc + +import ( + "github.com/davecgh/go-spew/spew" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + dockercl "github.com/docker/docker/client" + "github.com/docker/go-connections/nat" + "github.com/stretchr/testify/require" + "github.com/yugabyte/gocql" + "golang.org/x/net/context" + "io" + "os" + "testing" + "time" +) + +func SetupYugabyte(t *testing.T) { + ctx := context.Background() + cli, err := dockercl.NewClientWithOpts(dockercl.FromEnv) + require.NoError(t, err) + + imageName := "yugabytedb/yugabyte:2.17.2.0-b216" + out, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{}) + require.NoError(t, err) + + _, err = io.Copy(os.Stdout, out) + require.NoError(t, err) + + tlog.Info("yugabyte docker container create...") + resp, err := cli.ContainerCreate(ctx, &container.Config{ + Image: imageName, + ExposedPorts: nat.PortSet{ + "7000": struct{}{}, + "9000": struct{}{}, + "5433": struct{}{}, + "9042": struct{}{}, + }, + }, &container.HostConfig{ + PortBindings: map[nat.Port][]nat.PortBinding{ + "7000": {{HostIP: "127.0.0.1", HostPort: "7001"}}, + "9000": {{HostIP: "127.0.0.1", HostPort: "9000"}}, + "5433": {{HostIP: "127.0.0.1", HostPort: "5433"}}, + "9042": {{HostIP: "127.0.0.1", HostPort: "9042"}}, + }, + }, nil, nil, "") + require.NoError(t, err) + tlog.Info("yugabyte docker container created") + + tlog.Info("yugabyte docker container start...") + err = cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}) + require.NoError(t, err) + tlog.Info("yugabyte docker container started") + + inspect, err := cli.ContainerInspect(ctx, resp.ID) + require.NoError(t, err) + spew.Dump(inspect) + + t.Cleanup(func() { + tlog.Info("yugabyte docker container remove...") + err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true}) + require.NoError(t, err) + tlog.Info("yugabyte docker container removed") + }) + + tlog.Info("wait for yugabyte start...") + awaitYugabyteUp(t, time.Minute) + tlog.Info("yugabyte started") +} + +func awaitYugabyteUp(t *testing.T, duration time.Duration) { + start := time.Now() + cluster := gocql.NewCluster("127.0.0.1") + for { + session, err := cluster.CreateSession() + if err == nil { + return + } + _ = session + + tlog.Debugf("waiting for yugabyte: %s", err) + if time.Since(start) > duration { + t.Fatalf("failed to start yugabyte within %s", duration) + } + time.Sleep(time.Second) + } +} diff --git a/extern/boostd-data/svc/svc_test.go b/extern/boostd-data/svc/svc_test.go index d91ccf04f..1279eee53 100644 --- a/extern/boostd-data/svc/svc_test.go +++ b/extern/boostd-data/svc/svc_test.go @@ -85,11 +85,13 @@ func TestService(t *testing.T) { }) t.Run("yugabyte", func(t *testing.T) { - // Running couchbase tests may require download the docker container + // Running yugabyte tests may require download the docker container // so set a high timeout - //ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() + + SetupYugabyte(t) + bdsvc := NewYugabyte(testYugaSettings) addr := "localhost:8044" @@ -151,6 +153,24 @@ func testService(ctx context.Context, t *testing.T, bdsvc *Service, addr string) require.Len(t, dis, 1) require.Equal(t, di, dis[0]) + // Add a second deal + di2 := model.DealInfo{ + DealUuid: uuid.NewString(), + SectorID: abi.SectorNumber(11), + PieceOffset: 11, + PieceLength: 12, + CarLength: 13, + } + err = cl.AddDealForPiece(ctx, pieceCid, di2) + require.NoError(t, err) + + // There should now be two deals + dis, err = cl.GetPieceDeals(ctx, pieceCid) + require.NoError(t, err) + require.Len(t, dis, 2) + require.Contains(t, dis, di) + require.Contains(t, dis, di2) + b, err := hex.DecodeString("1220ff63d7689e2d9567d1a90a7a68425f430137142e1fbc28fe4780b9ee8a5ef842") require.NoError(t, err) @@ -197,24 +217,45 @@ func TestServiceFuzz(t *testing.T) { t.Run("level db", func(t *testing.T) { bdsvc, err := NewLevelDB("") require.NoError(t, err) - testServiceFuzz(ctx, t, bdsvc, "localhost:8042") + addr := "localhost:8042" + err = bdsvc.Start(ctx, addr) + require.NoError(t, err) + testServiceFuzz(ctx, t, addr) }) + t.Run("couchbase", func(t *testing.T) { // TODO: Unskip this test once the couchbase instance can be created // from a docker container in CI as part of the test t.Skip() SetupCouchbase(t, testCouchSettings) bdsvc := NewCouchbase(testCouchSettings) - testServiceFuzz(ctx, t, bdsvc, "localhost:8043") + addr := "localhost:8043" + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + testServiceFuzz(ctx, t, addr) }) -} -func testServiceFuzz(ctx context.Context, t *testing.T, bdsvc *Service, addr string) { - err := bdsvc.Start(ctx, addr) - require.NoError(t, err) + t.Run("yugabyte", func(t *testing.T) { + SetupYugabyte(t) + bdsvc := NewYugabyte(testYugaSettings) + + addr := "localhost:8044" + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + + ybstore := bdsvc.impl.(*yugabyte.Store) + err = ybstore.Drop(ctx) + require.NoError(t, err) + err = ybstore.Create(ctx) + require.NoError(t, err) + + testServiceFuzz(ctx, t, addr) + }) +} +func testServiceFuzz(ctx context.Context, t *testing.T, addr string) { cl := client.NewStore() - err = cl.Dial(context.Background(), "http://localhost:8042") + err := cl.Dial(context.Background(), "http://"+addr) require.NoError(t, err) defer cl.Close(ctx) From 8adfc0d6be68ec87482324ee30424a9bf544b200 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 25 Apr 2023 15:38:56 +0200 Subject: [PATCH 03/15] fix: use out own yugabyte docker image --- extern/boostd-data/svc/setup_yugabyte_test_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/boostd-data/svc/setup_yugabyte_test_util.go b/extern/boostd-data/svc/setup_yugabyte_test_util.go index 7ee2e2089..3e022adbb 100644 --- a/extern/boostd-data/svc/setup_yugabyte_test_util.go +++ b/extern/boostd-data/svc/setup_yugabyte_test_util.go @@ -20,7 +20,7 @@ func SetupYugabyte(t *testing.T) { cli, err := dockercl.NewClientWithOpts(dockercl.FromEnv) require.NoError(t, err) - imageName := "yugabytedb/yugabyte:2.17.2.0-b216" + imageName := "public.ecr.aws/n6b0k8i7/yugabyte-test:aarch64-2.13.0.0" out, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{}) require.NoError(t, err) From 024cdd25fd7fb4180d4b61decb24cbe8abb84048 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 25 Apr 2023 15:43:16 +0200 Subject: [PATCH 04/15] fix: use yugabyte 2.17.2.0 docker image --- extern/boostd-data/svc/setup_yugabyte_test_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/boostd-data/svc/setup_yugabyte_test_util.go b/extern/boostd-data/svc/setup_yugabyte_test_util.go index 3e022adbb..dd64b353c 100644 --- a/extern/boostd-data/svc/setup_yugabyte_test_util.go +++ b/extern/boostd-data/svc/setup_yugabyte_test_util.go @@ -20,7 +20,7 @@ func SetupYugabyte(t *testing.T) { cli, err := dockercl.NewClientWithOpts(dockercl.FromEnv) require.NoError(t, err) - imageName := "public.ecr.aws/n6b0k8i7/yugabyte-test:aarch64-2.13.0.0" + imageName := "public.ecr.aws/n6b0k8i7/yugabyte-test:aarch64-2.17.2.0" out, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{}) require.NoError(t, err) From 0cd53732ed26a96b5f5a7e150f9b40649b70b7f2 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 27 Apr 2023 17:49:10 +0200 Subject: [PATCH 05/15] feat: piece doctor yugabyte impl --- extern/boostd-data/go.mod | 9 + extern/boostd-data/go.sum | 82 ++++++ extern/boostd-data/svc/svc.go | 12 +- extern/boostd-data/svc/svc_test.go | 7 +- extern/boostd-data/yugabyte/create.cql | 7 +- extern/boostd-data/yugabyte/create.sql | 17 ++ extern/boostd-data/yugabyte/drop.cql | 10 + extern/boostd-data/yugabyte/drop.sql | 6 + extern/boostd-data/yugabyte/piecedoctor.go | 287 +++++++++++++++++++++ extern/boostd-data/yugabyte/service.go | 66 ++--- extern/boostd-data/yugabyte/setup.go | 51 ++-- go.mod | 9 + go.sum | 85 ++++++ piecedirectory/doctor_test.go | 80 ++++-- 14 files changed, 634 insertions(+), 94 deletions(-) create mode 100644 extern/boostd-data/yugabyte/create.sql create mode 100644 extern/boostd-data/yugabyte/drop.cql create mode 100644 extern/boostd-data/yugabyte/drop.sql create mode 100644 extern/boostd-data/yugabyte/piecedoctor.go diff --git a/extern/boostd-data/go.mod b/extern/boostd-data/go.mod index 35d9994de..34d6da92c 100644 --- a/extern/boostd-data/go.mod +++ b/extern/boostd-data/go.mod @@ -25,6 +25,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/urfave/cli/v2 v2.24.4 github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0 + github.com/yugabyte/pgx/v4 v4.14.5 go.opentelemetry.io/otel v1.13.0 go.opentelemetry.io/otel/exporters/jaeger v1.13.0 go.opentelemetry.io/otel/sdk v1.13.0 @@ -77,6 +78,13 @@ require ( github.com/ipfs/go-verifcid v0.0.1 // indirect github.com/ipld/go-codec-dagpb v1.3.2 // indirect github.com/ipld/go-ipld-prime v0.18.0 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.11.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.2.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.10.0 // indirect github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/cpuid/v2 v2.1.1 // indirect @@ -119,6 +127,7 @@ require ( golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect diff --git a/extern/boostd-data/go.sum b/extern/boostd-data/go.sum index 33f551568..2f56ec0dd 100644 --- a/extern/boostd-data/go.sum +++ b/extern/boostd-data/go.sum @@ -67,6 +67,7 @@ github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETF github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa/go.mod h1:WUmMvh9wMtqj1Xhf1hf3kp9RvL+y6odtdYxpyZjb90U= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= @@ -199,6 +200,8 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= @@ -218,6 +221,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= @@ -485,6 +489,8 @@ github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblf github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= @@ -945,6 +951,52 @@ github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1ar github.com/ipld/go-storethehash v0.1.7/go.mod h1:O2CgbSwJfXCrYsjA1g1M7zJmVzzg71BM00ds6pyMLAQ= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c h1:Dznn52SgVIVst9UyOT9brctYUgxs+CvVfPaC3jKrA50= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -1016,6 +1068,7 @@ github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdy github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/koalacxr/quantile v0.0.1/go.mod h1:bGN/mCZLZ4lrSDHRQ6Lglj9chowGux8sGUIND+DQeD0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -1029,11 +1082,16 @@ github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= @@ -1475,6 +1533,7 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -1811,6 +1870,8 @@ github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XF github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1819,12 +1880,16 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sercand/kuberesolver v2.1.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -1850,6 +1915,7 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745/go.mod h1:G81aIFAMS9ECrwBYR9YxhlPjWgrItd+Kje78O6+uqm8= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -1894,6 +1960,7 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= @@ -2006,12 +2073,15 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr github.com/ybbus/jsonrpc/v2 v2.1.6/go.mod h1:rIuG1+ORoiqocf9xs/v+ecaAVeo3zcZHQgInyKFMeg0= github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0 h1:68nrJsrWe0A6JiKnsjWChAaWhj20v+AwYJObtp86D1k= github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0/go.mod h1:LAokR6+vevDCrTxk52U7p6ki+4qELu4XU7JUGYa2O2M= +github.com/yugabyte/pgx/v4 v4.14.5 h1:XLzEEiO3d/kWzpyctO8l4kwMLhzPQ9n2er7ATH7CJVA= +github.com/yugabyte/pgx/v4 v4.14.5/go.mod h1:nFSvjsVq4CuA61TWGriwWG74ZVxeuJCcNu42Mkn+rgw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= github.com/zondax/ledger-go v0.12.1/go.mod h1:KatxXrVDzgWwbssUWsF5+cOJHXPvzQ09YSlzGNuhOEo= @@ -2101,6 +2171,7 @@ go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95a go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= @@ -2127,6 +2198,7 @@ golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -2135,6 +2207,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -2151,6 +2224,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -2333,6 +2407,7 @@ golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2349,6 +2424,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2444,6 +2520,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= @@ -2470,6 +2547,7 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -2477,6 +2555,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -2528,6 +2607,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2667,6 +2748,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= diff --git a/extern/boostd-data/svc/svc.go b/extern/boostd-data/svc/svc.go index d08ae3ae2..bf20b009e 100644 --- a/extern/boostd-data/svc/svc.go +++ b/extern/boostd-data/svc/svc.go @@ -23,15 +23,15 @@ var ( ) type Service struct { - impl types.ServiceImpl + Impl types.ServiceImpl } func NewYugabyte(settings yugabyte.DBSettings) *Service { - return &Service{impl: yugabyte.NewStore(settings)} + return &Service{Impl: yugabyte.NewStore(settings)} } func NewCouchbase(settings couchbase.DBSettings) *Service { - return &Service{impl: couchbase.NewStore(settings)} + return &Service{Impl: couchbase.NewStore(settings)} } func NewLevelDB(repoPath string) (*Service, error) { @@ -43,7 +43,7 @@ func NewLevelDB(repoPath string) (*Service, error) { } } - return &Service{impl: ldb.NewStore(repoPath)}, nil + return &Service{Impl: ldb.NewStore(repoPath)}, nil } func MakeLevelDBDir(repoPath string) (string, error) { @@ -60,13 +60,13 @@ func (s *Service) Start(ctx context.Context, addr string) error { return fmt.Errorf("setting up listener for local index directory service: %w", err) } - err = s.impl.Start(ctx) + err = s.Impl.Start(ctx) if err != nil { return fmt.Errorf("starting local index directory service: %w", err) } server := jsonrpc.NewServer() - server.Register("boostddata", s.impl) + server.Register("boostddata", s.Impl) router := mux.NewRouter() router.Handle("/", server) diff --git a/extern/boostd-data/svc/svc_test.go b/extern/boostd-data/svc/svc_test.go index 1279eee53..59843be08 100644 --- a/extern/boostd-data/svc/svc_test.go +++ b/extern/boostd-data/svc/svc_test.go @@ -47,7 +47,8 @@ var testCouchSettings = couchbase.DBSettings{ } var testYugaSettings = yugabyte.DBSettings{ - ConnectString: "127.0.0.1", + Hosts: []string{"127.0.0.1"}, + ConnectString: "postgresql://postgres:postgres@localhost", } func TestService(t *testing.T) { @@ -98,7 +99,7 @@ func TestService(t *testing.T) { err := bdsvc.Start(ctx, addr) require.NoError(t, err) - ybstore := bdsvc.impl.(*yugabyte.Store) + ybstore := bdsvc.Impl.(*yugabyte.Store) err = ybstore.Drop(ctx) require.NoError(t, err) err = ybstore.Create(ctx) @@ -243,7 +244,7 @@ func TestServiceFuzz(t *testing.T) { err := bdsvc.Start(ctx, addr) require.NoError(t, err) - ybstore := bdsvc.impl.(*yugabyte.Store) + ybstore := bdsvc.Impl.(*yugabyte.Store) err = ybstore.Drop(ctx) require.NoError(t, err) err = ybstore.Create(ctx) diff --git a/extern/boostd-data/yugabyte/create.cql b/extern/boostd-data/yugabyte/create.cql index 038a0be78..f0243039c 100644 --- a/extern/boostd-data/yugabyte/create.cql +++ b/extern/boostd-data/yugabyte/create.cql @@ -14,13 +14,16 @@ CREATE TABLE idx.PieceBlockOffsetSize ( ); CREATE TABLE idx.PieceMetadata ( - PieceCid BLOB PRIMARY KEY, + PieceCid TEXT PRIMARY KEY, Version TEXT, + CreatedAt TIMESTAMP, IndexedAt TIMESTAMP, CompleteIndex BOOLEAN, Error TEXT, ErrorType Text -); +) WITH transactions = { 'enabled' : true }; + +CREATE INDEX PieceMetadataCreatedAt ON idx.PieceMetadata (CreatedAt); CREATE TABLE idx.PieceDeal ( DealUuid TEXT PRIMARY KEY, diff --git a/extern/boostd-data/yugabyte/create.sql b/extern/boostd-data/yugabyte/create.sql new file mode 100644 index 000000000..6b78af851 --- /dev/null +++ b/extern/boostd-data/yugabyte/create.sql @@ -0,0 +1,17 @@ +CREATE TABLE PieceTracker ( + PieceCid TEXT PRIMARY KEY, + CreatedAt TIMESTAMP, + UpdatedAt TIMESTAMP +); + +CREATE INDEX PieceTrackerCreatedAt ON PieceTracker (CreatedAt); +CREATE INDEX PieceTrackerUpdatedAt ON PieceTracker (UpdatedAt); + +CREATE TABLE PieceFlagged ( + PieceCid TEXT PRIMARY KEY, + CreatedAt TIMESTAMP, + UpdatedAt TIMESTAMP +); + +CREATE INDEX PieceFlaggedCreatedAt ON PieceFlagged (CreatedAt); +CREATE INDEX PieceFlaggedUpdatedAt ON PieceFlagged (UpdatedAt); diff --git a/extern/boostd-data/yugabyte/drop.cql b/extern/boostd-data/yugabyte/drop.cql new file mode 100644 index 000000000..e50b35d44 --- /dev/null +++ b/extern/boostd-data/yugabyte/drop.cql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS idx.PayloadToPieces; +DROP TABLE IF EXISTS idx.PieceBlockOffsetSize; +DROP TABLE IF EXISTS idx.PieceMetadata; +DROP INDEX IF EXISTS idx.PieceMetadataIndexedAt; +DROP TABLE IF EXISTS idx.PieceDeal; +DROP TABLE IF EXISTS idx.PieceTracker; +DROP INDEX IF EXISTS idx.PieceTrackerUpdatedAt; +DROP TABLE IF EXISTS idx.PieceFlagged; +DROP INDEX IF EXISTS idx.PieceFlaggedCreatedAt; +DROP INDEX IF EXISTS idx.PieceFlaggedUpdatedAt; diff --git a/extern/boostd-data/yugabyte/drop.sql b/extern/boostd-data/yugabyte/drop.sql new file mode 100644 index 000000000..92e483c0f --- /dev/null +++ b/extern/boostd-data/yugabyte/drop.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS PieceTracker; +DROP INDEX IF EXISTS PieceTrackerCreatedAt; +DROP INDEX IF EXISTS PieceTrackerUpdatedAt; +DROP TABLE IF EXISTS PieceFlagged; +DROP INDEX IF EXISTS PieceFlaggedCreatedAt; +DROP INDEX IF EXISTS PieceFlaggedUpdatedAt; diff --git a/extern/boostd-data/yugabyte/piecedoctor.go b/extern/boostd-data/yugabyte/piecedoctor.go new file mode 100644 index 000000000..93a6a9699 --- /dev/null +++ b/extern/boostd-data/yugabyte/piecedoctor.go @@ -0,0 +1,287 @@ +package yugabyte + +import ( + "context" + "fmt" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/tracing" + "github.com/ipfs/go-cid" + "github.com/jackc/pgtype" + "go.opentelemetry.io/otel/attribute" + "golang.org/x/sync/errgroup" + "time" +) + +var TrackerCheckBatchSize = 1024 + +const insertTrackerParallelism = 16 + +type pieceCreated struct { + PieceCid cid.Cid + CreatedAt time.Time +} + +// NextPiecesToCheck is periodically called by the piece doctor. +// It returns a selection of piece cids so the piece doctor can check the +// status of each piece. +// For each piece it saves the time at which the piece was checked, so that +// the piece won't be checked again for a while. +// The implementation uses a PieceTracker table to keep track of when each piece +// was last checked. +func (s *Store) NextPiecesToCheck(ctx context.Context) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.next_pieces_to_check") + defer span.End() + + // Get the time at which pieces were last copied from the piece metadata + // to the piece tracker table + var lastCopiedRes pgtype.Timestamptz + err := s.db.QueryRow(ctx, `SELECT MAX(CreatedAt) FROM PieceTracker`).Scan(&lastCopiedRes) + if err != nil { + return nil, fmt.Errorf("getting time piece tracker was last updated: %w", err) + } + + lastCopied := lastCopiedRes.Time + if lastCopiedRes.Status&pgtype.Present == 0 { + // If there are no results, set last updated to the zero value of time, + // so that we copy across all rows from piece metadata to piece tracker + lastCopied = time.UnixMilli(0) + } + log.Debugw("got tracker last updated", "updated at", lastCopied.String()) + + // Get the list of pieces that have been added since tracking information + // was last updated + qry := `SELECT PieceCid, CreatedAt from idx.PieceMetadata WHERE CreatedAt >= ?` + iter := s.session.Query(qry, lastCopied).WithContext(ctx).Iter() + var newPieces []pieceCreated + var createdAt time.Time + var pcidstr string + for iter.Scan(&pcidstr, &createdAt) { + c, err := cid.Parse(pcidstr) + if err != nil { + return nil, fmt.Errorf("getting new pieces: parsing piece cid %s: %w", pcidstr, err) + } + newPieces = append(newPieces, pieceCreated{PieceCid: c, CreatedAt: createdAt}) + } + if err := iter.Close(); err != nil { + return nil, fmt.Errorf("getting new pieces: %w", err) + } + + // Add any new pieces into the piece status tracking table + log.Debugw("inserting new pieces into tracker", "count", len(newPieces)) + err = s.execWithConcurrency(ctx, newPieces, insertTrackerParallelism, func(pc pieceCreated) error { + qry := `INSERT INTO PieceTracker (PieceCid, CreatedAt, UpdatedAt) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING` + _, err := s.db.Exec(ctx, qry, pc.PieceCid, pc.CreatedAt, time.UnixMilli(0)) + if err != nil { + return fmt.Errorf("inserting row into piece tracker: %w", err) + } + return nil + }) + if err != nil { + return nil, err + } + + // Work out how frequently to check each piece, based on how many pieces + // there are. + // Any pieces that have not been checked in the last pieceCheckPeriod + // will be checked now (eg check all pieces that haven't been checked + // for 10s) + pieceCheckPeriod, err := s.getPieceCheckPeriod(ctx) + if err != nil { + return nil, fmt.Errorf("getting piece check period: %w", err) + } + + // Get all pieces from the piece tracker table that have not been updated + // since the last piece check period. + // At the same time set the UpdatedAt field so that these pieces are marked + // as checked (and will not be returned until the next piece check period + // elapses again). + // Note that we limit the number of rows to fetch so as not to overload the + // system. Any rows beyond the limit will be fetched the next time + // NextPiecesToCheck is called. + now := time.Now() + qry = `WITH cte AS (SELECT PieceCid FROM PieceTracker WHERE UpdatedAt < $1 LIMIT $2)` + + `UPDATE PieceTracker pt SET UpdatedAt = $3 ` + + `FROM cte WHERE pt.PieceCid = cte.PieceCid RETURNING pt.PieceCid` + rows, err := s.db.Query(ctx, qry, now.Add(-pieceCheckPeriod), TrackerCheckBatchSize, now) + if err != nil { + return nil, fmt.Errorf("getting pieces from piece tracker: %w", err) + } + defer rows.Close() + + pcids := make([]cid.Cid, 0, TrackerCheckBatchSize) + var pcid string + for rows.Next() { + err := rows.Scan(&pcid) + if err != nil { + return nil, fmt.Errorf("scanning piece tracker row: %w", err) + } + + c, err := cid.Parse(pcid) + if err != nil { + return nil, fmt.Errorf("parsing tracker piece cid %s as cid: %w", pcid, err) + } + pcids = append(pcids, c) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("getting pieces to check: %w", err) + } + + log.Debugw("got tracker pieces", "count", len(pcids), + "last updated", lastCopied.String(), "now", now.String(), "piece-check-period", pieceCheckPeriod.String()) + return pcids, nil +} + +func (s *Store) execWithConcurrency(ctx context.Context, pcids []pieceCreated, concurrency int, exec func(created pieceCreated) error) error { + queue := make(chan pieceCreated, len(pcids)) + for _, pc := range pcids { + queue <- pc + } + close(queue) + + var eg errgroup.Group + for i := 0; i < concurrency; i++ { + eg.Go(func() error { + for ctx.Err() == nil { + select { + case <-ctx.Done(): + return ctx.Err() + case pc, ok := <-queue: + if !ok { + // Finished adding all the queued items, exit the thread + return nil + } + + err := exec(pc) + if err != nil { + return err + } + } + } + + return ctx.Err() + }) + } + return eg.Wait() +} + +// The minimum frequency with which to check pieces for errors (eg bad index) +var MinPieceCheckPeriod = 30 * time.Second + +// Work out how frequently to check each piece, based on how many pieces +// there are: if there are many pieces, each piece will be checked +// less frequently +func (s *Store) getPieceCheckPeriod(ctx context.Context) (time.Duration, error) { + var count int + err := s.db.QueryRow(ctx, `SELECT Count(*) FROM PieceTracker`).Scan(&count) + if err != nil { + return 0, fmt.Errorf("getting count of rows in piece tracker: %w", err) + } + + // Check period: + // - 1k pieces; every 10s + // - 100k pieces; every 15m + // - 1m pieces; every 2 hours + period := time.Duration(count*10) * time.Millisecond + if period < MinPieceCheckPeriod { + period = MinPieceCheckPeriod + } + + return period, nil +} + +func (s *Store) FlagPiece(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "store.flag_piece") + span.SetAttributes(attribute.String("pieceCid", pieceCid.String())) + defer span.End() + + now := time.Now() + qry := `INSERT INTO PieceFlagged (PieceCid, CreatedAt, UpdatedAt) ` + + `VALUES ($1, $2, $3) ` + + `ON CONFLICT (PieceCid) DO UPDATE SET UpdatedAt = excluded.UpdatedAt` + _, err := s.db.Exec(ctx, qry, pieceCid.String(), now, now) + if err != nil { + return fmt.Errorf("flagging piece %s: %w", pieceCid, err) + } + return nil +} + +func (s *Store) UnflagPiece(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "store.unflag_piece") + span.SetAttributes(attribute.String("pieceCid", pieceCid.String())) + defer span.End() + + qry := `DELETE FROM PieceFlagged WHERE PieceCid = $1` + _, err := s.db.Exec(ctx, qry, pieceCid.String()) + if err != nil { + return fmt.Errorf("unflagging piece %s: %w", pieceCid, err) + } + + return nil +} + +func (s *Store) FlaggedPiecesList(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.flagged_pieces") + var spanCursor int + if cursor != nil { + spanCursor = int(cursor.UnixMilli()) + } + span.SetAttributes(attribute.Int("cursor", spanCursor)) + span.SetAttributes(attribute.Int("offset", offset)) + span.SetAttributes(attribute.Int("limit", limit)) + defer span.End() + + var args []interface{} + idx := 0 + qry := `SELECT PieceCid, CreatedAt from PieceFlagged ` + if cursor != nil { + qry += `WHERE CreatedAt < $1 ` + args = append(args, cursor) + idx++ + } + qry += `ORDER BY CreatedAt desc ` + + qry += fmt.Sprintf(`LIMIT $%d OFFSET $%d`, idx+1, idx+2) + args = append(args, limit, offset) + + rows, err := s.db.Query(ctx, qry, args...) + if err != nil { + return nil, fmt.Errorf("getting flagged pieces: %w", err) + } + defer rows.Close() + + var pieces []model.FlaggedPiece + var pcid string + var createdAt time.Time + for rows.Next() { + err := rows.Scan(&pcid, &createdAt) + if err != nil { + return nil, fmt.Errorf("scanning flagged piece: %w", err) + } + + c, err := cid.Parse(pcid) + if err != nil { + return nil, fmt.Errorf("parsing flagged piece cid %s: %w", pcid, err) + } + pieces = append(pieces, model.FlaggedPiece{PieceCid: c, CreatedAt: createdAt}) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("getting new pieces: %w", err) + } + + return pieces, nil +} + +func (s *Store) FlaggedPiecesCount(ctx context.Context) (int, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.flagged_pieces_count") + defer span.End() + + var count int + qry := `SELECT COUNT(*) FROM PieceFlagged` + err := s.db.QueryRow(ctx, qry).Scan(&count) + if err != nil { + return 0, fmt.Errorf("getting flagged pieces count: %w", err) + } + + return count, nil +} diff --git a/extern/boostd-data/yugabyte/service.go b/extern/boostd-data/yugabyte/service.go index 46dcdbdc3..883734785 100644 --- a/extern/boostd-data/yugabyte/service.go +++ b/extern/boostd-data/yugabyte/service.go @@ -13,7 +13,7 @@ import ( "github.com/multiformats/go-multihash" mh "github.com/multiformats/go-multihash" "github.com/yugabyte/gocql" - "go.opentelemetry.io/otel/attribute" + "github.com/yugabyte/pgx/v4/pgxpool" "golang.org/x/sync/errgroup" "time" ) @@ -25,7 +25,11 @@ var log = logging.Logger("boostd-data-yb") const pieceMetadataVersion = "1" type DBSettings struct { - ConnectString string + // The cassandra hosts to connect to + Hosts []string + // The postgres connect string + ConnectString string + // The number of threads to use when inserting into the PayloadToPieces index PayloadPiecesParallelism int } @@ -33,6 +37,7 @@ type Store struct { settings DBSettings cluster *gocql.ClusterConfig session *gocql.Session + db *pgxpool.Pool } var _ types.ServiceImpl = (*Store)(nil) @@ -42,7 +47,7 @@ func NewStore(settings DBSettings) *Store { settings.PayloadPiecesParallelism = 16 } - cluster := gocql.NewCluster(settings.ConnectString) + cluster := gocql.NewCluster(settings.Hosts...) //cluster.Timeout = 30 * time.Second //cluster.ConnectTimeout = 30 * time.Second return &Store{ @@ -57,6 +62,13 @@ func (s *Store) Start(ctx context.Context) error { return fmt.Errorf("creating yugabyte cluster: %w", err) } s.session = session + + db, err := pgxpool.Connect(context.Background(), s.settings.ConnectString) + if err != nil { + return fmt.Errorf("connecting to database: %w", err) + } + s.db = db + return nil } @@ -85,8 +97,8 @@ func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo } func (s *Store) createPieceMetadata(ctx context.Context, pieceCid cid.Cid) error { - qry := `INSERT INTO idx.PieceMetadata (PieceCid, Version) VALUES (?, ?) IF NOT EXISTS` - err := s.session.Query(qry, pieceCid.String(), pieceMetadataVersion).WithContext(ctx).Exec() + qry := `INSERT INTO idx.PieceMetadata (PieceCid, Version, CreatedAt) VALUES (?, ?, ?) IF NOT EXISTS` + err := s.session.Query(qry, pieceCid.String(), pieceMetadataVersion, time.Now()).WithContext(ctx).Exec() if err != nil { return fmt.Errorf("inserting piece metadata for piece %s: %w", pieceCid, err) } @@ -415,50 +427,6 @@ func (s *Store) ListPieces(ctx context.Context) ([]cid.Cid, error) { return pcids, nil } -func (s *Store) NextPiecesToCheck(ctx context.Context) ([]cid.Cid, error) { - ctx, span := tracing.Tracer.Start(ctx, "store.next_pieces_to_check") - defer span.End() - - return nil, nil -} - -func (s *Store) FlagPiece(ctx context.Context, pieceCid cid.Cid) error { - ctx, span := tracing.Tracer.Start(ctx, "store.flag_piece") - span.SetAttributes(attribute.String("pieceCid", pieceCid.String())) - defer span.End() - - return nil -} - -func (s *Store) UnflagPiece(ctx context.Context, pieceCid cid.Cid) error { - ctx, span := tracing.Tracer.Start(ctx, "store.unflag_piece") - span.SetAttributes(attribute.String("pieceCid", pieceCid.String())) - defer span.End() - - return nil -} - -func (s *Store) FlaggedPiecesList(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) { - ctx, span := tracing.Tracer.Start(ctx, "store.flagged_pieces") - var spanCursor int - if cursor != nil { - spanCursor = int(cursor.UnixMilli()) - } - span.SetAttributes(attribute.Int("cursor", spanCursor)) - span.SetAttributes(attribute.Int("offset", offset)) - span.SetAttributes(attribute.Int("limit", limit)) - defer span.End() - - return nil, nil -} - -func (s *Store) FlaggedPiecesCount(ctx context.Context) (int, error) { - ctx, span := tracing.Tracer.Start(ctx, "store.flagged_pieces_count") - defer span.End() - - return 0, nil -} - // RemoveDealForPiece removes Single deal for pieceCID. If []Deals is empty then Metadata is removed as well func (s *Store) RemoveDealForPiece(ctx context.Context, pieceCid cid.Cid, dealId string) error { ctx, span := tracing.Tracer.Start(ctx, "store.remove_deal_for_piece") diff --git a/extern/boostd-data/yugabyte/setup.go b/extern/boostd-data/yugabyte/setup.go index 9cb217ed3..de699a42f 100644 --- a/extern/boostd-data/yugabyte/setup.go +++ b/extern/boostd-data/yugabyte/setup.go @@ -10,32 +10,53 @@ import ( //go:embed create.cql var createCQL string +//go:embed create.sql +var createSQL string + func (s *Store) Create(ctx context.Context) error { - createTablesLines := strings.Split(createCQL, ";") - for _, line := range createTablesLines { + err := s.execScript(ctx, createCQL, s.execCQL) + if err != nil { + return err + } + return s.execScript(ctx, createSQL, s.execSQL) +} + +//go:embed drop.cql +var dropCQL string + +//go:embed drop.sql +var dropSQL string + +func (s *Store) Drop(ctx context.Context) error { + err := s.execScript(ctx, dropCQL, s.execCQL) + if err != nil { + return err + } + return s.execScript(ctx, dropSQL, s.execSQL) +} + +func (s *Store) execScript(ctx context.Context, cqlstr string, exec func(context.Context, string) error) error { + lines := strings.Split(cqlstr, ";") + for _, line := range lines { line = strings.Trim(line, "\n \t") if line == "" { continue } log.Debug(line) - err := s.session.Query(line).WithContext(ctx).Exec() + err := exec(ctx, line) if err != nil { - return fmt.Errorf("creating tables: executing\n%s\n%w", line, err) + return fmt.Errorf("executing\n%s\n%w", line, err) } } return nil } -func (s *Store) Drop(ctx context.Context) error { - tables := []string{`PayloadToPieces`, `PieceBlockOffsetSize`, `PieceMetadata`, `PieceDeal`} - for _, tbl := range tables { - qry := `drop table if exists idx.` + tbl - log.Debug(qry) - err := s.session.Query(qry).WithContext(ctx).Exec() - if err != nil { - return err - } - } - return nil +func (s *Store) execCQL(ctx context.Context, query string) error { + return s.session.Query(query).WithContext(ctx).Exec() +} + +func (s *Store) execSQL(ctx context.Context, query string) error { + _, err := s.db.Exec(ctx, query) + return err } diff --git a/go.mod b/go.mod index 32276bc5a..669fd8128 100644 --- a/go.mod +++ b/go.mod @@ -372,6 +372,14 @@ require ( github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.11.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.2.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.10.0 // indirect + github.com/jackc/puddle v1.2.1 // indirect github.com/onsi/ginkgo/v2 v2.5.1 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-19 v0.2.1 // indirect @@ -381,6 +389,7 @@ require ( github.com/tidwall/gjson v1.14.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0 // indirect + github.com/yugabyte/pgx/v4 v4.14.5 // indirect github.com/zyedidia/generic v1.2.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect nhooyr.io/websocket v1.8.7 // indirect diff --git a/go.sum b/go.sum index e45216233..c17988896 100644 --- a/go.sum +++ b/go.sum @@ -74,6 +74,7 @@ github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa h1:1PPxEyGdIGVkX/kqMv github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa/go.mod h1:WUmMvh9wMtqj1Xhf1hf3kp9RvL+y6odtdYxpyZjb90U= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= @@ -201,6 +202,8 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= @@ -218,6 +221,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= @@ -534,6 +538,8 @@ github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -980,6 +986,53 @@ github.com/ipni/storetheindex v0.5.10 h1:r97jIZsXPuwQvePJQuStu2a/kn+Zn8X4MAdA0rU github.com/ipni/storetheindex v0.5.10/go.mod h1:SJKFCnSx4X/4ekQuZvq8pVU/7tmxkEv632Qmgu3m2bQ= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c h1:Dznn52SgVIVst9UyOT9brctYUgxs+CvVfPaC3jKrA50= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -1055,6 +1108,7 @@ github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8t github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -1068,6 +1122,7 @@ github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NB github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -1075,6 +1130,10 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= @@ -1330,6 +1389,7 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -1337,6 +1397,7 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -1670,6 +1731,8 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1678,6 +1741,7 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -1687,6 +1751,9 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -1711,6 +1778,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -1754,6 +1822,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= @@ -1880,6 +1949,8 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRT github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0 h1:68nrJsrWe0A6JiKnsjWChAaWhj20v+AwYJObtp86D1k= github.com/yugabyte/gocql v0.0.0-20221110041640-6fc475c5aeb0/go.mod h1:LAokR6+vevDCrTxk52U7p6ki+4qELu4XU7JUGYa2O2M= +github.com/yugabyte/pgx/v4 v4.14.5 h1:XLzEEiO3d/kWzpyctO8l4kwMLhzPQ9n2er7ATH7CJVA= +github.com/yugabyte/pgx/v4 v4.14.5/go.mod h1:nFSvjsVq4CuA61TWGriwWG74ZVxeuJCcNu42Mkn+rgw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1887,6 +1958,7 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= github.com/zondax/hid v0.9.1 h1:gQe66rtmyZ8VeGFcOpbuH3r7erYtNEAezCAYu8LdkJo= @@ -1961,6 +2033,7 @@ go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95a go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= @@ -1984,6 +2057,7 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1992,6 +2066,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -2005,10 +2080,12 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -2175,6 +2252,7 @@ golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2188,6 +2266,7 @@ golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2285,6 +2364,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -2309,6 +2389,7 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -2316,6 +2397,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -2363,6 +2445,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2485,6 +2569,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= diff --git a/piecedirectory/doctor_test.go b/piecedirectory/doctor_test.go index 8e6ec2110..0d6f46f7d 100644 --- a/piecedirectory/doctor_test.go +++ b/piecedirectory/doctor_test.go @@ -7,11 +7,13 @@ import ( "testing" "time" + "github.com/filecoin-project/boost/testutil" "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/couchbase" "github.com/filecoin-project/boostd-data/ldb" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/svc" + "github.com/filecoin-project/boostd-data/yugabyte" "github.com/filecoin-project/go-state-types/abi" "github.com/google/uuid" "github.com/ipfs/go-cid" @@ -28,9 +30,6 @@ func TestPieceDoctor(t *testing.T) { prev := ldb.MinPieceCheckPeriod ldb.MinPieceCheckPeriod = 1 * time.Second - prevp := ldb.PiecesToTrackerBatchSize - ldb.PiecesToTrackerBatchSize = 4 - bdsvc, err := svc.NewLevelDB("") require.NoError(t, err) @@ -44,7 +43,11 @@ func TestPieceDoctor(t *testing.T) { defer cl.Close(ctx) t.Run("next pieces pagination", func(t *testing.T) { - testNextPiecesPagination(ctx, t, cl, ldb.MinPieceCheckPeriod) + prevp := ldb.PiecesToTrackerBatchSize + testNextPiecesPagination(ctx, t, cl, func(pageSize int) { + ldb.PiecesToTrackerBatchSize = pageSize + }) + ldb.PiecesToTrackerBatchSize = prevp }) t.Run("check pieces", func(t *testing.T) { @@ -52,8 +55,8 @@ func TestPieceDoctor(t *testing.T) { }) ldb.MinPieceCheckPeriod = prev - ldb.PiecesToTrackerBatchSize = prevp }) + t.Run("couchbase", func(t *testing.T) { // TODO: Unskip this test once the couchbase instance can be created // from a docker container in CI @@ -84,6 +87,51 @@ func TestPieceDoctor(t *testing.T) { couchbase.MinPieceCheckPeriod = prev }) + + t.Run("yugabyte", func(t *testing.T) { + prev := yugabyte.MinPieceCheckPeriod + yugabyte.MinPieceCheckPeriod = 1 * time.Second + + svc.SetupYugabyte(t) + + bdsvc := svc.NewYugabyte(yugabyte.DBSettings{ + Hosts: []string{"127.0.0.1"}, + ConnectString: "postgresql://postgres:postgres@localhost", + }) + + addr := "localhost:8044" + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + + ybstore := bdsvc.Impl.(*yugabyte.Store) + err = ybstore.Drop(ctx) + require.NoError(t, err) + err = ybstore.Create(ctx) + require.NoError(t, err) + + cl := client.NewStore() + err = cl.Dial(ctx, fmt.Sprintf("http://%s", addr)) + require.NoError(t, err) + defer cl.Close(ctx) + + t.Run("next pieces", func(t *testing.T) { + testNextPieces(ctx, t, cl, yugabyte.MinPieceCheckPeriod) + }) + + t.Run("next pieces pagination", func(t *testing.T) { + prevp := yugabyte.TrackerCheckBatchSize + testNextPiecesPagination(ctx, t, cl, func(pageSize int) { + yugabyte.TrackerCheckBatchSize = pageSize + }) + yugabyte.TrackerCheckBatchSize = prevp + }) + + t.Run("check pieces", func(t *testing.T) { + testCheckPieces(ctx, t, cl) + }) + + yugabyte.MinPieceCheckPeriod = prev + }) } // Verify that after a new piece is added @@ -92,7 +140,6 @@ func TestPieceDoctor(t *testing.T) { func testNextPieces(ctx context.Context, t *testing.T, cl *client.Store, pieceCheckPeriod time.Duration) { // Add a new piece pieceCid := blocks.NewBlock([]byte(fmt.Sprintf("%d", time.Now().UnixMilli()))).Cid() - fmt.Println(pieceCid) di := model.DealInfo{ DealUuid: uuid.New().String(), ChainDealID: 1, @@ -127,14 +174,14 @@ func testNextPieces(ctx context.Context, t *testing.T, cl *client.Store, pieceCh require.Contains(t, pcids, pieceCid) } -func testNextPiecesPagination(ctx context.Context, t *testing.T, cl *client.Store, pieceCheckPeriod time.Duration) { - // Add 8 pieces - allPcids := make(map[cid.Cid]struct{}) +func testNextPiecesPagination(ctx context.Context, t *testing.T, cl *client.Store, setPageSize func(int)) { + setPageSize(4) + + // Add 9 pieces seen := make(map[cid.Cid]int) for i := 1; i <= 9; i++ { - pieceCid := blocks.NewBlock([]byte(fmt.Sprintf("%d%d", time.Now().UnixMilli(), i))).Cid() - fmt.Println(pieceCid) + pieceCid := testutil.GenerateCid() di := model.DealInfo{ DealUuid: uuid.New().String(), ChainDealID: abi.DealID(i), @@ -144,8 +191,6 @@ func testNextPiecesPagination(ctx context.Context, t *testing.T, cl *client.Stor } err := cl.AddDealForPiece(ctx, pieceCid, di) require.NoError(t, err) - - allPcids[pieceCid] = struct{}{} } // expect to get 4 pieces @@ -191,19 +236,16 @@ func testNextPiecesPagination(ctx context.Context, t *testing.T, cl *client.Stor // Add 1 more piece for i := 1; i <= 1; i++ { - pieceCid := blocks.NewBlock([]byte(fmt.Sprintf("%d%d", time.Now().UnixMilli(), i))).Cid() - fmt.Println(pieceCid) + pieceCid := testutil.GenerateCid() di := model.DealInfo{ DealUuid: uuid.New().String(), - ChainDealID: abi.DealID(i), - SectorID: abi.SectorNumber(i), + ChainDealID: abi.DealID(100), + SectorID: abi.SectorNumber(100), PieceOffset: 0, PieceLength: 2048, } err := cl.AddDealForPiece(ctx, pieceCid, di) require.NoError(t, err) - - allPcids[pieceCid] = struct{}{} } // wait to reset the interval and start from scratch From 3c61ea9ad9e2a603e746beedb184343bcfb79308 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 3 May 2023 10:29:48 +0200 Subject: [PATCH 06/15] fix: go mod tidy --- extern/boostd-data/go.mod | 3 ++- extern/boostd-data/go.sum | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/extern/boostd-data/go.mod b/extern/boostd-data/go.mod index 34d6da92c..32dff3ec5 100644 --- a/extern/boostd-data/go.mod +++ b/extern/boostd-data/go.mod @@ -18,6 +18,7 @@ require ( github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-car/v2 v2.4.2-0.20220707083113-89de8134e58e + github.com/jackc/pgtype v1.10.0 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-multicodec v0.6.0 github.com/multiformats/go-multihash v0.2.1 @@ -84,7 +85,7 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.2.0 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.10.0 // indirect + github.com/jackc/puddle v1.2.1 // indirect github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/cpuid/v2 v2.1.1 // indirect diff --git a/extern/boostd-data/go.sum b/extern/boostd-data/go.sum index 2f56ec0dd..cc22c597f 100644 --- a/extern/boostd-data/go.sum +++ b/extern/boostd-data/go.sum @@ -996,6 +996,7 @@ github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgS github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= From acdde47b04565cf6e265243a1c475e9f80cab823 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 3 May 2023 11:58:52 +0200 Subject: [PATCH 07/15] refactor: remove SetCarSize as its not longer being used --- extern/boostd-data/client/client.go | 5 - extern/boostd-data/couchbase/db.go | 17 ---- extern/boostd-data/couchbase/service.go | 14 --- extern/boostd-data/ldb/db.go | 17 ---- extern/boostd-data/ldb/service.go | 17 ---- extern/boostd-data/svc/types/types.go | 4 +- extern/boostd-data/yugabyte/err.go | 20 +++- extern/boostd-data/yugabyte/service.go | 39 +++++--- piecedirectory/doctor_test.go | 17 ++-- piecedirectory/piecedirectory.go | 79 ---------------- piecedirectory/piecedirectory_test.go | 96 +++++++------------- piecedirectory/types/mocks/piecedirectory.go | 14 --- piecedirectory/types/types.go | 1 - 13 files changed, 89 insertions(+), 251 deletions(-) diff --git a/extern/boostd-data/client/client.go b/extern/boostd-data/client/client.go index e41a7952d..fee11386c 100644 --- a/extern/boostd-data/client/client.go +++ b/extern/boostd-data/client/client.go @@ -26,7 +26,6 @@ type Store struct { ListPieces func(ctx context.Context) ([]cid.Cid, error) GetPieceMetadata func(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) GetPieceDeals func(context.Context, cid.Cid) ([]model.DealInfo, error) - SetCarSize func(ctx context.Context, pieceCid cid.Cid, size uint64) error MarkIndexErrored func(context.Context, cid.Cid, string) error IndexedAt func(context.Context, cid.Cid) (time.Time, error) PiecesContainingMultihash func(context.Context, mh.Multihash) ([]cid.Cid, error) @@ -115,10 +114,6 @@ func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo return s.client.AddDealForPiece(ctx, pieceCid, dealInfo) } -func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { - return s.client.SetCarSize(ctx, pieceCid, size) -} - func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error { log.Debugw("add-index", "piece-cid", pieceCid, "records", len(records)) diff --git a/extern/boostd-data/couchbase/db.go b/extern/boostd-data/couchbase/db.go index c624b5df7..e679b2869 100644 --- a/extern/boostd-data/couchbase/db.go +++ b/extern/boostd-data/couchbase/db.go @@ -337,23 +337,6 @@ func (db *DB) setPieceCidsForMultihash(ctx context.Context, mh multihash.Multiha return nil } -func (db *DB) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { - ctx, span := tracing.Tracer.Start(ctx, "db.set_car_size") - defer span.End() - - return db.mutatePieceMetadata(ctx, pieceCid, "set-car-size", func(metadata CouchbaseMetadata) *CouchbaseMetadata { - // Set the car size on each deal (should be the same for all deals) - var deals []model.DealInfo - for _, dl := range metadata.Deals { - dl.CarLength = size - - deals = append(deals, dl) - } - metadata.Deals = deals - return &metadata - }) -} - func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr error) error { ctx, span := tracing.Tracer.Start(ctx, "db.mark_piece_index_errored") defer span.End() diff --git a/extern/boostd-data/couchbase/service.go b/extern/boostd-data/couchbase/service.go index 35259b04a..1c9625a01 100644 --- a/extern/boostd-data/couchbase/service.go +++ b/extern/boostd-data/couchbase/service.go @@ -56,20 +56,6 @@ func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo return s.db.AddDealForPiece(ctx, pieceCid, dealInfo) } -func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { - log.Debugw("handle.set-car-size", "piece-cid", pieceCid, "size", size) - - ctx, span := tracing.Tracer.Start(ctx, "store.set-car-size") - defer span.End() - - defer func(now time.Time) { - log.Debugw("handled.set-car-size", "took", time.Since(now).String()) - }(time.Now()) - - err := s.db.SetCarSize(ctx, pieceCid, size) - return normalizePieceCidError(pieceCid, err) -} - func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr string) error { log.Debugw("handle.mark-piece-index-errored", "piece-cid", pieceCid, "err", idxErr) diff --git a/extern/boostd-data/ldb/db.go b/extern/boostd-data/ldb/db.go index 5489f5bce..bd8fb3277 100644 --- a/extern/boostd-data/ldb/db.go +++ b/extern/boostd-data/ldb/db.go @@ -290,23 +290,6 @@ func (db *DB) GetPieceCidToMetadata(ctx context.Context, pieceCid cid.Cid) (Leve return metadata, nil } -func (db *DB) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { - ctx, span := tracing.Tracer.Start(ctx, "db.set_car_size") - defer span.End() - - md, err := db.GetPieceCidToMetadata(ctx, pieceCid) - if err != nil { - return fmt.Errorf("getting piece metadata for piece %s: %w", pieceCid, err) - } - - // Set the car size on each deal (should be the same for all deals) - for _, dl := range md.Deals { - dl.CarLength = size - } - - return db.SetPieceCidToMetadata(ctx, pieceCid, md) -} - func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, sourceErr error) error { ctx, span := tracing.Tracer.Start(ctx, "db.mark_piece_index_errored") defer span.End() diff --git a/extern/boostd-data/ldb/service.go b/extern/boostd-data/ldb/service.go index e3915857d..e93e9c8b1 100644 --- a/extern/boostd-data/ldb/service.go +++ b/extern/boostd-data/ldb/service.go @@ -121,23 +121,6 @@ func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo return nil } -func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { - log.Debugw("handle.set-car-size", "piece-cid", pieceCid, "size", size) - - ctx, span := tracing.Tracer.Start(ctx, "store.set-car-size") - defer span.End() - - defer func(now time.Time) { - log.Debugw("handled.set-car-size", "took", time.Since(now).String()) - }(time.Now()) - - s.Lock() - defer s.Unlock() - - err := s.db.SetCarSize(ctx, pieceCid, size) - return normalizePieceCidError(pieceCid, err) -} - func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr string) error { log.Debugw("handle.mark-piece-index-errored", "piece-cid", pieceCid, "err", idxErr) diff --git a/extern/boostd-data/svc/types/types.go b/extern/boostd-data/svc/types/types.go index 631cf27d7..76106df32 100644 --- a/extern/boostd-data/svc/types/types.go +++ b/extern/boostd-data/svc/types/types.go @@ -18,6 +18,9 @@ var ErrNotFound = errors.New("not found") // We have to do string matching so that it can be used on errors that // cross the RPC boundary (we can't use errors.Is) func IsNotFound(err error) bool { + if err == nil { + return false + } return strings.Contains(err.Error(), ErrNotFound.Error()) } @@ -31,7 +34,6 @@ type Service interface { ListPieces(ctx context.Context) ([]cid.Cid, error) GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) GetPieceDeals(context.Context, cid.Cid) ([]model.DealInfo, error) - SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error IndexedAt(context.Context, cid.Cid) (time.Time, error) PiecesContainingMultihash(context.Context, mh.Multihash) ([]cid.Cid, error) RemoveDealForPiece(context.Context, cid.Cid, string) error diff --git a/extern/boostd-data/yugabyte/err.go b/extern/boostd-data/yugabyte/err.go index acf4a7391..176679a4e 100644 --- a/extern/boostd-data/yugabyte/err.go +++ b/extern/boostd-data/yugabyte/err.go @@ -1,10 +1,12 @@ package yugabyte import ( + "errors" "fmt" "github.com/filecoin-project/boostd-data/svc/types" "github.com/ipfs/go-cid" mh "github.com/multiformats/go-multihash" + "github.com/yugabyte/gocql" "strings" ) @@ -13,7 +15,7 @@ func normalizePieceCidError(pieceCid cid.Cid, err error) error { return nil } if isNotFoundErr(err) { - return fmt.Errorf("piece %s: %s", pieceCid, types.ErrNotFound) + return fmt.Errorf("piece %s: %w", pieceCid, types.ErrNotFound) } return err } @@ -23,13 +25,21 @@ func normalizeMultihashError(m mh.Multihash, err error) error { return nil } if isNotFoundErr(err) { - return fmt.Errorf("multihash %s: %s", m, types.ErrNotFound) + return fmt.Errorf("multihash %s: %w", m, types.ErrNotFound) } return err } func isNotFoundErr(err error) bool { - // TODO: is there some other way to know if the error is a not found error? - // Check yugabyte cassandra driver docs - return strings.Contains(err.Error(), "not found") + if err == nil { + return false + } + + if errors.Is(err, gocql.ErrNotFound) { + return true + } + + // Unfortunately it seems like the Cassandra driver doesn't always return + // a specific not found error type, so we need to rely on string parsing + return strings.Contains(strings.ToLower(err.Error()), "not found") } diff --git a/extern/boostd-data/yugabyte/service.go b/extern/boostd-data/yugabyte/service.go index 883734785..4c6def54b 100644 --- a/extern/boostd-data/yugabyte/service.go +++ b/extern/boostd-data/yugabyte/service.go @@ -48,8 +48,6 @@ func NewStore(settings DBSettings) *Store { } cluster := gocql.NewCluster(settings.Hosts...) - //cluster.Timeout = 30 * time.Second - //cluster.ConnectTimeout = 30 * time.Second return &Store{ settings: settings, cluster: cluster, @@ -105,17 +103,6 @@ func (s *Store) createPieceMetadata(ctx context.Context, pieceCid cid.Cid) error return nil } -// TODO: I don't think we're using this functionality anymore, we can probably -// just remove it from the interface and data model -func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { - ctx, span := tracing.Tracer.Start(ctx, "store.set-car-size") - defer span.End() - - //err := s.db.SetCarSize(ctx, pieceCid, size) - //return normalizePieceCidError(pieceCid, err) - return nil -} - // TODO: Do we need this? func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr string) error { ctx, span := tracing.Tracer.Start(ctx, "store.mark-piece-index-errored") @@ -149,7 +136,7 @@ func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Mul func (s *Store) GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) { md, err := s.getPieceMetadata(ctx, pieceCid) if err != nil { - return md, nil + return md, err } deals, err := s.GetPieceDeals(ctx, pieceCid) @@ -208,6 +195,15 @@ func (s *Store) GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.De return nil, fmt.Errorf("getting piece deals: %w", err) } + // For correctness, we should always return a not found error if there is + // no piece with the piece cid + if len(deals) == 0 { + _, err := s.getPieceMetadata(ctx, pieceCid) + if err != nil { + return nil, err + } + } + return deals, nil } @@ -255,6 +251,15 @@ func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) ([]model.Record, return nil, fmt.Errorf("getting piece index for piece %s: %w", pieceCid, err) } + // For correctness, we should always return a not found error if there is + // no piece with the piece cid + if len(records) == 0 { + _, err := s.getPieceMetadata(ctx, pieceCid) + if err != nil { + return nil, err + } + } + return records, nil } @@ -388,6 +393,9 @@ func (s *Store) IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, er func (s *Store) IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) { t, err := s.IndexedAt(ctx, pieceCid) if err != nil { + if isNotFoundErr(err) { + return false, nil + } return false, err } return !t.IsZero(), nil @@ -399,6 +407,9 @@ func (s *Store) IndexedAt(ctx context.Context, pieceCid cid.Cid) (time.Time, err md, err := s.getPieceMetadata(ctx, pieceCid) if err != nil { + if isNotFoundErr(err) { + return time.Time{}, nil + } return time.Time{}, err } diff --git a/piecedirectory/doctor_test.go b/piecedirectory/doctor_test.go index 0d6f46f7d..08904ef7d 100644 --- a/piecedirectory/doctor_test.go +++ b/piecedirectory/doctor_test.go @@ -103,22 +103,26 @@ func TestPieceDoctor(t *testing.T) { err := bdsvc.Start(ctx, addr) require.NoError(t, err) - ybstore := bdsvc.Impl.(*yugabyte.Store) - err = ybstore.Drop(ctx) - require.NoError(t, err) - err = ybstore.Create(ctx) - require.NoError(t, err) - cl := client.NewStore() err = cl.Dial(ctx, fmt.Sprintf("http://%s", addr)) require.NoError(t, err) defer cl.Close(ctx) + resetTables := func() { + ybstore := bdsvc.Impl.(*yugabyte.Store) + err = ybstore.Drop(ctx) + require.NoError(t, err) + err = ybstore.Create(ctx) + require.NoError(t, err) + } + t.Run("next pieces", func(t *testing.T) { + resetTables() testNextPieces(ctx, t, cl, yugabyte.MinPieceCheckPeriod) }) t.Run("next pieces pagination", func(t *testing.T) { + resetTables() prevp := yugabyte.TrackerCheckBatchSize testNextPiecesPagination(ctx, t, cl, func(pageSize int) { yugabyte.TrackerCheckBatchSize = pageSize @@ -127,6 +131,7 @@ func TestPieceDoctor(t *testing.T) { }) t.Run("check pieces", func(t *testing.T) { + resetTables() testCheckPieces(ctx, t, cl) }) diff --git a/piecedirectory/piecedirectory.go b/piecedirectory/piecedirectory.go index 5b3aecef6..3e2c02fe5 100644 --- a/piecedirectory/piecedirectory.go +++ b/piecedirectory/piecedirectory.go @@ -3,7 +3,6 @@ package piecedirectory import ( "bufio" "context" - "encoding/binary" "errors" "fmt" "io" @@ -111,84 +110,6 @@ func (ps *PieceDirectory) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, h return ps.store.GetOffsetSize(ctx, pieceCid, hash) } -func (ps *PieceDirectory) GetCarSize(ctx context.Context, pieceCid cid.Cid) (uint64, error) { - // Get the deals for the piece - dls, err := ps.GetPieceDeals(ctx, pieceCid) - if err != nil { - return 0, fmt.Errorf("getting piece deals for piece %s: %w", pieceCid, err) - } - - if len(dls) == 0 { - return 0, fmt.Errorf("no deals for piece %s in index: piece not found", pieceCid) - } - - // The size of the CAR should be the same for any deal, so just return the - // first non-zero CAR size - for _, dl := range dls { - if dl.CarLength > 0 { - return dl.CarLength, nil - } - } - - // There are no deals with a non-zero CAR size. - // The CAR size is zero if it's been imported from the dagstore (the - // dagstore doesn't store CAR size information). So instead work out the - // size of the CAR by getting the offset of the last section in the CAR - // file, then reading the section information. - - // Get the offset of the last section in the CAR file from the index. - var lastSectionOffset uint64 - idx, err := ps.GetIterableIndex(ctx, pieceCid) - if err != nil { - return 0, fmt.Errorf("getting index for piece %s: %w", pieceCid, err) - } - err = idx.ForEach(func(_ mh.Multihash, offset uint64) error { - if offset > lastSectionOffset { - lastSectionOffset = offset - } - return nil - }) - if err != nil { - return 0, fmt.Errorf("iterating index for piece %s: %w", pieceCid, err) - } - - // Get a reader over the piece - pieceReader, err := ps.GetPieceReader(ctx, pieceCid) - if err != nil { - return 0, fmt.Errorf("getting piece reader for piece %s: %w", pieceCid, err) - } - - // Seek to the last section - _, err = pieceReader.Seek(int64(lastSectionOffset), io.SeekStart) - if err != nil { - return 0, fmt.Errorf("seeking to offset %d in piece data: %w", lastSectionOffset, err) - } - - // A section consists of - // - - // Get - cr := &countReader{r: bufio.NewReader(pieceReader)} - dataLength, err := binary.ReadUvarint(cr) - if err != nil { - return 0, fmt.Errorf("reading CAR section length: %w", err) - } - - // The number of bytes in the uvarint that records - dataLengthUvarSize := cr.count - - // Get the size of the (unpadded) CAR file - unpaddedCarSize := lastSectionOffset + dataLengthUvarSize + dataLength - - // Write the CAR size back to the store so that it's cached for next time - err = ps.store.SetCarSize(ctx, pieceCid, unpaddedCarSize) - if err != nil { - log.Errorw("writing CAR size to local index directory store", "pieceCid", pieceCid, "err", err) - } - - return unpaddedCarSize, nil -} - func (ps *PieceDirectory) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { ctx, span := tracing.Tracer.Start(ctx, "pm.add_deal_for_piece") defer span.End() diff --git a/piecedirectory/piecedirectory_test.go b/piecedirectory/piecedirectory_test.go index ea4017d7a..dbbcbf65d 100644 --- a/piecedirectory/piecedirectory_test.go +++ b/piecedirectory/piecedirectory_test.go @@ -14,6 +14,7 @@ import ( "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/svc" "github.com/filecoin-project/boostd-data/svc/types" + "github.com/filecoin-project/boostd-data/yugabyte" "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/ipfs/go-cid" @@ -29,25 +30,51 @@ func TestPieceDirectory(t *testing.T) { t.Run("leveldb", func(t *testing.T) { bdsvc, err := svc.NewLevelDB("") require.NoError(t, err) - testPieceDirectory(ctx, t, bdsvc) + addr := "localhost:8042" + err = bdsvc.Start(ctx, addr) + require.NoError(t, err) + + testPieceDirectory(ctx, t, addr) }) + t.Run("couchbase", func(t *testing.T) { // TODO: Unskip this test once the couchbase instance can be created // from a docker container in CI as part of the test t.Skip() svc.SetupCouchbase(t, testCouchSettings) bdsvc := svc.NewCouchbase(testCouchSettings) - testPieceDirectory(ctx, t, bdsvc) + addr := "localhost:8043" + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + + testPieceDirectory(ctx, t, addr) }) -} -func testPieceDirectory(ctx context.Context, t *testing.T, bdsvc *svc.Service) { - addr := "localhost:8044" - err := bdsvc.Start(ctx, addr) - require.NoError(t, err) + t.Run("yugabyte", func(t *testing.T) { + svc.SetupYugabyte(t) + bdsvc := svc.NewYugabyte(yugabyte.DBSettings{ + Hosts: []string{"127.0.0.1"}, + ConnectString: "postgresql://postgres:postgres@localhost", + }) + + addr := "localhost:8044" + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + + ybstore := bdsvc.Impl.(*yugabyte.Store) + err = ybstore.Drop(ctx) + require.NoError(t, err) + err = ybstore.Create(ctx) + require.NoError(t, err) + + testPieceDirectory(ctx, t, addr) + }) +} + +func testPieceDirectory(ctx context.Context, t *testing.T, addr string) { cl := client.NewStore() - err = cl.Dial(ctx, fmt.Sprintf("http://%s", addr)) + err := cl.Dial(ctx, fmt.Sprintf("http://%s", addr)) require.NoError(t, err) defer cl.Close(ctx) @@ -63,10 +90,6 @@ func testPieceDirectory(ctx context.Context, t *testing.T, bdsvc *svc.Service) { testImportedIndex(ctx, t, cl) }) - t.Run("car file size", func(t *testing.T) { - testCarFileSize(ctx, t, cl) - }) - t.Run("flagging pieces", func(t *testing.T) { testFlaggingPieces(ctx, t, cl) }) @@ -252,55 +275,6 @@ func testImportedIndex(ctx context.Context, t *testing.T, cl *client.Store) { require.Equal(t, len(blk.RawData()), sz) } -// Verify that if the deal info has been imported from the DAG store, meaning -// it does not have CAR size information, GetCarSize will correctly calculate -// the CAR size from the index + piece data -func testCarFileSize(ctx context.Context, t *testing.T, cl *client.Store) { - // Create a random CAR file - carFilePath := CreateCarFile(t) - carFile, err := os.Open(carFilePath) - require.NoError(t, err) - defer carFile.Close() - - carReader, err := car.OpenReader(carFilePath) - require.NoError(t, err) - defer carReader.Close() - carv1Reader, err := carReader.DataReader() - require.NoError(t, err) - - // Read the CAR bytes - carBytes, err := io.ReadAll(carv1Reader) - require.NoError(t, err) - - // Any calls to get a reader over data should return a reader over the random CAR file - pr := CreateMockPieceReader(t, carv1Reader) - - recs := GetRecords(t, carv1Reader) - commpCalc := CalculateCommp(t, carv1Reader) - err = cl.AddIndex(ctx, commpCalc.PieceCID, recs, false) - require.NoError(t, err) - - // Add deal info for the piece without a CAR file - di := model.DealInfo{ - DealUuid: uuid.New().String(), - ChainDealID: 1, - SectorID: 1, - PieceOffset: 0, - PieceLength: commpCalc.PieceSize, - } - err = cl.AddDealForPiece(ctx, commpCalc.PieceCID, di) - require.NoError(t, err) - - // Verify that getting the size of the CAR file works correctly: - // There is no CAR size information in the deal info, so the piece - // directory should work it out from the index and piece data. - pm := NewPieceDirectory(cl, pr, 1) - pm.Start(ctx) - size, err := pm.GetCarSize(ctx, commpCalc.PieceCID) - require.NoError(t, err) - require.Equal(t, len(carBytes), int(size)) -} - func testFlaggingPieces(ctx context.Context, t *testing.T, cl *client.Store) { // Create a random CAR file carFilePath := CreateCarFile(t) diff --git a/piecedirectory/types/mocks/piecedirectory.go b/piecedirectory/types/mocks/piecedirectory.go index 70b61ff8e..60b1b25f8 100644 --- a/piecedirectory/types/mocks/piecedirectory.go +++ b/piecedirectory/types/mocks/piecedirectory.go @@ -424,20 +424,6 @@ func (mr *MockStoreMockRecorder) RemovePieceMetadata(arg0, arg1 interface{}) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemovePieceMetadata", reflect.TypeOf((*MockStore)(nil).RemovePieceMetadata), arg0, arg1) } -// SetCarSize mocks base method. -func (m *MockStore) SetCarSize(arg0 context.Context, arg1 cid.Cid, arg2 uint64) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetCarSize", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// SetCarSize indicates an expected call of SetCarSize. -func (mr *MockStoreMockRecorder) SetCarSize(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetCarSize", reflect.TypeOf((*MockStore)(nil).SetCarSize), arg0, arg1, arg2) -} - // UnflagPiece mocks base method. func (m *MockStore) UnflagPiece(arg0 context.Context, arg1 cid.Cid) error { m.ctrl.T.Helper() diff --git a/piecedirectory/types/types.go b/piecedirectory/types/types.go index 8221dd5d3..ddeda8abb 100644 --- a/piecedirectory/types/types.go +++ b/piecedirectory/types/types.go @@ -39,7 +39,6 @@ type Store interface { GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) ListPieces(ctx context.Context) ([]cid.Cid, error) GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.DealInfo, error) - SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error PiecesContainingMultihash(ctx context.Context, m multihash.Multihash) ([]cid.Cid, error) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err string) error RemoveDealForPiece(context.Context, cid.Cid, string) error From 681902e6197582f858666d23de5cd1ad8c7e34f6 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 3 May 2023 14:21:27 +0200 Subject: [PATCH 08/15] refactor: remove functionality to mark index as errored (not being used) --- api/api.go | 3 +- api/proxy_gen.go | 13 ------ build/openrpc/boost.json.gz | Bin 5365 -> 5321 bytes cmd/boostd/piecedir.go | 43 ------------------- documentation/en/api-v1-methods.md | 18 -------- extern/boostd-data/client/client.go | 5 --- extern/boostd-data/couchbase/db.go | 21 --------- extern/boostd-data/couchbase/service.go | 19 -------- extern/boostd-data/ldb/db.go | 20 --------- extern/boostd-data/ldb/service.go | 23 ---------- extern/boostd-data/model/model.go | 2 - extern/boostd-data/yugabyte/service.go | 21 ++------- gql/resolver_piece.go | 3 -- node/impl/boost.go | 4 -- piecedirectory/doctor.go | 10 ----- piecedirectory/piecedirectory.go | 18 -------- piecedirectory/types/mocks/piecedirectory.go | 14 ------ piecedirectory/types/types.go | 1 - 18 files changed, 4 insertions(+), 234 deletions(-) diff --git a/api/api.go b/api/api.go index 5877f52be..56e71b603 100644 --- a/api/api.go +++ b/api/api.go @@ -55,8 +55,7 @@ type Boost interface { BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) //perm:read // MethodGroup: PieceDirectory - PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error //perm:admin - PdMarkIndexErrored(ctx context.Context, piececid cid.Cid, err string) error //perm:admin + PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error //perm:admin // RuntimeSubsystems returns the subsystems that are enabled // in this instance. diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 1a82a57ec..83f0ce039 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -129,8 +129,6 @@ type BoostStruct struct { PdBuildIndexForPieceCid func(p0 context.Context, p1 cid.Cid) error `perm:"admin"` - PdMarkIndexErrored func(p0 context.Context, p1 cid.Cid, p2 string) error `perm:"admin"` - RuntimeSubsystems func(p0 context.Context) (lapi.MinerSubsystems, error) `perm:"read"` SectorsRefs func(p0 context.Context) (map[string][]lapi.SealedRef, error) `perm:"read"` @@ -777,17 +775,6 @@ func (s *BoostStub) PdBuildIndexForPieceCid(p0 context.Context, p1 cid.Cid) erro return ErrNotSupported } -func (s *BoostStruct) PdMarkIndexErrored(p0 context.Context, p1 cid.Cid, p2 string) error { - if s.Internal.PdMarkIndexErrored == nil { - return ErrNotSupported - } - return s.Internal.PdMarkIndexErrored(p0, p1, p2) -} - -func (s *BoostStub) PdMarkIndexErrored(p0 context.Context, p1 cid.Cid, p2 string) error { - return ErrNotSupported -} - func (s *BoostStruct) RuntimeSubsystems(p0 context.Context) (lapi.MinerSubsystems, error) { if s.Internal.RuntimeSubsystems == nil { return *new(lapi.MinerSubsystems), ErrNotSupported diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index 2c5e19aee0e25958fdc6c3bc1049ea72708beadf..966bf3abb61f6b165e9be759e9603f297470359e 100644 GIT binary patch delta 4970 zcmV-w6P4`sDak2+ABzY8000000RQYg`*WK(xBm*m{iW?t*RMEDX7-09efit8>l?TG zec!g3TNz|q8xRzUG;YTK`yC0ec}O5IPE#+toz}+E;Yd1f9Ua~4t_QL2nfj^T>-0M# z-Ov%9yZY(9P7$@CeyU%(G$7j5uRQpDZHbdi|}7V^LiSvEFTimISc=NsyzWS)YmJ(nOp!_m*F#qoN%Wynl_VG1Mm#au!MFtUJm!0~Lz;OX;j zuRj>-*9Pm(UkB`o?|RUuh_#pln?OUSD-W{f?(A=9QvG^u=sx@wARk)#DfI)$knwzI z0?MB7Uh8Ythy*reFOBd{uRwkb*`Fmth$$Nf&J6lFFdz714G|4!n%PkOSQoXp0QVi)bh$T2}3iD)mucD!A$d^<5H;E2;b<29{9F?sxK+^$mxq0 z3*Mw-y3UJv#D32`-({2jV#e6&#mshRFC0K9^j`oEz3_bZZuNf)h&v1Se=hr@p1xii zc^*EuUGs)e*N1N*t;WHrt}^~Lk=40C)>G|&$>;0YiWc%~3Sa@ZhL9?8X%^SV#-q_R zl@m6fTTHdF1u;c)Bp~t&WNAL5LBcTx8iDrwdDv!#@FH9%Ok>#!Ee&m;b+j>M{4sZZ z2U^-}r2*~FP%Ri5&@NmHn%a}^OJpuJge}DAoV~*dY>G_w>vIp7H_+O=g>lhS4PZ-u z1KNAmY6-})ljgg7{RNvhFkkg&KY$6r;5)IFqtyTmMszkHH_-9^cJID>FI36>?)M|iP=JPK(nH@BO|3L^Bs*zW!{JztvmUrYGEzx}4y z#qW`dl;ggFNU3fZ4$Llj zIVck%?*OSQK;8jTE<9$gYpY3*cYuf`M>w0cq(_KiD>?pYl7mrQZf2v0<+cTTQvw^- z4+8vaOM;W3L=v#hufT${%PUBz@2;jx;9C`?f((pILWE2FG?WZN!61Vu77`VIq(cF! zdzTauWh+^@VO#AM*a56<-HN#5Lty8*BYK#SSsOiQ+rKpJ54uMRwEy;E2a11tA)s^a zmB%eOgaFrJrqbRWZv87yY!wGj?O@O*>S=C@cS zt=EDeVM0^@-v_H-n|XU91?C-NL=mvj54fxD%0OyHePw_)s=@LIw@U1P5S7^Ar2-#Z zB0@PCRCVgaK$Mi)7|&g0m9~YguG-iO>MOR?XdXhrF?7q+Th!)(l^X?=;ma}LU(Rj; zvcb%T`lC^GA&$9U4qwj=PZaojf}jZrV?}^4!i$YZz5}~UVajx$vk3!insTWSMM0Eh zf%L_Nq9G0~)fNavJ-eKnjSs>9&T}L`LIr~GZ)*e$wOi>xDA74gwp4qIZNuWE1$rSED}(rWuPr39(UgKZ9y(>mQ=6e=JKUNmIfC zF#x+0e*#WZ)Sm#^F!`sEZR!6I>HnoH_koiF1o_EOAa@9C8FdGL0kS%uK8;^J)t)>f z4x32QnfUH4vY^j8a<`=1u!X=DQ#(d>b$c~Lc<&Ux8N3>icaOvWp9}tq|KE?H>VUg( z2(&jcbO+(Yx2B9(`ReEJE^Jypk@q@l|0zJ0eyTR9hJN8gHh?qAx^%m}=lsvdZugY` z`HMBN5yYH!n0M}f5@56Uram+O=VJVdeQv_gHbRIz73s9s9rO+TPv|pi*y5}j&3}lP zej4Y#8hSi$$!C{f2JLXxrWE+}wda~k2DaDj8TxB%r5{YG>w^WH_{fA4=tn>5-30Ut z*R}zLKEs6n5F(O!ypEqw(E@{zneO$EoT~bU{t8f_pXPIa&em%~ACo`ka~ol}9VrAj z@WMXH!A~Ic*}Ut${;R?7uKVw|Kkk0~{?~tR;k*A(@9pS}_@DlDZvOl3-T9ko{~dYl zzWsRTzrVcw{XcZB8=PPm)GODe&8^-T&)fi8Ovp|EU2b$UY1_$oLg?Uc&mRYKUY{Zdm~s0*i!UXkoId*%7F;<2zt>gu1!GL&N%_;2sATzvs(TfsW#e|0}pHv^dmDC3-sgEQj9Qg*tg^LMWw@46e zvMg)t)|A5&7j7-Pi=!NFLl<-?4PY;|THA{uvP63iqSSobbk zMW{c9td5qhGjx5_m6UBrt>-6v8^uZ+qHQt22IyN1cpo$_9YQORAB1pwl4ComMf$T@ zvK4`AfM`VxVD~|?67*YweGr1}OOvq+A%9QzyR;V@v=R()%P2n@O0p^?q_CrjRt&IN)+o2rc4mwW@-15& z3|kyYvH)^~&`j{N1Y%Q+p&SFjg9<|>(psM__Np#KsDHOC zaWE|LvS?E`K?jXRXiH%>#2Qrswe{IUI)s)*4u(Zeinfvyw9rs2xCCND?4ceKuFo72 z^jp?A6xJAZi#DyNkTwx@7J-p5!8S--&IyXa=j!Ay!glX8-v*c;bAY*(6_=x)BoW{} z@NXc!0N8~5oYUs`P9;$xgCdg@7Js0CMr}HAY>WpjOrT13O~yPm@c1NuEc@f^Q)`BR6$qQP@+w zMgEM0Cb7d?cEXEUA~X4n+!Sq`CxJI|=qZEkUN*){#w)mZq^!Nt0Tyx-RwP+KCXA{K z#4btl*?WX@r4rtQyZkeR(^U~ZA_o!*9B*3*;m(tD4K{zSvP$y|g;hgLbpHDntF&-f zC$&aHNe;$FYHs$6>Z^$)2#LP%<0g?rR8~K3gpcdMe(TC!jz*HypO8f_5uwRgQpL?F2@qKhc(T#O z5?~Aq&nrlv7bb)YUQsDITt1>bWrqt876kIet(1cLi74IafS0|Z0pT|Uo&(Zsb-^(i zV?+_4t{?Nsm`ow&HNL{{f4YIWgn~N<-$O#c0_K1CgD#;zfxJ)6Wg8^pDd7CeOtmQm zi_p}HKYhN}KZHw?dHpMa2-_C+3oig;?x6gN#ZR79-ssX@gjigDLV1Q1>1-LYL75QpLRVos>z_4)&D zVNzfL3W{4zZ9takB^F+GEwxwqtX)9@eSYOAU+^n9=ZhI>5^!D zE(q=pZ7y34^APye=Ft_k7iU+KWe`CYAAJe*oQYF$O%irX^;^2<6shUzEN=sq-8M3#qpJQ?N>hbwpd^&j5@x26tLU|G2(zW$jL#@QV zpBVZd^ErVu4i{ZQyr9eQ+xj{YQ0}sUn-JaZwEyH%U{ATh%vL6HBRJGJe?lM)CtL)+ zpQDRQxB%vA;=9x}T|2(>sjhdLqLfFj7zGoep+Ju&@ zuwq=H)(Sf9x<|Otq)HCPe^$J!G;cUt@}S&;(ANa|$hA1u^39r5 zTm5v@-TYTEeSIe_{fq8Xv~lz{j{fN5=t>r4S_&vlzCg&_>vPw4e*g{7jKk*?Ik3PC zJ&4mE$302ulTdA6LBi%{*Lns5Fb`7DK=9sXj){l|)Rr`8OB%E#4Vsfp{lZc!)7m8s zrm~U-TmrHABb{@!7-P!CPUgWt12fdQSOSE(7Bn<($RACBWkG9W1sIxlBzj&dwWb9{ z5vmHR%}|rFiDAsJf7`2!O3s?%9k6aAhJo)vuxe$_pOU&;%KT_g>7eEopG* zB@J3pb9p?J+*cKr96tx(dZIt zWc!#d0p&6Ve_}AR+1n8g#R|fN0&K$07u>{tduH6?{q}q#g&bHt^jnB7F3(-XvCg^C zhn0YGEUA|hW^R!yIIqZ5iH>4+a)9?alz`WtulW*BA+Vu!1?NRvm4I8oPX)j$u~m8$ zdr3JdKs7hx#cfrb;**!%ZkK=B=H&Id-DG0$@a56*IH+srary}VzUBn)0m>$3lJapo o&*~;U>??&Z#=VJT!vou^eR-m<*VpU+4*&rF|EpeN`HT4g04D8%N&o-= delta 5011 zcmV;E6Kw3sDfKCTABzY8000000RQYg`*YejxBrUA_m{TgQUZiLX7-0X`rzBN%MI;* z-?we%R@efxv1Qki)6fk6`yI(P_#xS{F=@(rx6^`=j*q1C*3r>}?s^dWo~fVe2c2GL zs2e)MGgm)-&?%xe)KB#*mj*<;9$#o@;U6;@a^#hYW z*D{@-d0owPF@^WEa}oY)Xx@PaEX#+4bk4$mpKH&+6m>TJ3=R6e>p#~XzThtu%{0;( zVG4Z=>@0qN+Hb#U7};UVFVu%L@Uiv+!|xfG&kgOh?|*RVB=9`fr_f?OR_w2+ll&#m zye`B0`4_zUg1>(Ksv9~990&MI{Z#+BfIifK4>jykZ3(Fc@KQ5f$AOpTWr%MmA=I-=wXj1)VW#~Tq79by5`YH7T$dK`T zXadTf@LsDM)`$c)WiO5JPOm_I4B4M0Lx?FG2hJ4wIWQmiV+|1vXzM14VfP39!;_=_ zaesJ!e6tx`)KYD5f@ka%A+=n(CSho%v3iT>BA9AEbX*EG8{u2s&;#GrPxS?*9yxvW zYR;QiY04q}4b$)m6s7CbBvg$a=1SJ^Oq!UD85+O#v+6)(}!9F3sZl)Oa+S zrgFmObBn1qvLL2th6F@@fh^5uG)OqcKqJtey$st-5uS(ZglQ~Wp{1cUw2n5Sj6Y_s z??6kNE;XS28L96N^an5@=zk~HVz}&s{*X@lq$%SV4sT7u(&xCg%$i|7!vN_qX5l zy7)a&k#aOp5GmCSgIxD=(Sk5lmMe09OQ2g@pp+{9R!Z!rlt4JIGs9=Qu}=)LA4;~gMk$q~+GE$I=W*h-Fnn&eVYzL=-ju+G z^@9Na+LGX8D3JuL^Cg&bcDaOv`tEYF0KQdGD#*adBt*Ex&qK)|6bv$mVj)p~K{^zm zx^qbpQMQtW8@AQ%fE~cv)~$#;J_L52JEDgPnYGq~w*5=fe!qL9K>KemwxIa87Xmux zUU}StQ`j|Mi9&UlhFa*R3qV)%+H)h=n)JoKo&ZsS>nCni&L^KA^xxrx9|JTP4iBwa@Wbzan~}lP0oKsE?;|v~zUhWO2}~1` zO#tpSTPXch{}Ob&eUmR6{|NQf&5fZW3L}m$4C8r;M-MT~n89LpUK!yq1kcx4t$&ME z(rP6L5++0i@O`lSwVC(eSb=#*7*Pal^aF0IyE2fPQC}ILjcTww!mSd2+ealf=qm8R z6(W?AK~<+t3`9w(jquD>R%sj9>Z*;spuS>Djpi{F97DHEy+v&vS-DX_8NM6?{^jfr zARA0=s6QE17vh-vW&icu;ei5wj}bH>VXO!cMtHvV$hTm3DNLE}bJk&ibyF@Cq9};6 zERepqP&CA$rP>0aXuLarSDk{nQ`4j8-@`4A4IkDCcIIN6HF-!Z1~*}_tne!hkflXl zBjF_BWbIJ^E@w&YmpmNy6y)JGG~GMsZ)@}?!8Bv>Cm}X!?q^VKZT)?;^^aufBxy>R zBL-l5;!nV7iuw~E8z%oWvMv4ZBmE!CavwM;K#-pd1#*YLmQiC@=)x%TV@ zao9wX&e(VFkOh6#k-H`3hAjlPnA$P2tJ|v~!aJw%_2AWryn7V>|5ET*{QqtYRR`RU zLZAmjLw68Pd~3>xm9Ksd@4}|#6ZxRC_MZY|>8EOwYUmd}WCJ*(tV_3h@RI-e*zKP3 zKYy_%HiDSb4)e}`T>@{hl2KCx?X>+SL!c#ZE789~#Ko@JB7wIuW|L6luX3*cvNVY{d zJU+C{FXKpm0_CU9&_jwD^w``l0K#l|494Mr(a<=U+K4PTp)!qT=wXI3`+j2Rf6QhC z(wN|`AfD4j{Ow`^&0Eh!jNA^AUyB&$weRQAzlM}EF$k!u8~TUyC57Y~nh@PFIdWp? zZvdgJ6@qsgT0g_s4X_D6x}))jQTTLKmh7Dlon=9P9vXBrRZOXw@^k^m>dDCR_Glg=0FSa4^xDJzkH2N*J49pOhw?3 zGe>T4Ap?|8-llI9uUr}WR--NLlzUxX&{c)7k+&+;63OnWFG=d7!Z1y4EfRW11;35H zsf?n3K$ld7kyK|Y*oswK)wiZKal!uax{ru()9OnQwoe!)s!}4W%&IN@#&&v$EzPW> zBDO0_=e^v@jEu}xq%#E5kx>dF8JN?$0x${7LIfgKdMebE)m$olVil{zTM2DlbuA+$ zE7}cOYkU){ zaGYh`M1{DoG#4jw-?$yZm-)H&>_yU%#Sx@mfu8k;j~+*mbJqceTDr>zlN` z$uH!aG~jQYZCN##xCSaFukaO0rv-muSL2|le#Hn|@E-ax>zvs(TfsW#e}gA1&Ui6RLy$NrCKp4{l-2 z0ASqpv6FlXDSsRm75g0{%QE{ZH{2-oZV{NgGJZ2CW`K71>w_f}F5B_$mB2E~Po30t>F5UsN; zYwXsP!xI;7ExU`O9BxAwbSMpAC$(DJiy^W^dnBo|Cx2R_$bo8zWRuXEBiJO=9g=GX z`IcB8g;>(XO%OZQjA}2*#$q8C$C>C4-u_5+Q4++<24hi}# zYwQba47){}))PpZh&qeF$e3UgBrfL!#o%*w@)u#dbDD1h%uhMM+{%ip;ejL(;63nf zA-w?Dg#4V-`uI*IQ6YmOlN1(zpnyhgI&tIZtbzxj-{GJ8WolLS+HGY?!Ehe!B!Xm$ z%4#!JR4xOk+jz0v^|avN48SwJR?stm*~ zN%Glygma}5-h=!6GlbJs5k4XZ5(*q|QwibTlX49(xqeBu8CUa;7GzrgYm6 z(Qqiq1wRQ{WaN`94j+Hq*K&kU>%e~N%3ckRB&k0ki(Vl@ld+_Vn^O`XvK;Vaql*Q= z7#5yakU%d?2p7DfQgXO_L_5k37a%MM zB0ya~=93YbK+J1=h2Q^l19JfdcMiUXgn&8B|L_N0Kz|H*pPFSiNXApZ`Inh$6AI>` zsTF_v{Gfjbmn8H0OMwX66!r@*0Auc;{EEd-o>kuH%AJQ;T)Ff7dlW#dZdNy|%il$d+&mIp(q7 zD-4K4Qmx8`X;m*+^Hpz){B?qklw)1 zufcbojg%-eeRc4H$mhQxD2qp~gt z?hdUlTMhFN_|@jo6}1;<*OO!rK^C8U3G|$aQ*liac1!hJx@Q$xW21O?le-Wr5!NP4 z%}Vb36(~5795UPZz}u7e5FvkV2|R`23L*|FebizPU}?L+YId)Kxk60R*<98!z}Mir z%tK$>6Ol8PPco^RQX)!AVfq=ex3HgMad-Ljd@ep6yz2N)fYA<{-}zM6J55o_6IYCa2~l$H(oRh90ULOzlW10U z_fQ^gfz>YpUc=1KjAAD{gCbOOGcIYQdVgx1aNTYU?&iYNn|0`G41MHU9BcVzO{%Sa zI_$3htC+sNm6rZR_bJ*qdK*W7@^N$}i!v<*lqO#wWFGXH>pOpdhG)j%ONty=V219+ z>5roWN$HbNZC^ve=4RV^1_CgTQqVx~-e!)8hzHb`G-yj2v?UFilTE$CQY(|%B@HIB zk_KD?vH2sNb2J}e%EeCRL0ns+FAUMjUF1w|36 z3aa%`ld_3n%&>pksfdYR2$Pw^JXpr?psUgV?+Zf9bidw~sz`hjo&88+DA zDEhqy266by4x&{ZS4aKAYB-ba>m5u=>m6`qB0ANtq5#SZ9@NkT8H9IU%&#qJu<=Xs zg~rx-fNX1oEqMRN_1CGTl{i2Gq@;LA0XIkhZp#>QJf9kmv?WG+qv*o0dh{Iytaq$vPR>m6CNtV-Sm`JX#?%qR|D^$mTI!0LrZ$h{4om zZ$~&Za1bUGU=wz};3hJEGUFC!{^T1eWaH~$1cm70ivp^I_;LZg{Yr&9k(AO1b4bZG zoK$3oQk$}Uery$+Ep@hjC&Jl d&-;}%a@_6dtJTfw{{sL3|Nky9$(R-X001$fja~o% diff --git a/cmd/boostd/piecedir.go b/cmd/boostd/piecedir.go index da8f04628..54436b0aa 100644 --- a/cmd/boostd/piecedir.go +++ b/cmd/boostd/piecedir.go @@ -16,7 +16,6 @@ var pieceDirCmd = &cli.Command{ Usage: "Manage Local Index Directory", Subcommands: []*cli.Command{ pdIndexGenerate, - pdIndexMarkErroredCmd, }, } @@ -56,45 +55,3 @@ var pdIndexGenerate = &cli.Command{ return nil }, } - -var pdIndexMarkErroredCmd = &cli.Command{ - Name: "mark-index", - Usage: "Mark an index errored for a given piece in the local index directory", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "piece-cid", - Usage: "piece-cid of the index that will be marked as errored", - Required: true, - }, - &cli.StringFlag{ - Name: "error", - Usage: "error message", - Required: true, - }, - }, - Action: func(cctx *cli.Context) error { - ctx := lcli.ReqContext(cctx) - - // parse piececid - piececid, err := cid.Decode(cctx.String("piece-cid")) - if err != nil { - return err - } - - boostApi, ncloser, err := bcli.GetBoostAPI(cctx) - if err != nil { - return fmt.Errorf("getting boost api: %w", err) - } - defer ncloser() - - errMsg := cctx.String("error") - err = boostApi.PdMarkIndexErrored(ctx, piececid, errMsg) - if err != nil { - return err - } - - fmt.Printf("Marked %s as errored with \"%s\"\n", piececid, errMsg) - - return nil - }, -} diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md index 26fdba8d5..8c9d0c610 100644 --- a/documentation/en/api-v1-methods.md +++ b/documentation/en/api-v1-methods.md @@ -85,7 +85,6 @@ * [OnlineBackup](#onlinebackup) * [Pd](#pd) * [PdBuildIndexForPieceCid](#pdbuildindexforpiececid) - * [PdMarkIndexErrored](#pdmarkindexerrored) * [Runtime](#runtime) * [RuntimeSubsystems](#runtimesubsystems) * [Sectors](#sectors) @@ -1765,23 +1764,6 @@ Inputs: Response: `{}` -### PdMarkIndexErrored - - -Perms: admin - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "string value" -] -``` - -Response: `{}` - ## Runtime diff --git a/extern/boostd-data/client/client.go b/extern/boostd-data/client/client.go index fee11386c..98fcb8609 100644 --- a/extern/boostd-data/client/client.go +++ b/extern/boostd-data/client/client.go @@ -26,7 +26,6 @@ type Store struct { ListPieces func(ctx context.Context) ([]cid.Cid, error) GetPieceMetadata func(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) GetPieceDeals func(context.Context, cid.Cid) ([]model.DealInfo, error) - MarkIndexErrored func(context.Context, cid.Cid, string) error IndexedAt func(context.Context, cid.Cid) (time.Time, error) PiecesContainingMultihash func(context.Context, mh.Multihash) ([]cid.Cid, error) RemoveDealForPiece func(context.Context, cid.Cid, string) error @@ -106,10 +105,6 @@ func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ( return s.client.PiecesContainingMultihash(ctx, m) } -func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err string) error { - return s.client.MarkIndexErrored(ctx, pieceCid, err) -} - func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { return s.client.AddDealForPiece(ctx, pieceCid, dealInfo) } diff --git a/extern/boostd-data/couchbase/db.go b/extern/boostd-data/couchbase/db.go index e679b2869..bc80f118a 100644 --- a/extern/boostd-data/couchbase/db.go +++ b/extern/boostd-data/couchbase/db.go @@ -337,25 +337,6 @@ func (db *DB) setPieceCidsForMultihash(ctx context.Context, mh multihash.Multiha return nil } -func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr error) error { - ctx, span := tracing.Tracer.Start(ctx, "db.mark_piece_index_errored") - defer span.End() - - return db.mutatePieceMetadata(ctx, pieceCid, "mark-index-errored", func(metadata CouchbaseMetadata) *CouchbaseMetadata { - // If the error was already set, don't overwrite it - if metadata.Error != "" { - // If the error state has already been set, don't over-write the existing error - return nil - } - - // Set the error state - metadata.Error = idxErr.Error() - metadata.ErrorType = fmt.Sprintf("%T", idxErr) - - return &metadata - }) -} - func (db *DB) MarkIndexingComplete(ctx context.Context, pieceCid cid.Cid, blockCount int, isCompleteIndex bool) error { ctx, span := tracing.Tracer.Start(ctx, "db.mark_indexing_complete") defer span.End() @@ -365,8 +346,6 @@ func (db *DB) MarkIndexingComplete(ctx context.Context, pieceCid cid.Cid, blockC metadata.IndexedAt = time.Now() metadata.CompleteIndex = isCompleteIndex metadata.BlockCount = blockCount - metadata.Error = "" - metadata.ErrorType = "" if metadata.Deals == nil { metadata.Deals = []model.DealInfo{} } diff --git a/extern/boostd-data/couchbase/service.go b/extern/boostd-data/couchbase/service.go index 1c9625a01..38c9d637b 100644 --- a/extern/boostd-data/couchbase/service.go +++ b/extern/boostd-data/couchbase/service.go @@ -2,7 +2,6 @@ package couchbase import ( "context" - "errors" "fmt" "time" @@ -56,24 +55,6 @@ func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo return s.db.AddDealForPiece(ctx, pieceCid, dealInfo) } -func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr string) error { - log.Debugw("handle.mark-piece-index-errored", "piece-cid", pieceCid, "err", idxErr) - - ctx, span := tracing.Tracer.Start(ctx, "store.mark-piece-index-errored") - defer span.End() - - defer func(now time.Time) { - log.Debugw("handled.mark-piece-index-errored", "took", time.Since(now).String()) - }(time.Now()) - - err := s.db.MarkIndexErrored(ctx, pieceCid, errors.New(idxErr)) - if err != nil { - return normalizePieceCidError(pieceCid, err) - } - - return s.FlagPiece(ctx, pieceCid) -} - func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) { log.Debugw("handle.get-offset-size", "piece-cid", pieceCid) diff --git a/extern/boostd-data/ldb/db.go b/extern/boostd-data/ldb/db.go index bd8fb3277..2ca08ad41 100644 --- a/extern/boostd-data/ldb/db.go +++ b/extern/boostd-data/ldb/db.go @@ -290,26 +290,6 @@ func (db *DB) GetPieceCidToMetadata(ctx context.Context, pieceCid cid.Cid) (Leve return metadata, nil } -func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, sourceErr error) error { - ctx, span := tracing.Tracer.Start(ctx, "db.mark_piece_index_errored") - defer span.End() - - md, err := db.GetPieceCidToMetadata(ctx, pieceCid) - if err != nil { - return fmt.Errorf("getting piece metadata for piece %s: %w", pieceCid, err) - } - - if md.Error != "" { - // If the error state has already been set, don't over-write the existing error - return nil - } - - md.Error = sourceErr.Error() - md.ErrorType = fmt.Sprintf("%T", sourceErr) - - return db.SetPieceCidToMetadata(ctx, pieceCid, md) -} - // AllRecords func (db *DB) AllRecords(ctx context.Context, cursor uint64) ([]model.Record, error) { ctx, span := tracing.Tracer.Start(ctx, "db.all_records") diff --git a/extern/boostd-data/ldb/service.go b/extern/boostd-data/ldb/service.go index e93e9c8b1..f41abe3a4 100644 --- a/extern/boostd-data/ldb/service.go +++ b/extern/boostd-data/ldb/service.go @@ -121,27 +121,6 @@ func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo return nil } -func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr string) error { - log.Debugw("handle.mark-piece-index-errored", "piece-cid", pieceCid, "err", idxErr) - - ctx, span := tracing.Tracer.Start(ctx, "store.mark-piece-index-errored") - defer span.End() - - defer func(now time.Time) { - log.Debugw("handled.mark-piece-index-errored", "took", time.Since(now).String()) - }(time.Now()) - - s.Lock() - defer s.Unlock() - - err := s.db.MarkIndexErrored(ctx, pieceCid, errors.New(idxErr)) - if err != nil { - return normalizePieceCidError(pieceCid, err) - } - - return s.FlagPiece(ctx, pieceCid) -} - func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) { log.Debugw("handle.get-offset-size", "piece-cid", pieceCid) @@ -337,8 +316,6 @@ func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model. md.Cursor = cursor md.IndexedAt = time.Now() md.CompleteIndex = isCompleteIndex - md.Error = "" - md.ErrorType = "" err = s.db.SetPieceCidToMetadata(ctx, pieceCid, md) if err != nil { diff --git a/extern/boostd-data/model/model.go b/extern/boostd-data/model/model.go index fa919b3a4..7ab0004a2 100644 --- a/extern/boostd-data/model/model.go +++ b/extern/boostd-data/model/model.go @@ -43,8 +43,6 @@ type Metadata struct { // offsets). CompleteIndex bool `json:"c"` Deals []DealInfo `json:"d"` - Error string `json:"e"` - ErrorType string `json:"t"` } // Record is the information stored in the index for each block in a piece diff --git a/extern/boostd-data/yugabyte/service.go b/extern/boostd-data/yugabyte/service.go index 4c6def54b..fd857d3a6 100644 --- a/extern/boostd-data/yugabyte/service.go +++ b/extern/boostd-data/yugabyte/service.go @@ -103,20 +103,6 @@ func (s *Store) createPieceMetadata(ctx context.Context, pieceCid cid.Cid) error return nil } -// TODO: Do we need this? -func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr string) error { - ctx, span := tracing.Tracer.Start(ctx, "store.mark-piece-index-errored") - defer span.End() - - //err := s.db.MarkIndexErrored(ctx, pieceCid, errors.New(idxErr)) - //if err != nil { - // return normalizePieceCidError(pieceCid, err) - //} - // - //return s.FlagPiece(ctx, pieceCid) - return nil -} - func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) { ctx, span := tracing.Tracer.Start(ctx, "store.get_offset_size") defer span.End() @@ -155,10 +141,9 @@ func (s *Store) getPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.M // Get piece metadata var md model.Metadata - qry := `SELECT Version, IndexedAt, CompleteIndex, Error, ErrorType ` + - `FROM idx.PieceMetadata WHERE PieceCid = ?` + qry := `SELECT Version, IndexedAt, CompleteIndex FROM idx.PieceMetadata WHERE PieceCid = ?` err := s.session.Query(qry, pieceCid.String()).WithContext(ctx). - Scan(&md.Version, &md.IndexedAt, &md.CompleteIndex, &md.Error, &md.ErrorType) + Scan(&md.Version, &md.IndexedAt, &md.CompleteIndex) if err != nil { err = normalizePieceCidError(pieceCid, err) return md, fmt.Errorf("getting piece metadata: %w", err) @@ -293,7 +278,7 @@ func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, recs []model.Rec // Mark indexing as complete for the piece qry := `UPDATE idx.PieceMetadata ` + - `SET IndexedAt = ?, CompleteIndex = ?, Error = '', ErrorType = '' ` + + `SET IndexedAt = ?, CompleteIndex = ? ` + `WHERE PieceCid = ?` err = s.session.Query(qry, time.Now(), isCompleteIndex, pieceCid.String()).WithContext(ctx).Exec() if err != nil { diff --git a/gql/resolver_piece.go b/gql/resolver_piece.go index 6842bcf89..65fc7a569 100644 --- a/gql/resolver_piece.go +++ b/gql/resolver_piece.go @@ -362,9 +362,6 @@ func (r *resolver) getIndexStatus(ctx context.Context, pieceCid cid.Cid, md pdty idxerr = mdErr.Error() case md.Indexing: idxst = IndexStatusIndexing - case md.Error != "": - idxst = IndexStatusFailed - idxerr = md.Error case md.IndexedAt.IsZero(): idxst = IndexStatusRegistered default: diff --git a/node/impl/boost.go b/node/impl/boost.go index 5b2886f31..fb2d24d57 100644 --- a/node/impl/boost.go +++ b/node/impl/boost.go @@ -543,10 +543,6 @@ func (sm *BoostAPI) PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Ci return sm.Pd.BuildIndexForPiece(ctx, piececid) } -func (sm *BoostAPI) PdMarkIndexErrored(ctx context.Context, piececid cid.Cid, err string) error { - return sm.Pd.MarkIndexErrored(ctx, piececid, err) -} - func (sm *BoostAPI) OnlineBackup(ctx context.Context, dstDir string) error { return sm.Bkp.Backup(ctx, dstDir) } diff --git a/piecedirectory/doctor.go b/piecedirectory/doctor.go index 805b5359f..ec6cfe77e 100644 --- a/piecedirectory/doctor.go +++ b/piecedirectory/doctor.go @@ -86,16 +86,6 @@ func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { return fmt.Errorf("failed to get piece %s from local index directory: %w", pieceCid, err) } - // Check if the piece is in an error state - if md.Error != "" { - err = d.store.FlagPiece(ctx, pieceCid) - if err != nil { - return fmt.Errorf("failed to flag piece in error state %s: %w", pieceCid, err) - } - doclog.Debugw("piece is in error state", "err", md.Error) - return nil - } - // Check if piece has been indexed isIndexed, err := d.store.IsIndexed(ctx, pieceCid) if err != nil { diff --git a/piecedirectory/piecedirectory.go b/piecedirectory/piecedirectory.go index 3e2c02fe5..1e41cab14 100644 --- a/piecedirectory/piecedirectory.go +++ b/piecedirectory/piecedirectory.go @@ -269,10 +269,6 @@ func (ps *PieceDirectory) RemoveDealForPiece(ctx context.Context, pieceCid cid.C return nil } -func (ps *PieceDirectory) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err string) error { - return ps.store.MarkIndexErrored(ctx, pieceCid, err) -} - //func (ps *piecedirectory) deleteIndexForPiece(pieceCid cid.Cid) interface{} { // TODO: Maybe mark for GC instead of deleting immediately @@ -507,20 +503,6 @@ func (ps *PieceDirectory) GetBlockstore(ctx context.Context, pieceCid cid.Cid) ( return bs, nil } -// countReader just counts the number of bytes read -type countReader struct { - r *bufio.Reader - count uint64 -} - -func (c *countReader) ReadByte() (byte, error) { - b, err := c.r.ReadByte() - if err == nil { - c.count++ - } - return b, err -} - type SectorAccessorAsPieceReader struct { dagstore.SectorAccessor } diff --git a/piecedirectory/types/mocks/piecedirectory.go b/piecedirectory/types/mocks/piecedirectory.go index 60b1b25f8..d44e2031d 100644 --- a/piecedirectory/types/mocks/piecedirectory.go +++ b/piecedirectory/types/mocks/piecedirectory.go @@ -338,20 +338,6 @@ func (mr *MockStoreMockRecorder) ListPieces(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPieces", reflect.TypeOf((*MockStore)(nil).ListPieces), arg0) } -// MarkIndexErrored mocks base method. -func (m *MockStore) MarkIndexErrored(arg0 context.Context, arg1 cid.Cid, arg2 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MarkIndexErrored", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// MarkIndexErrored indicates an expected call of MarkIndexErrored. -func (mr *MockStoreMockRecorder) MarkIndexErrored(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkIndexErrored", reflect.TypeOf((*MockStore)(nil).MarkIndexErrored), arg0, arg1, arg2) -} - // NextPiecesToCheck mocks base method. func (m *MockStore) NextPiecesToCheck(arg0 context.Context) ([]cid.Cid, error) { m.ctrl.T.Helper() diff --git a/piecedirectory/types/types.go b/piecedirectory/types/types.go index ddeda8abb..8ad5dae8a 100644 --- a/piecedirectory/types/types.go +++ b/piecedirectory/types/types.go @@ -40,7 +40,6 @@ type Store interface { ListPieces(ctx context.Context) ([]cid.Cid, error) GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.DealInfo, error) PiecesContainingMultihash(ctx context.Context, m multihash.Multihash) ([]cid.Cid, error) - MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err string) error RemoveDealForPiece(context.Context, cid.Cid, string) error RemovePieceMetadata(context.Context, cid.Cid) error RemoveIndexes(context.Context, cid.Cid) error From 162c94b0718e308d65b38fc35331014e7d9decb7 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 4 May 2023 11:07:32 +0200 Subject: [PATCH 09/15] feat: implement delete commands --- .../svc/setup_yugabyte_test_util.go | 17 ++ extern/boostd-data/svc/svc_test.go | 83 ++++---- extern/boostd-data/yugabyte/create.cql | 10 +- extern/boostd-data/yugabyte/encode.go | 17 -- extern/boostd-data/yugabyte/service.go | 199 ++++++++++++------ piecedirectory/doctor_test.go | 14 +- piecedirectory/piecedirectory_test.go | 27 +-- 7 files changed, 217 insertions(+), 150 deletions(-) diff --git a/extern/boostd-data/svc/setup_yugabyte_test_util.go b/extern/boostd-data/svc/setup_yugabyte_test_util.go index dd64b353c..3411d893f 100644 --- a/extern/boostd-data/svc/setup_yugabyte_test_util.go +++ b/extern/boostd-data/svc/setup_yugabyte_test_util.go @@ -6,6 +6,7 @@ import ( "github.com/docker/docker/api/types/container" dockercl "github.com/docker/docker/client" "github.com/docker/go-connections/nat" + "github.com/filecoin-project/boostd-data/yugabyte" "github.com/stretchr/testify/require" "github.com/yugabyte/gocql" "golang.org/x/net/context" @@ -66,6 +67,22 @@ func SetupYugabyte(t *testing.T) { tlog.Info("wait for yugabyte start...") awaitYugabyteUp(t, time.Minute) tlog.Info("yugabyte started") + + store := yugabyte.NewStore(yugabyte.DBSettings{ + Hosts: []string{"127.0.0.1"}, + ConnectString: "postgresql://postgres:postgres@localhost", + }) + err = store.Start(ctx) + require.NoError(t, err) + + RecreateTables(ctx, t, store) +} + +func RecreateTables(ctx context.Context, t *testing.T, store *yugabyte.Store) { + err := store.Drop(ctx) + require.NoError(t, err) + err = store.Create(ctx) + require.NoError(t, err) } func awaitYugabyteUp(t *testing.T, duration time.Duration) { diff --git a/extern/boostd-data/svc/svc_test.go b/extern/boostd-data/svc/svc_test.go index 59843be08..b5280e2e5 100644 --- a/extern/boostd-data/svc/svc_test.go +++ b/extern/boostd-data/svc/svc_test.go @@ -60,10 +60,6 @@ func TestService(t *testing.T) { bdsvc, err := NewLevelDB("") require.NoError(t, err) - addr := "localhost:8042" - err = bdsvc.Start(ctx, addr) - require.NoError(t, err) - testService(ctx, t, bdsvc, "localhost:8042") }) @@ -79,9 +75,6 @@ func TestService(t *testing.T) { bdsvc := NewCouchbase(testCouchSettings) addr := "localhost:8043" - err := bdsvc.Start(ctx, addr) - require.NoError(t, err) - testService(ctx, t, bdsvc, addr) }) @@ -96,22 +89,16 @@ func TestService(t *testing.T) { bdsvc := NewYugabyte(testYugaSettings) addr := "localhost:8044" - err := bdsvc.Start(ctx, addr) - require.NoError(t, err) - - ybstore := bdsvc.Impl.(*yugabyte.Store) - err = ybstore.Drop(ctx) - require.NoError(t, err) - err = ybstore.Create(ctx) - require.NoError(t, err) - testService(ctx, t, bdsvc, addr) }) } func testService(ctx context.Context, t *testing.T, bdsvc *Service, addr string) { + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + cl := client.NewStore() - err := cl.Dial(context.Background(), fmt.Sprintf("http://%s", addr)) + err = cl.Dial(context.Background(), fmt.Sprintf("http://%s", addr)) require.NoError(t, err) defer cl.Close(ctx) @@ -244,12 +231,6 @@ func TestServiceFuzz(t *testing.T) { err := bdsvc.Start(ctx, addr) require.NoError(t, err) - ybstore := bdsvc.Impl.(*yugabyte.Store) - err = ybstore.Drop(ctx) - require.NoError(t, err) - err = ybstore.Create(ctx) - require.NoError(t, err) - testServiceFuzz(ctx, t, addr) }) } @@ -461,6 +442,7 @@ func TestCleanup(t *testing.T) { require.NoError(t, err) testCleanup(ctx, t, bdsvc, "localhost:8042") }) + t.Run("couchbase", func(t *testing.T) { // TODO: Unskip this test once the couchbase instance can be created // from a docker container in CI as part of the test @@ -469,6 +451,18 @@ func TestCleanup(t *testing.T) { bdsvc := NewCouchbase(testCouchSettings) testCleanup(ctx, t, bdsvc, "localhost:8043") }) + + t.Run("yugabyte", func(t *testing.T) { + // Running yugabyte tests may require download the docker container + // so set a high timeout + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + SetupYugabyte(t) + + bdsvc := NewYugabyte(testYugaSettings) + testCleanup(ctx, t, bdsvc, "localhost:8044") + }) } func testCleanup(ctx context.Context, t *testing.T, bdsvc *Service, addr string) { @@ -491,29 +485,41 @@ func testCleanup(ctx context.Context, t *testing.T, bdsvc *Service, addr string) records, err := getRecords(subject) require.NoError(t, err) - randomuuid, err := uuid.Parse("4d8f5ce6-dbfd-40dc-8b03-29308e97357b") - require.NoError(t, err) - err = cl.AddIndex(ctx, pieceCid, records, true) require.NoError(t, err) di := model.DealInfo{ - DealUuid: randomuuid.String(), + DealUuid: uuid.NewString(), SectorID: abi.SectorNumber(1), PieceOffset: 1, PieceLength: 2, CarLength: 3, } + di2 := model.DealInfo{ + DealUuid: uuid.NewString(), + SectorID: abi.SectorNumber(10), + PieceOffset: 11, + PieceLength: 12, + CarLength: 13, + } - // Add a deal for the piece + // Add two deals for the piece err = cl.AddDealForPiece(ctx, pieceCid, di) require.NoError(t, err) + err = cl.AddDealForPiece(ctx, pieceCid, di2) + require.NoError(t, err) - // There should only be one deal dis, err := cl.GetPieceDeals(ctx, pieceCid) require.NoError(t, err) + require.Len(t, dis, 2) + + // Remove one deal for the piece + err = cl.RemoveDealForPiece(ctx, pieceCid, di.DealUuid) + require.NoError(t, err) + + dis, err = cl.GetPieceDeals(ctx, pieceCid) + require.NoError(t, err) require.Len(t, dis, 1) - require.Equal(t, di, dis[0]) b, err := hex.DecodeString("1220ff63d7689e2d9567d1a90a7a68425f430137142e1fbc28fe4780b9ee8a5ef842") require.NoError(t, err) @@ -524,7 +530,6 @@ func testCleanup(ctx context.Context, t *testing.T, bdsvc *Service, addr string) offset, err := cl.GetOffsetSize(ctx, pieceCid, mhash) require.NoError(t, err) require.EqualValues(t, 3039040395, offset.Offset) - require.EqualValues(t, 0, offset.Size) pcids, err := cl.PiecesContainingMultihash(ctx, mhash) require.NoError(t, err) @@ -535,18 +540,10 @@ func testCleanup(ctx context.Context, t *testing.T, bdsvc *Service, addr string) require.NoError(t, err) require.True(t, indexed) - recs, err := cl.GetRecords(ctx, pieceCid) - require.NoError(t, err) - require.Equal(t, len(records), len(recs)) - - loadedSubject, err := cl.GetIndex(ctx, pieceCid) - require.NoError(t, err) - - ok, err := compareIndices(subject, loadedSubject) - require.NoError(t, err) - require.True(t, ok) - - err = cl.RemoveDealForPiece(ctx, pieceCid, di.DealUuid) + // Remove the other deal for the piece. + // After this call there are no deals left, so it should also cause the + // piece metadata and indexes to be removed. + err = cl.RemoveDealForPiece(ctx, pieceCid, di2.DealUuid) require.NoError(t, err) _, err = cl.GetPieceDeals(ctx, pieceCid) diff --git a/extern/boostd-data/yugabyte/create.cql b/extern/boostd-data/yugabyte/create.cql index f0243039c..a2a32fec1 100644 --- a/extern/boostd-data/yugabyte/create.cql +++ b/extern/boostd-data/yugabyte/create.cql @@ -1,8 +1,14 @@ create keyspace if not exists idx with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; CREATE TABLE idx.PayloadToPieces ( - PayloadMultihash BLOB PRIMARY KEY, - PieceCids BLOB + PayloadMultihash BLOB, + PieceCid BLOB, + -- PayloadMultihash is the partition key: the key that determines which + -- node the row is sent to. + -- PieceCid is the clustering key: the data is sorted by this key. + -- We want both of these to be in the primary key because + -- the piece cids should be a unique set. + PRIMARY KEY (PayloadMultihash, PieceCid) ); CREATE TABLE idx.PieceBlockOffsetSize ( diff --git a/extern/boostd-data/yugabyte/encode.go b/extern/boostd-data/yugabyte/encode.go index 71da9de7d..33cee6084 100644 --- a/extern/boostd-data/yugabyte/encode.go +++ b/extern/boostd-data/yugabyte/encode.go @@ -1,26 +1,9 @@ package yugabyte import ( - "fmt" - "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" ) -func bytesToCids(bz []byte) ([]cid.Cid, error) { - var bytesIdx int - var pieceCids []cid.Cid - for bytesIdx < len(bz) { - readCount, pcid, err := cid.CidFromBytes(bz[bytesIdx:]) - if err != nil { - return nil, fmt.Errorf("parsing bytes to cid: %w", err) - } - - bytesIdx += readCount - pieceCids = append(pieceCids, pcid) - } - return pieceCids, nil -} - // Probability of a collision in two 24 byte hashes (birthday problem): // 2^(24*8/2) = 8 x 10^28 const multihashLimitBytes = 24 diff --git a/extern/boostd-data/yugabyte/service.go b/extern/boostd-data/yugabyte/service.go index fd857d3a6..1c996ced4 100644 --- a/extern/boostd-data/yugabyte/service.go +++ b/extern/boostd-data/yugabyte/service.go @@ -15,6 +15,7 @@ import ( "github.com/yugabyte/gocql" "github.com/yugabyte/pgx/v4/pgxpool" "golang.org/x/sync/errgroup" + "sync" "time" ) @@ -34,10 +35,11 @@ type DBSettings struct { } type Store struct { - settings DBSettings - cluster *gocql.ClusterConfig - session *gocql.Session - db *pgxpool.Pool + settings DBSettings + cluster *gocql.ClusterConfig + session *gocql.Session + db *pgxpool.Pool + startOnce sync.Once } var _ types.ServiceImpl = (*Store)(nil) @@ -54,20 +56,25 @@ func NewStore(settings DBSettings) *Store { } } -func (s *Store) Start(ctx context.Context) error { - session, err := s.cluster.CreateSession() - if err != nil { - return fmt.Errorf("creating yugabyte cluster: %w", err) - } - s.session = session +func (s *Store) Start(_ context.Context) error { + var startErr error + s.startOnce.Do(func() { + session, err := s.cluster.CreateSession() + if err != nil { + startErr = fmt.Errorf("creating yugabyte cluster: %w", err) + return + } + s.session = session - db, err := pgxpool.Connect(context.Background(), s.settings.ConnectString) - if err != nil { - return fmt.Errorf("connecting to database: %w", err) - } - s.db = db + db, err := pgxpool.Connect(context.Background(), s.settings.ConnectString) + if err != nil { + startErr = fmt.Errorf("connecting to database: %w", err) + return + } + s.db = db + }) - return nil + return startErr } func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { @@ -197,15 +204,28 @@ func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ( ctx, span := tracing.Tracer.Start(ctx, "store.pieces_containing_multihash") defer span.End() + // Get all piece cids referred to by the multihash + pcids := make([]cid.Cid, 0, 1) var bz []byte - qry := `SELECT PieceCids FROM idx.PayloadToPieces WHERE PayloadMultihash = ?` - err := s.session.Query(qry, trimMultihash(m)).WithContext(ctx).Scan(&bz) - if err != nil { - err = normalizeMultihashError(m, err) - return nil, fmt.Errorf("getting pieces containing multihash: %w", err) + qry := `SELECT PieceCid FROM idx.PayloadToPieces WHERE PayloadMultihash = ?` + iter := s.session.Query(qry, trimMultihash(m)).WithContext(ctx).Iter() + for iter.Scan(&bz) { + pcid, err := cid.Parse(bz) + if err != nil { + return nil, fmt.Errorf("parsing piece cid: %w", err) + } + pcids = append(pcids, pcid) + } + if err := iter.Close(); err != nil { + return nil, fmt.Errorf("getting pieces containing multihash %s: %w", m, err) + } + + // No pieces found for multihash, return a "not found" error + if len(pcids) == 0 { + return nil, normalizeMultihashError(m, types.ErrNotFound) } - return bytesToCids(bz) + return pcids, nil } func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) ([]model.Record, error) { @@ -292,37 +312,15 @@ func (s *Store) addMultihashesToPieces(ctx context.Context, pieceCid cid.Cid, re ctx, span := tracing.Tracer.Start(ctx, "store.add_index.payloadpiece") defer span.End() - queue := make(chan []byte, len(recs)) - for _, rec := range recs { - queue <- rec.Cid.Hash() - } - close(queue) - - var eg errgroup.Group - for i := 0; i < s.settings.PayloadPiecesParallelism; i++ { - eg.Go(func() error { - for ctx.Err() == nil { - select { - case <-ctx.Done(): - return ctx.Err() - case multihashBytes, ok := <-queue: - if !ok { - // Finished adding all the queued items, exit the thread - return nil - } - - q := `INSERT INTO idx.PayloadToPieces (PayloadMultihash, PieceCids) VALUES (?, ?)` - err := s.session.Query(q, trimMultihash(multihashBytes), pieceCid.Bytes()).Exec() - if err != nil { - return fmt.Errorf("inserting into PayloadToPieces: %w", err) - } - } - } - - return ctx.Err() - }) - } - return eg.Wait() + return s.execParallel(ctx, recs, s.settings.PayloadPiecesParallelism, func(rec model.Record) error { + multihashBytes := rec.Cid.Hash() + q := `INSERT INTO idx.PayloadToPieces (PayloadMultihash, PieceCid) VALUES (?, ?)` + err := s.session.Query(q, trimMultihash(multihashBytes), pieceCid.Bytes()).Exec() + if err != nil { + return fmt.Errorf("inserting into PayloadToPieces: %w", err) + } + return nil + }) } func (s *Store) addPieceInfos(ctx context.Context, pieceCid cid.Cid, recs []model.Record) error { @@ -423,29 +421,112 @@ func (s *Store) ListPieces(ctx context.Context) ([]cid.Cid, error) { return pcids, nil } -// RemoveDealForPiece removes Single deal for pieceCID. If []Deals is empty then Metadata is removed as well +// RemoveDealForPiece removes Single deal for pieceCID. func (s *Store) RemoveDealForPiece(ctx context.Context, pieceCid cid.Cid, dealId string) error { ctx, span := tracing.Tracer.Start(ctx, "store.remove_deal_for_piece") defer span.End() + qry := `DELETE FROM idx.PieceDeal WHERE DealUuid = ?` + err := s.session.Query(qry, dealId).WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("removing deal %s for piece %s: %w", dealId, pieceCid, err) + } + + dls, err := s.GetPieceDeals(ctx, pieceCid) + if err != nil { + return fmt.Errorf("getting remaining deals in remove deal for piece %s: %w", pieceCid, err) + } + + if len(dls) > 0 { + return nil + } + + err = s.RemoveIndexes(ctx, pieceCid) + if err != nil { + return fmt.Errorf("removing deal: %w", err) + } + + err = s.RemovePieceMetadata(ctx, pieceCid) + if err != nil { + return fmt.Errorf("removing deal: %w", err) + } + return nil } -// RemovePieceMetadata removes all Metadata for pieceCID. To be used manually in case of failure -// in RemoveDealForPiece +// RemovePieceMetadata removes all Metadata for pieceCID. func (s *Store) RemovePieceMetadata(ctx context.Context, pieceCid cid.Cid) error { ctx, span := tracing.Tracer.Start(ctx, "store.remove_piece_metadata") defer span.End() + qry := `DELETE FROM idx.PieceMetadata WHERE PieceCid = ?` + err := s.session.Query(qry, pieceCid.String()).WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("removing piece metadata for piece %s: %w", pieceCid, err) + } + return nil } -// RemoveIndexes removes all MultiHashes for pieceCID. To be used manually in case of failure -// in RemoveDealForPiece or RemovePieceMetadata. Metadata for the piece must be -// present in the database +// RemoveIndexes removes all multihash -> piece cid mappings, and all +// offset / size information for the piece. func (s *Store) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error { ctx, span := tracing.Tracer.Start(ctx, "store.remove_indexes") defer span.End() + recs, err := s.GetIndex(ctx, pieceCid) + if err != nil { + return fmt.Errorf("removing indexes for piece %s: getting recs: %w", pieceCid, err) + } + + err = s.execParallel(ctx, recs, s.settings.PayloadPiecesParallelism, func(rec model.Record) error { + multihashBytes := rec.Cid.Hash() + q := `DELETE FROM idx.PayloadToPieces WHERE PayloadMultihash = ? AND PieceCid = ?` + err := s.session.Query(q, trimMultihash(multihashBytes), pieceCid.Bytes()).Exec() + if err != nil { + return fmt.Errorf("inserting into PayloadToPieces: %w", err) + } + return nil + }) + + qry := `DELETE FROM idx.PieceBlockOffsetSize WHERE PieceCid = ?` + err = s.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Exec() + if err != nil { + return fmt.Errorf("removing indexes for piece %s: deleting offset / size info: %w", pieceCid, err) + } + return nil } + +func (s *Store) execParallel(ctx context.Context, recs []model.Record, parallelism int, f func(record model.Record) error) error { + queue := make(chan model.Record, len(recs)) + for _, rec := range recs { + queue <- rec + } + close(queue) + + var eg errgroup.Group + for i := 0; i < parallelism; i++ { + eg.Go(func() error { + for ctx.Err() == nil { + select { + case <-ctx.Done(): + return ctx.Err() + case rec, ok := <-queue: + if !ok { + // Finished adding all the queued items, exit the thread + return nil + } + + err := f(rec) + if err != nil { + return err + } + } + } + + return ctx.Err() + }) + } + return eg.Wait() +} diff --git a/piecedirectory/doctor_test.go b/piecedirectory/doctor_test.go index 08904ef7d..5c471be16 100644 --- a/piecedirectory/doctor_test.go +++ b/piecedirectory/doctor_test.go @@ -108,21 +108,15 @@ func TestPieceDoctor(t *testing.T) { require.NoError(t, err) defer cl.Close(ctx) - resetTables := func() { - ybstore := bdsvc.Impl.(*yugabyte.Store) - err = ybstore.Drop(ctx) - require.NoError(t, err) - err = ybstore.Create(ctx) - require.NoError(t, err) - } + ybstore := bdsvc.Impl.(*yugabyte.Store) t.Run("next pieces", func(t *testing.T) { - resetTables() + svc.RecreateTables(ctx, t, ybstore) testNextPieces(ctx, t, cl, yugabyte.MinPieceCheckPeriod) }) t.Run("next pieces pagination", func(t *testing.T) { - resetTables() + svc.RecreateTables(ctx, t, ybstore) prevp := yugabyte.TrackerCheckBatchSize testNextPiecesPagination(ctx, t, cl, func(pageSize int) { yugabyte.TrackerCheckBatchSize = pageSize @@ -131,7 +125,7 @@ func TestPieceDoctor(t *testing.T) { }) t.Run("check pieces", func(t *testing.T) { - resetTables() + svc.RecreateTables(ctx, t, ybstore) testCheckPieces(ctx, t, cl) }) diff --git a/piecedirectory/piecedirectory_test.go b/piecedirectory/piecedirectory_test.go index dbbcbf65d..e06613ad1 100644 --- a/piecedirectory/piecedirectory_test.go +++ b/piecedirectory/piecedirectory_test.go @@ -31,10 +31,8 @@ func TestPieceDirectory(t *testing.T) { bdsvc, err := svc.NewLevelDB("") require.NoError(t, err) addr := "localhost:8042" - err = bdsvc.Start(ctx, addr) - require.NoError(t, err) - testPieceDirectory(ctx, t, addr) + testPieceDirectory(ctx, t, bdsvc, addr) }) t.Run("couchbase", func(t *testing.T) { @@ -44,10 +42,7 @@ func TestPieceDirectory(t *testing.T) { svc.SetupCouchbase(t, testCouchSettings) bdsvc := svc.NewCouchbase(testCouchSettings) addr := "localhost:8043" - err := bdsvc.Start(ctx, addr) - require.NoError(t, err) - - testPieceDirectory(ctx, t, addr) + testPieceDirectory(ctx, t, bdsvc, addr) }) t.Run("yugabyte", func(t *testing.T) { @@ -59,22 +54,16 @@ func TestPieceDirectory(t *testing.T) { }) addr := "localhost:8044" - err := bdsvc.Start(ctx, addr) - require.NoError(t, err) - - ybstore := bdsvc.Impl.(*yugabyte.Store) - err = ybstore.Drop(ctx) - require.NoError(t, err) - err = ybstore.Create(ctx) - require.NoError(t, err) - - testPieceDirectory(ctx, t, addr) + testPieceDirectory(ctx, t, bdsvc, addr) }) } -func testPieceDirectory(ctx context.Context, t *testing.T, addr string) { +func testPieceDirectory(ctx context.Context, t *testing.T, bdsvc *svc.Service, addr string) { + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + cl := client.NewStore() - err := cl.Dial(ctx, fmt.Sprintf("http://%s", addr)) + err = cl.Dial(ctx, fmt.Sprintf("http://%s", addr)) require.NoError(t, err) defer cl.Close(ctx) From 625197ce85179a4ccac67a11803a5eb1b2f99c8b Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 4 May 2023 15:20:01 +0200 Subject: [PATCH 10/15] refactor: consolidate test params --- extern/boostd-data/svc/setup_yugabyte_test_util.go | 10 ++++++---- extern/boostd-data/svc/svc_test.go | 12 +++--------- piecedirectory/doctor_test.go | 5 +---- piecedirectory/piecedirectory_test.go | 7 +------ 4 files changed, 11 insertions(+), 23 deletions(-) diff --git a/extern/boostd-data/svc/setup_yugabyte_test_util.go b/extern/boostd-data/svc/setup_yugabyte_test_util.go index 3411d893f..a6f9306d6 100644 --- a/extern/boostd-data/svc/setup_yugabyte_test_util.go +++ b/extern/boostd-data/svc/setup_yugabyte_test_util.go @@ -16,6 +16,11 @@ import ( "time" ) +var TestYugabyteSettings = yugabyte.DBSettings{ + Hosts: []string{"127.0.0.1"}, + ConnectString: "postgresql://postgres:postgres@localhost", +} + func SetupYugabyte(t *testing.T) { ctx := context.Background() cli, err := dockercl.NewClientWithOpts(dockercl.FromEnv) @@ -68,10 +73,7 @@ func SetupYugabyte(t *testing.T) { awaitYugabyteUp(t, time.Minute) tlog.Info("yugabyte started") - store := yugabyte.NewStore(yugabyte.DBSettings{ - Hosts: []string{"127.0.0.1"}, - ConnectString: "postgresql://postgres:postgres@localhost", - }) + store := yugabyte.NewStore(TestYugabyteSettings) err = store.Start(ctx) require.NoError(t, err) diff --git a/extern/boostd-data/svc/svc_test.go b/extern/boostd-data/svc/svc_test.go index b5280e2e5..87ccc0fbb 100644 --- a/extern/boostd-data/svc/svc_test.go +++ b/extern/boostd-data/svc/svc_test.go @@ -15,7 +15,6 @@ import ( "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/couchbase" "github.com/filecoin-project/boostd-data/model" - "github.com/filecoin-project/boostd-data/yugabyte" "github.com/filecoin-project/go-state-types/abi" "github.com/google/uuid" "github.com/ipfs/go-cid" @@ -46,11 +45,6 @@ var testCouchSettings = couchbase.DBSettings{ TestMode: true, } -var testYugaSettings = yugabyte.DBSettings{ - Hosts: []string{"127.0.0.1"}, - ConnectString: "postgresql://postgres:postgres@localhost", -} - func TestService(t *testing.T) { _ = logging.SetLogLevel("*", "debug") @@ -86,7 +80,7 @@ func TestService(t *testing.T) { SetupYugabyte(t) - bdsvc := NewYugabyte(testYugaSettings) + bdsvc := NewYugabyte(TestYugabyteSettings) addr := "localhost:8044" testService(ctx, t, bdsvc, addr) @@ -225,7 +219,7 @@ func TestServiceFuzz(t *testing.T) { t.Run("yugabyte", func(t *testing.T) { SetupYugabyte(t) - bdsvc := NewYugabyte(testYugaSettings) + bdsvc := NewYugabyte(TestYugabyteSettings) addr := "localhost:8044" err := bdsvc.Start(ctx, addr) @@ -460,7 +454,7 @@ func TestCleanup(t *testing.T) { SetupYugabyte(t) - bdsvc := NewYugabyte(testYugaSettings) + bdsvc := NewYugabyte(TestYugabyteSettings) testCleanup(ctx, t, bdsvc, "localhost:8044") }) } diff --git a/piecedirectory/doctor_test.go b/piecedirectory/doctor_test.go index 5c471be16..d0198a410 100644 --- a/piecedirectory/doctor_test.go +++ b/piecedirectory/doctor_test.go @@ -94,10 +94,7 @@ func TestPieceDoctor(t *testing.T) { svc.SetupYugabyte(t) - bdsvc := svc.NewYugabyte(yugabyte.DBSettings{ - Hosts: []string{"127.0.0.1"}, - ConnectString: "postgresql://postgres:postgres@localhost", - }) + bdsvc := svc.NewYugabyte(svc.TestYugabyteSettings) addr := "localhost:8044" err := bdsvc.Start(ctx, addr) diff --git a/piecedirectory/piecedirectory_test.go b/piecedirectory/piecedirectory_test.go index e06613ad1..7cf447d60 100644 --- a/piecedirectory/piecedirectory_test.go +++ b/piecedirectory/piecedirectory_test.go @@ -14,7 +14,6 @@ import ( "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/svc" "github.com/filecoin-project/boostd-data/svc/types" - "github.com/filecoin-project/boostd-data/yugabyte" "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/ipfs/go-cid" @@ -48,11 +47,7 @@ func TestPieceDirectory(t *testing.T) { t.Run("yugabyte", func(t *testing.T) { svc.SetupYugabyte(t) - bdsvc := svc.NewYugabyte(yugabyte.DBSettings{ - Hosts: []string{"127.0.0.1"}, - ConnectString: "postgresql://postgres:postgres@localhost", - }) - + bdsvc := svc.NewYugabyte(svc.TestYugabyteSettings) addr := "localhost:8044" testPieceDirectory(ctx, t, bdsvc, addr) }) From b02d3d7d922e96b8344c34f059ef26b3e5c83aa3 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Thu, 4 May 2023 16:31:13 +0200 Subject: [PATCH 11/15] feat: add lid yugabyte config --- extern/boostd-data/cmd/run.go | 31 ++++++++++++++++++++++++++ extern/boostd-data/yugabyte/create.cql | 10 ++++----- extern/boostd-data/yugabyte/create.sql | 12 +++++----- extern/boostd-data/yugabyte/service.go | 4 +++- extern/boostd-data/yugabyte/setup.go | 3 +++ node/config/def.go | 3 +++ node/config/types.go | 9 ++++++++ node/modules/piecedirectory.go | 19 +++++++++++++--- 8 files changed, 76 insertions(+), 15 deletions(-) diff --git a/extern/boostd-data/cmd/run.go b/extern/boostd-data/cmd/run.go index 109d804d3..a41bb35cd 100644 --- a/extern/boostd-data/cmd/run.go +++ b/extern/boostd-data/cmd/run.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/boostd-data/shared/cliutil" "github.com/filecoin-project/boostd-data/shared/tracing" "github.com/filecoin-project/boostd-data/svc" + "github.com/filecoin-project/boostd-data/yugabyte" "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" ) @@ -18,6 +19,7 @@ var runCmd = &cli.Command{ Subcommands: []*cli.Command{ leveldbCmd, couchbaseCmd, + yugabyteCmd, }, } @@ -106,6 +108,35 @@ var couchbaseCmd = &cli.Command{ }, } +var yugabyteCmd = &cli.Command{ + Name: "yugabyte", + Usage: "Run boostd-data with a yugabyte database", + Before: before, + Flags: append([]cli.Flag{ + &cli.StringSliceFlag{ + Name: "hosts", + Usage: "yugabyte hosts to connect to over cassandra interface eg '127.0.0.1'", + Required: true, + }, + &cli.StringFlag{ + Name: "connect-string", + Usage: "postgres connect string eg 'postgresql://postgres:postgres@localhost'", + Required: true, + }}, + runFlags..., + ), + Action: func(cctx *cli.Context) error { + // Create a yugabyte data service + settings := yugabyte.DBSettings{ + Hosts: cctx.StringSlice("hosts"), + ConnectString: cctx.String("connect-string"), + } + + bdsvc := svc.NewYugabyte(settings) + return runAction(cctx, "yugabyte", bdsvc) + }, +} + func runAction(cctx *cli.Context, dbType string, store *svc.Service) error { ctx := cliutil.ReqContext(cctx) diff --git a/extern/boostd-data/yugabyte/create.cql b/extern/boostd-data/yugabyte/create.cql index a2a32fec1..0e7c01e48 100644 --- a/extern/boostd-data/yugabyte/create.cql +++ b/extern/boostd-data/yugabyte/create.cql @@ -1,6 +1,6 @@ create keyspace if not exists idx with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; -CREATE TABLE idx.PayloadToPieces ( +CREATE TABLE IF NOT EXISTS idx.PayloadToPieces ( PayloadMultihash BLOB, PieceCid BLOB, -- PayloadMultihash is the partition key: the key that determines which @@ -11,7 +11,7 @@ CREATE TABLE idx.PayloadToPieces ( PRIMARY KEY (PayloadMultihash, PieceCid) ); -CREATE TABLE idx.PieceBlockOffsetSize ( +CREATE TABLE IF NOT EXISTS idx.PieceBlockOffsetSize ( PieceCid BLOB, PayloadMultihash BLOB, BlockOffset BIGINT, @@ -19,7 +19,7 @@ CREATE TABLE idx.PieceBlockOffsetSize ( PRIMARY KEY (PieceCid, PayloadMultihash) ); -CREATE TABLE idx.PieceMetadata ( +CREATE TABLE IF NOT EXISTS idx.PieceMetadata ( PieceCid TEXT PRIMARY KEY, Version TEXT, CreatedAt TIMESTAMP, @@ -29,9 +29,9 @@ CREATE TABLE idx.PieceMetadata ( ErrorType Text ) WITH transactions = { 'enabled' : true }; -CREATE INDEX PieceMetadataCreatedAt ON idx.PieceMetadata (CreatedAt); +CREATE INDEX IF NOT EXISTS PieceMetadataCreatedAt ON idx.PieceMetadata (CreatedAt); -CREATE TABLE idx.PieceDeal ( +CREATE TABLE IF NOT EXISTS idx.PieceDeal ( DealUuid TEXT PRIMARY KEY, PieceCid BLOB, IsLegacy BOOLEAN, diff --git a/extern/boostd-data/yugabyte/create.sql b/extern/boostd-data/yugabyte/create.sql index 6b78af851..d296143af 100644 --- a/extern/boostd-data/yugabyte/create.sql +++ b/extern/boostd-data/yugabyte/create.sql @@ -1,17 +1,17 @@ -CREATE TABLE PieceTracker ( +CREATE TABLE IF NOT EXISTS PieceTracker ( PieceCid TEXT PRIMARY KEY, CreatedAt TIMESTAMP, UpdatedAt TIMESTAMP ); -CREATE INDEX PieceTrackerCreatedAt ON PieceTracker (CreatedAt); -CREATE INDEX PieceTrackerUpdatedAt ON PieceTracker (UpdatedAt); +CREATE INDEX IF NOT EXISTS PieceTrackerCreatedAt ON PieceTracker (CreatedAt); +CREATE INDEX IF NOT EXISTS PieceTrackerUpdatedAt ON PieceTracker (UpdatedAt); -CREATE TABLE PieceFlagged ( +CREATE TABLE IF NOT EXISTS PieceFlagged ( PieceCid TEXT PRIMARY KEY, CreatedAt TIMESTAMP, UpdatedAt TIMESTAMP ); -CREATE INDEX PieceFlaggedCreatedAt ON PieceFlagged (CreatedAt); -CREATE INDEX PieceFlaggedUpdatedAt ON PieceFlagged (UpdatedAt); +CREATE INDEX IF NOT EXISTS PieceFlaggedCreatedAt ON PieceFlagged (CreatedAt); +CREATE INDEX IF NOT EXISTS PieceFlaggedUpdatedAt ON PieceFlagged (UpdatedAt); diff --git a/extern/boostd-data/yugabyte/service.go b/extern/boostd-data/yugabyte/service.go index 1c996ced4..b529cc7e6 100644 --- a/extern/boostd-data/yugabyte/service.go +++ b/extern/boostd-data/yugabyte/service.go @@ -56,7 +56,7 @@ func NewStore(settings DBSettings) *Store { } } -func (s *Store) Start(_ context.Context) error { +func (s *Store) Start(ctx context.Context) error { var startErr error s.startOnce.Do(func() { session, err := s.cluster.CreateSession() @@ -72,6 +72,8 @@ func (s *Store) Start(_ context.Context) error { return } s.db = db + + startErr = s.Create(ctx) }) return startErr diff --git a/extern/boostd-data/yugabyte/setup.go b/extern/boostd-data/yugabyte/setup.go index de699a42f..849db882b 100644 --- a/extern/boostd-data/yugabyte/setup.go +++ b/extern/boostd-data/yugabyte/setup.go @@ -14,10 +14,13 @@ var createCQL string var createSQL string func (s *Store) Create(ctx context.Context) error { + log.Infow("creating cassandra tables") err := s.execScript(ctx, createCQL, s.execCQL) if err != nil { return err } + + log.Infow("creating postgres tables") return s.execScript(ctx, createSQL, s.execSQL) } diff --git a/node/config/def.go b/node/config/def.go index 7c29ceba4..74daa704d 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -76,6 +76,9 @@ func DefaultBoost() *Boost { }, LocalIndexDirectory: LocalIndexDirectoryConfig{ + Yugabyte: LocalIndexDirectoryYugabyteConfig{ + Enabled: false, + }, Couchbase: LocalIndexDirectoryCouchbaseConfig{ ConnectString: "", Username: "", diff --git a/node/config/types.go b/node/config/types.go index 34a52c2cd..2b6db413e 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -323,7 +323,16 @@ type LocalIndexDirectoryCouchbaseConfig struct { PieceOffsetsBucket LocalIndexDirectoryCouchbaseBucketConfig } +type LocalIndexDirectoryYugabyteConfig struct { + Enabled bool + // The yugabyte postgres connect string eg "postgresql://postgres:postgres@localhost" + ConnectString string + // The yugabyte cassandra hosts eg ["127.0.0.1"] + Hosts []string +} + type LocalIndexDirectoryConfig struct { + Yugabyte LocalIndexDirectoryYugabyteConfig Couchbase LocalIndexDirectoryCouchbaseConfig // The maximum number of add index operations allowed to execute in parallel. // The add index operation is executed when a new deal is created - it fetches diff --git a/node/modules/piecedirectory.go b/node/modules/piecedirectory.go index 625b7ccf1..18185c50f 100644 --- a/node/modules/piecedirectory.go +++ b/node/modules/piecedirectory.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/boostd-data/couchbase" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/svc" + "github.com/filecoin-project/boostd-data/yugabyte" "github.com/filecoin-project/dagstore" "github.com/filecoin-project/dagstore/shard" "github.com/filecoin-project/go-address" @@ -50,9 +51,20 @@ func NewPieceDirectoryStore(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_rep } svcCtx, cancel = context.WithCancel(ctx) - var bdsvc *svc.Service - if cfg.LocalIndexDirectory.Couchbase.ConnectString != "" { + switch { + case cfg.LocalIndexDirectory.Yugabyte.Enabled: + log.Infow("local index directory: connecting to yugabyte server", + "connect-string", cfg.LocalIndexDirectory.Yugabyte.ConnectString, + "hosts", cfg.LocalIndexDirectory.Yugabyte.Hosts) + + // Set up a local index directory service that connects to the yugabyte db + bdsvc = svc.NewYugabyte(yugabyte.DBSettings{ + Hosts: cfg.LocalIndexDirectory.Yugabyte.Hosts, + ConnectString: cfg.LocalIndexDirectory.Yugabyte.ConnectString, + }) + + case cfg.LocalIndexDirectory.Couchbase.ConnectString != "": log.Infow("local index directory: connecting to couchbase server", "connect-string", cfg.LocalIndexDirectory.Couchbase.ConnectString) @@ -74,7 +86,8 @@ func NewPieceDirectoryStore(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_rep RAMQuotaMB: cfg.LocalIndexDirectory.Couchbase.PieceOffsetsBucket.RAMQuotaMB, }, }) - } else { + + default: log.Infow("local index directory: connecting to leveldb instance") // Setup a local index directory service that connects to the leveldb From f0ee52b6f091e72bb7e50f16db6185fca2bdff09 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 8 May 2023 16:05:56 +0200 Subject: [PATCH 12/15] fix: port map yugabyte postgres to standard port --- .../boostd-data/svc/setup_yugabyte_test_util.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/extern/boostd-data/svc/setup_yugabyte_test_util.go b/extern/boostd-data/svc/setup_yugabyte_test_util.go index a6f9306d6..72b8131f6 100644 --- a/extern/boostd-data/svc/setup_yugabyte_test_util.go +++ b/extern/boostd-data/svc/setup_yugabyte_test_util.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/boostd-data/yugabyte" "github.com/stretchr/testify/require" "github.com/yugabyte/gocql" + "github.com/yugabyte/pgx/v4/pgxpool" "golang.org/x/net/context" "io" "os" @@ -46,7 +47,9 @@ func SetupYugabyte(t *testing.T) { PortBindings: map[nat.Port][]nat.PortBinding{ "7000": {{HostIP: "127.0.0.1", HostPort: "7001"}}, "9000": {{HostIP: "127.0.0.1", HostPort: "9000"}}, - "5433": {{HostIP: "127.0.0.1", HostPort: "5433"}}, + // Yugabyte's postgres interface in docker runs on 5433 + // whereas the standard postgres port is 5432 + "5433": {{HostIP: "127.0.0.1", HostPort: "5432"}}, "9042": {{HostIP: "127.0.0.1", HostPort: "9042"}}, }, }, nil, nil, "") @@ -89,13 +92,15 @@ func RecreateTables(ctx context.Context, t *testing.T, store *yugabyte.Store) { func awaitYugabyteUp(t *testing.T, duration time.Duration) { start := time.Now() - cluster := gocql.NewCluster("127.0.0.1") + cluster := gocql.NewCluster(TestYugabyteSettings.Hosts[0]) for { - session, err := cluster.CreateSession() + _, err := cluster.CreateSession() if err == nil { - return + _, err = pgxpool.Connect(context.Background(), TestYugabyteSettings.ConnectString) + if err == nil { + return + } } - _ = session tlog.Debugf("waiting for yugabyte: %s", err) if time.Since(start) > duration { From 79f178ac22b286c9056b29cf975bebf3d4fd6ae8 Mon Sep 17 00:00:00 2001 From: dirkmc Date: Mon, 8 May 2023 16:52:00 +0200 Subject: [PATCH 13/15] Fix yugabyte CI (#1433) * fix: yugabyte tests in CI * docker-compose.yml ; Dockerfile.test ; connect to `yugabyte` and not localhost * add tag * test lid * make gen * fixup * move couchbase settings under build tag --------- Co-authored-by: Anton Evangelatov --- .circleci/config.yml | 19 ++++-- Makefile | 4 ++ extern/boostd-data/Dockerfile.test | 8 +++ extern/boostd-data/docker-compose.yml | 17 +++++ .../svc/setup_yugabyte_test_util.go | 65 ++----------------- extern/boostd-data/svc/svc_test.go | 3 + node/config/doc_gen.go | 26 ++++++++ piecedirectory/doctor_test.go | 3 + piecedirectory/piecedirectory_test.go | 21 ++++++ piecedirectory/test_util.go | 19 ------ 10 files changed, 103 insertions(+), 82 deletions(-) create mode 100644 extern/boostd-data/Dockerfile.test create mode 100644 extern/boostd-data/docker-compose.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 751ece814..123823f43 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,6 +83,19 @@ jobs: paths: - linux + lid-docker-compose: + description: 'Run LID integration tests' + machine: + image: ubuntu-2004:202104-01 + resource_class: xlarge + steps: + - checkout + - run: + name: local index directory docker compose tests + command: | + set -x + make test-lid + test: description: | Run go tests @@ -322,7 +335,5 @@ workflows: suite: all target: "`go list ./... | grep -v boost/itests`" - - test: - name: local index directory - suite: all - cwd: "./extern/boostd-data" + - lid-docker-compose + diff --git a/Makefile b/Makefile index 8c32e638c..81ed6a2bc 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ unexport GOFLAGS GOCC?=go +ARCH?=$(shell arch) GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}') ifeq ($(shell expr $(GOVERSION) \< 1016000), 1) $(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000)) @@ -259,6 +260,9 @@ docker/all: $(lotus_build_cmd) docker/boost docker/booster-http docker/booster-b docker/lotus docker/lotus-miner .PHONY: docker/all +test-lid: + cd ./extern/boostd-data && ARCH=$(ARCH) docker-compose up --build --exit-code-from go-tests + devnet/up: rm -rf ./docker/devnet/data && docker compose -f ./docker/devnet/docker-compose.yaml up -d diff --git a/extern/boostd-data/Dockerfile.test b/extern/boostd-data/Dockerfile.test new file mode 100644 index 000000000..24972f7b8 --- /dev/null +++ b/extern/boostd-data/Dockerfile.test @@ -0,0 +1,8 @@ +FROM golang:1.18-alpine + +WORKDIR /go/src/ + +ENV CGO_ENABLED=0 + +ENTRYPOINT ["go", "test"] +CMD ["-v", "./..."] diff --git a/extern/boostd-data/docker-compose.yml b/extern/boostd-data/docker-compose.yml new file mode 100644 index 000000000..5ac7b7368 --- /dev/null +++ b/extern/boostd-data/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3.7' + +services: + + yugabyte: + image: public.ecr.aws/n6b0k8i7/yugabyte-test:${ARCH}-2.17.2.0 + restart: on-failure + + go-tests: + build: + context: . + dockerfile: ./Dockerfile.test + environment: + YUGABYTE_HOST: yugabyte + volumes: + - ./:/go/src/ + command: -tags=test_lid -v -count=1 -p=1 ./... diff --git a/extern/boostd-data/svc/setup_yugabyte_test_util.go b/extern/boostd-data/svc/setup_yugabyte_test_util.go index 72b8131f6..ff7f5e61f 100644 --- a/extern/boostd-data/svc/setup_yugabyte_test_util.go +++ b/extern/boostd-data/svc/setup_yugabyte_test_util.go @@ -1,83 +1,30 @@ package svc import ( - "github.com/davecgh/go-spew/spew" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - dockercl "github.com/docker/docker/client" - "github.com/docker/go-connections/nat" + "testing" + "time" + "github.com/filecoin-project/boostd-data/yugabyte" "github.com/stretchr/testify/require" "github.com/yugabyte/gocql" "github.com/yugabyte/pgx/v4/pgxpool" "golang.org/x/net/context" - "io" - "os" - "testing" - "time" ) var TestYugabyteSettings = yugabyte.DBSettings{ - Hosts: []string{"127.0.0.1"}, - ConnectString: "postgresql://postgres:postgres@localhost", + Hosts: []string{"yugabyte"}, + ConnectString: "postgresql://postgres:postgres@yugabyte:5433", } func SetupYugabyte(t *testing.T) { ctx := context.Background() - cli, err := dockercl.NewClientWithOpts(dockercl.FromEnv) - require.NoError(t, err) - - imageName := "public.ecr.aws/n6b0k8i7/yugabyte-test:aarch64-2.17.2.0" - out, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{}) - require.NoError(t, err) - - _, err = io.Copy(os.Stdout, out) - require.NoError(t, err) - - tlog.Info("yugabyte docker container create...") - resp, err := cli.ContainerCreate(ctx, &container.Config{ - Image: imageName, - ExposedPorts: nat.PortSet{ - "7000": struct{}{}, - "9000": struct{}{}, - "5433": struct{}{}, - "9042": struct{}{}, - }, - }, &container.HostConfig{ - PortBindings: map[nat.Port][]nat.PortBinding{ - "7000": {{HostIP: "127.0.0.1", HostPort: "7001"}}, - "9000": {{HostIP: "127.0.0.1", HostPort: "9000"}}, - // Yugabyte's postgres interface in docker runs on 5433 - // whereas the standard postgres port is 5432 - "5433": {{HostIP: "127.0.0.1", HostPort: "5432"}}, - "9042": {{HostIP: "127.0.0.1", HostPort: "9042"}}, - }, - }, nil, nil, "") - require.NoError(t, err) - tlog.Info("yugabyte docker container created") - - tlog.Info("yugabyte docker container start...") - err = cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}) - require.NoError(t, err) - tlog.Info("yugabyte docker container started") - - inspect, err := cli.ContainerInspect(ctx, resp.ID) - require.NoError(t, err) - spew.Dump(inspect) - - t.Cleanup(func() { - tlog.Info("yugabyte docker container remove...") - err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true}) - require.NoError(t, err) - tlog.Info("yugabyte docker container removed") - }) tlog.Info("wait for yugabyte start...") awaitYugabyteUp(t, time.Minute) tlog.Info("yugabyte started") store := yugabyte.NewStore(TestYugabyteSettings) - err = store.Start(ctx) + err := store.Start(ctx) require.NoError(t, err) RecreateTables(ctx, t, store) diff --git a/extern/boostd-data/svc/svc_test.go b/extern/boostd-data/svc/svc_test.go index 87ccc0fbb..2ec2d83f7 100644 --- a/extern/boostd-data/svc/svc_test.go +++ b/extern/boostd-data/svc/svc_test.go @@ -1,3 +1,6 @@ +//go:build test_lid +// +build test_lid + package svc import ( diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index b4f8e2acd..c0da49c90 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -443,6 +443,12 @@ for any other deal.`, }, }, "LocalIndexDirectoryConfig": []DocField{ + { + Name: "Yugabyte", + Type: "LocalIndexDirectoryYugabyteConfig", + + Comment: ``, + }, { Name: "Couchbase", Type: "LocalIndexDirectoryCouchbaseConfig", @@ -521,6 +527,26 @@ If empty, a leveldb database is used instead.`, Comment: ``, }, }, + "LocalIndexDirectoryYugabyteConfig": []DocField{ + { + Name: "Enabled", + Type: "bool", + + Comment: ``, + }, + { + Name: "ConnectString", + Type: "string", + + Comment: `The yugabyte postgres connect string eg "postgresql://postgres:postgres@localhost"`, + }, + { + Name: "Hosts", + Type: "[]string", + + Comment: `The yugabyte cassandra hosts eg ["127.0.0.1"]`, + }, + }, "LotusDealmakingConfig": []DocField{ { Name: "PieceCidBlocklist", diff --git a/piecedirectory/doctor_test.go b/piecedirectory/doctor_test.go index d0198a410..eb8338d0f 100644 --- a/piecedirectory/doctor_test.go +++ b/piecedirectory/doctor_test.go @@ -1,3 +1,6 @@ +//go:build test_lid +// +build test_lid + package piecedirectory import ( diff --git a/piecedirectory/piecedirectory_test.go b/piecedirectory/piecedirectory_test.go index 7cf447d60..567f4c8bc 100644 --- a/piecedirectory/piecedirectory_test.go +++ b/piecedirectory/piecedirectory_test.go @@ -1,3 +1,6 @@ +//go:build test_lid +// +build test_lid + package piecedirectory import ( @@ -22,6 +25,24 @@ import ( "github.com/stretchr/testify/require" ) +var testCouchSettings = couchbase.DBSettings{ + ConnectString: "couchbase://localhost", + Auth: couchbase.DBSettingsAuth{ + Username: "Administrator", + Password: "boostdemo", + }, + PieceMetadataBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: 128, + }, + MultihashToPiecesBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: 128, + }, + PieceOffsetsBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: 128, + }, + TestMode: true, +} + func TestPieceDirectory(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) defer cancel() diff --git a/piecedirectory/test_util.go b/piecedirectory/test_util.go index f3aac7840..cc4bef77c 100644 --- a/piecedirectory/test_util.go +++ b/piecedirectory/test_util.go @@ -9,7 +9,6 @@ import ( "github.com/filecoin-project/boost/piecedirectory/types" mock_piecedirectory "github.com/filecoin-project/boost/piecedirectory/types/mocks" "github.com/filecoin-project/boost/testutil" - "github.com/filecoin-project/boostd-data/couchbase" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/go-commp-utils/writer" "github.com/filecoin-project/go-state-types/abi" @@ -18,24 +17,6 @@ import ( "github.com/stretchr/testify/require" ) -var testCouchSettings = couchbase.DBSettings{ - ConnectString: "couchbase://localhost", - Auth: couchbase.DBSettingsAuth{ - Username: "Administrator", - Password: "boostdemo", - }, - PieceMetadataBucket: couchbase.DBSettingsBucket{ - RAMQuotaMB: 128, - }, - MultihashToPiecesBucket: couchbase.DBSettingsBucket{ - RAMQuotaMB: 128, - }, - PieceOffsetsBucket: couchbase.DBSettingsBucket{ - RAMQuotaMB: 128, - }, - TestMode: true, -} - // Get the index records from the CAR file func GetRecords(t *testing.T, reader car.SectionReader) []model.Record { _, err := reader.Seek(0, io.SeekStart) From 53b61feac148399c132ae0608e932d1a6e8d5b9b Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 10 May 2023 10:08:49 +0200 Subject: [PATCH 14/15] wip: service GetIndex returns channel of records instead of array --- extern/boostd-data/client/client.go | 15 ++++- extern/boostd-data/couchbase/service.go | 16 ++++- extern/boostd-data/ldb/service.go | 15 ++++- extern/boostd-data/svc/svc_test.go | 60 +++++++++++++++++-- extern/boostd-data/svc/types/types.go | 8 ++- extern/boostd-data/yugabyte/service.go | 77 ++++++++++++++++++------- 6 files changed, 157 insertions(+), 34 deletions(-) diff --git a/extern/boostd-data/client/client.go b/extern/boostd-data/client/client.go index 98fcb8609..55626aa2e 100644 --- a/extern/boostd-data/client/client.go +++ b/extern/boostd-data/client/client.go @@ -3,6 +3,7 @@ package client import ( "context" "fmt" + "github.com/filecoin-project/boostd-data/svc/types" "time" "github.com/filecoin-project/boostd-data/model" @@ -21,7 +22,7 @@ type Store struct { AddIndex func(context.Context, cid.Cid, []model.Record, bool) error IsIndexed func(ctx context.Context, pieceCid cid.Cid) (bool, error) IsCompleteIndex func(ctx context.Context, pieceCid cid.Cid) (bool, error) - GetIndex func(context.Context, cid.Cid) ([]model.Record, error) + GetIndex func(context.Context, cid.Cid) (<-chan types.IndexRecord, error) GetOffsetSize func(context.Context, cid.Cid, mh.Multihash) (*model.OffsetSize, error) ListPieces func(ctx context.Context) ([]cid.Cid, error) GetPieceMetadata func(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) @@ -66,7 +67,7 @@ func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) (index.Index, er } var records []index.Record - for _, r := range resp { + for r := range resp { records = append(records, index.Record{ Cid: r.Cid, Offset: r.Offset, @@ -90,7 +91,15 @@ func (s *Store) GetRecords(ctx context.Context, pieceCid cid.Cid) ([]model.Recor log.Debugw("get-records", "piece-cid", pieceCid, "records", len(resp)) - return resp, nil + var records []model.Record + for r := range resp { + records = append(records, model.Record{ + Cid: r.Cid, + OffsetSize: model.OffsetSize{Offset: r.Offset}, + }) + } + + return records, nil } func (s *Store) GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) { diff --git a/extern/boostd-data/couchbase/service.go b/extern/boostd-data/couchbase/service.go index 38c9d637b..8308635b4 100644 --- a/extern/boostd-data/couchbase/service.go +++ b/extern/boostd-data/couchbase/service.go @@ -3,6 +3,7 @@ package couchbase import ( "context" "fmt" + "github.com/ipld/go-car/v2/index" "time" "github.com/filecoin-project/boostd-data/model" @@ -128,7 +129,7 @@ func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ( return pcids, normalizeMultihashError(m, err) } -func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) ([]model.Record, error) { +func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) (<-chan types.IndexRecord, error) { log.Debugw("handle.get-index", "pieceCid", pieceCid) ctx, span := tracing.Tracer.Start(ctx, "store.get_index") @@ -152,7 +153,18 @@ func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) ([]model.Record, log.Debugw("handle.get-index.records", "len(records)", len(records)) - return records, nil + recs := make(chan types.IndexRecord, len(records)) + for _, r := range records { + recs <- types.IndexRecord{ + Record: index.Record{ + Cid: r.Cid, + Offset: r.Offset, + }, + } + } + close(recs) + + return recs, nil } func (s *Store) IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) { diff --git a/extern/boostd-data/ldb/service.go b/extern/boostd-data/ldb/service.go index f41abe3a4..c69af7038 100644 --- a/extern/boostd-data/ldb/service.go +++ b/extern/boostd-data/ldb/service.go @@ -201,7 +201,7 @@ func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ( return pcs, normalizeMultihashError(m, err) } -func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) ([]model.Record, error) { +func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) (<-chan types.IndexRecord, error) { log.Warnw("handle.get-index", "pieceCid", pieceCid) ctx, span := tracing.Tracer.Start(ctx, "store.get_index") @@ -227,7 +227,18 @@ func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) ([]model.Record, log.Warnw("handle.get-index.records", "len(records)", len(records)) - return records, nil + recs := make(chan types.IndexRecord, len(records)) + for _, r := range records { + recs <- types.IndexRecord{ + Record: carindex.Record{ + Cid: r.Cid, + Offset: r.Offset, + }, + } + } + close(recs) + + return recs, nil } func (s *Store) IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) { diff --git a/extern/boostd-data/svc/svc_test.go b/extern/boostd-data/svc/svc_test.go index 2ec2d83f7..16e9765bb 100644 --- a/extern/boostd-data/svc/svc_test.go +++ b/extern/boostd-data/svc/svc_test.go @@ -49,7 +49,7 @@ var testCouchSettings = couchbase.DBSettings{ } func TestService(t *testing.T) { - _ = logging.SetLogLevel("*", "debug") + _ = logging.SetLogLevel("cbtest", "debug") t.Run("leveldb", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -88,6 +88,22 @@ func TestService(t *testing.T) { addr := "localhost:8044" testService(ctx, t, bdsvc, addr) }) + + t.Run("yugabyte", func(t *testing.T) { + _ = logging.SetLogLevel("boostd-data-yb", "debug") + + // Running yugabyte tests may require download the docker container + // so set a high timeout + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + SetupYugabyte(t) + + bdsvc := NewYugabyte(TestYugabyteSettings) + + addr := "localhost:8044" + testService(ctx, t, bdsvc, addr) + }) } func testService(ctx context.Context, t *testing.T, bdsvc *Service, addr string) { @@ -95,7 +111,7 @@ func testService(ctx context.Context, t *testing.T, bdsvc *Service, addr string) require.NoError(t, err) cl := client.NewStore() - err = cl.Dial(context.Background(), fmt.Sprintf("http://%s", addr)) + err = cl.Dial(context.Background(), fmt.Sprintf("ws://%s", addr)) require.NoError(t, err) defer cl.Close(ctx) @@ -423,9 +439,45 @@ func compareIndices(subject, subjectDb index.Index) (bool, error) { return false, err } - res := bytes.Compare(b.Bytes(), b2.Bytes()) + equal := bytes.Equal(b.Bytes(), b2.Bytes()) + + if !equal { + a, oka := toEntries(subject) + b, okb := toEntries(subjectDb) + if oka && okb { + if len(a) != len(b) { + fmt.Println("index length mismatch", "first length", len(a), "second length", len(b)) + } + for mh, oa := range a { + ob, ok := b[mh] + if !ok { + fmt.Println("second index missing multihash", "multihash", mh) + } + if oa != ob { + fmt.Println("offset mismatch", "multihash", mh, "first offset", oa, "second offset", ob) + } + } + } + } - return res == 0, nil + return equal, nil +} + +func toEntries(idx index.Index) (map[string]uint64, bool) { + it, ok := idx.(index.IterableIndex) + if !ok { + return nil, false + } + + entries := make(map[string]uint64) + err := it.ForEach(func(mh multihash.Multihash, o uint64) error { + entries[mh.String()] = o + return nil + }) + if err != nil { + return nil, false + } + return entries, true } func TestCleanup(t *testing.T) { diff --git a/extern/boostd-data/svc/types/types.go b/extern/boostd-data/svc/types/types.go index 76106df32..7be4171f4 100644 --- a/extern/boostd-data/svc/types/types.go +++ b/extern/boostd-data/svc/types/types.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/boostd-data/model" "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2/index" mh "github.com/multiformats/go-multihash" ) @@ -24,10 +25,15 @@ func IsNotFound(err error) bool { return strings.Contains(err.Error(), ErrNotFound.Error()) } +type IndexRecord struct { + index.Record + Error error +} + type Service interface { AddDealForPiece(context.Context, cid.Cid, model.DealInfo) error AddIndex(context.Context, cid.Cid, []model.Record, bool) error - GetIndex(context.Context, cid.Cid) ([]model.Record, error) + GetIndex(context.Context, cid.Cid) (<-chan IndexRecord, error) IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, error) GetOffsetSize(context.Context, cid.Cid, mh.Multihash) (*model.OffsetSize, error) diff --git a/extern/boostd-data/yugabyte/service.go b/extern/boostd-data/yugabyte/service.go index b529cc7e6..320e914af 100644 --- a/extern/boostd-data/yugabyte/service.go +++ b/extern/boostd-data/yugabyte/service.go @@ -10,6 +10,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" + "github.com/ipld/go-car/v2/index" "github.com/multiformats/go-multihash" mh "github.com/multiformats/go-multihash" "github.com/yugabyte/gocql" @@ -230,37 +231,61 @@ func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ( return pcids, nil } -func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) ([]model.Record, error) { +func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) (<-chan types.IndexRecord, error) { ctx, span := tracing.Tracer.Start(ctx, "store.get_index") defer span.End() qry := `SELECT PayloadMultihash, BlockOffset, BlockSize FROM idx.PieceBlockOffsetSize WHERE PieceCid = ?` iter := s.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Iter() - var records []model.Record - var payloadMHBz []byte - var offset, size uint64 - for iter.Scan(&payloadMHBz, &offset, &size) { - _, pmh, err := multihash.MHFromBytes(payloadMHBz) - if err != nil { - return nil, fmt.Errorf("scanning mulithash: %w", err) + scannedRecordCh := make(chan struct{}, 1) + records := make(chan types.IndexRecord) + go func() { + defer close(scannedRecordCh) + defer close(records) + + var payloadMHBz []byte + var offset, size uint64 + for iter.Scan(&payloadMHBz, &offset, &size) { + // The scan was successful, which means there is at least one + // record + select { + case scannedRecordCh <- struct{}{}: + default: + } + + // Parse the multihash bytes + _, pmh, err := multihash.MHFromBytes(payloadMHBz) + if err != nil { + records <- types.IndexRecord{Error: err} + return + } + + records <- types.IndexRecord{ + Record: index.Record{ + Cid: cid.NewCidV1(cid.Raw, pmh), + Offset: offset, + }, + } } + if err := iter.Close(); err != nil { + err = fmt.Errorf("getting piece index for piece %s: %w", pieceCid, err) + records <- types.IndexRecord{Error: err} + } + }() - records = append(records, model.Record{ - Cid: cid.NewCidV1(cid.Raw, pmh), - OffsetSize: model.OffsetSize{ - Offset: offset, - Size: size, - }, - }) - } - if err := iter.Close(); err != nil { - return nil, fmt.Errorf("getting piece index for piece %s: %w", pieceCid, err) + // Check if there were any records for this piece cid + var pieceHasRecords bool + select { + case <-ctx.Done(): + return nil, ctx.Err() + case _, pieceHasRecords = <-scannedRecordCh: } - // For correctness, we should always return a not found error if there is - // no piece with the piece cid - if len(records) == 0 { + if !pieceHasRecords { + // For correctness, we should always return a not found error if there + // is no piece with the piece cid. Call getPieceMetadata which returns + // not found if it can't find the piece. _, err := s.getPieceMetadata(ctx, pieceCid) if err != nil { return nil, err @@ -481,7 +506,15 @@ func (s *Store) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error { return fmt.Errorf("removing indexes for piece %s: getting recs: %w", pieceCid, err) } - err = s.execParallel(ctx, recs, s.settings.PayloadPiecesParallelism, func(rec model.Record) error { + var records []model.Record + for r := range recs { + records = append(records, model.Record{ + Cid: r.Cid, + OffsetSize: model.OffsetSize{Offset: r.Offset}, + }) + } + + err = s.execParallel(ctx, records, s.settings.PayloadPiecesParallelism, func(rec model.Record) error { multihashBytes := rec.Cid.Hash() q := `DELETE FROM idx.PayloadToPieces WHERE PayloadMultihash = ? AND PieceCid = ?` err := s.session.Query(q, trimMultihash(multihashBytes), pieceCid.Bytes()).Exec() From 8929f33e02b164ac8f50179cd818e00e12e833a6 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 16 May 2023 14:10:18 +0200 Subject: [PATCH 15/15] feat: return channel from AddIndex and GetIndex --- cmd/migrate-lid/couch-to-yuga.go | 19 ++- cmd/migrate-lid/migrate_lid.go | 13 +- extern/boostd-data/client/client.go | 33 +++-- extern/boostd-data/couchbase/service.go | 48 ++++--- extern/boostd-data/ldb/service.go | 123 ++++++++-------- extern/boostd-data/model/model.go | 6 +- extern/boostd-data/svc/svc_size_test.go | 121 ++++++++++++++++ extern/boostd-data/svc/svc_test.go | 10 +- extern/boostd-data/svc/types/types.go | 12 +- extern/boostd-data/yugabyte/service.go | 183 +++++++++++++++++------- node/config/def.go | 1 + node/config/doc_gen.go | 8 +- node/config/types.go | 4 +- node/modules/piecedirectory.go | 9 +- piecedirectory/doctor_test.go | 6 +- piecedirectory/piecedirectory.go | 5 +- piecedirectory/piecedirectory_test.go | 2 +- 17 files changed, 435 insertions(+), 168 deletions(-) create mode 100644 extern/boostd-data/svc/svc_size_test.go diff --git a/cmd/migrate-lid/couch-to-yuga.go b/cmd/migrate-lid/couch-to-yuga.go index 863af5db2..f52938480 100644 --- a/cmd/migrate-lid/couch-to-yuga.go +++ b/cmd/migrate-lid/couch-to-yuga.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/filecoin-project/boostd-data/couchbase" + "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/yugabyte" lcli "github.com/filecoin-project/lotus/cli" "github.com/ipfs/go-cid" @@ -211,16 +212,26 @@ func migrateLidToLidIndex(ctx context.Context, pieceCid cid.Cid, source StoreMig } // Load the index from the source store - records, err := source.GetIndex(ctx, pieceCid) + idx, err := source.GetIndex(ctx, pieceCid) if err != nil { return false, fmt.Errorf("loading index %s: %w", pieceCid, err) } + var records []model.Record + for r := range idx { + if r.Error != nil { + return false, r.Error + } + records = append(records, r.Record) + } + // Add the index to the destination store addStart := time.Now() - err = dest.AddIndex(ctx, pieceCid, records, true) - if err != nil { - return false, fmt.Errorf("adding index %s to store: %w", pieceCid, err) + respch := dest.AddIndex(ctx, pieceCid, records, true) + for resp := range respch { + if resp.Err != "" { + return false, fmt.Errorf("adding index %s to store: %s", pieceCid, err) + } } log.Debugw("AddIndex", "took", time.Since(addStart).String()) diff --git a/cmd/migrate-lid/migrate_lid.go b/cmd/migrate-lid/migrate_lid.go index eff532113..92e2d04bd 100644 --- a/cmd/migrate-lid/migrate_lid.go +++ b/cmd/migrate-lid/migrate_lid.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/boostd-data/ldb" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/svc" + "github.com/filecoin-project/boostd-data/svc/types" "github.com/filecoin-project/go-address" vfsm "github.com/filecoin-project/go-ds-versioning/pkg/fsm" "github.com/filecoin-project/go-fil-markets/piecestore" @@ -43,8 +44,8 @@ import ( type StoreMigrationApi interface { Start(ctx context.Context) error IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) - GetIndex(context.Context, cid.Cid) ([]model.Record, error) - AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error + GetIndex(context.Context, cid.Cid) (<-chan types.IndexRecord, error) + AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) <-chan types.AddIndexProgress AddDealForPiece(ctx context.Context, pcid cid.Cid, info model.DealInfo) error ListPieces(ctx context.Context) ([]cid.Cid, error) GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) @@ -333,9 +334,11 @@ func migrateIndex(ctx context.Context, ipath idxPath, store StoreMigrationApi, f // Add the index to the store addStart := time.Now() - err = store.AddIndex(ctx, pieceCid, records, false) - if err != nil { - return false, fmt.Errorf("adding index %s to store: %w", ipath.path, err) + respch := store.AddIndex(ctx, pieceCid, records, false) + for resp := range respch { + if resp.Err != "" { + return false, fmt.Errorf("adding index %s to store: %s", ipath.path, err) + } } log.Debugw("AddIndex", "took", time.Since(addStart).String()) diff --git a/extern/boostd-data/client/client.go b/extern/boostd-data/client/client.go index 55626aa2e..84bfe8107 100644 --- a/extern/boostd-data/client/client.go +++ b/extern/boostd-data/client/client.go @@ -3,10 +3,10 @@ package client import ( "context" "fmt" - "github.com/filecoin-project/boostd-data/svc/types" "time" "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/svc/types" "github.com/filecoin-project/go-jsonrpc" "github.com/ipfs/go-cid" logger "github.com/ipfs/go-log/v2" @@ -19,7 +19,7 @@ var log = logger.Logger("boostd-data-client") type Store struct { client struct { AddDealForPiece func(context.Context, cid.Cid, model.DealInfo) error - AddIndex func(context.Context, cid.Cid, []model.Record, bool) error + AddIndex func(context.Context, cid.Cid, []model.Record, bool) <-chan types.AddIndexProgress IsIndexed func(ctx context.Context, pieceCid cid.Cid) (bool, error) IsCompleteIndex func(ctx context.Context, pieceCid cid.Cid) (bool, error) GetIndex func(context.Context, cid.Cid) (<-chan types.IndexRecord, error) @@ -38,16 +38,17 @@ type Store struct { FlaggedPiecesList func(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) FlaggedPiecesCount func(ctx context.Context) (int, error) } - closer jsonrpc.ClientCloser + closer jsonrpc.ClientCloser + dialOpts []jsonrpc.Option } -func NewStore() *Store { - return &Store{} +func NewStore(dialOpts ...jsonrpc.Option) *Store { + return &Store{dialOpts: dialOpts} } func (s *Store) Dial(ctx context.Context, addr string) error { var err error - s.closer, err = jsonrpc.NewClient(ctx, addr, "boostddata", &s.client, nil) + s.closer, err = jsonrpc.NewMergeClient(ctx, addr, "boostddata", []interface{}{&s.client}, nil, s.dialOpts...) if err != nil { return fmt.Errorf("dialing local index directory server: %w", err) } @@ -68,6 +69,9 @@ func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) (index.Index, er var records []index.Record for r := range resp { + if r.Error != nil { + return nil, r.Error + } records = append(records, index.Record{ Cid: r.Cid, Offset: r.Offset, @@ -93,10 +97,10 @@ func (s *Store) GetRecords(ctx context.Context, pieceCid cid.Cid) ([]model.Recor var records []model.Record for r := range resp { - records = append(records, model.Record{ - Cid: r.Cid, - OffsetSize: model.OffsetSize{Offset: r.Offset}, - }) + if r.Error != nil { + return nil, r.Error + } + records = append(records, r.Record) } return records, nil @@ -121,7 +125,14 @@ func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error { log.Debugw("add-index", "piece-cid", pieceCid, "records", len(records)) - return s.client.AddIndex(ctx, pieceCid, records, isCompleteIndex) + respch := s.client.AddIndex(ctx, pieceCid, records, isCompleteIndex) + for resp := range respch { + if resp.Err != "" { + return fmt.Errorf("add index with piece cid %s: %s", pieceCid, resp.Err) + } + //fmt.Printf("%s: Percent complete: %f%%\n", time.Now(), resp.Progress*100) + } + return nil } func (s *Store) IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) { diff --git a/extern/boostd-data/couchbase/service.go b/extern/boostd-data/couchbase/service.go index 8308635b4..cba674025 100644 --- a/extern/boostd-data/couchbase/service.go +++ b/extern/boostd-data/couchbase/service.go @@ -3,7 +3,6 @@ package couchbase import ( "context" "fmt" - "github.com/ipld/go-car/v2/index" "time" "github.com/filecoin-project/boostd-data/model" @@ -155,12 +154,7 @@ func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) (<-chan types.In recs := make(chan types.IndexRecord, len(records)) for _, r := range records { - recs <- types.IndexRecord{ - Record: index.Record{ - Cid: r.Cid, - Offset: r.Offset, - }, - } + recs <- types.IndexRecord{Record: r} } close(recs) @@ -193,7 +187,7 @@ func (s *Store) IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, er return md.CompleteIndex, nil } -func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error { +func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) <-chan types.AddIndexProgress { log.Debugw("handle.add-index", "records", len(records)) ctx, span := tracing.Tracer.Start(ctx, "store.add_index") @@ -209,22 +203,32 @@ func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model. mhs = append(mhs, r.Cid.Hash()) } - setMhStart := time.Now() - err := s.db.SetMultihashesToPieceCid(ctx, mhs, pieceCid) - if err != nil { - return fmt.Errorf("failed to add entry from mh to pieceCid: %w", err) - } - log.Debugw("handled.add-index SetMultihashesToPieceCid", "took", time.Since(setMhStart).String()) + progress := make(chan types.AddIndexProgress, 1) + go func() { + defer close(progress) + progress <- types.AddIndexProgress{Progress: 0} - // Add a mapping from piece cid -> offset / size of each block so that - // clients can get the block info for all blocks in a piece - addOffsetsStart := time.Now() - if err := s.db.AddIndexRecords(ctx, pieceCid, records); err != nil { - return err - } - log.Debugw("handled.add-index AddIndexRecords", "took", time.Since(addOffsetsStart).String()) + setMhStart := time.Now() + err := s.db.SetMultihashesToPieceCid(ctx, mhs, pieceCid) + if err != nil { + progress <- types.AddIndexProgress{Err: err.Error()} + return + } + log.Debugw("handled.add-index SetMultihashesToPieceCid", "took", time.Since(setMhStart).String()) + progress <- types.AddIndexProgress{Progress: 0.5} + + // Add a mapping from piece cid -> offset / size of each block so that + // clients can get the block info for all blocks in a piece + addOffsetsStart := time.Now() + if err := s.db.AddIndexRecords(ctx, pieceCid, records); err != nil { + progress <- types.AddIndexProgress{Err: err.Error()} + return + } + log.Debugw("handled.add-index AddIndexRecords", "took", time.Since(addOffsetsStart).String()) + progress <- types.AddIndexProgress{Progress: 1} + }() - return s.db.MarkIndexingComplete(ctx, pieceCid, len(records), isCompleteIndex) + return progress } func (s *Store) IndexedAt(ctx context.Context, pieceCid cid.Cid) (time.Time, error) { diff --git a/extern/boostd-data/ldb/service.go b/extern/boostd-data/ldb/service.go index c69af7038..0c412cd21 100644 --- a/extern/boostd-data/ldb/service.go +++ b/extern/boostd-data/ldb/service.go @@ -229,12 +229,7 @@ func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) (<-chan types.In recs := make(chan types.IndexRecord, len(records)) for _, r := range records { - recs <- types.IndexRecord{ - Record: carindex.Record{ - Cid: r.Cid, - Offset: r.Offset, - }, - } + recs <- types.IndexRecord{Record: r} } close(recs) @@ -267,7 +262,7 @@ func (s *Store) IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, er return md.CompleteIndex, nil } -func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error { +func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) <-chan types.AddIndexProgress { log.Debugw("handle.add-index", "records", len(records)) ctx, span := tracing.Tracer.Start(ctx, "store.add_index") @@ -277,68 +272,84 @@ func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model. log.Debugw("handled.add-index", "took", time.Since(now).String()) }(time.Now()) - s.Lock() - defer s.Unlock() + progress := make(chan types.AddIndexProgress, 1) + go func() { + defer close(progress) - var recs []carindex.Record - for _, r := range records { - recs = append(recs, carindex.Record{ - Cid: r.Cid, - Offset: r.Offset, - }) - } + s.Lock() + defer s.Unlock() - err := s.db.SetMultihashesToPieceCid(ctx, recs, pieceCid) - if err != nil { - return fmt.Errorf("failed to add entry from mh to pieceCid: %w", err) - } + var recs []carindex.Record + for _, r := range records { + recs = append(recs, carindex.Record{ + Cid: r.Cid, + Offset: r.Offset, + }) + } - // get and set next cursor (handle synchronization, maybe with CAS) - cursor, keyCursorPrefix, err := s.db.NextCursor(ctx) - if err != nil { - return fmt.Errorf("couldnt generate next cursor: %w", err) - } + err := s.db.SetMultihashesToPieceCid(ctx, recs, pieceCid) + if err != nil { + progress <- types.AddIndexProgress{Err: err.Error()} + return + } + progress <- types.AddIndexProgress{Progress: 0.45} - // allocate metadata for pieceCid - err = s.db.SetNextCursor(ctx, cursor+1) - if err != nil { - return err - } + // get and set next cursor (handle synchronization, maybe with CAS) + cursor, keyCursorPrefix, err := s.db.NextCursor(ctx) + if err != nil { + progress <- types.AddIndexProgress{Err: err.Error()} + return + } - // process index and store entries - for _, rec := range records { - err := s.db.AddIndexRecord(ctx, keyCursorPrefix, rec) + // allocate metadata for pieceCid + err = s.db.SetNextCursor(ctx, cursor+1) if err != nil { - return err + progress <- types.AddIndexProgress{Err: err.Error()} + return } - } - // get the metadata for the piece - md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) - if err != nil { - if !errors.Is(err, ds.ErrNotFound) { - return fmt.Errorf("getting piece cid metadata for piece %s: %w", pieceCid, err) + // process index and store entries + for _, rec := range records { + err := s.db.AddIndexRecord(ctx, keyCursorPrefix, rec) + if err != nil { + progress <- types.AddIndexProgress{Err: err.Error()} + return + } } - // there isn't yet any metadata, so create new metadata - md = newLeveldbMetadata() - } + progress <- types.AddIndexProgress{Progress: 0.9} - // mark indexing as complete - md.Cursor = cursor - md.IndexedAt = time.Now() - md.CompleteIndex = isCompleteIndex + // get the metadata for the piece + md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) + if err != nil { + if !errors.Is(err, ds.ErrNotFound) { + progress <- types.AddIndexProgress{Err: err.Error()} + return + } + // there isn't yet any metadata, so create new metadata + md = newLeveldbMetadata() + } - err = s.db.SetPieceCidToMetadata(ctx, pieceCid, md) - if err != nil { - return err - } + // mark indexing as complete + md.Cursor = cursor + md.IndexedAt = time.Now() + md.CompleteIndex = isCompleteIndex - err = s.db.Sync(ctx, ds.NewKey(fmt.Sprintf("%d", cursor))) - if err != nil { - return err - } + err = s.db.SetPieceCidToMetadata(ctx, pieceCid, md) + if err != nil { + progress <- types.AddIndexProgress{Err: err.Error()} + return + } + progress <- types.AddIndexProgress{Progress: 0.95} - return nil + err = s.db.Sync(ctx, ds.NewKey(fmt.Sprintf("%d", cursor))) + if err != nil { + progress <- types.AddIndexProgress{Err: err.Error()} + return + } + progress <- types.AddIndexProgress{Progress: 1} + }() + + return progress } func (s *Store) IndexedAt(ctx context.Context, pieceCid cid.Cid) (time.Time, error) { diff --git a/extern/boostd-data/model/model.go b/extern/boostd-data/model/model.go index 7ab0004a2..b8452a766 100644 --- a/extern/boostd-data/model/model.go +++ b/extern/boostd-data/model/model.go @@ -47,16 +47,16 @@ type Metadata struct { // Record is the information stored in the index for each block in a piece type Record struct { - Cid cid.Cid + Cid cid.Cid `json:"c"` OffsetSize } type OffsetSize struct { // Offset is the offset into the CAR file of the section, where a section // is
- Offset uint64 + Offset uint64 `json:"o"` // Size is the size of the block data (not the whole section) - Size uint64 + Size uint64 `json:"s"` } func (ofsz *OffsetSize) MarshallBase64() string { diff --git a/extern/boostd-data/svc/svc_size_test.go b/extern/boostd-data/svc/svc_size_test.go new file mode 100644 index 000000000..5ea8ae253 --- /dev/null +++ b/extern/boostd-data/svc/svc_size_test.go @@ -0,0 +1,121 @@ +package svc + +import ( + "context" + "encoding/json" + "fmt" + "github.com/filecoin-project/boost/testutil" + "github.com/filecoin-project/boostd-data/client" + "github.com/filecoin-project/boostd-data/model" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "github.com/stretchr/testify/require" + "math" + "math/rand" + "testing" + "time" +) + +var tlg = logging.Logger("tlg") + +func TestSizeLimit(t *testing.T) { + // This test takes an hour so don't run as part of regular + // test suite + t.Skip() + + _ = logging.SetLogLevel("cbtest", "debug") + _ = logging.SetLogLevel("tlg", "info") + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + t.Run("leveldb", func(t *testing.T) { + bdsvc, err := NewLevelDB("") + require.NoError(t, err) + + testSizeLimit(ctx, t, bdsvc, "localhost:8042") + }) + + t.Run("yugabyte", func(t *testing.T) { + _ = logging.SetLogLevel("boostd-data-yb", "debug") + + SetupYugabyte(t) + + bdsvc := NewYugabyte(TestYugabyteSettings) + + addr := "localhost:8044" + testSizeLimit(ctx, t, bdsvc, addr) + }) +} + +func testSizeLimit(ctx context.Context, t *testing.T, bdsvc *Service, addr string) { + err := bdsvc.Start(ctx, addr) + require.NoError(t, err) + + cl := client.NewStore() + err = cl.Dial(context.Background(), fmt.Sprintf("ws://%s", addr)) + require.NoError(t, err) + defer cl.Close(ctx) + + pieceCid, err := cid.Parse("baga6ea4seaqnfhocd544oidrgsss2ahoaomvxuaqxfmlsizljtzsuivjl5hamka") + require.NoError(t, err) + + // 32GB file with 1k block size + recordCount := 32 * 1024 * 1024 + baseCid := testutil.GenerateCid().Bytes() + recStart := time.Now() + tlg.Infof("generating %d records", recordCount) + var records []model.Record + for i := 0; i < recordCount; i++ { + c, err := generateRandomCid(baseCid) + require.NoError(t, err) + records = append(records, model.Record{ + Cid: c, + OffsetSize: model.OffsetSize{ + Offset: math.MaxUint64, + Size: math.MaxUint64, + }, + }) + } + tlg.Infof("generated %d records in %s", recordCount, time.Since(recStart)) + + bz, err := json.Marshal(records[0]) + tlg.Infof("%s (%d bytes)", bz, len(bz)) + + addStart := time.Now() + tlg.Infof("adding index") + err = cl.AddIndex(ctx, pieceCid, records, true) + require.NoError(t, err) + tlg.Infof("added index in %s", time.Since(addStart)) + + getStart := time.Now() + tlg.Infof("getting index") + idx, err := cl.GetIndex(ctx, pieceCid) + require.NoError(t, err) + tlg.Infof("got index in %s", time.Since(getStart)) + + entries, ok := toEntries(idx) + require.True(t, ok) + tlg.Infof("got %d entries back", len(entries)) + require.Len(t, entries, recordCount) + + _, has := entries[records[recordCount/2].Cid.Hash().String()] + require.True(t, has) +} + +func generateRandomCid(baseCid []byte) (cid.Cid, error) { + buff := make([]byte, len(baseCid)) + copy(buff, baseCid) + + _, err := rand.Read(buff[len(buff)-8:]) + if err != nil { + return cid.Undef, err + } + + _, c, err := cid.CidFromBytes(buff) + if err != nil { + return cid.Undef, fmt.Errorf("generating cid: %w", err) + } + + return c, nil +} diff --git a/extern/boostd-data/svc/svc_test.go b/extern/boostd-data/svc/svc_test.go index 03f7f45b9..bc4f37e7b 100644 --- a/extern/boostd-data/svc/svc_test.go +++ b/extern/boostd-data/svc/svc_test.go @@ -236,7 +236,7 @@ func TestServiceFuzz(t *testing.T) { func testServiceFuzz(ctx context.Context, t *testing.T, addr string) { cl := client.NewStore() - err := cl.Dial(context.Background(), "http://"+addr) + err := cl.Dial(context.Background(), "ws://"+addr) require.NoError(t, err) defer cl.Close(ctx) @@ -432,15 +432,15 @@ func compareIndices(subject, subjectDb index.Index) (bool, error) { b, okb := toEntries(subjectDb) if oka && okb { if len(a) != len(b) { - fmt.Println("index length mismatch", "first length", len(a), "second length", len(b)) + return false, fmt.Errorf("index length mismatch: first %d / second %d", len(a), len(b)) } for mh, oa := range a { ob, ok := b[mh] if !ok { - fmt.Println("second index missing multihash", "multihash", mh) + return false, fmt.Errorf("second index missing multihash %s", mh) } if oa != ob { - fmt.Println("offset mismatch", "multihash", mh, "first offset", oa, "second offset", ob) + return false, fmt.Errorf("offset mismatch for multihash %s: first %d / second %d", mh, oa, ob) } } } @@ -505,7 +505,7 @@ func testCleanup(ctx context.Context, t *testing.T, bdsvc *Service, addr string) require.NoError(t, err) cl := client.NewStore() - err = cl.Dial(context.Background(), fmt.Sprintf("http://%s", addr)) + err = cl.Dial(context.Background(), fmt.Sprintf("ws://%s", addr)) require.NoError(t, err) defer cl.Close(ctx) diff --git a/extern/boostd-data/svc/types/types.go b/extern/boostd-data/svc/types/types.go index 7be4171f4..424ba1277 100644 --- a/extern/boostd-data/svc/types/types.go +++ b/extern/boostd-data/svc/types/types.go @@ -8,7 +8,6 @@ import ( "github.com/filecoin-project/boostd-data/model" "github.com/ipfs/go-cid" - "github.com/ipld/go-car/v2/index" mh "github.com/multiformats/go-multihash" ) @@ -26,13 +25,18 @@ func IsNotFound(err error) bool { } type IndexRecord struct { - index.Record - Error error + model.Record + Error error `json:"e,omitempty"` +} + +type AddIndexProgress struct { + Progress float64 `json:"p"` + Err string `json:"e,omitempty"` } type Service interface { AddDealForPiece(context.Context, cid.Cid, model.DealInfo) error - AddIndex(context.Context, cid.Cid, []model.Record, bool) error + AddIndex(context.Context, cid.Cid, []model.Record, bool) <-chan AddIndexProgress GetIndex(context.Context, cid.Cid) (<-chan IndexRecord, error) IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, error) diff --git a/extern/boostd-data/yugabyte/service.go b/extern/boostd-data/yugabyte/service.go index 4f7317be2..d3e5784e4 100644 --- a/extern/boostd-data/yugabyte/service.go +++ b/extern/boostd-data/yugabyte/service.go @@ -10,7 +10,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - "github.com/ipld/go-car/v2/index" "github.com/multiformats/go-multihash" mh "github.com/multiformats/go-multihash" "github.com/yugabyte/gocql" @@ -262,9 +261,12 @@ func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) (<-chan types.In } records <- types.IndexRecord{ - Record: index.Record{ - Cid: cid.NewCidV1(cid.Raw, pmh), - Offset: offset, + Record: model.Record{ + Cid: cid.NewCidV1(cid.Raw, pmh), + OffsetSize: model.OffsetSize{ + Offset: offset, + Size: size, + }, }, } } @@ -295,50 +297,116 @@ func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) (<-chan types.In return records, nil } -func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, recs []model.Record, isCompleteIndex bool) error { +func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, recs []model.Record, isCompleteIndex bool) <-chan types.AddIndexProgress { ctx, span := tracing.Tracer.Start(ctx, "store.add_index") defer span.End() + // Set up the progress channel + progress := make(chan types.AddIndexProgress, 2) if len(recs) == 0 { - return nil + // If there are no records, set progress to 100% and close the channel + progress <- types.AddIndexProgress{Progress: 1} + close(progress) + return progress } - // Add a mapping from multihash -> piece cid so that clients can look up - // which pieces contain a multihash - err := s.addMultihashesToPieces(ctx, pieceCid, recs) - if err != nil { - return err - } + // Start by sending a progress update of zero + progress <- types.AddIndexProgress{Progress: 0} + lastUpdateTime := time.Now() - // Add a mapping from piece cid -> offset / size of each block so that - // clients can get the block info for all blocks in a piece - err = s.addPieceInfos(ctx, pieceCid, recs) - if err != nil { - return err - } + var lastUpdateValue *float64 + updateProgress := func(prg float64) { + // Don't send updates more than once every few seconds + if time.Since(lastUpdateTime) < 5*time.Second { + lastUpdateValue = &prg + return + } - // Ensure the piece metadata exists - err = s.createPieceMetadata(ctx, pieceCid) - if err != nil { - return err + // If the channel is full, don't send this progress update, just + // wait for the next one. + select { + case progress <- types.AddIndexProgress{Progress: prg}: + lastUpdateTime = time.Now() + lastUpdateValue = nil + default: + lastUpdateValue = &prg + } } - // Mark indexing as complete for the piece - qry := `UPDATE idx.PieceMetadata ` + - `SET IndexedAt = ?, CompleteIndex = ? ` + - `WHERE PieceCid = ?` - err = s.session.Query(qry, time.Now(), isCompleteIndex, pieceCid.String()).WithContext(ctx).Exec() - if err != nil { - return fmt.Errorf("marking indexing as complete for piece %s", pieceCid) + completeProgress := func(err error) { + var lastProg *types.AddIndexProgress + if err != nil { + // If there was an error, send it as the last progress update + lastProg = &types.AddIndexProgress{Err: err.Error()} + } else if lastUpdateValue != nil { + // If there is an outstanding update that hasn't been sent out + // yet, make sure it gets sent + lastProg = &types.AddIndexProgress{Progress: *lastUpdateValue} + } + + if lastProg != nil { + select { + case progress <- *lastProg: + case <-time.After(5 * time.Second): + } + } + + // Close the channel + close(progress) } - return nil + go func() { + // Add a mapping from multihash -> piece cid so that clients can look up + // which pieces contain a multihash + err := s.addMultihashesToPieces(ctx, pieceCid, recs, func(addProgress float64) { + // The first 45% of progress is for adding multihash -> pieces index + updateProgress(0.45 * addProgress) + }) + if err != nil { + completeProgress(err) + return + } + + // Add a mapping from piece cid -> offset / size of each block so that + // clients can get the block info for all blocks in a piece + err = s.addPieceInfos(ctx, pieceCid, recs, func(addProgress float64) { + // From 45% - 90% of progress is for adding piece infos + updateProgress(0.45 + 0.45*addProgress) + }) + if err != nil { + completeProgress(err) + return + } + + // Ensure the piece metadata exists + err = s.createPieceMetadata(ctx, pieceCid) + if err != nil { + completeProgress(err) + return + } + updateProgress(0.95) + + // Mark indexing as complete for the piece + qry := `UPDATE idx.PieceMetadata ` + + `SET IndexedAt = ?, CompleteIndex = ? ` + + `WHERE PieceCid = ?` + err = s.session.Query(qry, time.Now(), isCompleteIndex, pieceCid.String()).WithContext(ctx).Exec() + if err != nil { + completeProgress(err) + return + } + updateProgress(1) + completeProgress(nil) + }() + + return progress } -func (s *Store) addMultihashesToPieces(ctx context.Context, pieceCid cid.Cid, recs []model.Record) error { +func (s *Store) addMultihashesToPieces(ctx context.Context, pieceCid cid.Cid, recs []model.Record, progress func(addProgress float64)) error { ctx, span := tracing.Tracer.Start(ctx, "store.add_index.payloadpiece") defer span.End() + var count float64 return s.execParallel(ctx, recs, s.settings.PayloadPiecesParallelism, func(rec model.Record) error { multihashBytes := rec.Cid.Hash() q := `INSERT INTO idx.PayloadToPieces (PayloadMultihash, PieceCid) VALUES (?, ?)` @@ -346,11 +414,14 @@ func (s *Store) addMultihashesToPieces(ctx context.Context, pieceCid cid.Cid, re if err != nil { return fmt.Errorf("inserting into PayloadToPieces: %w", err) } + + count++ + progress(count / float64(len(recs))) return nil }) } -func (s *Store) addPieceInfos(ctx context.Context, pieceCid cid.Cid, recs []model.Record) error { +func (s *Store) addPieceInfos(ctx context.Context, pieceCid cid.Cid, recs []model.Record, progress func(addProgress float64)) error { ctx, span := tracing.Tracer.Start(ctx, "store.add_index.pieceinfo") defer span.End() @@ -381,7 +452,8 @@ func (s *Store) addPieceInfos(ctx context.Context, pieceCid cid.Cid, recs []mode return fmt.Errorf("executing offset / size batch insert for piece %s: %w", pieceCid, err) } batch = nil - continue + + progress((float64(allIdx+1) / float64(len(batchEntries)))) } } @@ -501,29 +573,44 @@ func (s *Store) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error { ctx, span := tracing.Tracer.Start(ctx, "store.remove_indexes") defer span.End() + // Get multihashes for piece recs, err := s.GetIndex(ctx, pieceCid) if err != nil { return fmt.Errorf("removing indexes for piece %s: getting recs: %w", pieceCid, err) } - var records []model.Record - for r := range recs { - records = append(records, model.Record{ - Cid: r.Cid, - OffsetSize: model.OffsetSize{Offset: r.Offset}, + // Delete from multihash -> piece cids index + var eg errgroup.Group + for i := 0; i < s.settings.PayloadPiecesParallelism; i++ { + eg.Go(func() error { + for ctx.Err() == nil { + select { + case <-ctx.Done(): + return ctx.Err() + case rec, ok := <-recs: + if !ok { + // Finished adding all the queued items, exit the thread + return nil + } + + multihashBytes := rec.Cid.Hash() + q := `DELETE FROM idx.PayloadToPieces WHERE PayloadMultihash = ? AND PieceCid = ?` + err := s.session.Query(q, trimMultihash(multihashBytes), pieceCid.Bytes()).Exec() + if err != nil { + return fmt.Errorf("deleting from PayloadToPieces: %w", err) + } + } + } + + return ctx.Err() }) } + err = eg.Wait() + if err != nil { + return err + } - err = s.execParallel(ctx, records, s.settings.PayloadPiecesParallelism, func(rec model.Record) error { - multihashBytes := rec.Cid.Hash() - q := `DELETE FROM idx.PayloadToPieces WHERE PayloadMultihash = ? AND PieceCid = ?` - err := s.session.Query(q, trimMultihash(multihashBytes), pieceCid.Bytes()).Exec() - if err != nil { - return fmt.Errorf("inserting into PayloadToPieces: %w", err) - } - return nil - }) - + // Delete from piece offsets index qry := `DELETE FROM idx.PieceBlockOffsetSize WHERE PieceCid = ?` err = s.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Exec() if err != nil { diff --git a/node/config/def.go b/node/config/def.go index 74daa704d..656134488 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -90,6 +90,7 @@ func DefaultBoost() *Boost { ParallelAddIndexLimit: 4, EmbeddedServicePort: 8042, ServiceApiInfo: "", + ServiceRPCTimeout: Duration(15 * time.Minute), }, ContractDeals: ContractDealsConfig{ diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index c0da49c90..f5f7f9eaf 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -476,9 +476,15 @@ Set this value to zero to disable the embedded local index directory data servic Name: "ServiceApiInfo", Type: "string", - Comment: `The connect string for the local index directory data service RPC API eg "http://localhost:8042" + Comment: `The connect string for the local index directory data service RPC API eg "ws://localhost:8042" Set this value to "" if the local index directory data service is embedded.`, }, + { + Name: "ServiceRPCTimeout", + Type: "Duration", + + Comment: `The RPC timeout when making requests to the boostd-data service`, + }, }, "LocalIndexDirectoryCouchbaseBucketConfig": []DocField{ { diff --git a/node/config/types.go b/node/config/types.go index 2b6db413e..30a1b0702 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -343,7 +343,9 @@ type LocalIndexDirectoryConfig struct { // Set this value to zero to disable the embedded local index directory data service // (in that case the local index directory data service must be running externally) EmbeddedServicePort uint64 - // The connect string for the local index directory data service RPC API eg "http://localhost:8042" + // The connect string for the local index directory data service RPC API eg "ws://localhost:8042" // Set this value to "" if the local index directory data service is embedded. ServiceApiInfo string + // The RPC timeout when making requests to the boostd-data service + ServiceRPCTimeout Duration } diff --git a/node/modules/piecedirectory.go b/node/modules/piecedirectory.go index e1a73a760..bfa2ec57e 100644 --- a/node/modules/piecedirectory.go +++ b/node/modules/piecedirectory.go @@ -3,6 +3,7 @@ package modules import ( "context" "fmt" + "time" "github.com/filecoin-project/boost-gfm/piecestore" "github.com/filecoin-project/boost-gfm/shared" @@ -20,6 +21,7 @@ import ( "github.com/filecoin-project/dagstore" "github.com/filecoin-project/dagstore/shard" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/lotus/api/v1api" mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -35,7 +37,10 @@ import ( func NewPieceDirectoryStore(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_repo.LockedRepo) types.Store { return func(lc fx.Lifecycle, r lotus_repo.LockedRepo) types.Store { - client := piecedirectory.NewStore() + svcDialOpts := []jsonrpc.Option{ + jsonrpc.WithTimeout(time.Duration(cfg.LocalIndexDirectory.ServiceRPCTimeout)), + } + client := piecedirectory.NewStore(svcDialOpts...) var cancel context.CancelFunc var svcCtx context.Context @@ -108,7 +113,7 @@ func NewPieceDirectoryStore(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_rep } // Connect to the embedded service - return client.Dial(ctx, fmt.Sprintf("http://%s", addr)) + return client.Dial(ctx, fmt.Sprintf("ws://%s", addr)) }, OnStop: func(ctx context.Context) error { cancel() diff --git a/piecedirectory/doctor_test.go b/piecedirectory/doctor_test.go index eb8338d0f..105546b7f 100644 --- a/piecedirectory/doctor_test.go +++ b/piecedirectory/doctor_test.go @@ -41,7 +41,7 @@ func TestPieceDoctor(t *testing.T) { require.NoError(t, err) cl := client.NewStore() - err = cl.Dial(ctx, fmt.Sprintf("http://%s", addr)) + err = cl.Dial(ctx, fmt.Sprintf("ws://%s", addr)) require.NoError(t, err) defer cl.Close(ctx) @@ -76,7 +76,7 @@ func TestPieceDoctor(t *testing.T) { require.NoError(t, err) cl := client.NewStore() - err = cl.Dial(ctx, fmt.Sprintf("http://%s", addr)) + err = cl.Dial(ctx, fmt.Sprintf("ws://%s", addr)) require.NoError(t, err) defer cl.Close(ctx) @@ -104,7 +104,7 @@ func TestPieceDoctor(t *testing.T) { require.NoError(t, err) cl := client.NewStore() - err = cl.Dial(ctx, fmt.Sprintf("http://%s", addr)) + err = cl.Dial(ctx, fmt.Sprintf("ws://%s", addr)) require.NoError(t, err) defer cl.Close(ctx) diff --git a/piecedirectory/piecedirectory.go b/piecedirectory/piecedirectory.go index 1e41cab14..4758376a9 100644 --- a/piecedirectory/piecedirectory.go +++ b/piecedirectory/piecedirectory.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/boostd-data/client" "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/shared/tracing" + "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/markets/dagstore" "github.com/hashicorp/go-multierror" @@ -41,8 +42,8 @@ type PieceDirectory struct { addIdxOpByCid sync.Map } -func NewStore() *client.Store { - return client.NewStore() +func NewStore(dialOpts ...jsonrpc.Option) *client.Store { + return client.NewStore(dialOpts...) } func NewPieceDirectory(store types.Store, pr types.PieceReader, addIndexThrottleSize int) *PieceDirectory { diff --git a/piecedirectory/piecedirectory_test.go b/piecedirectory/piecedirectory_test.go index 567f4c8bc..36fe6fb61 100644 --- a/piecedirectory/piecedirectory_test.go +++ b/piecedirectory/piecedirectory_test.go @@ -79,7 +79,7 @@ func testPieceDirectory(ctx context.Context, t *testing.T, bdsvc *svc.Service, a require.NoError(t, err) cl := client.NewStore() - err = cl.Dial(ctx, fmt.Sprintf("http://%s", addr)) + err = cl.Dial(ctx, fmt.Sprintf("ws://%s", addr)) require.NoError(t, err) defer cl.Close(ctx)