-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #6393 from filecoin-project/feat/datamodel-selecto…
…r-retrieval Feat/datamodel selector retrieval
- Loading branch information
Showing
13 changed files
with
422 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,221 @@ | ||
| package itests | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "io" | ||
| "io/ioutil" | ||
| "os" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/filecoin-project/go-fil-markets/storagemarket" | ||
| "github.com/filecoin-project/go-state-types/abi" | ||
| "github.com/filecoin-project/go-state-types/big" | ||
| "github.com/filecoin-project/lotus/api" | ||
| "github.com/filecoin-project/lotus/chain/actors/policy" | ||
| "github.com/filecoin-project/lotus/itests/kit" | ||
| blocks "github.com/ipfs/go-block-format" | ||
| "github.com/ipfs/go-cid" | ||
| "github.com/ipld/go-car" | ||
| textselector "github.com/ipld/go-ipld-selector-text-lite" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // use the mainnet carfile as text fixture: it will always be here | ||
| // https://dweb.link/ipfs/bafy2bzacecnamqgqmifpluoeldx7zzglxcljo6oja4vrmtj7432rphldpdmm2/8/1/8/1/0/1/0 | ||
| var ( | ||
| sourceCar = "../build/genesis/mainnet.car" | ||
| carRoot, _ = cid.Parse("bafy2bzacecnamqgqmifpluoeldx7zzglxcljo6oja4vrmtj7432rphldpdmm2") | ||
| carCommp, _ = cid.Parse("baga6ea4seaqmrivgzei3fmx5qxtppwankmtou6zvigyjaveu3z2zzwhysgzuina") | ||
| carPieceSize = abi.PaddedPieceSize(2097152) | ||
| textSelector = textselector.Expression("8/1/8/1/0/1/0") | ||
| textSelectorNonLink = textselector.Expression("8/1/8/1/0/1") | ||
| textSelectorNonexistent = textselector.Expression("42") | ||
| expectedResult = "fil/1/storagepower" | ||
| ) | ||
|
|
||
| func TestPartialRetrieval(t *testing.T) { | ||
|
|
||
| ctx := context.Background() | ||
|
|
||
| policy.SetPreCommitChallengeDelay(2) | ||
| kit.EnableLargeSectors(t) | ||
| kit.QuietMiningLogs() | ||
| client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.MockProofs(), kit.SectorSize(512<<20)) | ||
| dh := kit.NewDealHarness(t, client, miner, miner) | ||
| ens.InterconnectAll().BeginMining(50 * time.Millisecond) | ||
|
|
||
| _, err := client.ClientImport(ctx, api.FileRef{Path: sourceCar, IsCAR: true}) | ||
| require.NoError(t, err) | ||
|
|
||
| caddr, err := client.WalletDefaultAddress(ctx) | ||
| require.NoError(t, err) | ||
|
|
||
| // first test retrieval from local car, then do an actual deal | ||
| for _, fullCycle := range []bool{false, true} { | ||
|
|
||
| var retOrder api.RetrievalOrder | ||
|
|
||
| if !fullCycle { | ||
|
|
||
| retOrder.FromLocalCAR = sourceCar | ||
| retOrder.Root = carRoot | ||
|
|
||
| } else { | ||
|
|
||
| dp := dh.DefaultStartDealParams() | ||
| dp.Data = &storagemarket.DataRef{ | ||
| // FIXME: figure out how to do this with an online partial transfer | ||
| TransferType: storagemarket.TTManual, | ||
| Root: carRoot, | ||
| PieceCid: &carCommp, | ||
| PieceSize: carPieceSize.Unpadded(), | ||
| } | ||
| proposalCid := dh.StartDeal(ctx, dp) | ||
|
|
||
| // Wait for the deal to reach StorageDealCheckForAcceptance on the client | ||
| cd, err := client.ClientGetDealInfo(ctx, *proposalCid) | ||
| require.NoError(t, err) | ||
| require.Eventually(t, func() bool { | ||
| cd, _ := client.ClientGetDealInfo(ctx, *proposalCid) | ||
| return cd.State == storagemarket.StorageDealCheckForAcceptance | ||
| }, 30*time.Second, 1*time.Second, "actual deal status is %s", storagemarket.DealStates[cd.State]) | ||
|
|
||
| err = miner.DealsImportData(ctx, *proposalCid, sourceCar) | ||
| require.NoError(t, err) | ||
|
|
||
| // Wait for the deal to be published, we should be able to start retrieval right away | ||
| dh.WaitDealPublished(ctx, proposalCid) | ||
|
|
||
| offers, err := client.ClientFindData(ctx, carRoot, nil) | ||
| require.NoError(t, err) | ||
| require.NotEmpty(t, offers, "no offers") | ||
|
|
||
| retOrder = offers[0].Order(caddr) | ||
| } | ||
|
|
||
| retOrder.DatamodelPathSelector = &textSelector | ||
|
|
||
| // test retrieval of either data or constructing a partial selective-car | ||
| for _, retrieveAsCar := range []bool{false, true} { | ||
| outFile, err := ioutil.TempFile(t.TempDir(), "ret-file") | ||
| require.NoError(t, err) | ||
| defer outFile.Close() //nolint:errcheck | ||
|
|
||
| require.NoError(t, testGenesisRetrieval( | ||
| ctx, | ||
| client, | ||
| retOrder, | ||
| &api.FileRef{ | ||
| Path: outFile.Name(), | ||
| IsCAR: retrieveAsCar, | ||
| }, | ||
| outFile, | ||
| )) | ||
|
|
||
| // UGH if I do not sleep here, I get things like: | ||
| /* | ||
| retrieval failed: Retrieve failed: there is an active retrieval deal with peer 12D3KooWK9fB9a3HZ4PQLVmEQ6pweMMn5CAyKtumB71CPTnuBDi6 for payload CID bafy2bzacecnamqgqmifpluoeldx7zzglxcljo6oja4vrmtj7432rphldpdmm2 (retrieval deal ID 1631259332180384709, state DealStatusFinalizingBlockstore) - existing deal must be cancelled before starting a new retrieval deal: | ||
| github.com/filecoin-project/lotus/node/impl/client.(*API).ClientRetrieve | ||
| /home/circleci/project/node/impl/client/client.go:774 | ||
| */ | ||
| time.Sleep(time.Second) | ||
| } | ||
| } | ||
|
|
||
| // ensure non-existent paths fail | ||
| require.EqualError( | ||
| t, | ||
| testGenesisRetrieval( | ||
| ctx, | ||
| client, | ||
| api.RetrievalOrder{ | ||
| FromLocalCAR: sourceCar, | ||
| Root: carRoot, | ||
| DatamodelPathSelector: &textSelectorNonexistent, | ||
| }, | ||
| &api.FileRef{}, | ||
| nil, | ||
| ), | ||
| fmt.Sprintf("retrieval failed: path selection '%s' does not match a node within %s", textSelectorNonexistent, carRoot), | ||
| ) | ||
|
|
||
| // ensure non-boundary retrievals fail | ||
| require.EqualError( | ||
| t, | ||
| testGenesisRetrieval( | ||
| ctx, | ||
| client, | ||
| api.RetrievalOrder{ | ||
| FromLocalCAR: sourceCar, | ||
| Root: carRoot, | ||
| DatamodelPathSelector: &textSelectorNonLink, | ||
| }, | ||
| &api.FileRef{}, | ||
| nil, | ||
| ), | ||
| fmt.Sprintf("retrieval failed: error while locating partial retrieval sub-root: unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", textSelectorNonLink), | ||
| ) | ||
| } | ||
|
|
||
| func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrder api.RetrievalOrder, retRef *api.FileRef, outFile *os.File) error { | ||
|
|
||
| if retOrder.Total.Nil() { | ||
| retOrder.Total = big.Zero() | ||
| } | ||
| if retOrder.UnsealPrice.Nil() { | ||
| retOrder.UnsealPrice = big.Zero() | ||
| } | ||
|
|
||
| err := client.ClientRetrieve(ctx, retOrder, retRef) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| var data []byte | ||
| if !retRef.IsCAR { | ||
|
|
||
| data, err = io.ReadAll(outFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| } else { | ||
|
|
||
| cr, err := car.NewCarReader(outFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if len(cr.Header.Roots) != 1 { | ||
| return fmt.Errorf("expected a single root in result car, got %d", len(cr.Header.Roots)) | ||
| } else if cr.Header.Roots[0].String() != carRoot.String() { | ||
| return fmt.Errorf("expected root cid '%s', got '%s'", carRoot.String(), cr.Header.Roots[0].String()) | ||
| } | ||
|
|
||
| blks := make([]blocks.Block, 0) | ||
| for { | ||
| b, err := cr.Next() | ||
| if err == io.EOF { | ||
| break | ||
| } else if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| blks = append(blks, b) | ||
| } | ||
|
|
||
| if len(blks) != 3 { | ||
| return fmt.Errorf("expected a car file with 3 blocks, got one with %d instead", len(blks)) | ||
| } | ||
|
|
||
| data = blks[2].RawData() | ||
| } | ||
|
|
||
| if string(data) != expectedResult { | ||
| return fmt.Errorf("retrieved data mismatch: expected '%s' got '%s'", expectedResult, data) | ||
| } | ||
|
|
||
| return nil | ||
| } |
Oops, something went wrong.