-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Expose basic text-based datamodel selector on retrieval
Syntaxt of selection is located at https://pkg.go.dev/github.com/ipld/go-ipld-selector-text-lite#SelectorSpecFromPath Example use, assuming that: - The root of the deal is a plain dag-pb unixfs directory - The directory is not sharded - The user wants to retrieve the first entry in that directory lotus client retrieve --miner f0XXXXX --datamodel-path-selector 'Links/0/Hash' bafyROOTCID ~/output For a much more elaborate example see the top of ./itests/deals_partial_retrieval_test.go
- Loading branch information
Showing
12 changed files
with
369 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,177 @@ | ||
| package itests | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "io" | ||
| "io/ioutil" | ||
| "os" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/filecoin-project/go-fil-markets/storagemarket" | ||
| "github.com/filecoin-project/go-state-types/abi" | ||
| "github.com/filecoin-project/go-state-types/big" | ||
| "github.com/filecoin-project/lotus/api" | ||
| "github.com/filecoin-project/lotus/chain/actors/policy" | ||
| "github.com/filecoin-project/lotus/itests/kit" | ||
| blocks "github.com/ipfs/go-block-format" | ||
| "github.com/ipfs/go-cid" | ||
| "github.com/ipld/go-car" | ||
| textselector "github.com/ipld/go-ipld-selector-text-lite" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // use the mainnet carfile as text fixture: it will always be here | ||
| // https://dweb.link/ipfs/bafy2bzacecnamqgqmifpluoeldx7zzglxcljo6oja4vrmtj7432rphldpdmm2/8/1/8/1/0/1/0 | ||
| var ( | ||
| sourceCar = "../build/genesis/mainnet.car" | ||
| carRoot, _ = cid.Parse("bafy2bzacecnamqgqmifpluoeldx7zzglxcljo6oja4vrmtj7432rphldpdmm2") | ||
| carCommp, _ = cid.Parse("baga6ea4seaqmrivgzei3fmx5qxtppwankmtou6zvigyjaveu3z2zzwhysgzuina") | ||
| carPieceSize = abi.PaddedPieceSize(2097152) | ||
| textSelector = textselector.Expression("8/1/8/1/0/1/0") | ||
| expectedResult = "fil/1/storagepo wer" | ||
| ) | ||
|
|
||
| func TestPartialRetrieval(t *testing.T) { | ||
|
|
||
| ctx := context.Background() | ||
|
|
||
| policy.SetPreCommitChallengeDelay(2) | ||
| kit.EnableLargeSectors(t) | ||
| kit.QuietMiningLogs() | ||
| client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.MockProofs(), kit.SectorSize(512<<20)) | ||
| dh := kit.NewDealHarness(t, client, miner, miner) | ||
| ens.InterconnectAll().BeginMining(50 * time.Millisecond) | ||
|
|
||
| _, err := client.ClientImport(ctx, api.FileRef{Path: sourceCar, IsCAR: true}) | ||
| require.NoError(t, err) | ||
|
|
||
| caddr, err := client.WalletDefaultAddress(ctx) | ||
| require.NoError(t, err) | ||
|
|
||
| // first test retrieval from local car, then do an actual deal | ||
| for _, fullCycle := range []bool{false, true} { | ||
|
|
||
| var retOrder api.RetrievalOrder | ||
|
|
||
| if !fullCycle { | ||
|
|
||
| retOrder.FromLocalCAR = sourceCar | ||
| retOrder.Root = carRoot | ||
|
|
||
| } else { | ||
|
|
||
| dp := dh.DefaultStartDealParams() | ||
| dp.Data = &storagemarket.DataRef{ | ||
| // FIXME: figure out how to do this with an online partial transfer | ||
| TransferType: storagemarket.TTManual, | ||
| Root: carRoot, | ||
| PieceCid: &carCommp, | ||
| PieceSize: carPieceSize.Unpadded(), | ||
| } | ||
| proposalCid := dh.StartDeal(ctx, dp) | ||
|
|
||
| // Wait for the deal to reach StorageDealCheckForAcceptance on the client | ||
| cd, err := client.ClientGetDealInfo(ctx, *proposalCid) | ||
| require.NoError(t, err) | ||
| require.Eventually(t, func() bool { | ||
| cd, _ := client.ClientGetDealInfo(ctx, *proposalCid) | ||
| return cd.State == storagemarket.StorageDealCheckForAcceptance | ||
| }, 30*time.Second, 1*time.Second, "actual deal status is %s", storagemarket.DealStates[cd.State]) | ||
|
|
||
| err = miner.DealsImportData(ctx, *proposalCid, sourceCar) | ||
| require.NoError(t, err) | ||
|
|
||
| // Wait for the deal to be published, we should be able to start retrieval right away | ||
| dh.WaitDealPublished(ctx, proposalCid) | ||
|
|
||
| offers, err := client.ClientFindData(ctx, carRoot, nil) | ||
| require.NoError(t, err) | ||
| require.NotEmpty(t, offers, "no offers") | ||
|
|
||
| retOrder = offers[0].Order(caddr) | ||
| } | ||
|
|
||
| retOrder.DatamodelPathSelector = &textSelector | ||
|
|
||
| // test retrieval of either data or constructing a partial selective-car | ||
| for _, retrieveAsCar := range []bool{false, true} { | ||
| outFile, err := ioutil.TempFile(t.TempDir(), "ret-file") | ||
| require.NoError(t, err) | ||
| defer outFile.Close() //nolint:errcheck | ||
|
|
||
| require.NoError(t, testGenesisRetrieval( | ||
| ctx, | ||
| client, | ||
| retOrder, | ||
| &api.FileRef{ | ||
| Path: outFile.Name(), | ||
| IsCAR: retrieveAsCar, | ||
| }, | ||
| outFile, | ||
| )) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrder api.RetrievalOrder, retRef *api.FileRef, outFile *os.File) error { | ||
|
|
||
| if retOrder.Total.Nil() { | ||
| retOrder.Total = big.Zero() | ||
| } | ||
| if retOrder.UnsealPrice.Nil() { | ||
| retOrder.UnsealPrice = big.Zero() | ||
| } | ||
|
|
||
| err := client.ClientRetrieve(ctx, retOrder, retRef) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| var data []byte | ||
| if !retRef.IsCAR { | ||
|
|
||
| data, err = io.ReadAll(outFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| } else { | ||
|
|
||
| cr, err := car.NewCarReader(outFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if len(cr.Header.Roots) != 1 { | ||
| return fmt.Errorf("expected a single root in result car, got %d", len(cr.Header.Roots)) | ||
| } else if cr.Header.Roots[0].String() != carRoot.String() { | ||
| return fmt.Errorf("expected root cid '%s', got '%s'", carRoot.String(), cr.Header.Roots[0].String()) | ||
| } | ||
|
|
||
| blks := make([]blocks.Block, 0) | ||
| for { | ||
| b, err := cr.Next() | ||
| if err == io.EOF { | ||
| break | ||
| } else if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| blks = append(blks, b) | ||
| } | ||
|
|
||
| if len(blks) != 3 { | ||
| return fmt.Errorf("expected a car file with 3 blocks, got one with %d instead", len(blks)) | ||
| } | ||
|
|
||
| data = blks[2].RawData() | ||
| } | ||
|
|
||
| if string(data) != expectedResult { | ||
| return fmt.Errorf("retrieved data mismatch: expected '%s' got '%s'", expectedResult, data) | ||
| } | ||
|
|
||
| return nil | ||
| } |
Oops, something went wrong.