-
Notifications
You must be signed in to change notification settings - Fork 1.3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Feat/datamodel selector retrieval #6393
Merged
Merged
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
0444435
Expose basic text-based datamodel selector on retrieval
ribasushi 16973bc
feat: use [email protected]
dirkmc f70ec02
feat: go-fil-markets-v1.13.0
dirkmc 2a1094f
Merge banch feat/go-fil-markets-v1.13.0 into feat/datamodel-selector-…
ribasushi af0d9b6
Error on corner cases with corresponding negative tests
ribasushi 5e6acee
Merge remote-tracking branch 'origin/master' into feat/datamodel-sele…
ribasushi f8ac98b
Adjust error wording
ribasushi File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,221 @@ | ||
| package itests | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "io" | ||
| "io/ioutil" | ||
| "os" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/filecoin-project/go-fil-markets/storagemarket" | ||
| "github.com/filecoin-project/go-state-types/abi" | ||
| "github.com/filecoin-project/go-state-types/big" | ||
| "github.com/filecoin-project/lotus/api" | ||
| "github.com/filecoin-project/lotus/chain/actors/policy" | ||
| "github.com/filecoin-project/lotus/itests/kit" | ||
| blocks "github.com/ipfs/go-block-format" | ||
| "github.com/ipfs/go-cid" | ||
| "github.com/ipld/go-car" | ||
| textselector "github.com/ipld/go-ipld-selector-text-lite" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // use the mainnet carfile as text fixture: it will always be here | ||
| // https://dweb.link/ipfs/bafy2bzacecnamqgqmifpluoeldx7zzglxcljo6oja4vrmtj7432rphldpdmm2/8/1/8/1/0/1/0 | ||
| var ( | ||
| sourceCar = "../build/genesis/mainnet.car" | ||
| carRoot, _ = cid.Parse("bafy2bzacecnamqgqmifpluoeldx7zzglxcljo6oja4vrmtj7432rphldpdmm2") | ||
| carCommp, _ = cid.Parse("baga6ea4seaqmrivgzei3fmx5qxtppwankmtou6zvigyjaveu3z2zzwhysgzuina") | ||
| carPieceSize = abi.PaddedPieceSize(2097152) | ||
| textSelector = textselector.Expression("8/1/8/1/0/1/0") | ||
| textSelectorNonLink = textselector.Expression("8/1/8/1/0/1") | ||
| textSelectorNonexistent = textselector.Expression("42") | ||
| expectedResult = "fil/1/storagepower" | ||
| ) | ||
|
|
||
| func TestPartialRetrieval(t *testing.T) { | ||
|
|
||
| ctx := context.Background() | ||
|
|
||
| policy.SetPreCommitChallengeDelay(2) | ||
| kit.EnableLargeSectors(t) | ||
| kit.QuietMiningLogs() | ||
| client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.MockProofs(), kit.SectorSize(512<<20)) | ||
| dh := kit.NewDealHarness(t, client, miner, miner) | ||
| ens.InterconnectAll().BeginMining(50 * time.Millisecond) | ||
|
|
||
| _, err := client.ClientImport(ctx, api.FileRef{Path: sourceCar, IsCAR: true}) | ||
| require.NoError(t, err) | ||
|
|
||
| caddr, err := client.WalletDefaultAddress(ctx) | ||
| require.NoError(t, err) | ||
|
|
||
| // first test retrieval from local car, then do an actual deal | ||
| for _, fullCycle := range []bool{false, true} { | ||
|
|
||
| var retOrder api.RetrievalOrder | ||
|
|
||
| if !fullCycle { | ||
|
|
||
| retOrder.FromLocalCAR = sourceCar | ||
| retOrder.Root = carRoot | ||
|
|
||
| } else { | ||
|
|
||
| dp := dh.DefaultStartDealParams() | ||
| dp.Data = &storagemarket.DataRef{ | ||
| // FIXME: figure out how to do this with an online partial transfer | ||
| TransferType: storagemarket.TTManual, | ||
| Root: carRoot, | ||
| PieceCid: &carCommp, | ||
| PieceSize: carPieceSize.Unpadded(), | ||
| } | ||
| proposalCid := dh.StartDeal(ctx, dp) | ||
|
|
||
| // Wait for the deal to reach StorageDealCheckForAcceptance on the client | ||
| cd, err := client.ClientGetDealInfo(ctx, *proposalCid) | ||
| require.NoError(t, err) | ||
| require.Eventually(t, func() bool { | ||
| cd, _ := client.ClientGetDealInfo(ctx, *proposalCid) | ||
| return cd.State == storagemarket.StorageDealCheckForAcceptance | ||
| }, 30*time.Second, 1*time.Second, "actual deal status is %s", storagemarket.DealStates[cd.State]) | ||
|
|
||
| err = miner.DealsImportData(ctx, *proposalCid, sourceCar) | ||
| require.NoError(t, err) | ||
|
|
||
| // Wait for the deal to be published, we should be able to start retrieval right away | ||
| dh.WaitDealPublished(ctx, proposalCid) | ||
|
|
||
| offers, err := client.ClientFindData(ctx, carRoot, nil) | ||
| require.NoError(t, err) | ||
| require.NotEmpty(t, offers, "no offers") | ||
|
|
||
| retOrder = offers[0].Order(caddr) | ||
| } | ||
|
|
||
| retOrder.DatamodelPathSelector = &textSelector | ||
|
|
||
| // test retrieval of either data or constructing a partial selective-car | ||
| for _, retrieveAsCar := range []bool{false, true} { | ||
| outFile, err := ioutil.TempFile(t.TempDir(), "ret-file") | ||
| require.NoError(t, err) | ||
| defer outFile.Close() //nolint:errcheck | ||
|
|
||
| require.NoError(t, testGenesisRetrieval( | ||
| ctx, | ||
| client, | ||
| retOrder, | ||
| &api.FileRef{ | ||
| Path: outFile.Name(), | ||
| IsCAR: retrieveAsCar, | ||
| }, | ||
| outFile, | ||
| )) | ||
|
|
||
| // UGH if I do not sleep here, I get things like: | ||
| /* | ||
| retrieval failed: Retrieve failed: there is an active retrieval deal with peer 12D3KooWK9fB9a3HZ4PQLVmEQ6pweMMn5CAyKtumB71CPTnuBDi6 for payload CID bafy2bzacecnamqgqmifpluoeldx7zzglxcljo6oja4vrmtj7432rphldpdmm2 (retrieval deal ID 1631259332180384709, state DealStatusFinalizingBlockstore) - existing deal must be cancelled before starting a new retrieval deal: | ||
| github.com/filecoin-project/lotus/node/impl/client.(*API).ClientRetrieve | ||
| /home/circleci/project/node/impl/client/client.go:774 | ||
| */ | ||
| time.Sleep(time.Second) | ||
| } | ||
| } | ||
|
|
||
| // ensure non-existent paths fail | ||
| require.EqualError( | ||
| t, | ||
| testGenesisRetrieval( | ||
| ctx, | ||
| client, | ||
| api.RetrievalOrder{ | ||
| FromLocalCAR: sourceCar, | ||
| Root: carRoot, | ||
| DatamodelPathSelector: &textSelectorNonexistent, | ||
| }, | ||
| &api.FileRef{}, | ||
| nil, | ||
| ), | ||
| fmt.Sprintf("retrieval failed: path selection '%s' does not match a node within %s", textSelectorNonexistent, carRoot), | ||
| ) | ||
|
|
||
| // ensure non-boundary retrievals fail | ||
| require.EqualError( | ||
| t, | ||
| testGenesisRetrieval( | ||
| ctx, | ||
| client, | ||
| api.RetrievalOrder{ | ||
| FromLocalCAR: sourceCar, | ||
| Root: carRoot, | ||
| DatamodelPathSelector: &textSelectorNonLink, | ||
| }, | ||
| &api.FileRef{}, | ||
| nil, | ||
| ), | ||
| fmt.Sprintf("retrieval failed: error while locating partial retrieval sub-root: unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", textSelectorNonLink), | ||
| ) | ||
| } | ||
|
|
||
| func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrder api.RetrievalOrder, retRef *api.FileRef, outFile *os.File) error { | ||
|
|
||
| if retOrder.Total.Nil() { | ||
| retOrder.Total = big.Zero() | ||
| } | ||
| if retOrder.UnsealPrice.Nil() { | ||
| retOrder.UnsealPrice = big.Zero() | ||
| } | ||
|
|
||
| err := client.ClientRetrieve(ctx, retOrder, retRef) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| var data []byte | ||
| if !retRef.IsCAR { | ||
|
|
||
| data, err = io.ReadAll(outFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| } else { | ||
|
|
||
| cr, err := car.NewCarReader(outFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if len(cr.Header.Roots) != 1 { | ||
| return fmt.Errorf("expected a single root in result car, got %d", len(cr.Header.Roots)) | ||
| } else if cr.Header.Roots[0].String() != carRoot.String() { | ||
| return fmt.Errorf("expected root cid '%s', got '%s'", carRoot.String(), cr.Header.Roots[0].String()) | ||
| } | ||
|
|
||
| blks := make([]blocks.Block, 0) | ||
| for { | ||
| b, err := cr.Next() | ||
| if err == io.EOF { | ||
| break | ||
| } else if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| blks = append(blks, b) | ||
| } | ||
|
|
||
| if len(blks) != 3 { | ||
| return fmt.Errorf("expected a car file with 3 blocks, got one with %d instead", len(blks)) | ||
| } | ||
|
|
||
| data = blks[2].RawData() | ||
| } | ||
|
|
||
| if string(data) != expectedResult { | ||
| return fmt.Errorf("retrieved data mismatch: expected '%s' got '%s'", expectedResult, data) | ||
| } | ||
|
|
||
| return nil | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@magik6k poke that this still needs your 👍 or a recipe how to do it correctly