-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #7673 from filecoin-project/chore/DM-level-tests_p…
…lus_merkle-proof-cars Chore/dm level tests plus merkle proof cars
- Loading branch information
Showing
5 changed files
with
278 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,252 @@ | ||
| package itests | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "io" | ||
| "io/ioutil" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/filecoin-project/go-fil-markets/storagemarket" | ||
| "github.com/filecoin-project/go-state-types/abi" | ||
| "github.com/filecoin-project/lotus/api" | ||
| api0 "github.com/filecoin-project/lotus/api/v0api" | ||
| "github.com/filecoin-project/lotus/chain/actors/policy" | ||
| "github.com/filecoin-project/lotus/itests/kit" | ||
| blocks "github.com/ipfs/go-block-format" | ||
| "github.com/ipfs/go-cid" | ||
| "github.com/ipld/go-car" | ||
| textselector "github.com/ipld/go-ipld-selector-text-lite" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // please talk to @ribasushi or @mikeal before modifying these test: there are | ||
| // downstream dependencies on ADL-less operation | ||
| var ( | ||
| adlFixtureCar = "fixtures/adl_test.car" | ||
| adlFixtureRoot, _ = cid.Parse("bafybeiaigxwanoxyeuzyiknhrg6io6kobfbm37ozcips6qdwumub2gaomy") | ||
| adlFixtureCommp, _ = cid.Parse("baga6ea4seaqjnmnrv4qsfz2rnda54mvo5al22dwpguhn2pmep63gl7bbqqqraai") | ||
| adlFixturePieceSize = abi.PaddedPieceSize(1024) | ||
| dmSelector = api.Selector("Links/0/Hash") | ||
| dmTextSelector = textselector.Expression(dmSelector) | ||
| dmExpectedResult = "NO ADL" | ||
| dmExpectedCarBlockCount = 4 | ||
| dmDagSpec = []api.DagSpec{{DataSelector: &dmSelector}} | ||
| ) | ||
|
|
||
| func TestDMLevelPartialRetrieval(t *testing.T) { | ||
|
|
||
| ctx := context.Background() | ||
|
|
||
| policy.SetPreCommitChallengeDelay(2) | ||
| kit.QuietMiningLogs() | ||
| client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.MockProofs()) | ||
| dh := kit.NewDealHarness(t, client, miner, miner) | ||
| ens.InterconnectAll().BeginMining(50 * time.Millisecond) | ||
|
|
||
| _, err := client.ClientImport(ctx, api.FileRef{Path: adlFixtureCar, IsCAR: true}) | ||
| require.NoError(t, err) | ||
|
|
||
| caddr, err := client.WalletDefaultAddress(ctx) | ||
| require.NoError(t, err) | ||
|
|
||
| // | ||
| // test retrieval from local car 1st | ||
| require.NoError(t, testDMExportAsCar( | ||
| ctx, client, api.ExportRef{ | ||
| FromLocalCAR: adlFixtureCar, | ||
| Root: adlFixtureRoot, | ||
| DAGs: dmDagSpec, | ||
| }, t.TempDir(), | ||
| )) | ||
| require.NoError(t, testDMExportAsFile( | ||
| ctx, client, api.ExportRef{ | ||
| FromLocalCAR: adlFixtureCar, | ||
| Root: adlFixtureRoot, | ||
| DAGs: dmDagSpec, | ||
| }, t.TempDir(), | ||
| )) | ||
|
|
||
| // | ||
| // ensure V0 continues functioning as expected | ||
| require.NoError(t, tesV0RetrievalAsCar( | ||
| ctx, client, api0.RetrievalOrder{ | ||
| FromLocalCAR: adlFixtureCar, | ||
| Root: adlFixtureRoot, | ||
| DatamodelPathSelector: &dmTextSelector, | ||
| }, t.TempDir(), | ||
| )) | ||
| require.NoError(t, testV0RetrievalAsFile( | ||
| ctx, client, api0.RetrievalOrder{ | ||
| FromLocalCAR: adlFixtureCar, | ||
| Root: adlFixtureRoot, | ||
| DatamodelPathSelector: &dmTextSelector, | ||
| }, t.TempDir(), | ||
| )) | ||
|
|
||
| // | ||
| // now perform a storage/retrieval deal as well, and retest | ||
| dp := dh.DefaultStartDealParams() | ||
| dp.Data = &storagemarket.DataRef{ | ||
| Root: adlFixtureRoot, | ||
| PieceCid: &adlFixtureCommp, | ||
| PieceSize: adlFixturePieceSize.Unpadded(), | ||
| } | ||
| proposalCid := dh.StartDeal(ctx, dp) | ||
|
|
||
| // Wait for the deal to reach StorageDealCheckForAcceptance on the client | ||
| cd, err := client.ClientGetDealInfo(ctx, *proposalCid) | ||
| require.NoError(t, err) | ||
| require.Eventually(t, func() bool { | ||
| cd, _ := client.ClientGetDealInfo(ctx, *proposalCid) | ||
| return cd.State == storagemarket.StorageDealCheckForAcceptance | ||
| }, 30*time.Second, 1*time.Second, "actual deal status is %s", storagemarket.DealStates[cd.State]) | ||
|
|
||
| dh.WaitDealSealed(ctx, proposalCid, false, false, nil) | ||
|
|
||
| offers, err := client.ClientFindData(ctx, adlFixtureRoot, nil) | ||
| require.NoError(t, err) | ||
| require.NotEmpty(t, offers, "no offers") | ||
|
|
||
| retOrder := offers[0].Order(caddr) | ||
| retOrder.DataSelector = &dmSelector | ||
|
|
||
| rr, err := client.ClientRetrieve(ctx, retOrder) | ||
| require.NoError(t, err) | ||
|
|
||
| err = client.ClientRetrieveWait(ctx, rr.DealID) | ||
| require.NoError(t, err) | ||
|
|
||
| require.NoError(t, testDMExportAsCar( | ||
| ctx, client, api.ExportRef{ | ||
| DealID: rr.DealID, | ||
| Root: adlFixtureRoot, | ||
| DAGs: dmDagSpec, | ||
| }, t.TempDir(), | ||
| )) | ||
| require.NoError(t, testDMExportAsFile( | ||
| ctx, client, api.ExportRef{ | ||
| DealID: rr.DealID, | ||
| Root: adlFixtureRoot, | ||
| DAGs: dmDagSpec, | ||
| }, t.TempDir(), | ||
| )) | ||
|
|
||
| } | ||
|
|
||
| func testDMExportAsFile(ctx context.Context, client *kit.TestFullNode, expDirective api.ExportRef, tempDir string) error { | ||
| out, err := ioutil.TempFile(tempDir, "exp-test") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer out.Close() //nolint:errcheck | ||
|
|
||
| fileDest := api.FileRef{ | ||
| Path: out.Name(), | ||
| } | ||
| err = client.ClientExport(ctx, expDirective, fileDest) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return validateDMUnixFile(out) | ||
| } | ||
| func testV0RetrievalAsFile(ctx context.Context, client *kit.TestFullNode, retOrder api0.RetrievalOrder, tempDir string) error { | ||
| out, err := ioutil.TempFile(tempDir, "exp-test") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer out.Close() //nolint:errcheck | ||
|
|
||
| cv0 := &api0.WrapperV1Full{client.FullNode} //nolint:govet | ||
| err = cv0.ClientRetrieve(ctx, retOrder, &api.FileRef{ | ||
| Path: out.Name(), | ||
| }) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return validateDMUnixFile(out) | ||
| } | ||
| func validateDMUnixFile(r io.Reader) error { | ||
| data, err := io.ReadAll(r) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if string(data) != dmExpectedResult { | ||
| return fmt.Errorf("retrieved data mismatch: expected '%s' got '%s'", dmExpectedResult, data) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func testDMExportAsCar(ctx context.Context, client *kit.TestFullNode, expDirective api.ExportRef, tempDir string) error { | ||
| out, err := ioutil.TempFile(tempDir, "exp-test") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer out.Close() //nolint:errcheck | ||
|
|
||
| carDest := api.FileRef{ | ||
| IsCAR: true, | ||
| Path: out.Name(), | ||
| } | ||
| err = client.ClientExport(ctx, expDirective, carDest) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return validateDMCar(out) | ||
| } | ||
| func tesV0RetrievalAsCar(ctx context.Context, client *kit.TestFullNode, retOrder api0.RetrievalOrder, tempDir string) error { | ||
| out, err := ioutil.TempFile(tempDir, "exp-test") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer out.Close() //nolint:errcheck | ||
|
|
||
| cv0 := &api0.WrapperV1Full{client.FullNode} //nolint:govet | ||
| err = cv0.ClientRetrieve(ctx, retOrder, &api.FileRef{ | ||
| Path: out.Name(), | ||
| IsCAR: true, | ||
| }) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return validateDMCar(out) | ||
| } | ||
| func validateDMCar(r io.Reader) error { | ||
| cr, err := car.NewCarReader(r) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if len(cr.Header.Roots) != 1 { | ||
| return fmt.Errorf("expected a single root in result car, got %d", len(cr.Header.Roots)) | ||
| } else if cr.Header.Roots[0].String() != adlFixtureRoot.String() { | ||
| return fmt.Errorf("expected root cid '%s', got '%s'", adlFixtureRoot.String(), cr.Header.Roots[0].String()) | ||
| } | ||
|
|
||
| blks := make([]blocks.Block, 0) | ||
| for { | ||
| b, err := cr.Next() | ||
| if err == io.EOF { | ||
| break | ||
| } else if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| blks = append(blks, b) | ||
| } | ||
|
|
||
| if len(blks) != dmExpectedCarBlockCount { | ||
| return fmt.Errorf("expected a car file with %d blocks, got one with %d instead", dmExpectedCarBlockCount, len(blks)) | ||
| } | ||
|
|
||
| data := fmt.Sprintf("%s%s", blks[2].RawData(), blks[3].RawData()) | ||
| if data != dmExpectedResult { | ||
| return fmt.Errorf("retrieved data mismatch: expected '%s' got '%s'", dmExpectedResult, data) | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters