Round-trip storage/retrieval test (#229)

* move TestPeerResolver to shared testutils * make everybody specify the path to their fixture file starting from root, so >1 package can use LoadUnixFSFile * export AllSelector, make everyone use it * please the linter
filecoin-project · May 8, 2020 · 32d301b · 32d301b
1 parent dde200e
commit 32d301b
Show file tree

Hide file tree

Showing 18 changed files with 498 additions and 128 deletions.
diff --git a/retrievalmarket/impl/blockio/reader_test.go b/retrievalmarket/impl/blockio/reader_test.go
@@ -6,24 +6,19 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
-	basicnode "github.com/ipld/go-ipld-prime/node/basic"
-	"github.com/ipld/go-ipld-prime/traversal/selector"
-	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/blockio"
+	"github.com/filecoin-project/go-fil-markets/shared"
 	tut "github.com/filecoin-project/go-fil-markets/shared_testutil"
 )
 
 func TestSelectorReader(t *testing.T) {
 	ctx := context.Background()
 	testdata := tut.NewTestIPLDTree()
 
-	ssb := builder.NewSelectorSpecBuilder(basicnode.Style.Any)
-	sel := ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
-
 	t.Run("reads correctly", func(t *testing.T) {
-		reader := blockio.NewSelectorBlockReader(testdata.RootNodeLnk, sel, testdata.Loader)
+		reader := blockio.NewSelectorBlockReader(testdata.RootNodeLnk, shared.AllSelector(), testdata.Loader)
 
 		checkReadSequence(ctx, t, reader, []blocks.Block{
 			testdata.RootBlock,

diff --git a/retrievalmarket/impl/blockio/traverser_test.go b/retrievalmarket/impl/blockio/traverser_test.go
@@ -7,24 +7,19 @@ import (
 
 	blocks "github.com/ipfs/go-block-format"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
-	basicnode "github.com/ipld/go-ipld-prime/node/basic"
-	"github.com/ipld/go-ipld-prime/traversal/selector"
-	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/blockio"
+	"github.com/filecoin-project/go-fil-markets/shared"
 	tut "github.com/filecoin-project/go-fil-markets/shared_testutil"
 )
 
 func TestTraverser(t *testing.T) {
 	ctx := context.Background()
 	testdata := tut.NewTestIPLDTree()
 
-	ssb := builder.NewSelectorSpecBuilder(basicnode.Style.Any)
-	sel := ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
-
 	t.Run("traverses correctly", func(t *testing.T) {
-		traverser := blockio.NewTraverser(testdata.RootNodeLnk, sel)
+		traverser := blockio.NewTraverser(testdata.RootNodeLnk, shared.AllSelector())
 		traverser.Start(ctx)
 		checkTraverseSequence(ctx, t, traverser, []blocks.Block{
 			testdata.RootBlock,

diff --git a/retrievalmarket/impl/blockio/verify_test.go b/retrievalmarket/impl/blockio/verify_test.go
@@ -5,22 +5,17 @@ import (
 	"testing"
 
 	blocks "github.com/ipfs/go-block-format"
-	basicnode "github.com/ipld/go-ipld-prime/node/basic"
-	"github.com/ipld/go-ipld-prime/traversal/selector"
-	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/blockio"
+	"github.com/filecoin-project/go-fil-markets/shared"
 	tut "github.com/filecoin-project/go-fil-markets/shared_testutil"
 )
 
 func TestSelectorVerifier(t *testing.T) {
 	ctx := context.Background()
 	testdata := tut.NewTestIPLDTree()
-
-	ssb := builder.NewSelectorSpecBuilder(basicnode.Style.Any)
-	sel := ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
-
+	sel := shared.AllSelector()
 	t.Run("verifies correctly", func(t *testing.T) {
 		verifier := blockio.NewSelectorVerifier(testdata.RootNodeLnk, sel)
 		checkVerifySequence(ctx, t, verifier, false, []blocks.Block{

diff --git a/retrievalmarket/impl/blockunsealing/blockunsealing_test.go b/retrievalmarket/impl/blockunsealing/blockunsealing_test.go
@@ -14,27 +14,22 @@ import (
 	bstore "github.com/ipfs/go-ipfs-blockstore"
 	"github.com/ipld/go-ipld-prime"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
-	basicnode "github.com/ipld/go-ipld-prime/node/basic"
-	"github.com/ipld/go-ipld-prime/traversal/selector"
-	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
 	"github.com/stretchr/testify/require"
 
 	"github.com/filecoin-project/go-fil-markets/pieceio/cario"
 	"github.com/filecoin-project/go-fil-markets/piecestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/blockunsealing"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/testnodes"
+	"github.com/filecoin-project/go-fil-markets/shared"
 	tut "github.com/filecoin-project/go-fil-markets/shared_testutil"
 )
 
 func TestNewLoaderWithUnsealing(t *testing.T) {
 	ctx := context.Background()
 	cio := cario.NewCarIO()
 	testdata := tut.NewTestIPLDTree()
-	ssb := builder.NewSelectorSpecBuilder(basicnode.Style.Any)
-	allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(),
-		ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
 	var carBuffer bytes.Buffer
-	err := cio.WriteCar(ctx, testdata, testdata.RootNodeLnk.(cidlink.Link).Cid, allSelector, &carBuffer)
+	err := cio.WriteCar(ctx, testdata, testdata.RootNodeLnk.(cidlink.Link).Cid, shared.AllSelector(), &carBuffer)
 	require.NoError(t, err)
 	carData := carBuffer.Bytes()
 

diff --git a/retrievalmarket/impl/client.go b/retrievalmarket/impl/client.go
@@ -22,6 +22,8 @@ import (
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/blockio"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/clientstates"
 	rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network"
+	"github.com/filecoin-project/go-fil-markets/shared"
+
 	"github.com/filecoin-project/go-storedcounter"
 )
 
@@ -151,7 +153,7 @@ func (c *client) Retrieve(ctx context.Context, payloadCID cid.Cid, params retrie
 
 	c.dealStreams[dealID] = s
 
-	sel := allSelector()
+	sel := shared.AllSelector()
 	if params.Selector != nil {
 		sel, err = retrievalmarket.DecodeNode(params.Selector)
 		if err != nil {
@@ -207,15 +209,15 @@ func (c *client) SubscribeToEvents(subscriber retrievalmarket.ClientSubscriber)
 }
 
 // V1
-func (c *client) AddMoreFunds(id retrievalmarket.DealID, amount abi.TokenAmount) error {
+func (c *client) AddMoreFunds(retrievalmarket.DealID, abi.TokenAmount) error {
 	panic("not implemented")
 }
 
-func (c *client) CancelDeal(id retrievalmarket.DealID) error {
+func (c *client) CancelDeal(retrievalmarket.DealID) error {
 	panic("not implemented")
 }
 
-func (c *client) RetrievalStatus(id retrievalmarket.DealID) {
+func (c *client) RetrievalStatus(retrievalmarket.DealID) {
 	panic("not implemented")
 }
 
@@ -237,12 +239,12 @@ func (c *client) ConsumeBlock(ctx context.Context, dealID retrievalmarket.DealID
 		return 0, false, err
 	}
 
-	cid, err := prefix.Sum(block.Data)
+	scid, err := prefix.Sum(block.Data)
 	if err != nil {
 		return 0, false, err
 	}
 
-	blk, err := blocks.NewBlockWithCid(block.Data, cid)
+	blk, err := blocks.NewBlockWithCid(block.Data, scid)
 	if err != nil {
 		return 0, false, err
 	}

diff --git a/retrievalmarket/impl/client_test.go b/retrievalmarket/impl/client_test.go
@@ -7,7 +7,6 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	dss "github.com/ipfs/go-datastore/sync"
 	bstore "github.com/ipfs/go-ipfs-blockstore"
@@ -65,7 +64,7 @@ func TestClient_Query(t *testing.T) {
 			net,
 			bs,
 			testnodes.NewTestRetrievalClientNode(testnodes.TestRetrievalClientNodeParams{}),
-			&testPeerResolver{},
+			&tut.TestPeerResolver{},
 			ds,
 			storedCounter)
 		require.NoError(t, err)
@@ -84,7 +83,7 @@ func TestClient_Query(t *testing.T) {
 			net,
 			bs,
 			testnodes.NewTestRetrievalClientNode(testnodes.TestRetrievalClientNodeParams{}),
-			&testPeerResolver{},
+			&tut.TestPeerResolver{},
 			ds,
 			storedCounter)
 		require.NoError(t, err)
@@ -110,7 +109,7 @@ func TestClient_Query(t *testing.T) {
 			net,
 			bs,
 			testnodes.NewTestRetrievalClientNode(testnodes.TestRetrievalClientNodeParams{}),
-			&testPeerResolver{},
+			&tut.TestPeerResolver{},
 			ds,
 			storedCounter)
 		require.NoError(t, err)
@@ -135,7 +134,7 @@ func TestClient_Query(t *testing.T) {
 			net,
 			bs,
 			testnodes.NewTestRetrievalClientNode(testnodes.TestRetrievalClientNodeParams{}),
-			&testPeerResolver{},
+			&tut.TestPeerResolver{},
 			ds,
 			storedCounter)
 		require.NoError(t, err)
@@ -164,7 +163,7 @@ func TestClient_FindProviders(t *testing.T) {
 
 	t.Run("when providers are found, returns providers", func(t *testing.T) {
 		peers := tut.RequireGenerateRetrievalPeers(t, 3)
-		testResolver := testPeerResolver{peers: peers}
+		testResolver := tut.TestPeerResolver{Peers: peers}
 
 		c, err := retrievalimpl.NewClient(net, bs, &testnodes.TestRetrievalClientNode{}, &testResolver, ds, storedCounter)
 		require.NoError(t, err)
@@ -174,7 +173,7 @@ func TestClient_FindProviders(t *testing.T) {
 	})
 
 	t.Run("when there is an error, returns empty provider list", func(t *testing.T) {
-		testResolver := testPeerResolver{peers: []retrievalmarket.RetrievalPeer{}, resolverError: errors.New("boom")}
+		testResolver := tut.TestPeerResolver{Peers: []retrievalmarket.RetrievalPeer{}, ResolverError: errors.New("boom")}
 		c, err := retrievalimpl.NewClient(net, bs, &testnodes.TestRetrievalClientNode{}, &testResolver, ds, storedCounter)
 		require.NoError(t, err)
 
@@ -183,22 +182,11 @@ func TestClient_FindProviders(t *testing.T) {
 	})
 
 	t.Run("when there are no providers", func(t *testing.T) {
-		testResolver := testPeerResolver{peers: []retrievalmarket.RetrievalPeer{}}
+		testResolver := tut.TestPeerResolver{Peers: []retrievalmarket.RetrievalPeer{}}
 		c, err := retrievalimpl.NewClient(net, bs, &testnodes.TestRetrievalClientNode{}, &testResolver, ds, storedCounter)
 		require.NoError(t, err)
 
 		testCid := tut.GenerateCids(1)[0]
 		assert.Len(t, c.FindProviders(testCid), 0)
 	})
 }
-
-type testPeerResolver struct {
-	peers         []retrievalmarket.RetrievalPeer
-	resolverError error
-}
-
-var _ retrievalmarket.PeerResolver = &testPeerResolver{}
-
-func (tpr testPeerResolver) GetPeers(cid.Cid) ([]retrievalmarket.RetrievalPeer, error) {
-	return tpr.peers, tpr.resolverError
-}
diff --git a/retrievalmarket/impl/integration_test.go b/retrievalmarket/impl/integration_test.go
@@ -3,6 +3,7 @@ package retrievalimpl_test
 import (
 	"bytes"
 	"context"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -14,7 +15,6 @@ import (
 	"github.com/ipld/go-ipld-prime"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	basicnode "github.com/ipld/go-ipld-prime/node/basic"
-	"github.com/ipld/go-ipld-prime/traversal/selector"
 	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -25,6 +25,7 @@ import (
 	retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/testnodes"
 	rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network"
+	"github.com/filecoin-project/go-fil-markets/shared"
 	tut "github.com/filecoin-project/go-fil-markets/shared_testutil"
 )
 
@@ -85,7 +86,7 @@ func requireSetupTestClientAndProvider(bgCtx context.Context, t *testing.T, payC
 		CreatePaychCID: cids[0],
 		AddFundsCID:    cids[1],
 	})
-	client, err := retrievalimpl.NewClient(nw1, testData.Bs1, rcNode1, &testPeerResolver{}, testData.Ds1, testData.RetrievalStoredCounter1)
+	client, err := retrievalimpl.NewClient(nw1, testData.Bs1, rcNode1, &tut.TestPeerResolver{}, testData.Ds1, testData.RetrievalStoredCounter1)
 	require.NoError(t, err)
 	nw2 := rmnet.NewFromLibp2pHost(testData.Host2)
 	providerNode := testnodes.NewTestRetrievalProviderNode()
@@ -135,9 +136,6 @@ func TestClientCanMakeDealWithProvider(t *testing.T) {
 
 	ssb := builder.NewSelectorSpecBuilder(basicnode.Style.Any)
 
-	allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(),
-		ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
-
 	partialSelector := ssb.ExploreFields(func(specBuilder builder.ExploreFieldsSpecBuilder) {
 		specBuilder.Insert("Links", ssb.ExploreIndex(0, ssb.ExploreFields(func(specBuilder builder.ExploreFieldsSpecBuilder) {
 			specBuilder.Insert("Hash", ssb.Matcher())
@@ -177,12 +175,12 @@ func TestClientCanMakeDealWithProvider(t *testing.T) {
 			filesize:    19000,
 			voucherAmts: []abi.TokenAmount{abi.NewTokenAmount(10136000), abi.NewTokenAmount(9784000)},
 			unsealing:   true},
-		{name: "multi-block file retrieval succeeds with V1 params and allSelector",
+		{name: "multi-block file retrieval succeeds with V1 params and AllSelector",
 			filename:    "lorem.txt",
 			filesize:    19000,
 			voucherAmts: []abi.TokenAmount{abi.NewTokenAmount(10136000), abi.NewTokenAmount(9784000)},
 			paramsV1:    true,
-			selector:    allSelector,
+			selector:    shared.AllSelector(),
 			unsealing:   false},
 		{name: "partial file retrieval succeeds with V1 params and selector recursion depth 1",
 			filename:    "lorem.txt",
@@ -203,7 +201,10 @@ func TestClientCanMakeDealWithProvider(t *testing.T) {
 
 			// Inject a unixFS file on the provider side to its blockstore
 			// obtained via `ls -laf` on this file
-			pieceLink := testData.LoadUnixFSFile(t, testCase.filename, true)
+
+			fpath := filepath.Join("retrievalmarket", "impl", "fixtures", testCase.filename)
+
+			pieceLink := testData.LoadUnixFSFile(t, fpath, true)
 			c, ok := pieceLink.(cidlink.Link)
 			require.True(t, ok)
 			payloadCID := c.Cid
@@ -226,7 +227,7 @@ func TestClientCanMakeDealWithProvider(t *testing.T) {
 			if testCase.unsealing {
 				cio := cario.NewCarIO()
 				var buf bytes.Buffer
-				err := cio.WriteCar(bgCtx, testData.Bs2, payloadCID, testData.AllSelector, &buf)
+				err := cio.WriteCar(bgCtx, testData.Bs2, payloadCID, shared.AllSelector(), &buf)
 				require.NoError(t, err)
 				carData := buf.Bytes()
 				sectorID := uint64(100000)
@@ -415,7 +416,7 @@ func setupClient(
 		CreatePaychCID:         cids[0],
 		AddFundsCID:            cids[1],
 	})
-	client, err := retrievalimpl.NewClient(nw1, testData.Bs1, clientNode, &testPeerResolver{}, testData.Ds1, testData.RetrievalStoredCounter1)
+	client, err := retrievalimpl.NewClient(nw1, testData.Bs1, clientNode, &tut.TestPeerResolver{}, testData.Ds1, testData.RetrievalStoredCounter1)
 	return &createdChan, &newLaneAddr, &createdVoucher, client, err
 }
 

diff --git a/retrievalmarket/impl/provider.go b/retrievalmarket/impl/provider.go
@@ -24,6 +24,7 @@ import (
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/blockunsealing"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket/impl/providerstates"
 	rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network"
+	"github.com/filecoin-project/go-fil-markets/shared"
 )
 
 // ProviderDsPrefix is the datastore for the provider key
@@ -247,7 +248,7 @@ func (p *provider) newProviderDeal(stream rmnet.RetrievalDealStream) error {
 			return xerrors.Errorf("selector is invalid: %w", err)
 		}
 	} else {
-		sel = allSelector()
+		sel = shared.AllSelector()
 	}
 
 	br := blockio.NewSelectorBlockReader(cidlink.Link{Cid: dealProposal.PayloadCID}, sel, loaderWithUnsealing.Load)