diff --git a/go.mod b/go.mod index 8b71eba..368b172 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/ipfs/go-graphsync v0.17.0 github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 + github.com/ipfs/go-test v0.0.2 github.com/ipfs/go-unixfsnode v1.9.0 github.com/ipld/go-car/v2 v2.13.1 github.com/ipld/go-codec-dagpb v1.6.0 @@ -19,7 +20,6 @@ require ( github.com/ipld/ipld/specs v0.0.0-20231012031213-54d3b21deda4 github.com/ipni/go-libipni v0.6.8 github.com/ipni/index-provider v0.15.3 - github.com/ipni/storetheindex v0.8.32 github.com/libp2p/go-libp2p v0.35.1 github.com/multiformats/go-multiaddr v0.12.4 github.com/multiformats/go-multihash v0.2.3 diff --git a/go.sum b/go.sum index caf5919..3139f15 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,8 @@ github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 h1:XM81BJ4/6h3FV0WfFjh74cIDIgq github.com/filecoin-project/go-amt-ipld/v4 v4.0.0/go.mod h1:gF053YQ4BIpzTNDoEwHZas7U3oAwncDVGvOHyY8oDpE= github.com/filecoin-project/go-cbor-util v0.0.1 h1:E1LYZYTtjfAQwCReho0VXvbu8t3CYAVPiMx8EiV/VAs= github.com/filecoin-project/go-cbor-util v0.0.1/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= -github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc8 h1:EWC89lM/tJAjyzaxZ624clq3oyHLoLjISfoyG+WIu9s= -github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc8/go.mod h1:mK3/NbSljx3Kr335+IXEe8gcdEPA2eZXJaNhodK9bAI= +github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 h1:v+zJS5B6pA3ptWZS4t8tbt1Hz9qENnN4nVr1w99aSWc= +github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7/go.mod h1:V3Y4KbttaCwyg1gwkP7iai8CbQx4mZUGjd3h9GZWLKE= github.com/filecoin-project/go-ds-versioning v0.1.2 h1:to4pTadv3IeV1wvgbCbN6Vqd+fu+7tveXgv/rCEZy6w= github.com/filecoin-project/go-ds-versioning v0.1.2/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4= github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 h1:rVVNq0x6RGQIzCo1iiJlGFm9AGIZzeifggxtKMU7zmI= @@ -174,15 +174,15 @@ github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.20.0 h1:umUl7q1v5g5AX8FPLTnZBvvagLmT+V0Tt61EigP81ec= -github.com/ipfs/boxo v0.20.0/go.mod h1:mwttn53Eibgska2DhVIj7ln3UViq7MVHRxOMb+ehSDM= +github.com/ipfs/boxo v0.19.0 h1:UbX9FBJQF19ACLqRZOgdEla6jR/sC4H1O+iGE0NToXA= +github.com/ipfs/boxo v0.19.0/go.mod h1:V5gJzbIMwKEXrg3IdvAxIdF7UPgU4RsXmNGS8MQ/0D4= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= -github.com/ipfs/go-blockservice v0.5.2 h1:in9Bc+QcXwd1apOVM7Un9t8tixPKdaHQFdLSUM1Xgk8= -github.com/ipfs/go-blockservice v0.5.2/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk= +github.com/ipfs/go-blockservice v0.5.1 h1:9pAtkyKAz/skdHTh0kH8VulzWp+qmSDD0aI17TYP/s0= +github.com/ipfs/go-blockservice v0.5.1/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= @@ -200,10 +200,10 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IW github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= -github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= -github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= -github.com/ipfs/go-ipfs-exchange-interface v0.2.1 h1:jMzo2VhLKSHbVe+mHNzYgs95n0+t0Q69GQ5WhRDZV/s= -github.com/ipfs/go-ipfs-exchange-interface v0.2.1/go.mod h1:MUsYn6rKbG6CTtsDp+lKJPmVt3ZrCViNyH3rfPGsZ2E= +github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= +github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= +github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= +github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= @@ -215,8 +215,8 @@ github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopo github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk= github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U= github.com/ipfs/go-ipld-format v0.6.0/go.mod h1:g4QVMTn3marU3qXchwjpKPKgJv+zF+OlaKMyhJ4LHPg= -github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= -github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM= +github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc= +github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= @@ -229,12 +229,14 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= -github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= -github.com/ipfs/go-unixfs v0.4.5/go.mod h1:BIznJNvt/gEx/ooRMI4Us9K8+qeGO7vx1ohnbk8gjFg= +github.com/ipfs/go-test v0.0.2 h1:Wdxl4bKEdjEM8SLiulXMHlAQwHYOhX3CSBoUoEvncmM= +github.com/ipfs/go-test v0.0.2/go.mod h1:qhIM1EluEfElKKM6fnWxGn822/z9knUGM1+I/OAQNKI= +github.com/ipfs/go-unixfs v0.4.4 h1:D/dLBOJgny5ZLIur2vIXVQVW0EyDHdOMBDEhgHrt6rY= +github.com/ipfs/go-unixfs v0.4.4/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM= github.com/ipfs/go-unixfsnode v1.9.0 h1:ubEhQhr22sPAKO2DNsyVBW7YB/zA8Zkif25aBvz8rc8= github.com/ipfs/go-unixfsnode v1.9.0/go.mod h1:HxRu9HYHOjK6HUqFBAi++7DVoWAHn0o4v/nZ/VA+0g8= -github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs= -github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw= +github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= +github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= github.com/ipld/go-car/v2 v2.13.1 h1:KnlrKvEPEzr5IZHKTXLAEub+tPrzeAFQVRlSQvuxBO4= github.com/ipld/go-car/v2 v2.13.1/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/JI1iDJdo= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= @@ -255,8 +257,6 @@ github.com/ipni/go-libipni v0.6.8 h1:bwyu7g+1jwgCptabQ6It2VtFucFJIsOckzQNBBObHoE github.com/ipni/go-libipni v0.6.8/go.mod h1:yXjq2JgqPMcubWpJie3OjyG0HNIoRdbmqxQMx+gF4TM= github.com/ipni/index-provider v0.15.3 h1:dq8pLyyf/vnyuPESnoc/ZOnIgFHZoBQiNwDOU8258WY= github.com/ipni/index-provider v0.15.3/go.mod h1:i5mhJPG1f2Ax+y+B48pamB9xNin5i2mvFLaQM5rCFMQ= -github.com/ipni/storetheindex v0.8.32 h1:Zyt9Uqa840X8kjBSin/2Hh9uQ7mgChDGDP2v9z7fHwQ= -github.com/ipni/storetheindex v0.8.32/go.mod h1:V/izSFH98NiECygKmFJ04qWND2ZgoT32yasn0tLSkic= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= diff --git a/internal/integration/e2e_test.go b/internal/integration/e2e_test.go index 5fbaf16..3342623 100644 --- a/internal/integration/e2e_test.go +++ b/internal/integration/e2e_test.go @@ -15,14 +15,16 @@ import ( "time" "github.com/ipfs/go-cid" + testcmd "github.com/ipfs/go-test/cmd" unixfs "github.com/ipfs/go-unixfsnode/testutil" "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/storage" cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipni/storetheindex/test" "github.com/stretchr/testify/require" ) +const indexerReadyMatch = "Indexer is ready" + const rseed = 1234 func TestIpniAndFetchIntegration(t *testing.T) { @@ -57,14 +59,14 @@ func TestIpniAndFetchIntegration(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() - tr := test.NewTestIpniRunner(t, ctx, t.TempDir()) + tr := testcmd.NewRunner(t, t.TempDir()) t.Log("Running in test directory:", tr.Dir) // install the frisbii cmd, when done in tr.Run() will use the GOPATH/GOBIN // in the test directory, so we get a localised `frisbii` executable frisbii := filepath.Join(tr.Dir, "frisbii") - tr.Run("go", "install", "../../cmd/frisbii/") + tr.Run(ctx, "go", "install", "../../cmd/frisbii/") cwd, err := os.Getwd() req.NoError(err) @@ -73,26 +75,23 @@ func TestIpniAndFetchIntegration(t *testing.T) { // install the indexer to announce to indexer := filepath.Join(tr.Dir, "storetheindex") - tr.Run("go", "install", "github.com/ipni/storetheindex@latest") + tr.Run(ctx, "go", "install", "github.com/ipni/storetheindex@latest") // install the ipni cli to inspect the indexer ipni := filepath.Join(tr.Dir, "ipni") - tr.Run("go", "install", "github.com/ipni/ipni-cli/cmd/ipni@latest") + tr.Run(ctx, "go", "install", "github.com/ipni/ipni-cli/cmd/ipni@latest") // install lassie to perform a fetch of our content lassie := filepath.Join(tr.Dir, "lassie") - tr.Run("go", "install", "github.com/filecoin-project/lassie/cmd/lassie@latest") + tr.Run(ctx, "go", "install", "github.com/filecoin-project/lassie/cmd/lassie@latest") err = os.Chdir(cwd) req.NoError(err) // initialise and start the indexer and adjust the config - tr.Run(indexer, "init", "--store", "pebble", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap") - indexerReady := test.NewStdoutWatcher(test.IndexerReadyMatch) - cmdIndexer := tr.Start(test.NewExecution(indexer, "daemon").WithWatcher(indexerReady)) - select { - case <-indexerReady.Signal: - case <-ctx.Done(): - t.Fatal("timed out waiting for indexer to start") - } + tr.Run(ctx, indexer, "init", "--store", "pebble", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap") + indexerReady := testcmd.NewStdoutWatcher(indexerReadyMatch) + cmdIndexer := tr.Start(ctx, testcmd.Args(indexer, "daemon"), indexerReady) + err = indexerReady.Wait(ctx) + req.NoError(err, "timed out waiting for indexer to start") /* We don't seem to need to give it explicit permission, but if we do, here it is @@ -106,7 +105,7 @@ func TestIpniAndFetchIntegration(t *testing.T) { req.NoError(err) // Allow provider advertisements, regardless of default policy. - tr.Run(indexer, "admin", "allow", "-i", "http://localhost:3002", "--peer", id.String()) + tr.Run(ctx, indexer, "admin", "allow", "-i", "http://localhost:3002", "--peer", id.String()) */ // setup the frisbii CLI args @@ -126,20 +125,16 @@ func TestIpniAndFetchIntegration(t *testing.T) { args = append(args, testCase.frisbiiFlags...) // start frisbii - frisbiiReady := test.NewStdoutWatcher("Announce() complete") - cmdFrisbii := tr.Start(test.NewExecution(frisbii, args...).WithWatcher(frisbiiReady)) - - select { - case <-frisbiiReady.Signal: - case <-ctx.Done(): - t.Fatal("timed out waiting for frisbii to announce") - } + frisbiiReady := testcmd.NewStderrWatcher("Announce() complete") + cmdFrisbii := tr.Start(ctx, testcmd.Args(frisbii, args...), frisbiiReady) + err = frisbiiReady.Wait(ctx) + req.NoError(err, "timed out waiting for frisbii to announce") // wait for the CARs to be indexed req.Eventually(func() bool { for root := range cars { mh := root.Hash().B58String() - findOutput := tr.Run(ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh) + findOutput := tr.Run(ctx, ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh) t.Logf("import output:\n%s\n", findOutput) if bytes.Contains(findOutput, []byte("not found")) { @@ -157,7 +152,7 @@ func TestIpniAndFetchIntegration(t *testing.T) { // fetch the data with lassie using the local indexer and make sure we // got the CAR content we expected for root, carPath := range cars { - tr.Run(lassie, + tr.Run(ctx, lassie, "fetch", "-vv", "--ipni-endpoint", "http://localhost:3000",