Add test script for car verify (#236)

* Add test script for car verify
* Add test script for filter

.gitattributes

@@ -0,0 +1,2 @@
+# To prevent CRLF breakages on Windows for fragile files, like testdata.
+* -text

cmd/car/car.go

@@ -31,6 +31,10 @@ func main1() int {
 						Usage:     "A file to read CIDs from",
 						TakesFile: true,
 					},
+					&cli.BoolFlag{
+						Name:  "append",
+						Usage: "Append cids to an existing output file",
+					},
 				},
 			},
 			{

cmd/car/filter.go

@@ -50,10 +50,36 @@ func FilterCar(c *cli.Context) error {
 			outRoots = append(outRoots, r)
 		}
 	}
+
+	outPath := c.Args().Get(1)
+	if !c.Bool("append") {
+		if _, err := os.Stat(outPath); err == nil || !os.IsNotExist(err) {
+			// output to an existing file.
+			if err := os.Truncate(outPath, 0); err != nil {
+				return err
+			}
+		}
+	} else {
+		// roots will need to be whatever is in the output already.
+		cv2r, err := carv2.OpenReader(outPath)
+		if err != nil {
+			return err
+		}
+		if cv2r.Version != 2 {
+			return fmt.Errorf("can only append to version 2 car files")
+		}
+		outRoots, err = cv2r.Roots()
+		if err != nil {
+			return err
+		}
+		_ = cv2r.Close()
+	}
+
 	if len(outRoots) == 0 {
 		fmt.Fprintf(os.Stderr, "warning: no roots defined after filtering\n")
 	}
-	bs, err := blockstore.OpenReadWrite(c.Args().Get(1), outRoots)
+
+	bs, err := blockstore.OpenReadWrite(outPath, outRoots)
 	if err != nil {
 		return err
 	}

cmd/car/testdata/script/filter.txt

@@ -0,0 +1,30 @@
+# basic filter
+stdin filteredcids.txt
+car filter ${INPUTS}/sample-wrapped-v2.car out.car
+stderr 'warning: no roots defined after filtering'
+car list out.car
+! stderr .
+cmp stdout filteredcids.txt
+
+# filter with root CID
+stdin filteredroot.txt
+car filter ${INPUTS}/sample-wrapped-v2.car out.car
+! stderr .
+car list out.car
+! stderr .
+cmp stdout filteredroot.txt
+
+# append other cids
+stdin filteredcids.txt
+car filter -append ${INPUTS}/sample-wrapped-v2.car out.car
+! stderr .
+car list out.car
+stdout -count=4 '^bafy'
+
+
+-- filteredcids.txt --
+bafy2bzacebohz654namrgmwjjx4qmtwgxixsd7pn4tlanyrc3g3hwj75hlxrw
+bafy2bzaceaqtiesyfqd2jibmofz22oolguzf5wscwh73rmeypglfu2xhkptri
+bafy2bzacebct3dm7izgyauijzkaf3yd7ylni725k66rq7dfp3jr5ywhpprj3k
+-- filteredroot.txt --
+bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy

cmd/car/testdata/script/verify.txt

@@ -0,0 +1,3 @@
+# "verify" should exit with code 0 on reasonable cars.
+car verify ${INPUTS}/sample-v1.car
+car verify ${INPUTS}/sample-wrapped-v2.car

cmd/car/verify.go

@@ -8,6 +8,7 @@ import (
 	"github.com/ipfs/go-cid"
 	carv2 "github.com/ipld/go-car/v2"
 	"github.com/ipld/go-car/v2/index"
+	"github.com/multiformats/go-multihash"
 	"github.com/urfave/cli/v2"
 )
 
@@ -44,12 +45,15 @@ func VerifyCar(c *cli.Context) error {
 		if err != nil {
 			return err
 		}
-		lengthToIndex := carv2.PragmaSize + carv2.HeaderSize + rx.Header.DataOffset + rx.Header.DataSize
+		lengthToIndex := carv2.PragmaSize + carv2.HeaderSize + rx.Header.DataSize
 		if uint64(flen.Size()) > lengthToIndex && rx.Header.IndexOffset == 0 {
 			return fmt.Errorf("header claims no index, but extra bytes in file beyond data size")
 		}
+		if rx.Header.DataOffset < carv2.PragmaSize+carv2.HeaderSize {
+			return fmt.Errorf("data offset places data within carv2 header")
+		}
 		if rx.Header.IndexOffset < lengthToIndex {
-			return fmt.Errorf("index offset overlaps with data")
+			return fmt.Errorf("index offset overlaps with data. data ends at %d. index offset of %d", lengthToIndex, rx.Header.IndexOffset)
 		}
 	}
 
@@ -87,6 +91,13 @@ func VerifyCar(c *cli.Context) error {
 			return err
 		}
 		for _, c := range cidList {
+			cidHash, err := multihash.Decode(c.Hash())
+			if err != nil {
+				return err
+			}
+			if cidHash.Code == multihash.IDENTITY {
+				continue
+			}
 			if err := idx.GetAll(c, func(_ uint64) bool {
 				return true
 			}); err != nil {

cmd/go.mod

@@ -7,10 +7,11 @@ require (
 	github.com/ipfs/go-cid v0.1.0
 	github.com/ipfs/go-ipfs-blockstore v1.0.3
 	github.com/ipld/go-car v0.3.1
-	github.com/ipld/go-car/v2 v2.0.2
+	github.com/ipld/go-car/v2 v2.0.3-0.20210914083849-7544041c42bb
 	github.com/ipld/go-codec-dagpb v1.2.0
 	github.com/ipld/go-ipld-prime v0.12.3-0.20210910135350-e6597215c6d5
 	github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61
+	github.com/multiformats/go-multihash v0.0.15
 	github.com/multiformats/go-varint v0.0.6
 	github.com/rogpeppe/go-internal v1.8.0
 	github.com/urfave/cli/v2 v2.3.0

cmd/go.sum

@@ -234,8 +234,8 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
 github.com/ipld/go-car v0.3.1 h1:WT+3cdmXlvmWOlGxk9webhj4auGO5QvgqC2vCCkFRXs=
 github.com/ipld/go-car v0.3.1/go.mod h1:dPkEWeAK8KaVvH5TahaCs6Mncpd4lDMpkbs0/SPzuVs=
-github.com/ipld/go-car/v2 v2.0.2 h1:R1oIAPwrGp26mEFzcGf5bfTZAAHDOkaVnZTEVebaWX4=
-github.com/ipld/go-car/v2 v2.0.2/go.mod h1:I2ACeeg6XNBe5pdh5TaR7Ambhfa7If9KXxmXgZsYENU=
+github.com/ipld/go-car/v2 v2.0.3-0.20210914083849-7544041c42bb h1:w86j9nGrl+kCsrWF2KAevmqnkT7gMWjr4uSZ/a5jwWY=
+github.com/ipld/go-car/v2 v2.0.3-0.20210914083849-7544041c42bb/go.mod h1:Xr6GwkDhv8dtOtgHzOynAkIOg0t0YiPc5DxBPppWqZA=
 github.com/ipld/go-codec-dagpb v1.2.0 h1:2umV7ud8HBMkRuJgd8gXw95cLhwmcYrihS3cQEy9zpI=
 github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s=
 github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
@@ -391,7 +391,6 @@ github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d
 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
 github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
 github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
-github.com/multiformats/go-multicodec v0.2.1-0.20210713081508-b421db6850ae/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ=
 github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ=
 github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61 h1:ZrUuMKNgJ52qHPoQ+bx0h0uBfcWmN7Px+4uKSZeesiI=
 github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ=