From 13fcc6c70e6f2af81e78f149c6e507109adc5563 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 15 Mar 2023 08:01:32 +0100 Subject: [PATCH] fix: use car block validator (#2241) Use new lib to validate blocks https://github.com/web3-storage/car-block-validator so that we can support all the hashers. Alternatively, we can block on linkdex `getDagStructure` which currently is done in `waitUntil`. Thoughts? --- package-lock.json | 149 ++++++++++++++++++++++++++++++---- packages/api/package.json | 1 + packages/api/src/car.js | 10 +-- packages/api/test/car.spec.js | 2 +- 4 files changed, 138 insertions(+), 24 deletions(-) diff --git a/package-lock.json b/package-lock.json index 229252e722..41eae0f74f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4433,6 +4433,15 @@ } } }, + "node_modules/@multiformats/blake2": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@multiformats/blake2/-/blake2-1.0.13.tgz", + "integrity": "sha512-T1Kzya0wjj85CaVeRSpJ858EnSvW1pw94GSitxYf84VsNdv5XYbJ6QG8y26Ft1bVALzrUCmqkQrR53QHSyu6RA==", + "dependencies": { + "blakejs": "^1.1.1", + "multiformats": "^9.5.4" + } + }, "node_modules/@multiformats/murmur3": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@multiformats/murmur3/-/murmur3-1.1.1.tgz", @@ -4443,6 +4452,15 @@ "murmurhash3js-revisited": "^3.0.0" } }, + "node_modules/@multiformats/sha3": { + "version": "2.0.15", + "resolved": "https://registry.npmjs.org/@multiformats/sha3/-/sha3-2.0.15.tgz", + "integrity": "sha512-il+PqbpPSFhj5HMmxSoy9Ns9wi7kvKI/ctNJ2TAve5Z4E5zIxzEFD6GWW1FIew3SgD9QraynuQbxjkIIVQSjHg==", + "dependencies": { + "js-sha3": "^0.8.0", + "multiformats": "^9.5.4" + } + }, "node_modules/@nftstorage/ipfs-cluster": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/@nftstorage/ipfs-cluster/-/ipfs-cluster-5.0.1.tgz", @@ -6043,6 +6061,52 @@ "resolved": "packages/api", "link": true }, + "node_modules/@web3-storage/car-block-validator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/car-block-validator/-/car-block-validator-1.0.0.tgz", + "integrity": "sha512-UJxDnN2Sj9NuNoM1yMNsVvGg2rec5AI519HGTD+E+xXle0v9jXyh912/eKx5XUdZuJuxxAA7VSrrckZA0JLgCg==", + "dependencies": { + "@multiformats/blake2": "^1.0.13", + "@multiformats/murmur3": "^2.1.3", + "@multiformats/sha3": "^2.0.15", + "multiformats": "^11.0.2", + "uint8arrays": "^4.0.3" + } + }, + "node_modules/@web3-storage/car-block-validator/node_modules/@multiformats/murmur3": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@multiformats/murmur3/-/murmur3-2.1.3.tgz", + "integrity": "sha512-YvLK1IrLnRckPsvXhOkZjaIGNonsEdD1dL3NPSaLilV/WjVYeBgnNZXTUsaPzFXGrIFM7motx+yCmmqzXO6gtQ==", + "dependencies": { + "multiformats": "^11.0.0", + "murmurhash3js-revisited": "^3.0.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@web3-storage/car-block-validator/node_modules/multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@web3-storage/car-block-validator/node_modules/uint8arrays": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-4.0.3.tgz", + "integrity": "sha512-b+aKlI2oTnxnfeSQWV1sMacqSNxqhtXySaH6bflvONGxF8V/fT3ZlYH7z2qgGfydsvpVo4JUgM/Ylyfl2YouCg==", + "dependencies": { + "multiformats": "^11.0.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/@web3-storage/cron": { "resolved": "packages/cron", "link": true @@ -15501,6 +15565,11 @@ "license": "MIT", "peer": true }, + "node_modules/js-sha3": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.8.0.tgz", + "integrity": "sha512-gF1cRrHhIzNfToc802P800N8PpXS+evLLXfsVpowqmAFR9uwbi89WvXg2QspOmXL8QL86J4T1EpFu+yUkwJY3Q==" + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -28174,6 +28243,7 @@ "@magic-ext/oauth": "^0.8.0", "@magic-sdk/admin": "^1.3.0", "@nftstorage/ipfs-cluster": "^5.0.1", + "@web3-storage/car-block-validator": "^1.0.0", "@web3-storage/db": "^4.0.0", "@web3-storage/multipart-parser": "^1.0.0", "cardex": "^1.0.0", @@ -30443,7 +30513,7 @@ }, "packages/w3": { "name": "@web3-storage/w3", - "version": "2.7.0", + "version": "2.8.0", "license": "(Apache-2.0 OR MIT)", "dependencies": { "@ipld/car": "^3.1.16", @@ -31418,7 +31488,7 @@ }, "packages/website": { "name": "@web3-storage/website", - "version": "2.35.1", + "version": "2.35.2", "dependencies": { "@docsearch/react": "^3.0.0", "@fortawesome/free-brands-svg-icons": "^6.1.2", @@ -35154,13 +35224,6 @@ "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", "license": "MIT" }, - "packages/website/node_modules/js-sha3": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.8.0.tgz", - "integrity": "sha512-gF1cRrHhIzNfToc802P800N8PpXS+evLLXfsVpowqmAFR9uwbi89WvXg2QspOmXL8QL86J4T1EpFu+yUkwJY3Q==", - "dev": true, - "license": "MIT" - }, "packages/website/node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -41552,6 +41615,15 @@ } } }, + "@multiformats/blake2": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@multiformats/blake2/-/blake2-1.0.13.tgz", + "integrity": "sha512-T1Kzya0wjj85CaVeRSpJ858EnSvW1pw94GSitxYf84VsNdv5XYbJ6QG8y26Ft1bVALzrUCmqkQrR53QHSyu6RA==", + "requires": { + "blakejs": "^1.1.1", + "multiformats": "^9.5.4" + } + }, "@multiformats/murmur3": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@multiformats/murmur3/-/murmur3-1.1.1.tgz", @@ -41561,6 +41633,15 @@ "murmurhash3js-revisited": "^3.0.0" } }, + "@multiformats/sha3": { + "version": "2.0.15", + "resolved": "https://registry.npmjs.org/@multiformats/sha3/-/sha3-2.0.15.tgz", + "integrity": "sha512-il+PqbpPSFhj5HMmxSoy9Ns9wi7kvKI/ctNJ2TAve5Z4E5zIxzEFD6GWW1FIew3SgD9QraynuQbxjkIIVQSjHg==", + "requires": { + "js-sha3": "^0.8.0", + "multiformats": "^9.5.4" + } + }, "@nftstorage/ipfs-cluster": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/@nftstorage/ipfs-cluster/-/ipfs-cluster-5.0.1.tgz", @@ -42685,6 +42766,7 @@ "@types/sinon": "^10.0.13", "@web-std/fetch": "^3.0.2", "@web-std/form-data": "^3.0.0", + "@web3-storage/car-block-validator": "^1.0.0", "@web3-storage/db": "^4.0.0", "@web3-storage/multipart-parser": "^1.0.0", "@web3-storage/tools": "^1.0.0", @@ -43355,6 +43437,42 @@ } } }, + "@web3-storage/car-block-validator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/car-block-validator/-/car-block-validator-1.0.0.tgz", + "integrity": "sha512-UJxDnN2Sj9NuNoM1yMNsVvGg2rec5AI519HGTD+E+xXle0v9jXyh912/eKx5XUdZuJuxxAA7VSrrckZA0JLgCg==", + "requires": { + "@multiformats/blake2": "^1.0.13", + "@multiformats/murmur3": "^2.1.3", + "@multiformats/sha3": "^2.0.15", + "multiformats": "^11.0.2", + "uint8arrays": "^4.0.3" + }, + "dependencies": { + "@multiformats/murmur3": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@multiformats/murmur3/-/murmur3-2.1.3.tgz", + "integrity": "sha512-YvLK1IrLnRckPsvXhOkZjaIGNonsEdD1dL3NPSaLilV/WjVYeBgnNZXTUsaPzFXGrIFM7motx+yCmmqzXO6gtQ==", + "requires": { + "multiformats": "^11.0.0", + "murmurhash3js-revisited": "^3.0.0" + } + }, + "multiformats": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-11.0.2.tgz", + "integrity": "sha512-b5mYMkOkARIuVZCpvijFj9a6m5wMVLC7cf/jIPd5D/ARDOfLC5+IFkbgDXQgcU2goIsTD/O9NY4DI/Mt4OGvlg==" + }, + "uint8arrays": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-4.0.3.tgz", + "integrity": "sha512-b+aKlI2oTnxnfeSQWV1sMacqSNxqhtXySaH6bflvONGxF8V/fT3ZlYH7z2qgGfydsvpVo4JUgM/Ylyfl2YouCg==", + "requires": { + "multiformats": "^11.0.0" + } + } + } + }, "@web3-storage/cron": { "version": "file:packages/cron", "requires": { @@ -43921,7 +44039,7 @@ "sade": "^1.7.4", "uint8arrays": "^3.0.0", "w3name": "^1.0.6", - "web3.storage": "4.5.4" + "web3.storage": "^4.5.4" }, "dependencies": { "ansi-regex": { @@ -47172,12 +47290,6 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, - "js-sha3": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.8.0.tgz", - "integrity": "sha512-gF1cRrHhIzNfToc802P800N8PpXS+evLLXfsVpowqmAFR9uwbi89WvXg2QspOmXL8QL86J4T1EpFu+yUkwJY3Q==", - "dev": true - }, "js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -56307,6 +56419,11 @@ "integrity": "sha512-/Ew+CJWHNddr7sjwgxaVeIORIH4AMVC9dy0hPf540ZGMVgS9d3ajwuVdyhDt6/QUvT8ATjR3yuYBKsS79F+H4A==", "peer": true }, + "js-sha3": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.8.0.tgz", + "integrity": "sha512-gF1cRrHhIzNfToc802P800N8PpXS+evLLXfsVpowqmAFR9uwbi89WvXg2QspOmXL8QL86J4T1EpFu+yUkwJY3Q==" + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", diff --git a/packages/api/package.json b/packages/api/package.json index 37cf6494e4..314fd735fd 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -69,6 +69,7 @@ "@magic-ext/oauth": "^0.8.0", "@magic-sdk/admin": "^1.3.0", "@nftstorage/ipfs-cluster": "^5.0.1", + "@web3-storage/car-block-validator": "^1.0.0", "@web3-storage/db": "^4.0.0", "@web3-storage/multipart-parser": "^1.0.0", "cardex": "^1.0.0", diff --git a/packages/api/src/car.js b/packages/api/src/car.js index 0703f87266..bda0922b15 100644 --- a/packages/api/src/car.js +++ b/packages/api/src/car.js @@ -3,13 +3,14 @@ import { PutObjectCommand } from '@aws-sdk/client-s3/dist-es/commands/PutObjectCommand.js' import { CarBlockIterator } from '@ipld/car' import { CarIndexer } from '@ipld/car/indexer' -import { toString, equals } from 'uint8arrays' +import { toString } from 'uint8arrays' import { LinkIndexer } from 'linkdex' import { maybeDecode } from 'linkdex/decode' import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' import * as raw from 'multiformats/codecs/raw' import * as pb from '@ipld/dag-pb' +import { validateBlock } from '@web3-storage/car-block-validator' import pRetry from 'p-retry' import { MultihashIndexSortedWriter } from 'cardex' import { InvalidCarError, LinkdexError } from './errors.js' @@ -438,12 +439,7 @@ async function carStat (carBytes) { if (blockSize > MAX_BLOCK_SIZE) { throw new InvalidCarError(`block too big: ${blockSize} > ${MAX_BLOCK_SIZE}`) } - if (block.cid.multihash.code === sha256.code) { - const ourHash = await sha256.digest(block.bytes) - if (!equals(ourHash.digest, block.cid.multihash.digest)) { - throw new InvalidCarError(`block data does not match CID for ${block.cid.toString()}`) - } - } + await validateBlock(block) if (!rawRootBlock && block.cid.equals(rootCid)) { rawRootBlock = block } diff --git a/packages/api/test/car.spec.js b/packages/api/test/car.spec.js index 89c5e4e665..7143646b6a 100644 --- a/packages/api/test/car.spec.js +++ b/packages/api/test/car.spec.js @@ -488,6 +488,6 @@ describe('POST /car', () => { assert.strictEqual(res.ok, false) const { message } = await res.json() - assert.strictEqual(message, `Invalid CAR file received: block data does not match CID for ${cid.toString()}`) + assert.strictEqual(message, 'CID hash does not match bytes') }) })