From d7899a011991ba23b4a77c2e2db231c9da8b5730 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Thu, 7 Apr 2022 14:05:43 +0100 Subject: [PATCH] feat: support ipfs-cluster@1.0 The IPFS Cluster v1 release has much improved perf, and some breaking API changes. In this PR we upgrade the ipfs-cluster client and local test environment to work with the v1 cluster API. When this PR is merged to main, the staging API credentials must be updated. The production release of this PR needs to be coordinated with the upgrade of the ipfs-cluster binaries on the prod cluster. There will be a second PR to update the pins cron, which should be merged to main once the prod cluster is upgraded. see: #1184 License: (Apache-2.0 AND MIT) Signed-off-by: Oli Evans --- package-lock.json | 18 ++++++++++++++---- packages/api/package.json | 2 +- packages/api/test/car.spec.js | 9 ++++++--- packages/api/wrangler.toml | 3 ++- .../tools/docker/cluster/docker-compose.yml | 2 +- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 75b96c1765..4640f332c6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28334,7 +28334,7 @@ }, "packages/api": { "name": "@web3-storage/api", - "version": "5.4.3", + "version": "5.5.0", "license": "(Apache-2.0 OR MIT)", "dependencies": { "@aws-sdk/client-s3": "^3.53.1", @@ -28345,7 +28345,7 @@ "@ipld/dag-pb": "^2.0.2", "@magic-ext/oauth": "^0.8.0", "@magic-sdk/admin": "^1.3.0", - "@nftstorage/ipfs-cluster": "^4.0.0", + "@nftstorage/ipfs-cluster": "^5.0.0", "@web3-storage/db": "^4.0.0", "@web3-storage/multipart-parser": "^1.0.0", "cborg": "^1.6.0", @@ -28401,6 +28401,11 @@ "crypto-js": "^3.3.0" } }, + "packages/api/node_modules/@nftstorage/ipfs-cluster": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@nftstorage/ipfs-cluster/-/ipfs-cluster-5.0.0.tgz", + "integrity": "sha512-i9VE3MF4PcBLWuerlTy0G5L4jdthESFg5rtxAz5FW9sFmTplpnCd+xYeIFkxrqSQyc9MAY4N8vyNf1IrEWIc/A==" + }, "packages/api/node_modules/@web-std/fetch": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/@web-std/fetch/-/fetch-3.0.2.tgz", @@ -30597,7 +30602,7 @@ }, "packages/website": { "name": "@web3-storage/website", - "version": "2.0.0", + "version": "2.1.0", "dependencies": { "@magic-ext/oauth": "^0.7.0", "clsx": "^1.1.1", @@ -43514,7 +43519,7 @@ "@ipld/dag-pb": "^2.0.2", "@magic-ext/oauth": "^0.8.0", "@magic-sdk/admin": "^1.3.0", - "@nftstorage/ipfs-cluster": "^4.0.0", + "@nftstorage/ipfs-cluster": "5.0.0", "@sentry/cli": "^1.72.1", "@types/mocha": "^9.0.0", "@web-std/fetch": "^3.0.2", @@ -43566,6 +43571,11 @@ "crypto-js": "^3.3.0" } }, + "@nftstorage/ipfs-cluster": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@nftstorage/ipfs-cluster/-/ipfs-cluster-5.0.0.tgz", + "integrity": "sha512-i9VE3MF4PcBLWuerlTy0G5L4jdthESFg5rtxAz5FW9sFmTplpnCd+xYeIFkxrqSQyc9MAY4N8vyNf1IrEWIc/A==" + }, "@web-std/fetch": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/@web-std/fetch/-/fetch-3.0.2.tgz", diff --git a/packages/api/package.json b/packages/api/package.json index e9dbd4f91c..e871b4f568 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -59,7 +59,7 @@ "@ipld/dag-pb": "^2.0.2", "@magic-ext/oauth": "^0.8.0", "@magic-sdk/admin": "^1.3.0", - "@nftstorage/ipfs-cluster": "^4.0.0", + "@nftstorage/ipfs-cluster": "^5.0.0", "@web3-storage/db": "^4.0.0", "@web3-storage/multipart-parser": "^1.0.0", "cborg": "^1.6.0", diff --git a/packages/api/test/car.spec.js b/packages/api/test/car.spec.js index a41e73c994..ede4df21fc 100644 --- a/packages/api/test/car.spec.js +++ b/packages/api/test/car.spec.js @@ -5,12 +5,16 @@ import { sha256, sha512 } from 'multiformats/hashes/sha2' import * as pb from '@ipld/dag-pb' import { CarWriter } from '@ipld/car' import fetch, { Blob } from '@web-std/fetch' +import { Cluster } from '@nftstorage/ipfs-cluster' import { endpoint, clusterApi, clusterApiAuthHeader } from './scripts/constants.js' import { createCar } from './scripts/car.js' import { MAX_BLOCK_SIZE } from '../src/constants.js' import { getTestJWT } from './scripts/helpers.js' import { PIN_OK_STATUS } from '../src/utils/pin.js' +// Cluster client needs global fetch +Object.assign(global, { fetch }) + describe('POST /car', () => { it('should add posted CARs to Cluster', async () => { const name = 'car' @@ -44,13 +48,12 @@ describe('POST /car', () => { const status = await statusRes.json() const pinInfo = status.pins.find(pin => PIN_OK_STATUS.includes(pin.status)) assert(pinInfo, `status is one of ${PIN_OK_STATUS}`) - - const clusterPeersRes = await fetch(new URL('peers', clusterApi), { + const cluster = new Cluster(clusterApi, { headers: { Authorization: clusterApiAuthHeader } }) - const clusterPeers = await clusterPeersRes.json() + const clusterPeers = await cluster.peerList() // assert that peerId from the status belongs to one of the cluster ipfs nodes. assert(clusterPeers.some(peer => peer.ipfs.id === pinInfo.peerId)) }) diff --git a/packages/api/wrangler.toml b/packages/api/wrangler.toml index ce69bed62f..65aa6baec8 100644 --- a/packages/api/wrangler.toml +++ b/packages/api/wrangler.toml @@ -53,7 +53,8 @@ bindings = [{ name = "NAME_ROOM", class_name = "NameRoom0" }] account_id = "fffa4b4363a7e5250af8357087263b3a" # Protocol Labs CF account zone_id = "7eee3323c1b35b6650568604c65f441e" # web3.storage zone route = "https://api-staging.web3.storage/*" -vars = { CLUSTER_API_URL = "https://web3.storage.ipfscluster.io/api/", ENV = "staging", PG_REST_URL = "https://web3-storage-pgrest-staging.herokuapp.com", GATEWAY_URL = "https://ipfs.io" } +# nft.storage.ipfscluster.io is the staging cluster +vars = { CLUSTER_API_URL = "https://nft.storage.ipfscluster.io/api/", ENV = "staging", PG_REST_URL = "https://web3-storage-pgrest-staging.herokuapp.com", GATEWAY_URL = "https://ipfs.io" } [env.staging.durable_objects] bindings = [{ name = "NAME_ROOM", class_name = "NameRoom0" }] diff --git a/packages/tools/docker/cluster/docker-compose.yml b/packages/tools/docker/cluster/docker-compose.yml index 6e843a9adb..feab75d622 100644 --- a/packages/tools/docker/cluster/docker-compose.yml +++ b/packages/tools/docker/cluster/docker-compose.yml @@ -47,7 +47,7 @@ services: cluster0: container_name: cluster0 - image: ipfs/ipfs-cluster:v0.14.5 + image: ipfs/ipfs-cluster:v1.0.0-rc3 depends_on: - ipfs0 environment: