Skip to content

Commit

Permalink
Merge branch 'main' into feat/dir-index-html
Browse files Browse the repository at this point in the history
  • Loading branch information
SgtPooki committed Jan 14, 2025
2 parents 65dbd3c + 0701c71 commit e47503a
Show file tree
Hide file tree
Showing 33 changed files with 868 additions and 214 deletions.
17 changes: 16 additions & 1 deletion .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,26 @@
version: 2
updates:
- package-ecosystem: npm
directory: "/"
directories:
- "/**/*"
schedule:
interval: daily
time: "10:00"
open-pull-requests-limit: 20
commit-message:
prefix: "deps"
prefix-development: "deps(dev)"
groups:
helia-deps: # group all deps that should be updated when Helia deps need updated
patterns:
- "*helia*"
- "*libp2p*"
- "*multiformats*"
store-deps: # group all blockstore and datastore updates (interface & impl)
patterns:
- "*blockstore*"
- "*datastore*"
kubo-deps: # group kubo, kubo-rpc-client, and ipfsd-ctl updates
patterns:
- "*kubo*"
- "ipfsd-ctl"
2 changes: 1 addition & 1 deletion .github/workflows/gateway-conformance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version: lts/*
- uses: ipfs/aegir/actions/cache-node-modules@master
- uses: ipfs/aegir/actions/cache-node-modules@main

# 2. Set up 'go' so we can install the gateway-conformance binary
- name: Setup Go
Expand Down
5 changes: 2 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@
[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech)
[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech)
[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-verified-fetch.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-verified-fetch)
[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-verified-fetch/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-verified-fetch/actions/workflows/js-test-and-release.yml?query=branch%3Amain)

[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-verified-fetch/js-test-and-release.yml?branch=main&style=flat-square)](https://github.com/ipfs/helia-verified-fetch/actions/workflows/js-test-and-release.yml?query=branch%3Amain)

## About

Expand All @@ -29,7 +28,7 @@ Learn more in the [announcement blog post](https://blog.ipfs.tech/verified-fetch

# API Docs

- <https://ipfs.github.io/helia-verified-fetch/modules.html>
- [https://ipfs.github.io/helia-verified-fetch/modules/\_helia_verified_fetch.html](https://ipfs.github.io/helia-verified-fetch/modules/_helia_verified_fetch.html)

# License

Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
"docs:no-publish": "aegir docs --publish false"
},
"devDependencies": {
"aegir": "^44.1.4",
"aegir": "^45.0.8",
"npm-run-all": "^4.1.5"
},
"type": "module",
Expand Down
33 changes: 17 additions & 16 deletions packages/gateway-conformance/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -52,29 +52,30 @@
"test": "aegir test -t node"
},
"dependencies": {
"@helia/block-brokers": "^3.0.1",
"@helia/http": "^1.0.8",
"@helia/interface": "^4.3.0",
"@helia/routers": "^1.1.0",
"@helia/verified-fetch": "1.5.0",
"@libp2p/kad-dht": "^12.0.17",
"@libp2p/logger": "^4.0.13",
"@libp2p/peer-id": "^4.1.2",
"@helia/block-brokers": "^4.0.2",
"@helia/http": "^2.0.2",
"@helia/interface": "^5.1.0",
"@helia/routers": "^2.2.0",
"@helia/verified-fetch": "^2.3.0",
"@libp2p/interface": "^2.2.1",
"@libp2p/kad-dht": "^14.1.3",
"@libp2p/logger": "^5.1.4",
"@libp2p/peer-id": "^5.0.8",
"@multiformats/dns": "^1.0.6",
"@sgtpooki/file-type": "^1.0.1",
"aegir": "^44.1.4",
"blockstore-core": "^4.4.1",
"datastore-core": "^9.2.9",
"aegir": "^45.0.1",
"blockstore-core": "^5.0.2",
"datastore-core": "^10.0.2",
"execa": "^9.1.0",
"fast-glob": "^3.3.2",
"interface-blockstore": "^5.2.10",
"interface-datastore": "^8.2.11",
"ipfsd-ctl": "^14.1.0",
"ipns": "^9.1.0",
"kubo": "^0.28.0",
"kubo-rpc-client": "^4.1.1",
"ipfsd-ctl": "^15.0.2",
"kubo": "^0.32.0",
"kubo-rpc-client": "^5.0.2",
"multiformats": "^13.3.1",
"uint8arrays": "^5.1.0",
"undici": "^6.18.1"
"undici": "^7.1.0"
},
"browser": {
"./dist/src/fixtures/create-kubo.js": "./dist/src/fixtures/create-kubo.browser.js",
Expand Down
11 changes: 9 additions & 2 deletions packages/gateway-conformance/src/conformance.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,15 @@ const tests: TestConfig[] = [
// },
{
name: 'TestGatewaySubdomainAndIPNS',
run: ['TestGatewaySubdomainAndIPNS'],
successRate: 31.58
run: [
'TestGatewaySubdomainAndIPNS'
],
skip: [
'TestGatewaySubdomainAndIPNS/request_for_a_ED25519_libp2p-key_.*',
'TestGatewaySubdomainAndIPNS/.*redirects_to_CID_with_libp2p-key_multicodec',
'TestGatewaySubdomainAndIPNS/.*redirects_to_CIDv1.*'
],
successRate: 46.15
},
{
// TODO: add directory listing support to verified-fetch
Expand Down
30 changes: 26 additions & 4 deletions packages/gateway-conformance/src/fixtures/kubo-mgmt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,30 @@
import { readFile } from 'node:fs/promises'
import { dirname, relative, posix, basename } from 'node:path'
import { fileURLToPath } from 'node:url'
import { type PeerId } from '@libp2p/interface'
import { Record as DhtRecord } from '@libp2p/kad-dht'
import { logger } from '@libp2p/logger'
import { peerIdFromString } from '@libp2p/peer-id'
import { peerIdFromCID, peerIdFromString } from '@libp2p/peer-id'
import { $ } from 'execa'
import fg from 'fast-glob'
import { Key } from 'interface-datastore'
import { peerIdToRoutingKey } from 'ipns'
import { path } from 'kubo'
import { CID } from 'multiformats/cid'
import { concat as uint8ArrayConcat } from 'uint8arrays/concat'
import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
import { GWC_IMAGE } from '../constants.js'
import { getIpnsRecordDatastore } from './ipns-record-datastore.js'

function getPeerIdFromString (peerIdString: string): PeerId {
if (peerIdString.charAt(0) === '1' || peerIdString.charAt(0) === 'Q') {
return peerIdFromString(peerIdString)
}

// try resolving as a base36 CID
return peerIdFromCID(CID.parse(peerIdString))
}

// eslint-disable-next-line @typescript-eslint/naming-convention
const __dirname = dirname(fileURLToPath(import.meta.url))

Expand Down Expand Up @@ -92,13 +104,23 @@ export async function loadFixtures (kuboRepoDir: string): Promise<string> {
}

const datastore = getIpnsRecordDatastore()
const IPNS_PREFIX = uint8ArrayFromString('/ipns/')

for (const fsIpnsRecord of await fg.glob([`${GWC_FIXTURES_PATH}/**/*.ipns-record`])) {
const peerIdString = basename(fsIpnsRecord, '.ipns-record').split('_')[0]
const relativePath = relative(GWC_FIXTURES_PATH, fsIpnsRecord)
log('Loading *.ipns-record fixture %s', relativePath)
const key = peerIdFromString(peerIdString)
const customRoutingKey = peerIdToRoutingKey(key)
const key = getPeerIdFromString(peerIdString)
let bytes: Uint8Array
if (['Ed25519', 'RSA'].includes(key.type)) {
bytes = key.toMultihash().bytes
} else {
throw new Error('Unsupported key type')
}
const customRoutingKey = uint8ArrayConcat([
IPNS_PREFIX,
bytes
])
const dhtKey = new Key('/dht/record/' + uint8ArrayToString(customRoutingKey, 'base32'), false)

const dhtRecord = new DhtRecord(customRoutingKey, await readFile(fsIpnsRecord, null), new Date(Date.now() + 9999999))
Expand Down
41 changes: 40 additions & 1 deletion packages/interop/.aegir.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,19 @@
import { resolve } from 'node:path'
import { tmpdir } from 'node:os'
import { createDelegatedRoutingV1HttpApiServer } from '@helia/delegated-routing-v1-http-api-server'
import { stubInterface } from 'sinon-ts'

const IPFS_PATH = resolve(tmpdir(), 'verified-fetch-interop-ipfs-repo')

/** @type {import('aegir').PartialOptions} */
export default {
dependencyCheck: {
ignore: [
'@helia/delegated-routing-v1-http-api-server',
'sinon-ts'
]

},
test: {
files: './dist/src/*.spec.js',
before: async () => {
Expand All @@ -19,12 +28,42 @@ export default {

await loadFixtures(IPFS_PATH)

const multiaddrs = (await kuboNode.api.id()).addresses
const id = (await kuboNode.api.id()).id

const helia = stubInterface({
routing: stubInterface({
findProviders: async function * findProviders () {
yield {
multiaddrs,
id,
protocols: ['transport-bitswap']
}
}
})
})
const routingServer = await createDelegatedRoutingV1HttpApiServer(helia, {
listen: {
host: '127.0.0.1',
port: 0
}
})
await routingServer.ready()

const address = routingServer.server.address()
const port = typeof address === 'string' ? address : address?.port

return {
kuboNode
kuboNode,
routingServer,
env: {
KUBO_DIRECT_RETRIEVAL_ROUTER: `http://127.0.0.1:${port}`
}
}
},
after: async (_options, beforeResult) => {
await beforeResult.kuboNode.stop()
await beforeResult.routingServer.close()
}
}
}
17 changes: 10 additions & 7 deletions packages/interop/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,18 @@
"test:electron-main": "aegir test -t electron-main"
},
"dependencies": {
"@helia/verified-fetch": "1.5.0",
"aegir": "^44.1.4",
"@helia/delegated-routing-v1-http-api-server": "^4.0.4",
"@helia/verified-fetch": "^2.1.3",
"aegir": "^45.0.1",
"execa": "^9.1.0",
"fast-glob": "^3.3.2",
"ipfsd-ctl": "^14.1.0",
"kubo": "^0.28.0",
"kubo-rpc-client": "^4.1.1",
"glob": "^11.0.0",
"ipfsd-ctl": "^15.0.2",
"kubo": "^0.32.0",
"kubo-rpc-client": "^5.0.2",
"magic-bytes.js": "^1.10.0",
"multiformats": "^13.1.0"
"multiformats": "^13.1.0",
"sinon-ts": "^2.0.0",
"wherearewe": "^2.0.1"
},
"browser": {
"./dist/src/fixtures/create-kubo.js": "./dist/src/fixtures/create-kubo.browser.js",
Expand Down
43 changes: 43 additions & 0 deletions packages/interop/src/direct-retrieval.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import { createVerifiedFetch, type CreateVerifiedFetchInit } from '@helia/verified-fetch'
import { expect } from 'aegir/chai'
import { isNode, isBrowser } from 'wherearewe'

/**
* Currently only testing browser and node
*/
const describe = isNode || isBrowser ? global.describe : global.describe.skip

describe('@helia/verified-fetch - direct retrieval', () => {
let directRetrievalRouterUrl: string
let createVerifiedFetchInit: CreateVerifiedFetchInit

beforeEach(async () => {
if (process.env.KUBO_DIRECT_RETRIEVAL_ROUTER == null || process.env.KUBO_DIRECT_RETRIEVAL_ROUTER === '') {
throw new Error('KUBO_DIRECT_RETRIEVAL_ROUTER environment variable is required')
}
directRetrievalRouterUrl = process.env.KUBO_DIRECT_RETRIEVAL_ROUTER
createVerifiedFetchInit = {
gateways: [],
routers: [directRetrievalRouterUrl]
}
if (!isNode) {
createVerifiedFetchInit.libp2pConfig = {
connectionGater: {
denyDialMultiaddr: () => false
}
}
}
})

it('can fetch content directly from another node', async () => {
const fetch = await createVerifiedFetch(createVerifiedFetchInit)

const res = await fetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1 - alt.txt')

expect(res.status).to.equal(200)
const body = await res.text()
expect(body).to.equal('Don\'t we all.')

await fetch.stop()
})
})
6 changes: 5 additions & 1 deletion packages/interop/src/fixtures/create-kubo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@ export async function createKuboNode (repoPath = undefined): Promise<KuboNode> {
Addresses: {
Swarm: [
'/ip4/0.0.0.0/tcp/4001',
'/ip4/0.0.0.0/tcp/4002/ws'
'/ip4/0.0.0.0/tcp/4002/ws',
'/ip4/0.0.0.0/udp/4001/webrtc-direct',
'/ip4/0.0.0.0/udp/4001/quic-v1/webtransport',
'/ip6/::/udp/4001/webrtc-direct',
'/ip6/::/udp/4001/quic-v1/webtransport'
],
Gateway: '/ip4/127.0.0.1/tcp/8180'
},
Expand Down
8 changes: 5 additions & 3 deletions packages/interop/src/fixtures/load-fixtures.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { $ } from 'execa'
import fg from 'fast-glob'
import { glob } from 'glob'
import { path as kuboPath } from 'kubo'

/**
Expand All @@ -9,7 +9,9 @@ import { path as kuboPath } from 'kubo'
export async function loadFixtures (IPFS_PATH = undefined): Promise<void> {
const kuboBinary = process.env.KUBO_BINARY ?? kuboPath()

for (const carFile of await fg.glob('**/fixtures/data/*.car')) {
const files = await glob('**/fixtures/data/*.car', { cwd: process.cwd() })

await Promise.allSettled(files.map(async (carFile) => {
await $({ env: { IPFS_PATH } })`${kuboBinary} dag import --pin-roots=false --offline ${carFile}`
}
}))
}
Loading

0 comments on commit e47503a

Please sign in to comment.