Skip to content
This repository has been archived by the owner on Feb 12, 2024. It is now read-only.

Commit

Permalink
feat: add HTTP Gateway support for /ipns/ paths (#2020)
Browse files Browse the repository at this point in the history
> Part of an effort to run embedded js-ipfs in Brave 🦁 ipfs/ipfs-companion#716
> Fixes #1918 

This PR will add support for `/ipns/` paths at HTTP Gateway.
Smoke test: [/ipns/tr.wikipedia-on-ipfs.org](http://127.0.0.1:9090/ipns/tr.wikipedia-on-ipfs.org/wiki/Anasayfa.html) (IPNS+DNSLink+HAMT-sharded website)
 
This PR depends on the following merged PRs:

  - Gateway Improvements from #1989
    (after merging #1989 I will rebase this PR, which will remove first two commits)
  - PeerID 
    eg. `/ipns/<PeerId-as-multihash-b58>`
    - requires #2002 to land first
  - `/ipns/<libp2p-key-in-cidv1>` 
    - requires multiformats/js-multicodec#45
  - DNSLink 
    eg. `/ipns/<fqdn>/path/file` like `/ipns/docs.ipfs.io/assets/logo.svg`
    - requires #2002 to land first
  - HAMT shard support 
    eg. `/ipns/tr.wikipedia-on-ipfs.org/wiki/Anasayfa.html` (`wiki` is a sharded directory)
    - requires ipfs/js-ipfs-http-response#22 and ipfs-inactive/js-ipfs-mfs#48  to land first
  - Tests for `/ipns/` 

License: MIT
Signed-off-by: Marcin Rataj <[email protected]>
  • Loading branch information
lidel authored and Alan Shaw committed Jul 4, 2019
1 parent 438f70a commit 43ac305
Show file tree
Hide file tree
Showing 6 changed files with 170 additions and 53 deletions.
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@
"ipfs-block": "~0.8.1",
"ipfs-block-service": "~0.15.1",
"ipfs-http-client": "^32.0.1",
"ipfs-http-response": "~0.3.0",
"ipfs-mfs": "~0.11.4",
"ipfs-http-response": "~0.3.1",
"ipfs-mfs": "~0.11.5",
"ipfs-multipart": "~0.1.0",
"ipfs-repo": "~0.26.6",
"ipfs-unixfs": "~0.1.16",
Expand Down
21 changes: 15 additions & 6 deletions src/http/api/routes/webui.js
Original file line number Diff line number Diff line change
@@ -1,17 +1,26 @@
'use strict'

const Joi = require('@hapi/joi')
const resources = require('../../gateway/resources')

module.exports = [
{
method: '*',
path: '/ipfs/{cid*}',
path: '/ipfs/{path*}',
options: {
pre: [
{ method: resources.gateway.checkCID, assign: 'args' }
]
},
handler: resources.gateway.handler
handler: resources.gateway.handler,
validate: {
params: {
path: Joi.string().required()
}
},
response: {
ranges: false // disable built-in support, handler does it manually
},
ext: {
onPostHandler: { method: resources.gateway.afterHandler }
}
}
},
{
method: '*',
Expand Down
70 changes: 40 additions & 30 deletions src/http/gateway/resources/gateway.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,23 @@ const Boom = require('boom')
const Ammo = require('@hapi/ammo') // HTTP Range processing utilities
const peek = require('buffer-peek-stream')

const multibase = require('multibase')
const { resolver } = require('ipfs-http-response')
const PathUtils = require('../utils/path')
const { cidToString } = require('../../../utils/cid')
const isIPFS = require('is-ipfs')

function detectContentType (ref, chunk) {
function detectContentType (path, chunk) {
let fileSignature

// try to guess the filetype based on the first bytes
// note that `file-type` doesn't support svgs, therefore we assume it's a svg if ref looks like it
if (!ref.endsWith('.svg')) {
if (!path.endsWith('.svg')) {
fileSignature = fileType(chunk)
}

// if we were unable to, fallback to the `ref` which might contain the extension
const mimeType = mime.lookup(fileSignature ? fileSignature.ext : ref)
// if we were unable to, fallback to the path which might contain the extension
const mimeType = mime.lookup(fileSignature ? fileSignature.ext : path)

return mime.contentType(mimeType)
}
Expand All @@ -45,44 +47,45 @@ class ResponseStream extends PassThrough {
}

module.exports = {
checkCID (request, h) {
if (!request.params.cid) {
throw Boom.badRequest('Path Resolve error: path must contain at least one component')
}

return { ref: `/ipfs/${request.params.cid}` }
},

async handler (request, h) {
const { ref } = request.pre.args
const { ipfs } = request.server.app
const path = request.path

// The resolver from ipfs-http-response supports only immutable /ipfs/ for now,
// so we convert /ipns/ to /ipfs/ before passing it to the resolver ¯\_(ツ)_/¯
// This could be removed if a solution proposed in
// https://github.com/ipfs/js-ipfs-http-response/issues/22 lands upstream
const ipfsPath = decodeURI(path.startsWith('/ipns/')
? await ipfs.name.resolve(path, { recursive: true })
: path)

let data
try {
data = await resolver.cid(ipfs, ref)
data = await resolver.cid(ipfs, ipfsPath)
} catch (err) {
const errorToString = err.toString()
log.error('err: ', errorToString, ' fileName: ', err.fileName)

// switch case with true feels so wrong.
switch (true) {
case (errorToString === 'Error: This dag node is a directory'):
data = await resolver.directory(ipfs, ref, err.cid)
data = await resolver.directory(ipfs, ipfsPath, err.cid)

if (typeof data === 'string') {
// no index file found
if (!ref.endsWith('/')) {
if (!path.endsWith('/')) {
// for a directory, if URL doesn't end with a /
// append / and redirect permanent to that URL
return h.redirect(`${ref}/`).permanent(true)
return h.redirect(`${path}/`).permanent(true)
}
// send directory listing
return h.response(data)
}

// found index file
// redirect to URL/<found-index-file>
return h.redirect(PathUtils.joinURLParts(ref, data[0].Name))
return h.redirect(PathUtils.joinURLParts(path, data[0].Name))
case (errorToString.startsWith('Error: no link named')):
throw Boom.boomify(err, { statusCode: 404 })
case (errorToString.startsWith('Error: multihash length inconsistent')):
Expand All @@ -94,9 +97,9 @@ module.exports = {
}
}

if (ref.endsWith('/')) {
if (path.endsWith('/')) {
// remove trailing slash for files
return h.redirect(PathUtils.removeTrailingSlash(ref)).permanent(true)
return h.redirect(PathUtils.removeTrailingSlash(path)).permanent(true)
}

// Support If-None-Match & Etag (Conditional Requests from RFC7232)
Expand All @@ -108,7 +111,7 @@ module.exports = {
}

// Immutable content produces 304 Not Modified for all values of If-Modified-Since
if (ref.startsWith('/ipfs/') && request.headers['if-modified-since']) {
if (path.startsWith('/ipfs/') && request.headers['if-modified-since']) {
return h.response().code(304) // Not Modified
}

Expand Down Expand Up @@ -150,7 +153,7 @@ module.exports = {
log.error(err)
return reject(err)
}
resolve({ peekedStream, contentType: detectContentType(ref, streamHead) })
resolve({ peekedStream, contentType: detectContentType(path, streamHead) })
})
})

Expand All @@ -163,11 +166,11 @@ module.exports = {
res.header('etag', etag)

// Set headers specific to the immutable namespace
if (ref.startsWith('/ipfs/')) {
if (path.startsWith('/ipfs/')) {
res.header('Cache-Control', 'public, max-age=29030400, immutable')
}

log('ref ', ref)
log('path ', path)
log('content-type ', contentType)

if (contentType) {
Expand Down Expand Up @@ -200,18 +203,25 @@ module.exports = {
const { response } = request
// Add headers to successfult responses (regular or range)
if (response.statusCode === 200 || response.statusCode === 206) {
const { ref } = request.pre.args
response.header('X-Ipfs-Path', ref)
if (ref.startsWith('/ipfs/')) {
const path = request.path
response.header('X-Ipfs-Path', path)
if (path.startsWith('/ipfs/')) {
// "set modtime to a really long time ago, since files are immutable and should stay cached"
// Source: https://github.com/ipfs/go-ipfs/blob/v0.4.20/core/corehttp/gateway_handler.go#L228-L229
response.header('Last-Modified', 'Thu, 01 Jan 1970 00:00:01 GMT')
// Suborigins: https://github.com/ipfs/in-web-browsers/issues/66
const rootCid = ref.split('/')[2]
// Suborigin for /ipfs/: https://github.com/ipfs/in-web-browsers/issues/66
const rootCid = path.split('/')[2]
const ipfsOrigin = cidToString(rootCid, { base: 'base32' })
response.header('Suborigin', 'ipfs000' + ipfsOrigin)
response.header('Suborigin', `ipfs000${ipfsOrigin}`)
} else if (path.startsWith('/ipns/')) {
// Suborigin for /ipns/: https://github.com/ipfs/in-web-browsers/issues/66
const root = path.split('/')[2]
// encode CID/FQDN in base32 (Suborigin allows only a-z)
const ipnsOrigin = isIPFS.cid(root)
? cidToString(root, { base: 'base32' })
: multibase.encode('base32', Buffer.from(root)).toString()
response.header('Suborigin', `ipns000${ipnsOrigin}`)
}
// TODO: we don't have case-insensitive solution for /ipns/ yet (https://github.com/ipfs/go-ipfs/issues/5287)
}
return h.continue
}
Expand Down
51 changes: 37 additions & 14 deletions src/http/gateway/routes/gateway.js
Original file line number Diff line number Diff line change
@@ -1,20 +1,43 @@
'use strict'

const Joi = require('@hapi/joi')
const resources = require('../resources')

module.exports = {
method: '*',
path: '/ipfs/{cid*}',
options: {
handler: resources.gateway.handler,
pre: [
{ method: resources.gateway.checkCID, assign: 'args' }
],
response: {
ranges: false // disable built-in support, we do it manually
},
ext: {
onPostHandler: { method: resources.gateway.afterHandler }
module.exports = [
{
method: '*',
path: '/ipfs/{path*}',
options: {
handler: resources.gateway.handler,
validate: {
params: {
path: Joi.string().required()
}
},
response: {
ranges: false // disable built-in support, handler does it manually
},
ext: {
onPostHandler: { method: resources.gateway.afterHandler }
}
}
},
{
method: '*',
path: '/ipns/{path*}',
options: {
handler: resources.gateway.handler,
validate: {
params: {
path: Joi.string().required()
}
},
response: {
ranges: false // disable built-in support, handler does it manually
},
ext: {
onPostHandler: { method: resources.gateway.afterHandler }
}
}
}
}
]
2 changes: 1 addition & 1 deletion src/http/gateway/routes/index.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
'use strict'

module.exports = [require('./gateway')]
module.exports = require('./gateway')
75 changes: 75 additions & 0 deletions test/gateway/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ const os = require('os')
const path = require('path')
const hat = require('hat')
const fileType = require('file-type')
const CID = require('cids')

const bigFile = loadFixture('test/fixtures/15mb.random', 'interface-ipfs-core')
const directoryContent = {
Expand All @@ -20,6 +21,7 @@ const directoryContent = {
'nested-folder/ipfs.txt': loadFixture('test/gateway/test-folder/nested-folder/ipfs.txt'),
'nested-folder/nested.html': loadFixture('test/gateway/test-folder/nested-folder/nested.html'),
'cat-folder/cat.jpg': loadFixture('test/gateway/test-folder/cat-folder/cat.jpg'),
'utf8/cat-with-óąśśł-and-أعظم._.jpg': loadFixture('test/gateway/test-folder/cat-folder/cat.jpg'),
'unsniffable-folder/hexagons-xml.svg': loadFixture('test/gateway/test-folder/unsniffable-folder/hexagons-xml.svg'),
'unsniffable-folder/hexagons.svg': loadFixture('test/gateway/test-folder/unsniffable-folder/hexagons.svg')
}
Expand Down Expand Up @@ -84,6 +86,10 @@ describe('HTTP Gateway', function () {
content('unsniffable-folder/hexagons-xml.svg'),
content('unsniffable-folder/hexagons.svg')
])
// QmaRdtkDark8TgXPdDczwBneadyF44JvFGbrKLTkmTUhHk
await http.api._ipfs.add([content('utf8/cat-with-óąśśł-and-أعظم._.jpg')])
// Publish QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ to IPNS using self key
await http.api._ipfs.name.publish('QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ', { resolve: false })
})

after(() => http.api.stop())
Expand Down Expand Up @@ -526,4 +532,73 @@ describe('HTTP Gateway', function () {
expect(res.headers.location).to.equal('/ipfs/QmbQD7EMEL1zeebwBsWEfA3ndgSS6F7S6iTuwuqasPgVRi/index.html')
expect(res.headers['x-ipfs-path']).to.equal(undefined)
})

it('test(gateway): load from URI-encoded path', async () => {
// non-ascii characters will be URI-encoded by the browser
const utf8path = '/ipfs/QmaRdtkDark8TgXPdDczwBneadyF44JvFGbrKLTkmTUhHk/cat-with-óąśśł-and-أعظم._.jpg'
const escapedPath = encodeURI(utf8path) // this is what will be actually requested
const res = await gateway.inject({
method: 'GET',
url: escapedPath
})

expect(res.statusCode).to.equal(200)
expect(res.headers['content-type']).to.equal('image/jpeg')
expect(res.headers['x-ipfs-path']).to.equal(escapedPath)
expect(res.headers['cache-control']).to.equal('public, max-age=29030400, immutable')
expect(res.headers['last-modified']).to.equal('Thu, 01 Jan 1970 00:00:01 GMT')
expect(res.headers['content-length']).to.equal(res.rawPayload.length)
expect(res.headers.etag).to.equal('"Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u"')
expect(res.headers.suborigin).to.equal('ipfs000bafybeiftsm4u7cn24bn2suwg3x7sldx2uplvfylsk3e4bgylyxwjdevhqm')
})

it('load a file from IPNS', async () => {
const { id } = await http.api._ipfs.id()
const ipnsPath = `/ipns/${id}/cat.jpg`

const res = await gateway.inject({
method: 'GET',
url: ipnsPath
})

const kittyDirectCid = 'Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u'

expect(res.statusCode).to.equal(200)
expect(res.headers['content-type']).to.equal('image/jpeg')
expect(res.headers['content-length']).to.equal(res.rawPayload.length).to.equal(443230)
expect(res.headers['x-ipfs-path']).to.equal(ipnsPath)
expect(res.headers['etag']).to.equal(`"${kittyDirectCid}"`)
expect(res.headers['cache-control']).to.equal('no-cache') // TODO: should be record TTL
expect(res.headers['last-modified']).to.equal(undefined)
expect(res.headers.etag).to.equal('"Qmd286K6pohQcTKYqnS1YhWrCiS4gz7Xi34sdwMe9USZ7u"')
expect(res.headers.suborigin).to.equal(`ipns000${new CID(id).toV1().toBaseEncodedString('base32')}`)

let fileSignature = fileType(res.rawPayload)
expect(fileSignature.mime).to.equal('image/jpeg')
expect(fileSignature.ext).to.equal('jpg')
})

it('load a directory from IPNS', async () => {
const { id } = await http.api._ipfs.id()
const ipnsPath = `/ipns/${id}/`

const res = await gateway.inject({
method: 'GET',
url: ipnsPath
})

expect(res.statusCode).to.equal(200)
expect(res.headers['content-type']).to.equal('text/html; charset=utf-8')
expect(res.headers['x-ipfs-path']).to.equal(ipnsPath)
expect(res.headers['cache-control']).to.equal('no-cache')
expect(res.headers['last-modified']).to.equal(undefined)
expect(res.headers['content-length']).to.equal(res.rawPayload.length)
expect(res.headers.etag).to.equal(undefined)
expect(res.headers.suborigin).to.equal(`ipns000${new CID(id).toV1().toBaseEncodedString('base32')}`)

// check if the cat picture is in the payload as a way to check
// if this is an index of this directory
let listedFile = res.payload.match(/\/cat\.jpg/g)
expect(listedFile).to.have.lengthOf(1)
})
})

0 comments on commit 43ac305

Please sign in to comment.