From 651f113645f9715b7a25d536e564bde17c429725 Mon Sep 17 00:00:00 2001 From: Pedro Teixeira Date: Mon, 30 Jan 2017 15:19:02 +0000 Subject: [PATCH] fix: stop export visitor from trying to resolve leaf object (#130) * fixing recursive get, stopping visitor from trying to resolve leaf objects * fixed typo on comment * test for exporting deeply nested dirs --- src/exporter/index.js | 11 +++ test/browser.js | 1 + test/node.js | 1 + test/test-nested-dir-import-export.js | 118 ++++++++++++++++++++++++++ 4 files changed, 131 insertions(+) create mode 100644 test/test-nested-dir-import-export.js diff --git a/src/exporter/index.js b/src/exporter/index.js index 63fba476..9be52dfe 100644 --- a/src/exporter/index.js +++ b/src/exporter/index.js @@ -21,6 +21,17 @@ module.exports = (hash, ipldResolver, options) => { options = options || {} function visitor (item) { + if (!item.hash) { + // having no hash means that this visitor got a file object + // which needs no further resolving. + // No further resolving means that the visitor does not + // need to do anyting else, so he's returning + // an empty stream + + // TODO: perhaps change the pull streams construct. + // Instead of traversing with a visitor, consider recursing. + return pull.empty() + } return pull( ipldResolver.getStream(new CID(item.hash)), pull.map((node) => switchType( diff --git a/test/browser.js b/test/browser.js index 2e1b6b5f..23bd5600 100644 --- a/test/browser.js +++ b/test/browser.js @@ -59,4 +59,5 @@ describe('IPFS data importing tests on the Browser', function () { require('./test-importer')(repo) require('./test-import-export')(repo) require('./test-hash-parity-with-go-ipfs')(repo) + require('./test-nested-dir-import-export')(repo) }) diff --git a/test/node.js b/test/node.js index e60daebc..982b0fd8 100644 --- a/test/node.js +++ b/test/node.js @@ -44,4 +44,5 @@ describe('IPFS UnixFS Engine', () => { require('./test-importer')(repo) require('./test-import-export')(repo) require('./test-hash-parity-with-go-ipfs')(repo) + require('./test-nested-dir-import-export')(repo) }) diff --git a/test/test-nested-dir-import-export.js b/test/test-nested-dir-import-export.js new file mode 100644 index 00000000..f58dd78f --- /dev/null +++ b/test/test-nested-dir-import-export.js @@ -0,0 +1,118 @@ +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect +const BlockService = require('ipfs-block-service') +const IPLDResolver = require('ipld-resolver') +const pull = require('pull-stream') +const mh = require('multihashes') +const map = require('async/map') + +const unixFSEngine = require('./../') + +module.exports = (repo) => { + describe('import adn export big nested dir', () => { + const rootHash = 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK' + let ipldResolver + + before(() => { + const bs = new BlockService(repo) + ipldResolver = new IPLDResolver(bs) + }) + + it('imports', (done) => { + pull( + pull.values([ + { path: 'a/b/c/d/e', content: pull.values([new Buffer('banana')]) }, + { path: 'a/b/c/d/f', content: pull.values([new Buffer('strawberry')]) }, + { path: 'a/b/g', content: pull.values([new Buffer('ice')]) }, + { path: 'a/b/h', content: pull.values([new Buffer('cream')]) } + ]), + unixFSEngine.importer(ipldResolver), + pull.collect((err, files) => { + expect(err).to.not.exist + expect(files.map(normalizeNode).sort(byPath)).to.be.eql([ + { path: 'a/b/h', + multihash: 'QmWHMpCtdNjemT2F3SjyrmnBXQXwEohaZd4apcbFBhbFRC' }, + { path: 'a/b/g', + multihash: 'QmQGwYzzTPcbqTiy2Nbp88gqqBqCWY4QZGfen45LFZkD5n' }, + { path: 'a/b/c/d/f', + multihash: 'QmNVHs2dy7AjGUotsubWVncRsD3SpRXm8MgmCCQTVdVACz' }, + { path: 'a/b/c/d/e', + multihash: 'QmYPbDKwc7oneCcEc6BcRSN5GXthTGWUCd19bTCyP9u3vH' }, + { path: 'a/b/c/d', + multihash: 'QmQGDXr3ysARM38n7h79Tx7yD3YxuzcnZ1naG71WMojPoj' }, + { path: 'a/b/c', + multihash: 'QmYTVcjYpN3hQLtJstCPE8hhEacAYjWAuTmmAAXoonamuE' }, + { path: 'a/b', + multihash: 'QmWyWYxq1GD9fEyckf5LrJv8hMW35CwfWwzDBp8bTw3NQj' }, + { path: 'a', + multihash: rootHash } + ]) + done() + }) + ) + }) + + it('exports', done => { + pull( + unixFSEngine.exporter(rootHash, ipldResolver), + pull.collect((err, files) => { + expect(err).to.not.exist + map( + files, + (file, callback) => { + if (file.content) { + pull( + file.content, + pull.collect(mapFile(file, callback)) + ) + } else { + callback(null, { path: file.path }) + } + }, + (err, files) => { + expect(err).to.not.exist + expect(files.filter(fileHasContent).sort(byPath)).to.eql([ + { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/h', + content: 'cream' }, + { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/g', + content: 'ice' }, + { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/c/d/f', + content: 'strawberry' }, + { path: 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK/b/c/d/e', + content: 'banana' } + ]) + done() + }) + }) + ) + + function mapFile (file, callback) { + return (err, fileContent) => { + callback(err, fileContent && { + path: file.path, + content: fileContent.toString() + }) + } + } + }) + }) +} + +function normalizeNode (node) { + return { + path: node.path, + multihash: mh.toB58String(node.multihash) + } +} + +function fileHasContent (file) { + return Boolean(file.content) +} + +function byPath (a, b) { + if (a.path > b.path) return -1 + if (a.path < b.path) return 1 + return 0 +}