feat: subtree support (#175)

* feat: subtree support

README.md

@@ -12,7 +12,7 @@ IPFS unixFS Engine
 ![](https://img.shields.io/badge/npm-%3E%3D3.0.0-orange.svg?style=flat-square)
 ![](https://img.shields.io/badge/Node.js-%3E%3D4.0.0-orange.svg?style=flat-square)
 
-> JavaScript implementation of the layout and chunking mechanisms used by IPFS
+> JavaScript implementation of the layout and chunking mechanisms used by IPFS to handle Files
 
 ## Table of Contents
 
@@ -35,9 +35,12 @@ IPFS unixFS Engine
 
 ## Usage
 
-### Example Importer
+### Importer
+
+#### Importer example
 
 Let's create a little directory to import:
+
 ```sh
 > cd /tmp
 > mkdir foo
@@ -46,6 +49,7 @@ Let's create a little directory to import:
 ```
 
 And write the importing logic:
+
 ```js
 const Importer = require('ipfs-unixfs-engine').Importer
 const filesAddStream = new Importer(<dag or ipld-resolver instance)
@@ -74,7 +78,8 @@ filesAddStream.end()
 ```
 
 When run, the stat of DAG Node is outputted for each file on data event until the root:
-```
+
+```js
 { multihash: <Buffer 12 20 bd e2 2b 57 3f 6f bd 7c cc 5a 11 7f 28 6c a2 9a 9f c0 90 e1 d4 16 d0 5f 42 81 ec 0c 2a 7f 7f 93>,
   size: 39243,
   path: '/tmp/foo/bar' }
@@ -93,15 +98,15 @@ When run, the stat of DAG Node is outputted for each file on data event until th
 
 ```
 
-### Importer API
+#### Importer API
 
 ```js
 const Importer = require('ipfs-unixfs-engine').Importer
 ```
 
-#### const add = new Importer(dag)
+#### const import = new Importer(dag [, options])
 
-The importer is a object Transform stream that accepts objects of the form
+The `import` object is a duplex pull stream that takes objects of the form:
 
 ```js
 {
@@ -110,50 +115,50 @@ The importer is a object Transform stream that accepts objects of the form
 }
 ```
 
-The stream will output IPFS DAG Node stats for the nodes as they are added to
-the DAG Service. When stats on a node are emitted they are guaranteed to have
-been written into the [DAG Service][]'s storage mechanism.
+`import` will outoyt file info objects as files get stored in IPFS. When stats on a node are emitted they are guaranteed to have been written.
 
-The input's file paths and directory structure will be preserved in the DAG
-Nodes.
+`dag` is an instance of the [`IPLD Resolver`](https://github.com/ipld/js-ipld-resolver) or the [`js-ipfs` `dag api`](https://github.com/ipfs/interface-ipfs-core/tree/master/API/dag)
 
-### Importer options
+The input's file paths and directory structure will be preserved in the [`dag-pb`](https://github.com/ipld/js-ipld-dag-pb) created nodes.
 
-In the second argument of the importer constructor you can specify the following options:
+`options` is an JavaScript option that might include the following keys:
 
-* `wrap` (boolean, defaults to false): if true, a wrapping node will be created
-* `shardSplitThreshold` (positive integer, defaults to 1000): the number of directory entries above which we decide to use a sharding directory builder (instead of the default flat one)
-* `chunker` (string, defaults to `"fixed"`): the chunking strategy. Now only supports `"fixed"`
-* `chunkerOptions` (object, optional): the options for the chunker. Defaults to an object with the following properties:
-  * `maxChunkSize` (positive integer, defaults to `262144`): the maximum chunk size for the `fixed` chunker.
-* `strategy` (string, defaults to `"balanced"`): the DAG builder strategy name. Supports:
-  * `flat`: flat list of chunks
-  * `balanced`: builds a balanced tree
-  * `trickle`: builds [a trickle tree](https://github.com/ipfs/specs/pull/57#issuecomment-265205384)
-* `maxChildrenPerNode` (positive integer, defaults to `174`): the maximum children per node for the `balanced` and `trickle` DAG builder strategies
-* `layerRepeat` (positive integer, defaults to 4): (only applicable to the `trickle` DAG builder strategy). The maximum repetition of parent nodes for each layer of the tree.
-* `reduceSingleLeafToSelf` (boolean, defaults to `false`): optimization for, when reducing a set of nodes with one node, reduce it to that node.
-* `dirBuilder` (object): the options for the directory builder
-  * `hamt` (object): the options for the HAMT sharded directory builder
-    * bits (positive integer, defaults to `5`): the number of bits at each bucket of the HAMT
+- `wrap` (boolean, defaults to false): if true, a wrapping node will be created
+- `shardSplitThreshold` (positive integer, defaults to 1000): the number of directory entries above which we decide to use a sharding directory builder (instead of the default flat one)
+- `chunker` (string, defaults to `"fixed"`): the chunking strategy. Now only supports `"fixed"`
+- `chunkerOptions` (object, optional): the options for the chunker. Defaults to an object with the following properties:
+  - `maxChunkSize` (positive integer, defaults to `262144`): the maximum chunk size for the `fixed` chunker.
+- `strategy` (string, defaults to `"balanced"`): the DAG builder strategy name. Supports:
+  - `flat`: flat list of chunks
+  - `balanced`: builds a balanced tree
+  - `trickle`: builds [a trickle tree](https://github.com/ipfs/specs/pull/57#issuecomment-265205384)
+- `maxChildrenPerNode` (positive integer, defaults to `174`): the maximum children per node for the `balanced` and `trickle` DAG builder strategies
+- `layerRepeat` (positive integer, defaults to 4): (only applicable to the `trickle` DAG builder strategy). The maximum repetition of parent nodes for each layer of the tree.
+- `reduceSingleLeafToSelf` (boolean, defaults to `false`): optimization for, when reducing a set of nodes with one node, reduce it to that node.
+- `dirBuilder` (object): the options for the directory builder
+  - `hamt` (object): the options for the HAMT sharded directory builder
+    - bits (positive integer, defaults to `5`): the number of bits at each bucket of the HAMT
 
-### Example Exporter
+### Exporter
 
-```
-// Create an export readable object stream with the hash you want to export and a dag service
-const filesStream = Exporter(<multihash>, <dag or ipld-resolver instance>)
+#### Exporter example
+
+```js
+// Create an export source pull-stream cid or ipfs path you want to export and a
+// <dag or ipld-resolver instance> to fetch the file from
+const filesStream = Exporter(<cid or ipfsPath>, <dag or ipld-resolver instance>)
 
 // Pipe the return stream to console
 filesStream.on('data', (file) => file.content.pipe(process.stdout))
 ```
 
-### Exporter: API
+#### Exporter API
 
 ```js
 const Exporter = require('ipfs-unixfs-engine').Exporter
 ```
 
-### new Exporter(<hash>, <dag or ipld-resolver>)
+### new Exporter(<cid or ipfsPath>, <dag or ipld-resolver>)
 
 Uses the given [dag API or an ipld-resolver instance][] to fetch an IPFS [UnixFS][] object(s) by their multiaddress.
 

circle.yml

@@ -8,5 +8,6 @@ dependencies:
     - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
     - sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
     - sudo apt-get update
+    - sudo apt-get install libpango-1.0-0=1.40.1-1ubuntu1 libpangocairo-1.0-0=1.40.1-1ubuntu1 libpangoft2-1.0-0=1.40.1-1ubuntu1 libpangoxft-1.0-0=1.40.1-1ubuntu1
     - sudo apt-get --only-upgrade install google-chrome-stable
     - google-chrome --version

package.json

@@ -40,9 +40,9 @@
   "homepage": "https://github.com/ipfs/js-ipfs-unixfs-engine#readme",
   "devDependencies": {
     "aegir": "^11.0.2",
-    "chai": "^3.5.0",
+    "chai": "^4.0.1",
     "dirty-chai": "^1.2.2",
-    "ipfs": "^0.24.0",
+    "ipfs": "^0.24.1",
     "ipfs-block-service": "^0.9.1",
     "ipfs-repo": "^0.13.1",
     "ncp": "^2.0.0",
@@ -54,26 +54,28 @@
   },
   "dependencies": {
     "async": "^2.4.1",
+    "bs58": "^4.0.1",
     "cids": "^0.5.0",
     "deep-extend": "^0.5.0",
     "ipfs-unixfs": "^0.1.11",
     "ipld-dag-pb": "^0.11.0",
     "ipld-resolver": "^0.11.1",
-    "is-ipfs": "^0.3.0",
     "left-pad": "^1.1.3",
     "lodash": "^4.17.4",
     "multihashes": "^0.4.5",
     "multihashing-async": "^0.4.5",
     "pull-batch": "^1.0.0",
     "pull-block": "^1.2.0",
     "pull-cat": "^1.1.11",
+    "pull-defer": "^0.2.2",
     "pull-pair": "^1.1.0",
     "pull-paramap": "^1.2.2",
     "pull-pause": "0.0.1",
     "pull-pushable": "^2.1.1",
     "pull-stream": "^3.6.0",
     "pull-traverse": "^1.0.3",
     "pull-write": "^1.1.2",
+    "safe-buffer": "^5.1.0",
     "sparse-array": "^1.3.1"
   },
   "contributors": [
@@ -88,4 +90,4 @@
     "jbenet <[email protected]>",
     "nginnever <[email protected]>"
   ]
-}
+}

src/exporter/dir-flat.js

@@ -9,28 +9,40 @@ const cat = require('pull-cat')
 // Logic to export a unixfs directory.
 module.exports = dirExporter
 
-function dirExporter (node, name, ipldResolver, resolve, parent) {
+function dirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
+  const accepts = pathRest[0]
+
   const dir = {
     path: name,
     hash: node.multihash
   }
 
-  return cat([
-    pull.values([dir]),
+  const streams = [
     pull(
       pull.values(node.links),
       pull.map((link) => ({
+        linkName: link.name,
         path: path.join(name, link.name),
         hash: link.multihash
       })),
+      pull.filter((item) => accepts === undefined || item.linkName === accepts),
       paramap((item, cb) => ipldResolver.get(new CID(item.hash), (err, n) => {
         if (err) {
           return cb(err)
         }
 
-        cb(null, resolve(n.value, item.path, ipldResolver, name, parent))
+        cb(null, resolve(n.value, accepts || item.path, pathRest, ipldResolver, name, parent))
       })),
       pull.flatten()
     )
-  ])
+  ]
+
+  // place dir before if not specifying subtree
+  if (!pathRest.length) {
+    streams.unshift(pull.values([dir]))
+  }
+
+  pathRest.shift()
+
+  return cat(streams)
 }

src/exporter/dir-hamt-sharded.js

@@ -10,7 +10,7 @@ const cleanHash = require('./clean-multihash')
 // Logic to export a unixfs directory.
 module.exports = shardedDirExporter
 
-function shardedDirExporter (node, name, ipldResolver, resolve, parent) {
+function shardedDirExporter (node, name, pathRest, ipldResolver, resolve, parent) {
   let dir
   if (!parent || parent.path !== name) {
     dir = [{
@@ -19,30 +19,54 @@ function shardedDirExporter (node, name, ipldResolver, resolve, parent) {
     }]
   }
 
-  return cat([
-    pull.values(dir),
+  const streams = [
     pull(
       pull.values(node.links),
       pull.map((link) => {
         // remove the link prefix (2 chars for the bucket index)
-        let p = link.name.substring(2)
-        // another sharded dir or file?
-        p = p ? path.join(name, p) : name
-
-        return {
-          name: link.name,
-          path: p,
-          hash: link.multihash
+        const p = link.name.substring(2)
+        const pp = p ? path.join(name, p) : name
+        let accept = true
+        let fromPathRest = false
+
+        if (p && pathRest.length) {
+          fromPathRest = true
+          accept = (p === pathRest[0])
+        }
+        if (accept) {
+          return {
+            fromPathRest: fromPathRest,
+            name: p,
+            path: pp,
+            hash: link.multihash,
+            pathRest: p ? pathRest.slice(1) : pathRest
+          }
+        } else {
+          return ''
         }
       }),
+      pull.filter(Boolean),
       paramap((item, cb) => ipldResolver.get(new CID(item.hash), (err, n) => {
         if (err) {
           return cb(err)
         }
 
-        cb(null, resolve(n.value, item.path, ipldResolver, (dir && dir[0]) || parent))
+        cb(
+          null,
+          resolve(
+            n.value,
+            item.fromPathRest ? item.name : item.path,
+            item.pathRest,
+            ipldResolver,
+            (dir && dir[0]) || parent))
       })),
       pull.flatten()
     )
-  ])
+  ]
+
+  if (!pathRest.length) {
+    streams.unshift(pull.values(dir))
+  }
+
+  return cat(streams)
 }

src/exporter/file.js

@@ -7,7 +7,7 @@ const pull = require('pull-stream')
 const paramap = require('pull-paramap')
 
 // Logic to export a single (possibly chunked) unixfs file.
-module.exports = (node, name, ipldResolver) => {
+module.exports = (node, name, pathRest, ipldResolver) => {
   function getData (node) {
     try {
       const file = UnixFS.unmarshal(node.data)
@@ -25,6 +25,12 @@ module.exports = (node, name, ipldResolver) => {
     )
   }
 
+  const accepts = pathRest.shift()
+
+  if (accepts !== undefined && accepts !== name) {
+    return pull.empty()
+  }
+
   let content = pull(
     traverse.depthFirst(node, visitor),
     pull.map(getData)