refactor: object APIs write methods now return CIDs (#1730)

For the back story on this change, please see: ipfs-inactive/interface-js-ipfs-core#388 (review)

BREAKING CHANGE: Object API refactor.

Object API methods that write DAG nodes now return a [CID](https://www.npmjs.com/package/cids) instead of a DAG node. Affected methods:

* `ipfs.object.new`
* `ipfs.object.patch.addLink`
* `ipfs.object.patch.appendData`
* `ipfs.object.patch.rmLink`
* `ipfs.object.patch.setData`
* `ipfs.object.put`

Example:

```js
// Before
const dagNode = await ipfs.object.new()
```

```js
// After
const cid = await ipfs.object.new() // now returns a CID
const dagNode = await ipfs.object.get(cid) // fetch the DAG node that was created
```

IMPORTANT: `DAGNode` instances, which are part of the IPLD dag-pb format have been refactored.

These instances no longer have `multihash`, `cid` or `serialized` properties.

This effects the following API methods that return these types of objects:

* `ipfs.object.get`
* `ipfs.dag.get`

See ipld/js-ipld-dag-pb#99 for more information.

License: MIT
Signed-off-by: Alan Shaw <[email protected]>

README.md

@@ -186,7 +186,7 @@ Commands:
 
 ### IPFS Daemon
 
-The IPFS Daemon exposes the API defined [`http-api-spec`](https://github.com/ipfs/http-api-spec). You can use any of the IPFS HTTP-API client libraries with it, such as: [js-ipfs-api](https://github.com/ipfs/js-ipfs-api).
+The IPFS Daemon exposes the API defined [`http-api-spec`](https://github.com/ipfs/http-api-spec). You can use any of the IPFS HTTP-API client libraries with it, such as: [js-ipfs-http-client](https://github.com/ipfs/js-ipfs-http-client).
 
 If you want a programmatic way to spawn a IPFS Daemon using JavaScript, check out [ipfsd-ctl module](https://github.com/ipfs/js-ipfsd-ctl)
 
@@ -884,7 +884,7 @@ Listing of the main packages used in the IPFS ecosystem. There are also three sp
 | [`libp2p-crypto`](//github.com/libp2p/js-libp2p-crypto) | [![npm](https://img.shields.io/npm/v/libp2p-crypto.svg?maxAge=86400&style=flat-square)](//github.com/libp2p/js-libp2p-crypto/releases) | [![Deps](https://david-dm.org/libp2p/js-libp2p-crypto.svg?style=flat-square)](https://david-dm.org/libp2p/js-libp2p-crypto) | [![jenkins](https://ci.ipfs.team/buildStatus/icon?job=libp2p/js-libp2p-crypto/master)](https://ci.ipfs.team/job/libp2p/job/js-libp2p-crypto/job/master/) | [![codecov](https://codecov.io/gh/libp2p/js-libp2p-crypto/branch/master/graph/badge.svg)](https://codecov.io/gh/libp2p/js-libp2p-crypto) | [Friedel Ziegelmayer](mailto:[email protected]) |
 | [`libp2p-keychain`](//github.com/libp2p/js-libp2p-keychain) | [![npm](https://img.shields.io/npm/v/libp2p-keychain.svg?maxAge=86400&style=flat-square)](//github.com/libp2p/js-libp2p-keychain/releases) | [![Deps](https://david-dm.org/libp2p/js-libp2p-keychain.svg?style=flat-square)](https://david-dm.org/libp2p/js-libp2p-keychain) | N/A | [![codecov](https://codecov.io/gh/libp2p/js-libp2p-keychain/branch/master/graph/badge.svg)](https://codecov.io/gh/libp2p/js-libp2p-keychain) | [Vasco Santos](mailto:[email protected]) |
 | **Generics/Utils** |
-| [`ipfs-api`](//github.com/ipfs/js-ipfs-api) | [![npm](https://img.shields.io/npm/v/ipfs-api.svg?maxAge=86400&style=flat-square)](//github.com/ipfs/js-ipfs-api/releases) | [![Deps](https://david-dm.org/ipfs/js-ipfs-api.svg?style=flat-square)](https://david-dm.org/ipfs/js-ipfs-api) | [![jenkins](https://ci.ipfs.team/buildStatus/icon?job=ipfs/js-ipfs-api/master)](https://ci.ipfs.team/job/ipfs/job/js-ipfs-api/job/master/) | [![codecov](https://codecov.io/gh/ipfs/js-ipfs-api/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs/js-ipfs-api) | [Alan Shaw](mailto:[email protected]) |
+| [`ipfs-http-client`](//github.com/ipfs/js-ipfs-http-client) | [![npm](https://img.shields.io/npm/v/ipfs-http-client.svg?maxAge=86400&style=flat-square)](//github.com/ipfs/js-ipfs-http-client/releases) | [![Deps](https://david-dm.org/ipfs/js-ipfs-http-client.svg?style=flat-square)](https://david-dm.org/ipfs/js-ipfs-http-client) | [![jenkins](https://ci.ipfs.team/buildStatus/icon?job=ipfs/js-ipfs-http-client/master)](https://ci.ipfs.team/job/ipfs/job/js-ipfs-http-client/job/master/) | [![codecov](https://codecov.io/gh/ipfs/js-ipfs-http-client/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs/js-ipfs-http-client) | [Alan Shaw](mailto:[email protected]) |
 | [`ipfs-multipart`](//github.com/ipfs/ipfs-multipart) | [![npm](https://img.shields.io/npm/v/ipfs-multipart.svg?maxAge=86400&style=flat-square)](//github.com/ipfs/ipfs-multipart/releases) | [![Deps](https://david-dm.org/ipfs/ipfs-multipart.svg?style=flat-square)](https://david-dm.org/ipfs/ipfs-multipart) | N/A | [![codecov](https://codecov.io/gh/ipfs/ipfs-multipart/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs/ipfs-multipart) | N/A |
 | [`is-ipfs`](//github.com/ipfs/is-ipfs) | [![npm](https://img.shields.io/npm/v/is-ipfs.svg?maxAge=86400&style=flat-square)](//github.com/ipfs/is-ipfs/releases) | [![Deps](https://david-dm.org/ipfs/is-ipfs.svg?style=flat-square)](https://david-dm.org/ipfs/is-ipfs) | [![jenkins](https://ci.ipfs.team/buildStatus/icon?job=ipfs/is-ipfs/master)](https://ci.ipfs.team/job/ipfs/job/is-ipfs/job/master/) | [![codecov](https://codecov.io/gh/ipfs/is-ipfs/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs/is-ipfs) | [Marcin Rataj](mailto:[email protected]) |
 | [`multihashing`](//github.com/multiformats/js-multihashing) | [![npm](https://img.shields.io/npm/v/multihashing.svg?maxAge=86400&style=flat-square)](//github.com/multiformats/js-multihashing/releases) | [![Deps](https://david-dm.org/multiformats/js-multihashing.svg?style=flat-square)](https://david-dm.org/multiformats/js-multihashing) | [![jenkins](https://ci.ipfs.team/buildStatus/icon?job=multiformats/js-multihashing/master)](https://ci.ipfs.team/job/multiformats/job/js-multihashing/job/master/) | [![codecov](https://codecov.io/gh/multiformats/js-multihashing/branch/master/graph/badge.svg)](https://codecov.io/gh/multiformats/js-multihashing) | N/A |

img/overview.txt

@@ -1,17 +1,17 @@
-                                                                                 
-                       offline mode - uses IPFS core directly                    
-                    ┌───────────────────────────────────────────┐                
-                    │                                           │                
-                    │                                           │                
-                    │ online mode - uses IPFS through http-api  │                
-   ┌────────────┐   │                  ┌────────────────────┐   │    ┌──────────┐
-   │            │   │ ┌ ─ ─ ─ ─ ─ ─    │                    │   │    │          │
-   │    CLI     │───┴──  ipfs-api  ├──▶│     HTTP-API       │───┴───▶│IPFS Core │
-   │            │     └ ─ ─ ─ ─ ─ ─    │                    │        │          │
-   └────────────┘                      └────────────────────┘        └──────────┘
-          △                                       △                        △     
-          ├───────────────────────────────────────┴────────────────────────┘     
-          │                                                                      
-   ┌────────────┐                                                                
-   │   Tests    │                                                                
-   └────────────┘                                                                
+
+                     offline mode - uses IPFS core directly
+                  ┌────────────────────────────────────────────┐
+                  │                                            │
+                  │                                            │
+                  │ online mode - uses IPFS through http-api   │
+ ┌────────────┐   │                          ┌─────────────┐   │    ┌─────────┐
+ │            │   │ ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─    │             │   │    │         │
+ │    CLI     │───┴──  ipfs-http-client  ├──▶│  HTTP-API   │───┴───▶│IPFS Core│
+ │            │     └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─    │             │        │         │
+ └────────────┘                              └─────────────┘        └─────────┘
+        △                                           △                    △
+        ├───────────────────────────────────────────┴────────────────────┘
+        │
+ ┌────────────┐
+ │   Tests    │
+ └────────────┘                                                                

package-list.json

@@ -50,7 +50,7 @@
     ["libp2p/js-libp2p-keychain", "libp2p-keychain"],
 
     "Generics/Utils",
-    ["ipfs/js-ipfs-api", "ipfs-api"],
+    ["ipfs/js-ipfs-http-client", "ipfs-http-client"],
     ["ipfs/ipfs-multipart", "ipfs-multipart"],
     ["ipfs/is-ipfs", "is-ipfs"],
     ["multiformats/js-multihashing", "multihashing"],

package.json

@@ -69,8 +69,8 @@
     "execa": "^1.0.0",
     "form-data": "^2.3.3",
     "hat": "0.0.3",
-    "interface-ipfs-core": "~0.86.0",
-    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#update-dag-pb-to-not-have-cid-property",
+    "interface-ipfs-core": "~0.88.0",
+    "ipfsd-ctl": "~0.40.1",
     "ncp": "^2.0.0",
     "qs": "^6.5.2",
     "rimraf": "^2.6.2",
@@ -86,7 +86,7 @@
     "boom": "^7.2.0",
     "bs58": "^4.0.1",
     "byteman": "^1.3.5",
-    "cid-tool": "~0.1.0",
+    "cid-tool": "~0.2.0",
     "cids": "~0.5.5",
     "datastore-core": "~0.6.0",
     "debug": "^4.1.0",
@@ -102,10 +102,10 @@
     "hoek": "^5.0.4",
     "human-to-milliseconds": "^1.0.0",
     "interface-datastore": "~0.6.0",
-    "ipfs-api": "ipfs/js-ipfs-api",
     "ipfs-bitswap": "~0.21.0",
     "ipfs-block": "~0.8.0",
     "ipfs-block-service": "~0.15.1",
+    "ipfs-http-client": "^27.0.0",
     "ipfs-http-response": "~0.2.1",
     "ipfs-mfs": "~0.5.2",
     "ipfs-multipart": "~0.1.0",
@@ -124,7 +124,7 @@
     "is-stream": "^1.1.0",
     "joi": "^13.4.0",
     "joi-browser": "^13.4.0",
-    "joi-multiaddr": "^2.0.0",
+    "joi-multiaddr": "^3.0.0",
     "libp2p": "~0.24.0",
     "libp2p-bootstrap": "~0.9.3",
     "libp2p-crypto": "~0.14.1",
@@ -144,7 +144,7 @@
     "mkdirp": "~0.5.1",
     "multiaddr": "^5.0.0",
     "multiaddr-to-uri": "^4.0.0",
-    "multibase": "~0.5.0",
+    "multibase": "~0.6.0",
     "multihashes": "~0.4.14",
     "multihashing-async": "~0.5.1",
     "once": "^1.4.0",

src/cli/commands/block/rm.js

@@ -13,7 +13,7 @@ module.exports = {
 
   handler (argv) {
     if (utils.isDaemonOn()) {
-      // TODO implement this once `js-ipfs-api` supports it
+      // TODO implement this once `js-ipfs-http-client` supports it
       throw new Error('rm block with daemon running is not yet implemented')
     }
 

src/cli/commands/object/get.js

@@ -1,11 +1,7 @@
 'use strict'
 
 const print = require('../../utils').print
-const {
-  util: {
-    cid
-  }
-} = require('ipld-dag-pb')
+const dagPB = require('ipld-dag-pb')
 
 module.exports = {
   command: 'get <key>',
@@ -29,7 +25,7 @@ module.exports = {
         throw err
       }
 
-      cid(node, (err, result) => {
+      dagPB.util.cid(node, (err, result) => {
         if (err) {
           throw err
         }

src/cli/commands/object/new.js

@@ -4,11 +4,6 @@ const debug = require('debug')
 const log = debug('cli:object')
 log.error = debug('cli:object:error')
 const print = require('../../utils').print
-const {
-  util: {
-    cid
-  }
-} = require('ipld-dag-pb')
 
 module.exports = {
   command: 'new [<template>]',
@@ -23,18 +18,12 @@ module.exports = {
   },
 
   handler (argv) {
-    argv.ipfs.object.new(argv.template, (err, node) => {
+    argv.ipfs.object.new(argv.template, (err, cid) => {
       if (err) {
         throw err
       }
 
-      cid(node, (err, cid) => {
-        if (err) {
-          throw err
-        }
-
-        print(cid.toBaseEncodedString(argv.cidBase))
-      })
+      print(cid.toBaseEncodedString(argv.cidBase))
     })
   }
 }

src/cli/commands/object/patch/add-link.js

@@ -3,11 +3,6 @@
 const dagPB = require('ipld-dag-pb')
 const DAGLink = dagPB.DAGLink
 const print = require('../../../utils').print
-const {
-  util: {
-    cid
-  }
-} = require('ipld-dag-pb')
 
 module.exports = {
   command: 'add-link <root> <name> <ref>',
@@ -30,7 +25,7 @@ module.exports = {
         throw err
       }
 
-      cid(nodeA, (err, result) => {
+      dagPB.util.cid(nodeA, (err, result) => {
         if (err) {
           throw err
         }
@@ -39,18 +34,12 @@ module.exports = {
 
         ipfs.object.patch.addLink(argv.root, link, {
           enc: 'base58'
-        }, (err, nodeB) => {
+        }, (err, cid) => {
           if (err) {
             throw err
           }
 
-          cid(nodeB, (err, result) => {
-            if (err) {
-              throw err
-            }
-
-            print(result.toBaseEncodedString(argv.cidBase))
-          })
+          print(cid.toBaseEncodedString(argv.cidBase))
         })
       })
     })

src/cli/commands/object/patch/append-data.js

@@ -6,27 +6,16 @@ const debug = require('debug')
 const log = debug('cli:object')
 log.error = debug('cli:object:error')
 const print = require('../../../utils').print
-const {
-  util: {
-    cid
-  }
-} = require('ipld-dag-pb')
 
 function appendData (key, data, ipfs) {
   ipfs.object.patch.appendData(key, data, {
     enc: 'base58'
-  }, (err, node) => {
+  }, (err, cid) => {
     if (err) {
       throw err
     }
 
-    cid(node, (err, cid) => {
-      if (err) {
-        throw err
-      }
-
-      print(cid.toBaseEncodedString())
-    })
+    print(cid.toBaseEncodedString())
   })
 }
 

src/cli/commands/object/patch/rm-link.js

@@ -4,11 +4,6 @@ const debug = require('debug')
 const log = debug('cli:object')
 log.error = debug('cli:object:error')
 const print = require('../../../utils').print
-const {
-  util: {
-    cid
-  }
-} = require('ipld-dag-pb')
 
 module.exports = {
   command: 'rm-link <root> <link>',
@@ -25,18 +20,12 @@ module.exports = {
   handler (argv) {
     argv.ipfs.object.patch.rmLink(argv.root, { name: argv.link }, {
       enc: 'base58'
-    }, (err, node) => {
+    }, (err, cid) => {
       if (err) {
         throw err
       }
 
-      cid(node, (err, cid) => {
-        if (err) {
-          throw err
-        }
-
-        print(cid.toBaseEncodedString(argv.cidBase))
-      })
+      print(cid.toBaseEncodedString(argv.cidBase))
     })
   }
 }

src/cli/commands/object/patch/set-data.js

@@ -6,27 +6,16 @@ const debug = require('debug')
 const log = debug('cli:object')
 log.error = debug('cli:object:error')
 const print = require('../../../utils').print
-const {
-  util: {
-    cid
-  }
-} = require('ipld-dag-pb')
 
 function parseAndAddNode (key, data, ipfs) {
   ipfs.object.patch.setData(key, data, {
     enc: 'base58'
-  }, (err, node) => {
+  }, (err, cid) => {
     if (err) {
       throw err
     }
 
-    cid(node, (err, cid) => {
-      if (err) {
-        throw err
-      }
-
-      print(cid.toBaseEncodedString())
-    })
+    print(cid.toBaseEncodedString())
   })
 }
 

src/cli/commands/object/put.js

@@ -3,25 +3,14 @@
 const bl = require('bl')
 const fs = require('fs')
 const print = require('../../utils').print
-const {
-  util: {
-    cid
-  }
-} = require('ipld-dag-pb')
 
 function putNode (buf, enc, ipfs, cidEnc) {
-  ipfs.object.put(buf, { enc: enc }, (err, node) => {
+  ipfs.object.put(buf, { enc: enc }, (err, cid) => {
     if (err) {
       throw err
     }
 
-    cid(node, (err, cid) => {
-      if (err) {
-        throw err
-      }
-
-      print(`added ${cid.toBaseEncodedString(cidEnc)}`)
-    })
+    print(`added ${cid.toBaseEncodedString(cidEnc)}`)
   })
 }
 

src/cli/commands/object/stat.js

@@ -17,7 +17,7 @@ module.exports = {
         throw err
       }
 
-      delete stats.Hash // only for js-ipfs-api output
+      delete stats.Hash // only for js-ipfs-http-client output
 
       Object.keys(stats).forEach((key) => {
         print(`${key}: ${stats[key]}`)

src/cli/utils.js

@@ -35,7 +35,7 @@ function getAPICtl (apiAddr) {
     apiAddr = multiaddr(fs.readFileSync(apiPath).toString()).toString()
   }
   // Required inline to reduce startup time
-  const APIctl = require('ipfs-api')
+  const APIctl = require('ipfs-http-client')
   return APIctl(apiAddr)
 }
 

src/core/components/files-regular.js

@@ -14,7 +14,7 @@ const waterfall = require('async/waterfall')
 const isStream = require('is-stream')
 const isSource = require('is-pull-stream').isSource
 const Duplex = require('readable-stream').Duplex
-const OtherBuffer = require('buffer').Buffer
+const isString = require('lodash/isString')
 const CID = require('cids')
 const toB58String = require('multihashes').toB58String
 const errCode = require('err-code')
@@ -292,15 +292,23 @@ module.exports = function (self) {
 
         options = options || {}
 
-        const ok = Buffer.isBuffer(data) ||
-                   isStream.readable(data) ||
-                   Array.isArray(data) ||
-                   OtherBuffer.isBuffer(data) ||
-                   typeof data === 'object' ||
-                   isSource(data)
+        // Buffer, pull stream or Node.js stream
+        const isBufferOrStream = obj => Buffer.isBuffer(obj) || isStream.readable(obj) || isSource(obj)
+        // An object like { content?, path? }, where content isBufferOrStream and path isString
+        const isContentObject = obj => {
+          if (typeof obj !== 'object') return false
+          // path is optional if content is present
+          if (obj.content) return isBufferOrStream(obj.content)
+          // path must be a non-empty string if no content
+          return Boolean(obj.path) && isString(obj.path)
+        }
+        // An input atom: a buffer, stream or content object
+        const isInput = obj => isBufferOrStream(obj) || isContentObject(obj)
+        // All is ok if data isInput or data is an array of isInput
+        const ok = isInput(data) || (Array.isArray(data) && data.every(isInput))
 
         if (!ok) {
-          return callback(new Error('first arg must be a buffer, readable stream, pull stream, an object or array of objects'))
+          return callback(new Error('invalid input: expected buffer, readable stream, pull stream, object or array of objects'))
         }
 
         // CID v0 is for multihashes encoded with sha2-256