diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 75250d3..90cb31c 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -49,21 +49,11 @@ "from": "any-promise@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz" }, - "aproba": { - "version": "1.1.1", - "from": "aproba@>=1.0.3 <2.0.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.1.1.tgz" - }, "archive-type": { "version": "3.2.0", "from": "archive-type@>=3.0.1 <4.0.0", "resolved": "https://registry.npmjs.org/archive-type/-/archive-type-3.2.0.tgz" }, - "are-we-there-yet": { - "version": "1.1.2", - "from": "are-we-there-yet@>=1.1.2 <1.2.0", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.2.tgz" - }, "arr-diff": { "version": "2.0.0", "from": "arr-diff@>=2.0.0 <3.0.0", @@ -139,16 +129,6 @@ "from": "bin-build@>=2.2.0 <3.0.0", "resolved": "https://registry.npmjs.org/bin-build/-/bin-build-2.2.0.tgz" }, - "bindings": { - "version": "1.2.1", - "from": "bindings@>=1.2.1 <2.0.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz" - }, - "bip66": { - "version": "1.1.4", - "from": "bip66@>=1.1.3 <2.0.0", - "resolved": "https://registry.npmjs.org/bip66/-/bip66-1.1.4.tgz" - }, "bl": { "version": "1.2.0", "from": "bl@>=1.0.0 <2.0.0", @@ -161,7 +141,7 @@ }, "bn.js": { "version": "4.11.6", - "from": "bn.js@>=4.8.0 <5.0.0", + "from": "bn.js@>=4.0.0 <5.0.0", "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.6.tgz" }, "borc": { @@ -198,11 +178,6 @@ } } }, - "brorand": { - "version": "1.0.7", - "from": "brorand@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.0.7.tgz" - }, "browserify-aes": { "version": "1.0.6", "from": "browserify-aes@>=1.0.6 <2.0.0", @@ -289,7 +264,7 @@ }, "cipher-base": { "version": "1.0.3", - "from": "cipher-base@>=1.0.1 <2.0.0", + "from": "cipher-base@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.3.tgz" }, "cliui": { @@ -332,11 +307,6 @@ "from": "concat-stream@>=1.4.6 <2.0.0", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.0.tgz" }, - "console-control-strings": { - "version": "1.1.0", - "from": "console-control-strings@>=1.1.0 <1.2.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz" - }, "convert-source-map": { "version": "1.3.0", "from": "convert-source-map@>=1.1.1 <2.0.0", @@ -359,23 +329,18 @@ }, "create-hash": { "version": "1.1.2", - "from": "create-hash@>=1.1.2 <2.0.0", + "from": "create-hash@>=1.1.0 <2.0.0", "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.1.2.tgz" }, - "create-hmac": { - "version": "1.1.4", - "from": "create-hmac@>=1.1.4 <2.0.0", - "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.4.tgz" - }, "dateformat": { "version": "2.0.0", "from": "dateformat@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.0.0.tgz" }, "debug": { - "version": "2.6.1", + "version": "2.6.0", "from": "debug@>=2.1.3 <3.0.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.1.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.0.tgz" }, "decamelize": { "version": "1.2.0", @@ -460,11 +425,6 @@ "from": "deferred-leveldown@>=1.2.1 <1.3.0", "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-1.2.1.tgz" }, - "delegates": { - "version": "1.0.0", - "from": "delegates@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz" - }, "delimit-stream": { "version": "0.1.0", "from": "delimit-stream@0.1.0", @@ -492,11 +452,6 @@ } } }, - "drbg.js": { - "version": "1.0.1", - "from": "drbg.js@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/drbg.js/-/drbg.js-1.0.1.tgz" - }, "duplex-child-process": { "version": "0.0.5", "from": "duplex-child-process@0.0.5", @@ -524,11 +479,6 @@ "from": "each-async@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/each-async/-/each-async-1.1.1.tgz" }, - "elliptic": { - "version": "6.3.3", - "from": "elliptic@>=6.2.3 <7.0.0", - "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.3.3.tgz" - }, "encoding": { "version": "0.1.12", "from": "encoding@>=0.1.11 <0.2.0", @@ -608,11 +558,6 @@ "from": "expand-range@>=1.8.1 <2.0.0", "resolved": "https://registry.npmjs.org/expand-range/-/expand-range-1.8.2.tgz" }, - "expand-template": { - "version": "1.0.3", - "from": "expand-template@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-1.0.3.tgz" - }, "expand-tilde": { "version": "1.2.2", "from": "expand-tilde@>=1.2.2 <2.0.0", @@ -754,18 +699,6 @@ "from": "functional-red-black-tree@>=1.0.1 <2.0.0", "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz" }, - "gauge": { - "version": "2.7.3", - "from": "gauge@>=2.7.1 <2.8.0", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.3.tgz", - "dependencies": { - "object-assign": { - "version": "4.1.1", - "from": "object-assign@>=4.1.0 <5.0.0", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz" - } - } - }, "generate-function": { "version": "2.0.0", "from": "generate-function@>=2.0.0 <3.0.0", @@ -796,11 +729,6 @@ "from": "get-stdin@>=4.0.1 <5.0.0", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz" }, - "github-from-package": { - "version": "0.0.0", - "from": "github-from-package@0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz" - }, "glob": { "version": "5.0.15", "from": "glob@>=5.0.3 <6.0.0", @@ -939,16 +867,6 @@ "from": "has-gulplog@>=0.1.0 <0.2.0", "resolved": "https://registry.npmjs.org/has-gulplog/-/has-gulplog-0.1.0.tgz" }, - "has-unicode": { - "version": "2.0.1", - "from": "has-unicode@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz" - }, - "hash.js": { - "version": "1.0.3", - "from": "hash.js@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.0.3.tgz" - }, "homedir-polyfill": { "version": "1.0.1", "from": "homedir-polyfill@>=1.0.0 <2.0.0", @@ -1294,9 +1212,9 @@ "resolved": "https://registry.npmjs.org/levelup/-/levelup-1.3.3.tgz" }, "libp2p-crypto": { - "version": "0.8.0", - "from": "libp2p-crypto@0.8.0", - "resolved": "https://registry.npmjs.org/libp2p-crypto/-/libp2p-crypto-0.8.0.tgz" + "version": "0.8.1", + "from": "libp2p-crypto@>=0.8.0 <0.9.0", + "resolved": "https://registry.npmjs.org/libp2p-crypto/-/libp2p-crypto-0.8.1.tgz" }, "libp2p-identify": { "version": "0.3.2", @@ -1306,36 +1224,12 @@ "libp2p-ping": { "version": "0.3.1", "from": "libp2p-ping@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/libp2p-ping/-/libp2p-ping-0.3.1.tgz", - "dependencies": { - "libp2p-crypto": { - "version": "0.8.5", - "from": "libp2p-crypto@>=0.8.1 <0.9.0", - "resolved": "https://registry.npmjs.org/libp2p-crypto/-/libp2p-crypto-0.8.5.tgz" - }, - "libp2p-crypto-secp256k1": { - "version": "0.1.4", - "from": "libp2p-crypto-secp256k1@>=0.1.3 <0.2.0", - "resolved": "https://registry.npmjs.org/libp2p-crypto-secp256k1/-/libp2p-crypto-secp256k1-0.1.4.tgz" - } - } + "resolved": "https://registry.npmjs.org/libp2p-ping/-/libp2p-ping-0.3.1.tgz" }, "libp2p-secio": { "version": "0.6.6", "from": "libp2p-secio@>=0.6.4 <0.7.0", - "resolved": "https://registry.npmjs.org/libp2p-secio/-/libp2p-secio-0.6.6.tgz", - "dependencies": { - "libp2p-crypto": { - "version": "0.8.5", - "from": "libp2p-crypto@>=0.8.1 <0.9.0", - "resolved": "https://registry.npmjs.org/libp2p-crypto/-/libp2p-crypto-0.8.5.tgz" - }, - "libp2p-crypto-secp256k1": { - "version": "0.1.4", - "from": "libp2p-crypto-secp256k1@^0.1.3", - "resolved": "https://registry.npmjs.org/libp2p-crypto-secp256k1/-/libp2p-crypto-secp256k1-0.1.4.tgz" - } - } + "resolved": "https://registry.npmjs.org/libp2p-secio/-/libp2p-secio-0.6.6.tgz" }, "libp2p-spdy": { "version": "0.10.4", @@ -1679,8 +1573,9 @@ }, "nan": { "version": "2.5.1", - "from": "nan@>=2.2.1 <3.0.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.5.1.tgz" + "from": "nan@>=2.4.0 <3.0.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.5.1.tgz", + "optional": true }, "ndjson": { "version": "1.5.0", @@ -1694,11 +1589,6 @@ } } }, - "node-abi": { - "version": "1.3.3", - "from": "node-abi@>=1.0.3 <2.0.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-1.3.3.tgz" - }, "node-fetch": { "version": "1.6.3", "from": "node-fetch@>=1.6.3 <2.0.0", @@ -1720,11 +1610,6 @@ "from": "nodeify@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/nodeify/-/nodeify-1.0.0.tgz" }, - "noop-logger": { - "version": "0.1.1", - "from": "noop-logger@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz" - }, "normalize-package-data": { "version": "2.3.5", "from": "normalize-package-data@>=2.3.2 <3.0.0", @@ -1735,11 +1620,6 @@ "from": "normalize-path@>=2.0.1 <3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.0.1.tgz" }, - "npmlog": { - "version": "4.0.2", - "from": "npmlog@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.0.2.tgz" - }, "number-is-nan": { "version": "1.0.1", "from": "number-is-nan@>=1.0.0 <2.0.0", @@ -1932,11 +1812,6 @@ "from": "pinkie-promise@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz" }, - "prebuild-install": { - "version": "2.1.0", - "from": "prebuild-install@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-2.1.0.tgz" - }, "prepend-http": { "version": "1.0.4", "from": "prepend-http@>=1.0.1 <2.0.0", @@ -2056,11 +1931,6 @@ "from": "pull-ws@>=3.2.8 <4.0.0", "resolved": "https://registry.npmjs.org/pull-ws/-/pull-ws-3.2.8.tgz" }, - "pump": { - "version": "1.0.2", - "from": "pump@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.2.tgz" - }, "quote-stream": { "version": "1.0.2", "from": "quote-stream@>=1.0.1 <2.0.0", @@ -2190,11 +2060,6 @@ "from": "safe-buffer@>=5.0.1 <6.0.0", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.0.1.tgz" }, - "secp256k1": { - "version": "3.2.5", - "from": "secp256k1@>=3.2.5 <4.0.0", - "resolved": "https://registry.npmjs.org/secp256k1/-/secp256k1-3.2.5.tgz" - }, "seek-bzip": { "version": "1.0.5", "from": "seek-bzip@>=1.0.3 <2.0.0", @@ -2207,7 +2072,7 @@ }, "set-blocking": { "version": "2.0.0", - "from": "set-blocking@>=2.0.0 <2.1.0", + "from": "set-blocking@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz" }, "set-immediate-shim": { @@ -2225,21 +2090,11 @@ "from": "shallow-copy@>=0.0.1 <0.1.0", "resolved": "https://registry.npmjs.org/shallow-copy/-/shallow-copy-0.0.1.tgz" }, - "signal-exit": { - "version": "3.0.2", - "from": "signal-exit@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz" - }, "signed-varint": { "version": "2.0.1", "from": "signed-varint@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/signed-varint/-/signed-varint-2.0.1.tgz" }, - "simple-get": { - "version": "1.4.3", - "from": "simple-get@>=1.4.2 <2.0.0", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-1.4.3.tgz" - }, "source-map": { "version": "0.1.43", "from": "source-map@>=0.1.33 <0.2.0", @@ -3229,7 +3084,7 @@ }, "string-width": { "version": "1.0.2", - "from": "string-width@>=1.0.1 <2.0.0", + "from": "string-width@>=1.0.2 <2.0.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz" }, "strip-ansi": { @@ -3272,14 +3127,9 @@ "from": "supports-color@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz" }, - "tar-fs": { - "version": "1.15.0", - "from": "tar-fs@>=1.13.0 <2.0.0", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.15.0.tgz" - }, "tar-stream": { "version": "1.5.2", - "from": "tar-stream@>=1.1.1 <2.0.0", + "from": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.5.2.tgz", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.5.2.tgz" }, "temp": { @@ -3544,11 +3394,6 @@ "from": "which-module@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz" }, - "wide-align": { - "version": "1.1.0", - "from": "wide-align@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.0.tgz" - }, "wordwrap": { "version": "0.0.3", "from": "wordwrap@>=0.0.2 <0.1.0", diff --git a/package.json b/package.json index ac14c07..65dc821 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "pull-stream": "^3.5.0", "pull-window": "^2.1.4", "sqlite3": "^3.1.8", + "tar-stream": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.5.2.tgz", "temp": "^0.8.3", "thenify-all": "^1.6.0", "tunnel-ssh": "^4.1.1", diff --git a/src/client/api/RestClient.js b/src/client/api/RestClient.js index d361925..7b488be 100644 --- a/src/client/api/RestClient.js +++ b/src/client/api/RestClient.js @@ -192,9 +192,15 @@ class RestClient { }) } - batchGetDataStream (objectIds: Array): Promise { - return this.postRequest('data/get', objectIds.join('\n'), false) + batchGetDataStream (objectIds: Array, decode: boolean = true): Promise { + const resultStream = this.postRequest('data/get', objectIds.join('\n'), false) .then(r => new NDJsonResponse(r).stream()) + + if (!decode) { + return resultStream + } + + return resultStream .then((stream: DuplexStream) => { const outputStream = mapStream((obj: Object, callback: Function) => { try { diff --git a/src/client/cli/commands/archive.js b/src/client/cli/commands/archive.js new file mode 100644 index 0000000..91a06aa --- /dev/null +++ b/src/client/cli/commands/archive.js @@ -0,0 +1,177 @@ +// @flow + +const fs = require('fs') +const zlib = require('zlib') +const tar = require('tar-stream') +const {subcommand, printlnErr} = require('../util') +const {Statement} = require('../../../model/statement') +import type {RestClient} from '../../api' + +const STMT_BATCH_SIZE = 1024 +const OBJECT_BATCH_SIZE = 1024 +const TAR_ENTRY_OPTS = { + uid: 500, + gid: 500, + uname: 'mediachain', + gname: 'staff' +} + +function leftpad (str, length, char = '0') { + return char.repeat(Math.max(0, length - str.length)) + str +} + +module.exports = { + command: 'archive ', + description: 'Create a gzipped tar archive of the statements and data objects returned for the given `queryString`\n', + builder: { + output: { + alias: 'o', + type: 'string', + description: 'Filename to output archive tarball to. If not given archive will be written to stdout.\n', + required: false, + default: null + }, + allowErrors: { + alias: ['warn', 'w'], + description: 'Warn if an error occurs when fetching data instead of aborting the archive generation.\n', + type: 'boolean', + default: false + } + }, + handler: subcommand((opts: {client: RestClient, queryString: string, output?: ?string, allowErrors: boolean}) => { + const {client, queryString, allowErrors, output} = opts + let outputStream + const tarball = tar.pack() + const gzip = zlib.createGzip() + const objectIds: Set = new Set() + + return client.queryStream(queryString) + .then(response => new Promise((resolve, reject) => { + const queryStream = response.stream() + const outStreamName = output || 'standard output' + outputStream = (output == null) ? process.stdout : fs.createWriteStream(output) + tarball.pipe(gzip).pipe(outputStream) + + let stmtBatch: Array = [] + let stmtBatchNumber = 0 + function writeStatementBatch (force: boolean = false) { + if (force || stmtBatch.length >= STMT_BATCH_SIZE) { + const content = Buffer.from(stmtBatch.join('\n'), 'utf-8') + const filename = `stmt/${leftpad(stmtBatchNumber.toString(), 8)}.ndjson` + writeToTarball(tarball, filename, content) + stmtBatchNumber += 1 + stmtBatch = [] + } + } + + outputStream.on('error', err => { + reject(new Error(`Error writing to ${outStreamName}: ${err.message}`)) + }) + + queryStream.on('error', err => { + reject(new Error(`Error reading from query result stream: ${err.message}`)) + }) + + queryStream.on('data', obj => { + let stmt + try { + stmt = Statement.fromProtobuf(obj) + } catch (err) { + // ignore non-statement results + return + } + + stmtBatch.push(JSON.stringify(obj)) + writeStatementBatch() + + for (const id of stmt.objectIds) { + objectIds.add(id) + } + for (const id of stmt.depsSet) { + objectIds.add(id) + } + }) + + queryStream.on('end', () => { + writeStatementBatch(true) + resolve() + }) + })) + .then(() => writeDataObjectsToTarball(client, tarball, objectIds, allowErrors)) + .then(() => new Promise(resolve => { + outputStream.on('end', () => resolve()) + tarball.finalize() + })) + .catch(err => { + // if we're not allowing errors, and are writing to a file, try to delete it on failure + if (!allowErrors && output != null) { + try { + fs.unlinkSync(output) + } catch (err) { + // ignore deletion failures + } + } + throw err + }) + }) +} + +function writeToTarball (tarball: Object, filename: string, content: Buffer) { + const header = Object.assign({}, {name: filename, size: content.length}, TAR_ENTRY_OPTS) + tarball.entry(header, content) +} + +function writeDataObjectsToTarball (client: RestClient, tarball: Object, objectIds: Set, allowErrors: boolean): Promise<*> { + if (objectIds.size < 1) return Promise.resolve() + + const batchPromises = [] + let batch = [] + for (const id of objectIds) { + batch.push(id) + if (batch.length >= OBJECT_BATCH_SIZE) { + const ids = batch + batch = [] + batchPromises.push(fetchObjectBatch(client, tarball, ids, allowErrors)) + } + } + + batchPromises.push(fetchObjectBatch(client, tarball, batch, allowErrors)) + return Promise.all(batchPromises) +} + +function fetchObjectBatch (client: RestClient, tarball: Object, objectIds: Array, allowErrors: boolean): Promise<*> { + if (objectIds.length < 1) return Promise.resolve() + + return client.batchGetDataStream(objectIds, false) + .then(stream => new Promise((resolve, reject) => { + function handleError (message: string) { + if (allowErrors) { + printlnErr(message) + } else { + reject(new Error(message)) + } + } + + stream.on('data', dataResult => { + const key = objectIds.shift() + if (dataResult == null || typeof dataResult !== 'object') { + return handleError(`Unexpected response from node when requesting ${key}: ${dataResult}`) + } + if (dataResult.error != null) { + return handleError(`Error fetching object for ${key}: ${dataResult.error}`) + } + if (dataResult.data == null) { + return handleError(`Object not found for key ${key}`) + } + + const bytes = Buffer.from(dataResult.data, 'base64') + writeToTarball(tarball, `data/${key}`, bytes) + }) + + stream.on('error', err => { + reject(new Error(`Error reading from data object stream: ${err.message}`)) + }) + + stream.on('end', () => resolve()) + })) +} diff --git a/src/model/statement.js b/src/model/statement.js index 9ec7b27..dd14862 100644 --- a/src/model/statement.js +++ b/src/model/statement.js @@ -65,6 +65,10 @@ class Statement { return this.body.refSet } + get depsSet (): Set { + return this.body.depsSet + } + get source (): string { if (!(this.body instanceof EnvelopeStatementBody)) { return this.publisher @@ -233,6 +237,10 @@ class StatementBody { return new Set() } + get depsSet (): Set { + return new Set() + } + get objectIds (): Array { return [] } @@ -283,6 +291,10 @@ class SimpleStatementBody extends StatementBody { return new Set(this.refs) } + get depsSet (): Set { + return new Set(this.deps) + } + inspect (_depth?: number, _opts?: Object) { return this.toSimpleProtobuf() } @@ -326,6 +338,10 @@ class CompoundStatementBody extends StatementBody { get refSet (): Set { return setUnion(...this.simpleBodies.map(b => b.refSet)) } + + get depsSet (): Set { + return setUnion(...this.simpleBodies.map(b => b.depsSet)) + } } class EnvelopeStatementBody extends StatementBody { @@ -364,6 +380,10 @@ class EnvelopeStatementBody extends StatementBody { get refSet (): Set { return setUnion(...this.statements.map(stmt => stmt.refSet)) } + + get depsSet (): Set { + return setUnion(...this.statements.map(stmt => stmt.depsSet)) + } } // Expanded statement bodies include the data that a SimpleStatementBody's `object` hash links to diff --git a/test/model/statement_test.js b/test/model/statement_test.js index 55d7824..907d44d 100644 --- a/test/model/statement_test.js +++ b/test/model/statement_test.js @@ -62,6 +62,16 @@ describe('Statements', () => { } }) + it('returns the correct deps', () => { + for (const type of STMT_TYPES) { + for (let i = 0; i < fixtures.statements[type].length; i++) { + const stmt = Statement.fromProtobuf(fixtures.statements[type][i]) + const expectedDeps = fixtures.expectedDeps[type][i] + expect(setEquals(stmt.depsSet, expectedDeps)).to.be.true + } + } + }) + it('returns the correct object ids', () => { for (const type of STMT_TYPES) { for (let i = 0; i < fixtures.statements[type].length; i++) { @@ -256,6 +266,7 @@ describe('StatementBody base class', () => { it('returns empty values for refSet, etc.', () => { const stmt = Object.create(StatementBody.prototype) expect(stmt.refSet.size).to.be.eql(0) + expect(stmt.depsSet.size).to.be.eql(0) expect(stmt.objectIds.length).to.be.eql(0) expect(stmt.expandObjects(new Map())).to.be.eql(stmt) }) diff --git a/test/resources/fixtures/test-statements.js b/test/resources/fixtures/test-statements.js index 73e095c..ef857b2 100644 --- a/test/resources/fixtures/test-statements.js +++ b/test/resources/fixtures/test-statements.js @@ -3,8 +3,8 @@ const SIMPLE_STMT_1 = { publisher: '4XTTM81cjwraTF9FW33DyCz2PbdQ9peqCXWTz9rBhU3bwm4TE', namespace: 'scratch.test', timestamp: 1485446977027, - body: { simple: { object: 'foo', refs: [ 'simple-1' ], deps: [], tags: [] } }, - signature: Buffer.from('rYo/HI3zfEO7JtlFzIF9r7bJbqV7p3SjLDoedYQgI3X8zutAUNwayhXJURHVB0Yz/CShfLn+7Mc94iLCCBtJDw==', 'base64') + body: { simple: { object: 'foo', refs: [ 'simple-1' ], deps: [ 'dep1', 'dep2' ], tags: [] } }, + signature: Buffer.from('4Xl7an0GdvCZtNR8Hw50RBOhfthNydlyMHBZeIoFnuk0fAtZE8BfQqltrVMXxWp9dabE8g5rR/F+3Fdzl5yyAQ==', 'base64') } const SIMPLE_STMT_2 = { @@ -12,8 +12,8 @@ const SIMPLE_STMT_2 = { publisher: '4XTTM81cjwraTF9FW33DyCz2PbdQ9peqCXWTz9rBhU3bwm4TE', namespace: 'scratch.test', timestamp: 1485447081587, - body: { simple: { object: 'foo', refs: [ 'simple-2' ], deps: [], tags: [] } }, - signature: Buffer.from('6uuCL0zQSuSBuN2a7FeJGp75P5FBJUAwuBzIjC7nZrgVmHFqkiaVPUhO2lGikMh+DaU/Okgrf+thjfFDEFyxCQ==', 'base64') + body: { simple: { object: 'foo', refs: [ 'simple-2' ], deps: [ 'dep1', 'dep3' ], tags: [] } }, + signature: Buffer.from('u+u8ICJbRHiAsGFeLFVBODX29DXYf4Wj6J2am2J7TbQqhIdhbMjBhQ1kXFWeAMxmXpdxfRt3CocDoxo3z3t7CQ==', 'base64') } const COMPOUND_STMT = { @@ -32,6 +32,22 @@ const COMPOUND_STMT = { signature: Buffer.from('eJlR+rsTdiZQ7Lt8oI7M+tvtQPshjOb50OyKtrNQBfZ2KDyTpBIZnTWlZ2CAIq15oYjHetzrfZBxj81Nfu1QCw==', 'base64') } +const COMPOUND_STMT_2 = { + id: '4XTTMDah7ai6vqk6yzAhDtW9ATaEmTDJPNK3kcPT4bLKRuotG:1485447651564:1', + publisher: '4XTTMDah7ai6vqk6yzAhDtW9ATaEmTDJPNK3kcPT4bLKRuotG', + namespace: 'scratch.test.compound-stmt', + timestamp: 1485447651564, + body: { + compound: { + body: [ + { object: 'foo', refs: [ 'compound-3' ], deps: [ 'dep1', 'dep2' ], tags: [] }, + { object: 'foo', refs: [ 'compound-4' ], deps: [ 'dep1', 'dep3', 'dep4' ], tags: [] } + ] + } + }, + signature: Buffer.from('8nBP5iUEJu0TeMSWr4+HTg6Gp9I3yzu7Q590+HvVG7zbcbjJvI3qPN9yrnmh2txuVXua7lPHF9ORpOWdByeyDA==', 'base64') +} + const ENVELOPE_EMPTY = { id: '4XTTM2hkDuu73NXYakvw2uD6QfNAxB5emTd1P11uYt7YkmcXv:1485448028036:0', publisher: '4XTTM2hkDuu73NXYakvw2uD6QfNAxB5emTd1P11uYt7YkmcXv', @@ -47,7 +63,7 @@ const ENVELOPE_STMT = { namespace: 'scratch.test.envelope-stmt', timestamp: 1485448141505, body: { envelope: { body: [ SIMPLE_STMT_1, SIMPLE_STMT_2 ] } }, - signature: Buffer.from('/94sZ6ETWTNCaHO78h+ifrqrViN4v95//Qx/+j3OmvLqJ3eLrK5damMqbQw06kstVC5II58udNR7zCJFqsYbDw==', 'base64') + signature: Buffer.from('dEhboo/dqqHK/hB/Jur/DBQSKDpnr3bLM1sJgmCaRSlEtJpZdBHKlLjvy2CPyy9gqRCtczOAiMwkwgkiYvYaAg==', 'base64') } module.exports = { @@ -67,25 +83,31 @@ module.exports = { }, statements: { simple: [ SIMPLE_STMT_1, SIMPLE_STMT_2 ], - compound: [ COMPOUND_STMT ], + compound: [ COMPOUND_STMT, COMPOUND_STMT_2 ], envelope: [ ENVELOPE_STMT ], envelopeEmpty: [ ENVELOPE_EMPTY ] }, expectedRefs: { simple: [ new Set(['simple-1']), new Set(['simple-2']) ], - compound: [ new Set(['compound-1', 'compound-2']) ], + compound: [ new Set(['compound-1', 'compound-2']), new Set(['compound-3', 'compound-4']) ], envelope: [ new Set(['simple-1', 'simple-2']) ], envelopeEmpty: [ new Set() ] }, expectedSources: { simple: [ SIMPLE_STMT_1.publisher, SIMPLE_STMT_2.publisher ], - compound: [ COMPOUND_STMT.publisher ], + compound: [ COMPOUND_STMT.publisher, COMPOUND_STMT_2.publisher ], envelope: [ SIMPLE_STMT_1.publisher ], envelopeEmpty: [ ENVELOPE_STMT.publisher ] }, + expectedDeps: { + simple: [ new Set(['dep1', 'dep2']), new Set(['dep1', 'dep3']) ], + compound: [ new Set(), new Set(['dep1', 'dep2', 'dep3', 'dep4']) ], + envelope: [ new Set(['dep1', 'dep2', 'dep3']) ], + envelopeEmpty: [ new Set() ] + }, objectIds: { simple: [ ['foo'], ['foo'] ], - compound: [ ['foo', 'foo'] ], + compound: [ ['foo', 'foo'], ['foo', 'foo'] ], envelope: [ ['foo', 'foo'] ], envelopeEmpty: [ [] ] }