From 519e8ec0c0825a3f7bd9158716e38e9870f1db07 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 22 Jul 2020 00:38:38 -0700 Subject: [PATCH 1/7] feat: accept Uint8Arrays in place of Buffers --- src/index.js | 72 +++++++++++++++++++++++++++------------------- src/util.js | 13 +++++++++ test/index.spec.js | 42 ++++++++++++++++----------- 3 files changed, 81 insertions(+), 46 deletions(-) create mode 100644 src/util.js diff --git a/src/index.js b/src/index.js index 8accc704..269b4892 100644 --- a/src/index.js +++ b/src/index.js @@ -1,3 +1,4 @@ +// @ts-check /* eslint-disable guard-for-in */ /** * Multihash implementation in JavaScript. @@ -10,6 +11,9 @@ const { Buffer } = require('buffer') const multibase = require('multibase') const varint = require('varint') const { names } = require('./constants') +const textDecoder = typeof TextDecoder !== 'undefined' + ? new TextDecoder() + : new (require('util').TextDecoder)() const codes = {} @@ -22,15 +26,19 @@ exports.codes = Object.freeze(codes) /** * Convert the given multihash to a hex encoded string. * - * @param {Buffer} hash + * @param {Uint8Array} hash * @returns {string} */ exports.toHexString = function toHexString (hash) { - if (!Buffer.isBuffer(hash)) { - throw new Error('must be passed a buffer') + if (!(hash instanceof Uint8Array)) { + throw new Error('must be passed an Uint8Array') } - return hash.toString('hex') + const buffer = Buffer.isBuffer(hash) + ? hash + : Buffer.from(hash.buffer, hash.byteOffset, hash.byteLength) + + return buffer.toString('hex') } /** @@ -46,28 +54,27 @@ exports.fromHexString = function fromHexString (hash) { /** * Convert the given multihash to a base58 encoded string. * - * @param {Buffer} hash + * @param {Uint8Array} hash * @returns {string} */ exports.toB58String = function toB58String (hash) { - if (!Buffer.isBuffer(hash)) { - throw new Error('must be passed a buffer') + if (!(hash instanceof Uint8Array)) { + throw new Error('must be passed an Uint8Array') } - return multibase.encode('base58btc', hash).toString().slice(1) + return textDecoder.decode(multibase.encode('base58btc', hash)).slice(1) } /** * Convert the given base58 encoded string to a multihash. * - * @param {string|Buffer} hash + * @param {string|Uint8Array} hash * @returns {Buffer} */ exports.fromB58String = function fromB58String (hash) { - let encoded = hash - if (Buffer.isBuffer(hash)) { - encoded = hash.toString() - } + const encoded = hash instanceof Uint8Array + ? textDecoder.decode(hash) + : hash return multibase.decode('z' + encoded) } @@ -75,13 +82,16 @@ exports.fromB58String = function fromB58String (hash) { /** * Decode a hash from the given multihash. * - * @param {Buffer} buf + * @param {Uint8Array} bytes * @returns {{code: number, name: string, length: number, digest: Buffer}} result */ -exports.decode = function decode (buf) { - if (!(Buffer.isBuffer(buf))) { - throw new Error('multihash must be a Buffer') +exports.decode = function decode (bytes) { + if (!(bytes instanceof Uint8Array)) { + throw new Error('multihash must be an Uint8Array') } + let buf = Buffer.isBuffer(bytes) + ? bytes + : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength) if (buf.length < 2) { throw new Error('multihash too short. must be > 2 bytes.') @@ -116,7 +126,7 @@ exports.decode = function decode (buf) { * * > **Note:** the length is derived from the length of the digest itself. * - * @param {Buffer} digest + * @param {Uint8Array} digest * @param {string|number} code * @param {number} [length] * @returns {Buffer} @@ -129,8 +139,8 @@ exports.encode = function encode (digest, code, length) { // ensure it's a hashfunction code. const hashfn = exports.coerceCode(code) - if (!(Buffer.isBuffer(digest))) { - throw new Error('digest should be a Buffer') + if (!(digest instanceof Uint8Array)) { + throw new Error('digest should be an Uint8Array') } if (length == null) { @@ -141,11 +151,13 @@ exports.encode = function encode (digest, code, length) { throw new Error('digest length should be equal to specified length.') } - return Buffer.concat([ - Buffer.from(varint.encode(hashfn)), - Buffer.from(varint.encode(length)), - digest - ]) + const hash = varint.encode(hashfn) + const len = varint.encode(length) + const buffer = Buffer.alloc(hash.length + len.length + digest.length) + buffer.set(hash, 0) + buffer.set(len, hash.length) + buffer.set(digest, hash.length + len.length) + return buffer } /** @@ -206,8 +218,8 @@ exports.isValidCode = function validCode (code) { /** * Check if the given buffer is a valid multihash. Throws an error if it is not valid. * - * @param {Buffer} multihash - * @returns {undefined} + * @param {Uint8Array} multihash + * @returns {void} * @throws {Error} */ function validate (multihash) { @@ -218,12 +230,12 @@ exports.validate = validate /** * Returns a prefix from a valid multihash. Throws an error if it is not valid. * - * @param {Buffer} multihash - * @returns {undefined} + * @param {Uint8Array} multihash + * @returns {Buffer} * @throws {Error} */ exports.prefix = function prefix (multihash) { validate(multihash) - return multihash.slice(0, 2) + return Buffer.from(multihash.buffer, multihash.byteOffset, 2) } diff --git a/src/util.js b/src/util.js new file mode 100644 index 00000000..976f2a7c --- /dev/null +++ b/src/util.js @@ -0,0 +1,13 @@ +'use strict' + +const textDecoder = typeof TextDecoder !== 'undefined' + ? new TextDecoder() + : new (require('util').TextDecoder)() + +/** + * @param {Uint8Arary} bytes + * @returns {string} + */ +const decodeText = (bytes) => textDecoder.decode(bytes) + +module.exports = { decodeText } diff --git a/test/index.spec.js b/test/index.spec.js index f7e11e5e..ece4964a 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -12,6 +12,9 @@ const mh = require('../src') const constants = require('../src/constants') const validCases = require('./fixtures/valid') const invalidCases = require('./fixtures/invalid') +const textEncoder = typeof TextEncoder !== 'undefined' + ? new TextEncoder() + : new (require('util').TextEncoder)() function sample (code, size, hex) { const toHex = (i) => { @@ -24,12 +27,19 @@ function sample (code, size, hex) { return Buffer.from(`${toHex(code)}${toHex(size)}${hex}`, 'hex') } +const encodeHex = string => { + const { buffer, byteOffset, byteLength } = Buffer.from(string, 'hex') + return new Uint8Array(buffer, byteOffset, byteLength) +} + +const encodeText = string => textEncoder.encode(string) + describe('multihash', () => { describe('toHexString', () => { it('valid', () => { validCases.forEach((test) => { const code = test.encoding.code - const buf = mh.encode(Buffer.from(test.hex, 'hex'), code) + const buf = mh.encode(encodeHex(test.hex), code) expect( mh.toHexString(buf) ).to.be.eql( @@ -42,7 +52,7 @@ describe('multihash', () => { expect( () => mh.toHexString('hello world') ).to.throw( - /must be passed a buffer/ + /must be passed an Uint8Array/ ) }) }) @@ -51,7 +61,7 @@ describe('multihash', () => { it('valid', () => { validCases.forEach((test) => { const code = test.encoding.code - const buf = mh.encode(Buffer.from(test.hex, 'hex'), code) + const buf = mh.encode(encodeHex(test.hex), code) expect( mh.fromHexString(buf.toString('hex')).toString('hex') ).to.be.eql( @@ -65,7 +75,7 @@ describe('multihash', () => { it('valid', () => { validCases.forEach((test) => { const code = test.encoding.code - const buf = mh.encode(Buffer.from(test.hex, 'hex'), code) + const buf = mh.encode(encodeHex(test.hex, 'hex'), code) expect( mh.toB58String(buf) ).to.be.eql( @@ -78,7 +88,7 @@ describe('multihash', () => { expect( () => mh.toB58String('hello world') ).to.throw( - /must be passed a buffer/ + /must be passed an Uint8Array/ ) }) }) @@ -86,7 +96,7 @@ describe('multihash', () => { describe('fromB58String', () => { it('valid', () => { const src = 'QmPfjpVaf593UQJ9a5ECvdh2x17XuJYG5Yanv5UFnH3jPE' - const expected = Buffer.from('122013bf801597d74a660453412635edd8c34271e5998f801fac5d700c6ce8d8e461', 'hex') + const expected = encodeHex('122013bf801597d74a660453412635edd8c34271e5998f801fac5d700c6ce8d8e461') expect( mh.fromB58String(src) @@ -95,7 +105,7 @@ describe('multihash', () => { ) expect( - mh.fromB58String(Buffer.from(src)) + mh.fromB58String(encodeText(src)) ).to.be.eql( expected ) @@ -125,7 +135,7 @@ describe('multihash', () => { expect( () => mh.decode('hello') ).to.throw( - /multihash must be a Buffer/ + /multihash must be an Uint8Array/ ) }) }) @@ -137,8 +147,8 @@ describe('multihash', () => { const name = test.encoding.name const buf = sample(test.encoding.varint || code, test.size, test.hex) const results = [ - mh.encode(Buffer.from(test.hex, 'hex'), code), - mh.encode(Buffer.from(test.hex, 'hex'), name) + mh.encode(encodeHex(test.hex), code), + mh.encode(encodeHex(test.hex), name) ] results.forEach((res) => { @@ -161,11 +171,11 @@ describe('multihash', () => { expect( () => mh.encode('hello', 0x11) ).to.throw( - /digest should be a Buffer/ + /digest should be an Uint8Array/ ) expect( - () => mh.encode(Buffer.from('hello'), 0x11, 2) + () => mh.encode(encodeText('hello'), 0x11, 2) ).to.throw( /length should be equal/ ) @@ -188,7 +198,7 @@ describe('multihash', () => { ).to.throw() }) - const longBuffer = Buffer.alloc(150, 'a') + const longBuffer = Uint8Array.from(Buffer.alloc(150, 'a')) expect( () => mh.validate(longBuffer) ).to.throw() @@ -293,7 +303,7 @@ describe('multihash', () => { }) expect( - () => mh.coerceCode(Buffer.from('hello')) + () => mh.coerceCode(encodeText('hello')) ).to.throw( /should be a number/ ) @@ -307,13 +317,13 @@ describe('multihash', () => { }) it('prefix', () => { - const multihash = mh.encode(Buffer.from('hey'), 0x11, 3) + const multihash = mh.encode(encodeText('hey'), 0x11, 3) const prefix = mh.prefix(multihash) expect(prefix.toString('hex')).to.eql('1103') }) it('prefix throws on invalid multihash', () => { - const multihash = Buffer.from('definitely not valid') + const multihash = encodeText('definitely not valid') expect(() => mh.prefix(multihash)).to.throw() }) From 2b8127b2592e89d42159930778e7105907845083 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 22 Jul 2020 15:27:25 -0700 Subject: [PATCH 2/7] chore: remove dependency on node util --- package.json | 5 +++-- src/index.js | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 0357e3f4..3705af0c 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,8 @@ "dependencies": { "buffer": "^5.6.0", "multibase": "^1.0.1", - "varint": "^5.0.0" + "varint": "^5.0.0", + "web-encoding": "^1.0.1" }, "devDependencies": { "aegir": "^24.0.0", @@ -67,4 +68,4 @@ "node": ">=10.0.0", "npm": ">=6.0.0" } -} +} \ No newline at end of file diff --git a/src/index.js b/src/index.js index 269b4892..1d91eaf7 100644 --- a/src/index.js +++ b/src/index.js @@ -11,10 +11,9 @@ const { Buffer } = require('buffer') const multibase = require('multibase') const varint = require('varint') const { names } = require('./constants') -const textDecoder = typeof TextDecoder !== 'undefined' - ? new TextDecoder() - : new (require('util').TextDecoder)() +const { TextDecoder } = require('web-encoding') +const textDecoder = new TextDecoder() const codes = {} for (const key in names) { From 8ddfa1a0893c0e539d1cfa3a9178d8d71559604f Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 22 Jul 2020 16:05:15 -0700 Subject: [PATCH 3/7] chore: test with both Buffer and Uint8Array --- test/index.spec.js | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/test/index.spec.js b/test/index.spec.js index ece4964a..717fc613 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -27,16 +27,24 @@ function sample (code, size, hex) { return Buffer.from(`${toHex(code)}${toHex(size)}${hex}`, 'hex') } -const encodeHex = string => { - const { buffer, byteOffset, byteLength } = Buffer.from(string, 'hex') - return new Uint8Array(buffer, byteOffset, byteLength) +const they = (description, test) => { + it(`${description} (Buffer)`, () => test({ + encodeText: Buffer.from, + encodeHex: (text) => Buffer.from(text, 'hex') + })) + + it(`${description} (Uint8Array)`, () => test({ + encodeText: (text) => textEncoder.encode(text), + encodeHex: (text) => { + const { buffer, byteOffset, byteLength } = Buffer.from(text, 'hex') + return new Uint8Array(buffer, byteOffset, byteLength) + } + })) } -const encodeText = string => textEncoder.encode(string) - describe('multihash', () => { describe('toHexString', () => { - it('valid', () => { + they('valid', ({ encodeHex }) => { validCases.forEach((test) => { const code = test.encoding.code const buf = mh.encode(encodeHex(test.hex), code) @@ -58,7 +66,7 @@ describe('multihash', () => { }) describe('fromHexString', () => { - it('valid', () => { + they('valid', ({ encodeHex }) => { validCases.forEach((test) => { const code = test.encoding.code const buf = mh.encode(encodeHex(test.hex), code) @@ -72,10 +80,10 @@ describe('multihash', () => { }) describe('toB58String', () => { - it('valid', () => { + they('valid', ({ encodeHex }) => { validCases.forEach((test) => { const code = test.encoding.code - const buf = mh.encode(encodeHex(test.hex, 'hex'), code) + const buf = mh.encode(encodeHex(test.hex), code) expect( mh.toB58String(buf) ).to.be.eql( @@ -94,7 +102,7 @@ describe('multihash', () => { }) describe('fromB58String', () => { - it('valid', () => { + they('valid', ({ encodeHex, encodeText }) => { const src = 'QmPfjpVaf593UQJ9a5ECvdh2x17XuJYG5Yanv5UFnH3jPE' const expected = encodeHex('122013bf801597d74a660453412635edd8c34271e5998f801fac5d700c6ce8d8e461') @@ -141,7 +149,7 @@ describe('multihash', () => { }) describe('encode', () => { - it('valid', () => { + they('valid', ({ encodeHex }) => { validCases.forEach((test) => { const code = test.encoding.code const name = test.encoding.name @@ -161,7 +169,7 @@ describe('multihash', () => { }) }) - it('invalid', () => { + they('invalid', ({ encodeText }) => { expect( () => mh.encode() ).to.throw( @@ -287,7 +295,7 @@ describe('multihash', () => { }) }) - it('invalid', () => { + they('invalid', ({ encodeText }) => { const invalidNames = [ 'sha256', 'sha9', @@ -316,13 +324,13 @@ describe('multihash', () => { }) }) - it('prefix', () => { + they('prefix', ({ encodeText }) => { const multihash = mh.encode(encodeText('hey'), 0x11, 3) const prefix = mh.prefix(multihash) expect(prefix.toString('hex')).to.eql('1103') }) - it('prefix throws on invalid multihash', () => { + they('prefix throws on invalid multihash', ({ encodeText }) => { const multihash = encodeText('definitely not valid') expect(() => mh.prefix(multihash)).to.throw() From c92ccaf6005f8f1eca77a6173693cc8263695ee7 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Thu, 23 Jul 2020 08:22:04 -0700 Subject: [PATCH 4/7] chore: address review comments --- src/index.js | 8 ++++---- src/util.js | 13 ------------- test/index.spec.js | 5 ++--- 3 files changed, 6 insertions(+), 20 deletions(-) delete mode 100644 src/util.js diff --git a/src/index.js b/src/index.js index 1d91eaf7..a095597c 100644 --- a/src/index.js +++ b/src/index.js @@ -30,7 +30,7 @@ exports.codes = Object.freeze(codes) */ exports.toHexString = function toHexString (hash) { if (!(hash instanceof Uint8Array)) { - throw new Error('must be passed an Uint8Array') + throw new Error('must be passed a Uint8Array') } const buffer = Buffer.isBuffer(hash) @@ -58,7 +58,7 @@ exports.fromHexString = function fromHexString (hash) { */ exports.toB58String = function toB58String (hash) { if (!(hash instanceof Uint8Array)) { - throw new Error('must be passed an Uint8Array') + throw new Error('must be passed a Uint8Array') } return textDecoder.decode(multibase.encode('base58btc', hash)).slice(1) @@ -86,7 +86,7 @@ exports.fromB58String = function fromB58String (hash) { */ exports.decode = function decode (bytes) { if (!(bytes instanceof Uint8Array)) { - throw new Error('multihash must be an Uint8Array') + throw new Error('multihash must be a Uint8Array') } let buf = Buffer.isBuffer(bytes) ? bytes @@ -139,7 +139,7 @@ exports.encode = function encode (digest, code, length) { const hashfn = exports.coerceCode(code) if (!(digest instanceof Uint8Array)) { - throw new Error('digest should be an Uint8Array') + throw new Error('digest should be a Uint8Array') } if (length == null) { diff --git a/src/util.js b/src/util.js deleted file mode 100644 index 976f2a7c..00000000 --- a/src/util.js +++ /dev/null @@ -1,13 +0,0 @@ -'use strict' - -const textDecoder = typeof TextDecoder !== 'undefined' - ? new TextDecoder() - : new (require('util').TextDecoder)() - -/** - * @param {Uint8Arary} bytes - * @returns {string} - */ -const decodeText = (bytes) => textDecoder.decode(bytes) - -module.exports = { decodeText } diff --git a/test/index.spec.js b/test/index.spec.js index 717fc613..fef2539f 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -12,9 +12,7 @@ const mh = require('../src') const constants = require('../src/constants') const validCases = require('./fixtures/valid') const invalidCases = require('./fixtures/invalid') -const textEncoder = typeof TextEncoder !== 'undefined' - ? new TextEncoder() - : new (require('util').TextEncoder)() +const { TextEncoder } = require('web-encoding') function sample (code, size, hex) { const toHex = (i) => { @@ -33,6 +31,7 @@ const they = (description, test) => { encodeHex: (text) => Buffer.from(text, 'hex') })) + const textEncoder = new TextEncoder() it(`${description} (Uint8Array)`, () => test({ encodeText: (text) => textEncoder.encode(text), encodeHex: (text) => { From b5f9dace06ee247164dd04d7312f8be73b382794 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Fri, 24 Jul 2020 10:49:10 -0700 Subject: [PATCH 5/7] fix: breaking tests --- test/index.spec.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/index.spec.js b/test/index.spec.js index fef2539f..32bc7f22 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -59,7 +59,7 @@ describe('multihash', () => { expect( () => mh.toHexString('hello world') ).to.throw( - /must be passed an Uint8Array/ + /must be passed a Uint8Array/ ) }) }) @@ -95,7 +95,7 @@ describe('multihash', () => { expect( () => mh.toB58String('hello world') ).to.throw( - /must be passed an Uint8Array/ + /must be passed a Uint8Array/ ) }) }) @@ -142,7 +142,7 @@ describe('multihash', () => { expect( () => mh.decode('hello') ).to.throw( - /multihash must be an Uint8Array/ + /multihash must be a Uint8Array/ ) }) }) @@ -178,7 +178,7 @@ describe('multihash', () => { expect( () => mh.encode('hello', 0x11) ).to.throw( - /digest should be an Uint8Array/ + /digest should be a Uint8Array/ ) expect( From 18d151519f2ba728d27df0cb3f4e345122fb1909 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Fri, 24 Jul 2020 18:24:45 -0700 Subject: [PATCH 6/7] chore: bump web-encoding version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3705af0c..3d949c1d 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "buffer": "^5.6.0", "multibase": "^1.0.1", "varint": "^5.0.0", - "web-encoding": "^1.0.1" + "web-encoding": "^1.0.2" }, "devDependencies": { "aegir": "^24.0.0", From f2e56ad03ad389381dd7ea2b59aec4f7103266ee Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Fri, 24 Jul 2020 18:25:20 -0700 Subject: [PATCH 7/7] chore: update multibase to 2.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3d949c1d..67965f4f 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "repository": "github:multiformats/js-multihash", "dependencies": { "buffer": "^5.6.0", - "multibase": "^1.0.1", + "multibase": "^2.0.0", "varint": "^5.0.0", "web-encoding": "^1.0.2" },