benchmark,test: add brotli

Co-authored-by: Hackzzila <[email protected]> Backport-PR-URL: #27681 PR-URL: #24938 Reviewed-By: Ruben Bridgewater <[email protected]> Reviewed-By: Matteo Collina <[email protected]> Reviewed-By: Jan Krems <[email protected]> Reviewed-By: James M Snell <[email protected]> Reviewed-By: Myles Borins <[email protected]> Reviewed-By: Ali Ijaz Sheikh <[email protected]> Reviewed-By: Daniel Bevenius <[email protected]>
nodejs · May 16, 2019 · 15b43e3 · 15b43e3
1 parent aab0da6
commit 15b43e3
Show file tree

Hide file tree

Showing 16 changed files with 267 additions and 49 deletions.
diff --git a/benchmark/zlib/creation.js b/benchmark/zlib/creation.js
@@ -4,7 +4,8 @@ const zlib = require('zlib');
 
 const bench = common.createBenchmark(main, {
   type: [
-    'Deflate', 'DeflateRaw', 'Inflate', 'InflateRaw', 'Gzip', 'Gunzip', 'Unzip'
+    'Deflate', 'DeflateRaw', 'Inflate', 'InflateRaw', 'Gzip', 'Gunzip', 'Unzip',
+    'BrotliCompress', 'BrotliDecompress'
   ],
   options: ['true', 'false'],
   n: [5e5]

diff --git a/benchmark/zlib/pipe.js b/benchmark/zlib/pipe.js
@@ -6,15 +6,18 @@ const zlib = require('zlib');
 const bench = common.createBenchmark(main, {
   inputLen: [1024],
   duration: [5],
-  type: ['string', 'buffer']
+  type: ['string', 'buffer'],
+  algorithm: ['gzip', 'brotli']
 });
 
-function main({ inputLen, duration, type }) {
+function main({ inputLen, duration, type, algorithm }) {
   const buffer = Buffer.alloc(inputLen, fs.readFileSync(__filename));
   const chunk = type === 'buffer' ? buffer : buffer.toString('utf8');
 
-  const input = zlib.createGzip();
-  const output = zlib.createGunzip();
+  const input = algorithm === 'gzip' ?
+    zlib.createGzip() : zlib.createBrotliCompress();
+  const output = algorithm === 'gzip' ?
+    zlib.createGunzip() : zlib.createBrotliDecompress();
 
   let readFromOutput = 0;
   input.pipe(output);

diff --git a/test/fixtures/person.jpg.br b/test/fixtures/person.jpg.br
diff --git a/test/parallel/test-zlib-brotli-flush.js b/test/parallel/test-zlib-brotli-flush.js
@@ -0,0 +1,27 @@
+'use strict';
+require('../common');
+const assert = require('assert');
+const zlib = require('zlib');
+const fixtures = require('../common/fixtures');
+
+const file = fixtures.readSync('person.jpg');
+const chunkSize = 16;
+const deflater = new zlib.BrotliCompress();
+
+const chunk = file.slice(0, chunkSize);
+const expectedFull = Buffer.from('iweA/9j/4AAQSkZJRgABAQEASA==', 'base64');
+let actualFull;
+
+deflater.write(chunk, function() {
+  deflater.flush(function() {
+    const bufs = [];
+    let buf;
+    while (buf = deflater.read())
+      bufs.push(buf);
+    actualFull = Buffer.concat(bufs);
+  });
+});
+
+process.once('exit', function() {
+  assert.deepStrictEqual(actualFull, expectedFull);
+});
diff --git a/test/parallel/test-zlib-brotli-from-brotli.js b/test/parallel/test-zlib-brotli-from-brotli.js
@@ -0,0 +1,32 @@
+'use strict';
+// Test unzipping a file that was created with a non-node brotli lib,
+// piped in as fast as possible.
+
+const common = require('../common');
+const assert = require('assert');
+const zlib = require('zlib');
+const path = require('path');
+const fixtures = require('../common/fixtures');
+
+const tmpdir = require('../common/tmpdir');
+tmpdir.refresh();
+
+const decompress = new zlib.BrotliDecompress();
+
+const fs = require('fs');
+
+const fixture = fixtures.path('person.jpg.br');
+const unzippedFixture = fixtures.path('person.jpg');
+const outputFile = path.resolve(tmpdir.path, 'person.jpg');
+const expect = fs.readFileSync(unzippedFixture);
+const inp = fs.createReadStream(fixture);
+const out = fs.createWriteStream(outputFile);
+
+inp.pipe(decompress).pipe(out);
+out.on('close', common.mustCall(() => {
+  const actual = fs.readFileSync(outputFile);
+  assert.strictEqual(actual.length, expect.length);
+  for (let i = 0, l = actual.length; i < l; i++) {
+    assert.strictEqual(actual[i], expect[i], `byte[${i}]`);
+  }
+}));
diff --git a/test/parallel/test-zlib-brotli-from-string.js b/test/parallel/test-zlib-brotli-from-string.js
@@ -0,0 +1,34 @@
+'use strict';
+// Test compressing and uncompressing a string with brotli
+
+const common = require('../common');
+const assert = require('assert');
+const zlib = require('zlib');
+
+const inputString = 'ΩΩLorem ipsum dolor sit amet, consectetur adipiscing eli' +
+                    't. Morbi faucibus, purus at gravida dictum, libero arcu ' +
+                    'convallis lacus, in commodo libero metus eu nisi. Nullam' +
+                    ' commodo, neque nec porta placerat, nisi est fermentum a' +
+                    'ugue, vitae gravida tellus sapien sit amet tellus. Aenea' +
+                    'n non diam orci. Proin quis elit turpis. Suspendisse non' +
+                    ' diam ipsum. Suspendisse nec ullamcorper odio. Vestibulu' +
+                    'm arcu mi, sodales non suscipit id, ultrices ut massa. S' +
+                    'ed ac sem sit amet arcu malesuada fermentum. Nunc sed. ';
+const expectedBase64Compress = 'G/gBQBwHdky2aHV5KK9Snf05//1pPdmNw/7232fnIm1IB' +
+                               'K1AA8RsN8OB8Nb7Lpgk3UWWUlzQXZyHQeBBbXMTQXC1j7' +
+                               'wg3LJs9LqOGHRH2bj/a2iCTLLx8hBOyTqgoVuD1e+Qqdn' +
+                               'f1rkUNyrWq6LtOhWgxP3QUwdhKGdZm3rJWaDDBV7+pDk1' +
+                               'MIkrmjp4ma2xVi5MsgJScA3tP1I7mXeby6MELozrwoBQD' +
+                               'mVTnEAicZNj4lkGqntJe2qSnGyeMmcFgraK94vCg/4iLu' +
+                               'Tw5RhKhnVY++dZ6niUBmRqIutsjf5TzwF5iAg8a9UkjF5' +
+                               '2eZ0tB2vo6v8SqVfNMkBmmhxr0NT9LkYF69aEjlYzj7IE' +
+                               'KmEUQf1HBogRYhFIt4ymRNEgHAIzOyNEsQM=';
+
+zlib.brotliCompress(inputString, common.mustCall((err, buffer) => {
+  assert.strictEqual(buffer.toString('base64'), expectedBase64Compress);
+}));
+
+const buffer = Buffer.from(expectedBase64Compress, 'base64');
+zlib.brotliDecompress(buffer, common.mustCall((err, buffer) => {
+  assert.strictEqual(buffer.toString(), inputString);
+}));
diff --git a/test/parallel/test-zlib-brotli-kmaxlength-rangeerror.js b/test/parallel/test-zlib-brotli-kmaxlength-rangeerror.js
@@ -0,0 +1,28 @@
+'use strict';
+require('../common');
+
+// This test ensures that zlib throws a RangeError if the final buffer needs to
+// be larger than kMaxLength and concatenation fails.
+// https://github.com/nodejs/node/pull/1811
+
+const assert = require('assert');
+
+// Change kMaxLength for zlib to trigger the error without having to allocate
+// large Buffers.
+const buffer = require('buffer');
+const oldkMaxLength = buffer.kMaxLength;
+buffer.kMaxLength = 128;
+const zlib = require('zlib');
+buffer.kMaxLength = oldkMaxLength;
+
+const encoded = Buffer.from('G38A+CXCIrFAIAM=', 'base64');
+
+// Async
+zlib.brotliDecompress(encoded, function(err) {
+  assert.ok(err instanceof RangeError);
+});
+
+// Sync
+assert.throws(function() {
+  zlib.brotliDecompressSync(encoded);
+}, RangeError);
diff --git a/test/parallel/test-zlib-brotli.js b/test/parallel/test-zlib-brotli.js
@@ -0,0 +1,73 @@
+'use strict';
+const common = require('../common');
+const fixtures = require('../common/fixtures');
+const assert = require('assert');
+const zlib = require('zlib');
+
+// Test some brotli-specific properties of the brotli streams that can not
+// be easily covered through expanding zlib-only tests.
+
+const sampleBuffer = fixtures.readSync('/pss-vectors.json');
+
+{
+  // Test setting the quality parameter at stream creation:
+  const sizes = [];
+  for (let quality = zlib.constants.BROTLI_MIN_QUALITY;
+    quality <= zlib.constants.BROTLI_MAX_QUALITY;
+    quality++) {
+    const encoded = zlib.brotliCompressSync(sampleBuffer, {
+      params: {
+        [zlib.constants.BROTLI_PARAM_QUALITY]: quality
+      }
+    });
+    sizes.push(encoded.length);
+  }
+
+  // Increasing quality should roughly correspond to decreasing compressed size:
+  for (let i = 0; i < sizes.length - 1; i++) {
+    assert(sizes[i + 1] <= sizes[i] * 1.05, sizes);  // 5 % margin of error.
+  }
+  assert(sizes[0] > sizes[sizes.length - 1], sizes);
+}
+
+{
+  // Test that setting out-of-bounds option values or keys fails.
+  common.expectsError(() => {
+    zlib.createBrotliCompress({
+      params: {
+        10000: 0
+      }
+    });
+  }, {
+    code: 'ERR_BROTLI_INVALID_PARAM',
+    type: RangeError,
+    message: '10000 is not a valid Brotli parameter'
+  });
+
+  // Test that accidentally using duplicate keys fails.
+  common.expectsError(() => {
+    zlib.createBrotliCompress({
+      params: {
+        '0': 0,
+        '00': 0
+      }
+    });
+  }, {
+    code: 'ERR_BROTLI_INVALID_PARAM',
+    type: RangeError,
+    message: '00 is not a valid Brotli parameter'
+  });
+
+  common.expectsError(() => {
+    zlib.createBrotliCompress({
+      params: {
+        // This is a boolean flag
+        [zlib.constants.BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING]: 42
+      }
+    });
+  }, {
+    code: 'ERR_ZLIB_INITIALIZATION_FAILED',
+    type: Error,
+    message: 'Initialization failed'
+  });
+}
diff --git a/test/parallel/test-zlib-bytes-read.js b/test/parallel/test-zlib-bytes-read.js
@@ -25,7 +25,8 @@ for (const method of [
   ['createGzip', 'createGunzip', false],
   ['createGzip', 'createUnzip', false],
   ['createDeflate', 'createInflate', true],
-  ['createDeflateRaw', 'createInflateRaw', true]
+  ['createDeflateRaw', 'createInflateRaw', true],
+  ['createBrotliCompress', 'createBrotliDecompress', true]
 ]) {
   let compWriter;
   let compData = Buffer.alloc(0);

diff --git a/test/parallel/test-zlib-convenience-methods.js b/test/parallel/test-zlib-convenience-methods.js
@@ -52,6 +52,8 @@ for (const [type, expect] of [
     ['gzip', 'unzip', 'Gzip', 'Unzip'],
     ['deflate', 'inflate', 'Deflate', 'Inflate'],
     ['deflateRaw', 'inflateRaw', 'DeflateRaw', 'InflateRaw'],
+    ['brotliCompress', 'brotliDecompress',
+     'BrotliCompress', 'BrotliDecompress'],
   ]) {
     zlib[method[0]](expect, opts, common.mustCall((err, result) => {
       zlib[method[1]](result, opts, common.mustCall((err, result) => {

diff --git a/test/parallel/test-zlib-empty-buffer.js b/test/parallel/test-zlib-empty-buffer.js
@@ -10,9 +10,11 @@ const emptyBuffer = Buffer.alloc(0);
     [ zlib.deflateRawSync, zlib.inflateRawSync, 'raw sync' ],
     [ zlib.deflateSync, zlib.inflateSync, 'deflate sync' ],
     [ zlib.gzipSync, zlib.gunzipSync, 'gzip sync' ],
+    [ zlib.brotliCompressSync, zlib.brotliDecompressSync, 'br sync' ],
     [ promisify(zlib.deflateRaw), promisify(zlib.inflateRaw), 'raw' ],
     [ promisify(zlib.deflate), promisify(zlib.inflate), 'deflate' ],
-    [ promisify(zlib.gzip), promisify(zlib.gunzip), 'gzip' ]
+    [ promisify(zlib.gzip), promisify(zlib.gunzip), 'gzip' ],
+    [ promisify(zlib.brotliCompress), promisify(zlib.brotliDecompress), 'br' ]
   ]) {
     const compressed = await compress(emptyBuffer);
     const decompressed = await decompress(compressed);

diff --git a/test/parallel/test-zlib-invalid-input.js b/test/parallel/test-zlib-invalid-input.js
@@ -38,7 +38,8 @@ const unzips = [
   zlib.Unzip(),
   zlib.Gunzip(),
   zlib.Inflate(),
-  zlib.InflateRaw()
+  zlib.InflateRaw(),
+  zlib.BrotliDecompress()
 ];
 
 nonStringInputs.forEach(common.mustCall((input) => {

diff --git a/test/parallel/test-zlib-random-byte-pipes.js b/test/parallel/test-zlib-random-byte-pipes.js
@@ -141,14 +141,18 @@ class HashStream extends Stream {
   }
 }
 
-
-const inp = new RandomReadStream({ total: 1024, block: 256, jitter: 16 });
-const out = new HashStream();
-const gzip = zlib.createGzip();
-const gunz = zlib.createGunzip();
-
-inp.pipe(gzip).pipe(gunz).pipe(out);
-
-out.on('data', common.mustCall((c) => {
-  assert.strictEqual(c, inp._hash, `Hash '${c}' equals '${inp._hash}'.`);
-}));
+for (const [ createCompress, createDecompress ] of [
+  [ zlib.createGzip, zlib.createGunzip ],
+  [ zlib.createBrotliCompress, zlib.createBrotliDecompress ],
+]) {
+  const inp = new RandomReadStream({ total: 1024, block: 256, jitter: 16 });
+  const out = new HashStream();
+  const gzip = createCompress();
+  const gunz = createDecompress();
+
+  inp.pipe(gzip).pipe(gunz).pipe(out);
+
+  out.on('data', common.mustCall((c) => {
+    assert.strictEqual(c, inp._hash, `Hash '${c}' equals '${inp._hash}'.`);
+  }));
+}
diff --git a/test/parallel/test-zlib-write-after-flush.js b/test/parallel/test-zlib-write-after-flush.js
@@ -24,22 +24,27 @@ const common = require('../common');
 const assert = require('assert');
 const zlib = require('zlib');
 
-const gzip = zlib.createGzip();
-const gunz = zlib.createUnzip();
+for (const [ createCompress, createDecompress ] of [
+  [ zlib.createGzip, zlib.createGunzip ],
+  [ zlib.createBrotliCompress, zlib.createBrotliDecompress ],
+]) {
+  const gzip = createCompress();
+  const gunz = createDecompress();
 
-gzip.pipe(gunz);
+  gzip.pipe(gunz);
 
-let output = '';
-const input = 'A line of data\n';
-gunz.setEncoding('utf8');
-gunz.on('data', (c) => output += c);
-gunz.on('end', common.mustCall(() => {
-  assert.strictEqual(output, input);
-  assert.strictEqual(gzip._nextFlush, -1);
-}));
+  let output = '';
+  const input = 'A line of data\n';
+  gunz.setEncoding('utf8');
+  gunz.on('data', (c) => output += c);
+  gunz.on('end', common.mustCall(() => {
+    assert.strictEqual(output, input);
+    assert.strictEqual(gzip._nextFlush, -1);
+  }));
 
-// make sure that flush/write doesn't trigger an assert failure
-gzip.flush();
-gzip.write(input);
-gzip.end();
-gunz.read(0);
+  // Make sure that flush/write doesn't trigger an assert failure
+  gzip.flush();
+  gzip.write(input);
+  gzip.end();
+  gunz.read(0);
+}
diff --git a/test/parallel/test-zlib-zero-byte.js b/test/parallel/test-zlib-zero-byte.js
@@ -22,18 +22,22 @@
 'use strict';
 const common = require('../common');
 const assert = require('assert');
-
 const zlib = require('zlib');
-const gz = zlib.Gzip();
-const emptyBuffer = Buffer.alloc(0);
-let received = 0;
-gz.on('data', function(c) {
-  received += c.length;
-});
 
-gz.on('end', common.mustCall(function() {
-  assert.strictEqual(received, 20);
-}));
-gz.on('finish', common.mustCall());
-gz.write(emptyBuffer);
-gz.end();
+for (const Compressor of [ zlib.Gzip, zlib.BrotliCompress ]) {
+  const gz = Compressor();
+  const emptyBuffer = Buffer.alloc(0);
+  let received = 0;
+  gz.on('data', function(c) {
+    received += c.length;
+  });
+
+  gz.on('end', common.mustCall(function() {
+    const expected = Compressor === zlib.Gzip ? 20 : 1;
+    assert.strictEqual(received, expected,
+                       `${received}, ${expected}, ${Compressor.name}`);
+  }));
+  gz.on('finish', common.mustCall());
+  gz.write(emptyBuffer);
+  gz.end();
+}
diff --git a/test/parallel/test-zlib.js b/test/parallel/test-zlib.js
@@ -32,7 +32,8 @@ let zlibPairs = [
   [zlib.Gzip, zlib.Gunzip],
   [zlib.Deflate, zlib.Unzip],
   [zlib.Gzip, zlib.Unzip],
-  [zlib.DeflateRaw, zlib.InflateRaw]
+  [zlib.DeflateRaw, zlib.InflateRaw],
+  [zlib.BrotliCompress, zlib.BrotliDecompress],
 ];
 
 // how fast to trickle through the slowstream