diff --git a/packages/cli/src/lib/Cli.ts b/packages/cli/src/lib/Cli.ts index 6a6f34f9..3f32f835 100644 --- a/packages/cli/src/lib/Cli.ts +++ b/packages/cli/src/lib/Cli.ts @@ -29,6 +29,7 @@ import {updateConfig} from './cmds/updateConfig'; import {doctor} from './cmds/doctor'; import {merge} from './cmds/merge'; import {fingerprint} from './cmds/fingerprint'; +import {fetchUtils} from '@bubblewrap/core'; export class Cli { async run(args: string[]): Promise { @@ -38,6 +39,10 @@ export class Cli { ' Node.js version 12 or above is required to run bubblewrap.'); } const parsedArgs = minimist(args); + if (parsedArgs.fetchEngine && + (parsedArgs.fetchEngine == 'node-fetch' || parsedArgs.fetchEngine == 'fetch-h2')) { + fetchUtils.setFetchEngine(parsedArgs.fetchEngine); + } const config = await loadOrCreateConfig(undefined, undefined, parsedArgs.config); diff --git a/packages/cli/src/lib/Prompt.ts b/packages/cli/src/lib/Prompt.ts index 833bce8f..40af64bb 100644 --- a/packages/cli/src/lib/Prompt.ts +++ b/packages/cli/src/lib/Prompt.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import {Result, util} from '@bubblewrap/core'; +import {Result, fetchUtils} from '@bubblewrap/core'; import {Presets, Bar} from 'cli-progress'; import {green} from 'colors'; import * as inquirer from 'inquirer'; @@ -193,7 +193,7 @@ export class InquirerPrompt implements Prompt { }, Presets.shades_classic); progressBar.start(Math.round(totalSize / KILOBYTE_SIZE), 0); - await util.downloadFile(url, filename, (current, total) => { + await fetchUtils.downloadFile(url, filename, (current, total) => { if (total > 0 && total !== totalSize) { progressBar.setTotal(Math.round(total / KILOBYTE_SIZE)); totalSize = total; diff --git a/packages/core/package-lock.json b/packages/core/package-lock.json index 9e3f42e6..7437b1e3 100644 --- a/packages/core/package-lock.json +++ b/packages/core/package-lock.json @@ -400,9 +400,9 @@ "integrity": "sha512-tCkE96/ZTO+cWbln2xfyvd6ngHLanvVlJ3e5BeirJ3BYI5GbAyubIrmV4JjjugDly5D9fHjOL5MNsqsCnqwW6g==" }, "@types/node-fetch": { - "version": "2.5.8", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.8.tgz", - "integrity": "sha512-fbjI6ja0N5ZA8TV53RUqzsKNkl9fv8Oj3T7zxW7FGv1GSH7gwJaNF8dzCjrqKaxKeUpTz4yT1DaJFq/omNpGfw==", + "version": "2.5.10", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.10.tgz", + "integrity": "sha512-IpkX0AasN44hgEad0gEF/V6EgR5n69VEqPEgnmoM8GsIGro3PowbWs4tR6IhxUTyPLpOn+fiGG6nrQhcmoCuIQ==", "requires": { "@types/node": "*", "form-data": "^3.0.0" @@ -430,6 +430,11 @@ "@types/node": "*" } }, + "@types/tough-cookie": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz", + "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==" + }, "@types/valid-url": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@types/valid-url/-/valid-url-1.0.3.tgz", @@ -475,6 +480,14 @@ "uri-js": "^4.2.2" } }, + "already": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/already/-/already-1.13.2.tgz", + "integrity": "sha512-GU0ZqMhSetZeDlivqttmAmd2UpCbPSucziaDJcCN2NdOTedzaJTqZZwHHuGJvp0Us1wzQG0vSqFqax1SqgH8Aw==", + "requires": { + "throat": "^5.0.0" + } + }, "ansi-escapes": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.1.tgz", @@ -629,6 +642,11 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, + "callguard": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/callguard/-/callguard-2.0.0.tgz", + "integrity": "sha512-I3nd+fuj20FK1qu00ImrbH+II+8ULS6ioYr9igqR1xyqySoqc3DiHEyUM0mkoAdKeLGg2CtGnO8R3VRQX5krpQ==" + }, "canvas": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.6.1.tgz", @@ -973,6 +991,32 @@ "pend": "~1.2.0" } }, + "fetch-h2": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/fetch-h2/-/fetch-h2-2.5.1.tgz", + "integrity": "sha512-U+LQ+fHwF6TMg82A88wjljC5L174eoJfrc+0g4e7JWqL7U0w0QAoOkPDCGkO9KGH9BY55s4n45gLGOtlTAoqmw==", + "requires": { + "@types/tough-cookie": "^4.0.0", + "already": "^1.13.1", + "callguard": "^2.0.0", + "get-stream": "^6.0.0", + "through2": "^4.0.2", + "to-arraybuffer": "^1.0.1", + "tough-cookie": "^4.0.0" + }, + "dependencies": { + "tough-cookie": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz", + "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==", + "requires": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.1.2" + } + } + } + }, "figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -1062,6 +1106,11 @@ } } }, + "get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==" + }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -2114,11 +2163,36 @@ } } }, + "throat": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/throat/-/throat-5.0.0.tgz", + "integrity": "sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA==" + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, + "through2": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "requires": { + "readable-stream": "3" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "timm": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/timm/-/timm-1.6.2.tgz", @@ -2137,6 +2211,11 @@ "os-tmpdir": "~1.0.2" } }, + "to-arraybuffer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz", + "integrity": "sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=" + }, "tough-cookie": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz", @@ -2191,6 +2270,11 @@ "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" + }, "uri-js": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.0.tgz", diff --git a/packages/core/package.json b/packages/core/package.json index d58c5cff..77310384 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -36,16 +36,17 @@ "@types/lodash.template": "^4.5.0", "@types/mime-types": "^2.1.0", "@types/node": "^12.20.1", - "@types/node-fetch": "^2.5.8", + "@types/node-fetch": "^2.5.10", "@types/tar": "^4.0.4", "@types/valid-url": "^1.0.3", "color": "^3.1.3", "extract-zip": "^1.7.0", + "fetch-h2": "^2.5.1", "inquirer": "^7.3.3", "jimp": "^0.14.0", "lodash.template": "^4.5.0", "mime-types": "^2.1.28", - "node-fetch": "^2.6.0", + "node-fetch": "^2.6.1", "svg2img": "^0.8.1", "tar": "^6.1.0", "valid-url": "^1.0.9" diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2f4885f0..413e0fcc 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -29,12 +29,14 @@ import {TwaGenerator} from './lib/TwaGenerator'; import {DigitalAssetLinks} from './lib/DigitalAssetLinks'; import * as util from './lib/util'; import {Result} from './lib/Result'; +import {fetchUtils} from './lib/FetchUtils'; export { AndroidSdkTools, BufferedLog, Config, DigitalAssetLinks, + fetchUtils, Fingerprint, GradleWrapper, JarSigner, diff --git a/packages/core/src/lib/FetchUtils.ts b/packages/core/src/lib/FetchUtils.ts new file mode 100644 index 00000000..4d7e6477 --- /dev/null +++ b/packages/core/src/lib/FetchUtils.ts @@ -0,0 +1,93 @@ +/* + * Copyright 2021 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as fs from 'fs'; +import {fetch as fetchh2, Response as FetchH2Response} from 'fetch-h2'; +import nodefetch, {Response as NodeFetchResponse} from 'node-fetch'; + +export type FetchEngine = 'node-fetch' | 'fetch-h2'; +const DEFAULT_FETCH_ENGINE: FetchEngine = 'fetch-h2'; + +export type NodeFetchOrFetchH2Response = FetchH2Response | NodeFetchResponse; + +class FetchUtils { + private fetchEngine = DEFAULT_FETCH_ENGINE; + + setFetchEngine(newFetchEngine: FetchEngine): void { + this.fetchEngine = newFetchEngine; + } + + async fetch(input: string): Promise { + if (this.fetchEngine == 'node-fetch') { + return await nodefetch(input, {redirect: 'follow'}); + } else { + return await fetchh2(input, {redirect: 'follow'}); + } + } + + /** + * Downloads a file from `url` and saves it to `path`. If a `progressCallback` function is passed, it + * will be invoked for every chunk received. If the value of `total` parameter is -1, it means we + * were unable to determine to total file size before starting the download. + */ + async downloadFile( + url: string, + path: string, + progressCallback?: (current: number, total: number) => void, + ): Promise { + let result; + let readableStream: NodeJS.ReadableStream; + + if (this.fetchEngine === 'node-fetch') { + result = await nodefetch(url); + readableStream = result.body; + } else { + result = await fetchh2(url, {redirect: 'follow'}); + readableStream = await result.readable(); + } + + // Try to determine the file size via the `Content-Length` header. This may not be available + // for all cases. + const contentLength = result.headers.get('content-length'); + const fileSize = contentLength ? parseInt(contentLength) : -1; + + const fileStream = fs.createWriteStream(path); + let received = 0; + + await new Promise((resolve, reject) => { + readableStream.pipe(fileStream); + + // Even though we're piping the chunks, we intercept them to check for the download progress. + if (progressCallback) { + readableStream.on('data', (chunk) => { + received = received + chunk.length; + progressCallback(received, fileSize); + }); + } + + readableStream.on('error', (err) => { + reject(err); + }); + + fileStream.on('finish', () => { + resolve(); + }); + }); + } +} + +const fetchUtils = new FetchUtils(); +export {fetchUtils}; diff --git a/packages/core/src/lib/ImageHelper.ts b/packages/core/src/lib/ImageHelper.ts index 04166576..6ab914cf 100644 --- a/packages/core/src/lib/ImageHelper.ts +++ b/packages/core/src/lib/ImageHelper.ts @@ -17,7 +17,7 @@ import * as path from 'path'; import * as fs from 'fs'; import * as Jimp from 'jimp'; -import fetch from 'node-fetch'; +import {fetchUtils} from './FetchUtils'; import Color = require('color'); import {promisify} from 'util'; import {svg2img} from './wrappers/svg2img'; @@ -109,7 +109,8 @@ export class ImageHelper { * @returns an Object containing the original URL and the icon image data. */ async fetchIcon(iconUrl: string): Promise { - const response = await fetch(iconUrl); + const response = await fetchUtils.fetch(iconUrl); + if (response.status !== 200) { throw new Error( `Failed to download icon ${iconUrl}. Responded with status ${response.status}`); @@ -120,14 +121,13 @@ export class ImageHelper { throw new Error(`Received icon "${iconUrl}" with invalid Content-Type.` + ` Responded with Content-Type "${contentType}"`); } - let body; - body = await response.buffer(); + let body = await response.arrayBuffer(); if (contentType.startsWith('image/svg')) { body = await svg2img(iconUrl); } return { url: iconUrl, - data: await Jimp.read(body), + data: await Jimp.read(Buffer.from(body)), }; } } diff --git a/packages/core/src/lib/TwaGenerator.ts b/packages/core/src/lib/TwaGenerator.ts index 49e482ff..836f5ef3 100644 --- a/packages/core/src/lib/TwaGenerator.ts +++ b/packages/core/src/lib/TwaGenerator.ts @@ -17,7 +17,6 @@ import * as path from 'path'; import * as fs from 'fs'; import * as Color from 'color'; -import fetch from 'node-fetch'; import {template} from 'lodash'; import {promisify} from 'util'; import {TwaManifest} from './TwaManifest'; @@ -26,6 +25,7 @@ import {Log} from './Log'; import {ImageHelper, IconDefinition} from './ImageHelper'; import {FeatureManager} from './features/FeatureManager'; import {rmdir, escapeJsonString, toAndroidScreenOrientation} from './util'; +import {fetchUtils} from './FetchUtils'; const COPY_FILE_LIST = [ 'settings.gradle', @@ -266,7 +266,7 @@ export class TwaGenerator { 'Unable to write the Web Manifest. The TWA Manifest does not have a webManifestUrl'); } - const response = await fetch(twaManifest.webManifestUrl); + const response = await fetchUtils.fetch(twaManifest.webManifestUrl.toString()); if (response.status !== 200) { throw new Error(`Failed to download Web Manifest ${twaManifest.webManifestUrl}.` + `Responded with status ${response.status}`); diff --git a/packages/core/src/lib/TwaManifest.ts b/packages/core/src/lib/TwaManifest.ts index d7301058..6c109edf 100644 --- a/packages/core/src/lib/TwaManifest.ts +++ b/packages/core/src/lib/TwaManifest.ts @@ -17,7 +17,7 @@ 'use strict'; import * as fs from 'fs'; -import fetch from 'node-fetch'; +import {fetchUtils} from './FetchUtils'; import {findSuitableIcon, generatePackageId, validateNotEmpty} from './util'; import Color = require('color'); import {ConsoleLog} from './Log'; @@ -354,7 +354,7 @@ export class TwaManifest { * @returns {TwaManifest} */ static async fromWebManifest(url: string): Promise { - const response = await fetch(url); + const response = await fetchUtils.fetch(url); const webManifest = await response.json(); const webManifestUrl: URL = new URL(url); return TwaManifest.fromWebManifestJson(webManifestUrl, webManifest); diff --git a/packages/core/src/lib/util.ts b/packages/core/src/lib/util.ts index ed46cbbb..fe36b851 100644 --- a/packages/core/src/lib/util.ts +++ b/packages/core/src/lib/util.ts @@ -15,7 +15,6 @@ */ import * as extractZip from 'extract-zip'; -import fetch from 'node-fetch'; import * as fs from 'fs'; import {join} from 'path'; import {promisify} from 'util'; @@ -24,6 +23,7 @@ import {x as extractTar} from 'tar'; import {WebManifestIcon, WebManifestJson} from './types/WebManifest'; import {Log} from './Log'; import {Orientation} from './TwaManifest'; +import {fetchUtils} from './FetchUtils'; const execPromise = promisify(exec); const execFilePromise = promisify(execFile); @@ -52,43 +52,6 @@ export async function executeFile( return await execFilePromise(cmd, args, {env: env, cwd: cwd}); } -/** - * Downloads a file from `url` and saves it to `path`. If a `progressCallback` function is passed, it - * will be invoked for every chunk received. If the value of `total` parameter is -1, it means we - * were unable to determine to total file size before starting the download. - */ -export async function downloadFile(url: string, path: string, - progressCallback?: (current: number, total: number) => void): Promise { - const result = await fetch(url); - - // Try to determine the file size via the `Content-Length` header. This may not be available - // for all cases. - const contentLength = result.headers.get('content-length'); - const fileSize = contentLength ? parseInt(contentLength) : -1; - - const fileStream = fs.createWriteStream(path); - let received = 0; - - await new Promise((resolve, reject) => { - result.body.pipe(fileStream); - - // Even though we're piping the chunks, we intercept them to check for the download progress. - if (progressCallback) { - result.body.on('data', (chunk) => { - received = received + chunk.length; - progressCallback(received, fileSize); - }); - } - - result.body.on('error', (err) => { - reject(err); - }); - fileStream.on('finish', () => { - resolve(); - }); - }); -} - export async function unzipFile( zipFile: string, destinationPath: string, deleteZipWhenDone = false): Promise { await extractZipPromise(zipFile, {dir: destinationPath}); @@ -265,13 +228,13 @@ export async function rmdir(path: string): Promise { }; /** - * Given a web manifest's URL, the function retrns the web manifest. - * - * @param {URL} webManifestUrl the URL where the webManifest is available. - * @returns {Promise { - const response = await fetch(webManifestUrl); + const response = await fetchUtils.fetch(webManifestUrl.toString()); if (response.status !== 200) { throw new Error(`Failed to download Web Manifest ${webManifestUrl}. ` + `Responded with status ${response.status}`); @@ -280,7 +243,7 @@ export async function getWebManifest(webManifestUrl: URL): Promise