diff --git a/.circleci/config.yml b/.circleci/config.yml index 70f0ec9..2862ddb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ executors: - image: cimg/base:2021.04 macos: macos: - xcode: 12 + xcode: 13.4.1 windows: win/default # Matrices work with jobs that take parameters diff --git a/cli.js b/cli.js index a6d71db..59c3a1e 100755 --- a/cli.js +++ b/cli.js @@ -7,6 +7,7 @@ const { prompts } = require('prompts') const cliProgress = require('cli-progress'); const fs = require('fs'); const { version } = require('./lib/client'); +const Queue = require('better-queue'); /* first - parse the main command */ @@ -20,7 +21,8 @@ const multibar = new cliProgress.MultiBar({ barCompleteChar: '\u2588', barIncompleteChar: '\u2591', clearOnComplete: false, - stopOnComplete: true + stopOnComplete: true, + forceRedraw: true }); if (Object.entries( mainOptions).length === 0 || mainOptions._unknown && (mainOptions._unknown[0] === '--help' || mainOptions._unknown[0] === '-h')) { @@ -96,10 +98,25 @@ if (mainOptions.commandOrFiles && mainOptions.commandOrFiles[0] === 'zip2png') { function invokeRemovebg(mainOptions, removebgOptions, expandedInputPaths) { + let batchSize = 5; + const queue = new Queue(function (batch, cb) { + let done = 0; + batch.forEach(inputPath => { + var bar = multibar.create(100, 0, {file: inputPath, message: 'Processing:'}) + removebg(inputPath, removebgOptions, bar, function () { + done ++; + if (done == batchSize) { + cb(); + } + }); + }) + }, { batchSize: batchSize }); + expandedInputPaths.forEach(inputPath => { - var bar = multibar.create(100, 0, {file: inputPath, message: 'Processing:'}) - removebg(inputPath, removebgOptions, bar); + queue.push(inputPath); }) + + } diff --git a/lib/functions.js b/lib/functions.js index 6a9553b..ebc0b6a 100644 --- a/lib/functions.js +++ b/lib/functions.js @@ -5,6 +5,7 @@ const fs = require('fs'); const superagent = require('superagent'); const path = require('path'); const { userAgent } = require('./client'); +const { decodeBitmap } = require('ag-psd/dist/helpers'); const defaultOutputExtension = ".png"; @@ -69,17 +70,17 @@ function zip2png(binary, params) { }) } -function removebg(file, params, bar) { +function removebg(file, params, bar, cb) { if (bar) bar.update(5); const inputPath = file; - processRemovebg(params, inputPath); + processRemovebg(params, inputPath, cb); - function processRemovebg(params, inputPath) { + function processRemovebg(params, inputPath, cb) { var headers = { 'X-Api-Key': params['api-key'], - 'User-Agent': userAgent(), + 'User-Agent': userAgent() } var bgInputStream, bgBasename; @@ -90,6 +91,7 @@ function removebg(file, params, bar) { bar.stop(); } console.error(`bg_image_file ${params['bg-image-file']} doesn't exist`); + cb(); return; } else { bgInputStream = fs.createReadStream(params['bg-image-file']); @@ -103,6 +105,7 @@ function removebg(file, params, bar) { bar.stop(); } console.error(`input file ${inputPath} doesn't exist`); + cb(); return; } @@ -120,6 +123,7 @@ function removebg(file, params, bar) { bar.update(0, { message: 'Skipped:' }); bar.stop(); } + cb(); return; } @@ -155,9 +159,6 @@ function removebg(file, params, bar) { if (bar && progress) bar.update(progress); // 40-70 }) .then(res => { - if (res.statusCode == 429) { // rate limit reached - retry after 'Retry-After' seconds - setTimeout(processRemovebg(params, inputPath), res.headers['Retry-After'] * 1000); - } if (res.statusCode != 200) return console.error('Error:', response.status, response.statusText); if (bar) bar.update(70, { file: `${inputPath} -> ${outFilePath}` }); @@ -176,14 +177,26 @@ function removebg(file, params, bar) { fs.writeFileSync(outFilePath, res.body); if (bar) bar.update(100, { message: 'Processed:' }); } + cb(); }).catch(e => { + if (e.status == 429) { // rate limit reached - retry after 'Retry-After' seconds + let timeout = parseInt(e.response.headers['retry-after']) * 1000; + setTimeout(() => { + if (bar) bar.update(0, { message: `Retrying in ${timeout}ms: ` }); + processRemovebg(params, inputPath, cb) + }, timeout); + return; + } var errorText = 'Processing Error'; try { errorText += ` '${e.response.body.errors[0].code}': "${e.response.body.errors[0].title} - ${e.response.body.errors[0].detail}"`; - } catch (e) { } + } catch (err) { + errorText += ` ${e.toString()}`; + } if (bar) bar.update(100, { message: errorText }); if (bar) bar.stop(); + cb(); }); } } diff --git a/lib/options.js b/lib/options.js index a09e695..47838f8 100644 --- a/lib/options.js +++ b/lib/options.js @@ -115,6 +115,7 @@ function validateRemovebgOptions(removebgOptions) { removebgOptions.size = removebgOptions.size || "auto"; removebgOptions.type = removebgOptions.type || "auto"; removebgOptions.channels = removebgOptions.channels || "rgba"; + removebgOptions.format = removebgOptions.format || 'png'; if (removebgOptions['skip-png-format-optimization'] && removebgOptions.format == 'png') { removebgOptions.transferFormat = 'zip'; diff --git a/package-lock.json b/package-lock.json index 6d35e5f..70c6519 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,15 @@ { "name": "remove-bg-cli", + "version": "2.0.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "remove-bg-cli", + "version": "2.0.1", "dependencies": { "ag-psd": "^14.3.2", + "better-queue": "^3.8.10", "canvas": "^2.8.0", "canvas-exif-orientation": "^0.4.0", "cli-progress": "^3.9.1", @@ -28,7 +31,7 @@ "jest": "^27.4.2" }, "engines": { - "node": ">=12" + "node": "14.18.1" } }, "node_modules/@babel/code-frame": { @@ -1399,6 +1402,21 @@ } ] }, + "node_modules/better-queue": { + "version": "3.8.10", + "resolved": "https://registry.npmjs.org/better-queue/-/better-queue-3.8.10.tgz", + "integrity": "sha512-e3gwNZgDCnNWl0An0Tz6sUjKDV9m6aB+K9Xg//vYeo8+KiH8pWhLFxkawcXhm6FpM//GfD9IQv/kmvWCAVVpKA==", + "dependencies": { + "better-queue-memory": "^1.0.1", + "node-eta": "^0.9.0", + "uuid": "^3.0.0" + } + }, + "node_modules/better-queue-memory": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/better-queue-memory/-/better-queue-memory-1.0.4.tgz", + "integrity": "sha512-SWg5wFIShYffEmJpI6LgbL8/3Dqhku7xI1oEiy6FroP9DbcZlG0ZDjxvPdP9t7hTGW40IpIcC6zVoGT1oxjOuA==" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -3707,6 +3725,11 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, + "node_modules/node-eta": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/node-eta/-/node-eta-0.9.0.tgz", + "integrity": "sha512-mTCTZk29tmX1OGfVkPt63H3c3VqXrI2Kvua98S7iUIB/Gbp0MNw05YtUomxQIxnnKMyRIIuY9izPcFixzhSBrA==" + }, "node_modules/node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -4691,6 +4714,15 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, + "node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } + }, "node_modules/v8-to-istanbul": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-8.1.0.tgz", @@ -6063,6 +6095,21 @@ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, + "better-queue": { + "version": "3.8.10", + "resolved": "https://registry.npmjs.org/better-queue/-/better-queue-3.8.10.tgz", + "integrity": "sha512-e3gwNZgDCnNWl0An0Tz6sUjKDV9m6aB+K9Xg//vYeo8+KiH8pWhLFxkawcXhm6FpM//GfD9IQv/kmvWCAVVpKA==", + "requires": { + "better-queue-memory": "^1.0.1", + "node-eta": "^0.9.0", + "uuid": "^3.0.0" + } + }, + "better-queue-memory": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/better-queue-memory/-/better-queue-memory-1.0.4.tgz", + "integrity": "sha512-SWg5wFIShYffEmJpI6LgbL8/3Dqhku7xI1oEiy6FroP9DbcZlG0ZDjxvPdP9t7hTGW40IpIcC6zVoGT1oxjOuA==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -7839,6 +7886,11 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, + "node-eta": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/node-eta/-/node-eta-0.9.0.tgz", + "integrity": "sha512-mTCTZk29tmX1OGfVkPt63H3c3VqXrI2Kvua98S7iUIB/Gbp0MNw05YtUomxQIxnnKMyRIIuY9izPcFixzhSBrA==" + }, "node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -8562,6 +8614,11 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + }, "v8-to-istanbul": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-8.1.0.tgz", diff --git a/package.json b/package.json index 817db55..461af12 100644 --- a/package.json +++ b/package.json @@ -3,9 +3,9 @@ "description": "Remove image backgrounds with remove.bg using the command line", "repository": "https://github.com/remove-bg/remove-bg-cli", "author": "Kaleido AI GmbH", - "email" : "hello@kaleido.ai", + "email": "hello@kaleido.ai", "private": true, - "version": "2.0.0", + "version": "2.0.1", "engines": { "node": "14.18.1" }, @@ -16,6 +16,7 @@ }, "dependencies": { "ag-psd": "^14.3.2", + "better-queue": "^3.8.10", "canvas": "^2.8.0", "canvas-exif-orientation": "^0.4.0", "cli-progress": "^3.9.1",