Improved handling of rate limit and processing a large amount of files

.circleci/config.yml

@@ -9,7 +9,7 @@ executors:
       - image: cimg/base:2021.04
   macos:
     macos:
-      xcode: 12
+      xcode: 13.4.1
   windows: win/default
 
 # Matrices work with jobs that take parameters

cli.js

@@ -7,6 +7,7 @@ const { prompts } = require('prompts')
 const cliProgress = require('cli-progress');
 const fs = require('fs');
 const { version } = require('./lib/client');
+const Queue = require('better-queue');
 
 /* first - parse the main command */
 
@@ -20,7 +21,8 @@ const multibar = new cliProgress.MultiBar({
 	barCompleteChar: '\u2588',
 	barIncompleteChar: '\u2591',
 	clearOnComplete: false,
-	stopOnComplete: true
+	stopOnComplete: true,
+	forceRedraw: true
 });
 
 if (Object.entries( mainOptions).length === 0 || mainOptions._unknown && (mainOptions._unknown[0] === '--help' || mainOptions._unknown[0] === '-h')) {
@@ -96,10 +98,25 @@ if (mainOptions.commandOrFiles && mainOptions.commandOrFiles[0] === 'zip2png') {
 
 
 function invokeRemovebg(mainOptions, removebgOptions, expandedInputPaths) {
+	let batchSize = 5;
+	const queue = new Queue(function (batch, cb) {
+		let done = 0;
+		batch.forEach(inputPath => {
+			var bar = multibar.create(100, 0, {file: inputPath, message: 'Processing:'})
+			removebg(inputPath, removebgOptions, bar, function () {
+				done ++;
+				if (done == batchSize) {
+					cb();
+				}
+			});
+		})
+	}, { batchSize: batchSize });
+
 	expandedInputPaths.forEach(inputPath => {
-		var bar = multibar.create(100, 0, {file: inputPath, message: 'Processing:'})
-		removebg(inputPath, removebgOptions, bar);
+		queue.push(inputPath);
 	})
+
+
 }
 
 

lib/functions.js

@@ -5,6 +5,7 @@ const fs = require('fs');
 const superagent = require('superagent');
 const path = require('path');
 const { userAgent } = require('./client');
+const { decodeBitmap } = require('ag-psd/dist/helpers');
 
 const defaultOutputExtension = ".png";
 
@@ -69,17 +70,17 @@ function zip2png(binary, params) {
 	})
 }
 
-function removebg(file, params, bar) {
+function removebg(file, params, bar, cb) {
 	if (bar) bar.update(5);
 	const inputPath = file;
 
-	processRemovebg(params, inputPath);
+	processRemovebg(params, inputPath, cb);
 
-	function processRemovebg(params, inputPath) {
+	function processRemovebg(params, inputPath, cb) {
 
 		var headers = {
 			'X-Api-Key': params['api-key'],
-			'User-Agent': userAgent(),
+			'User-Agent': userAgent()
 		}
 
 		var bgInputStream, bgBasename;
@@ -90,6 +91,7 @@ function removebg(file, params, bar) {
 						bar.stop();
 				}
 				console.error(`bg_image_file ${params['bg-image-file']} doesn't exist`);
+				cb();
 				return;
 			} else {
 				bgInputStream = fs.createReadStream(params['bg-image-file']);
@@ -103,6 +105,7 @@ function removebg(file, params, bar) {
 					bar.stop();
 			}
 			console.error(`input file ${inputPath} doesn't exist`);
+			cb();
 			return;
 		}
 
@@ -120,6 +123,7 @@ function removebg(file, params, bar) {
 				bar.update(0, { message: 'Skipped:' });
 				bar.stop();
 			}
+			cb();
 			return;
 		}
 
@@ -155,9 +159,6 @@ function removebg(file, params, bar) {
 				if (bar && progress) bar.update(progress); // 40-70
 			})
 			.then(res => {
-				if (res.statusCode == 429) { // rate limit reached - retry after 'Retry-After' seconds
-					setTimeout(processRemovebg(params, inputPath), res.headers['Retry-After'] * 1000);
-				}
 				if (res.statusCode != 200) return console.error('Error:', response.status, response.statusText);
 
 				if (bar) bar.update(70, { file: `${inputPath} -> ${outFilePath}` });
@@ -176,14 +177,26 @@ function removebg(file, params, bar) {
 					fs.writeFileSync(outFilePath, res.body);
 					if (bar) bar.update(100, { message: 'Processed:' });
 				}
+				cb();
 			}).catch(e => {
+				if (e.status == 429) { // rate limit reached - retry after 'Retry-After' seconds
+					let timeout = parseInt(e.response.headers['retry-after']) * 1000;
+					setTimeout(() => { 
+						if (bar) bar.update(0, { message: `Retrying in ${timeout}ms: ` });
+						processRemovebg(params, inputPath, cb)
+					}, timeout);
+					return;
+				}
 				var errorText = 'Processing Error';
 				try {
 					errorText += ` '${e.response.body.errors[0].code}': "${e.response.body.errors[0].title} - ${e.response.body.errors[0].detail}"`;
-				} catch (e) { }
+				} catch (err) {
+					errorText += ` ${e.toString()}`;
+				}
 
 				if (bar) bar.update(100, { message: errorText });
 				if (bar) bar.stop();
+				cb();
 			});
 	}
 }

lib/options.js

@@ -115,6 +115,7 @@ function validateRemovebgOptions(removebgOptions) {
 	removebgOptions.size = removebgOptions.size || "auto";
 	removebgOptions.type = removebgOptions.type || "auto";
 	removebgOptions.channels = removebgOptions.channels || "rgba";
+	removebgOptions.format = removebgOptions.format || 'png';
 
 	if (removebgOptions['skip-png-format-optimization'] && removebgOptions.format == 'png') {
 		removebgOptions.transferFormat = 'zip';

package.json

@@ -3,9 +3,9 @@
 	"description": "Remove image backgrounds with remove.bg using the command line",
 	"repository": "https://github.com/remove-bg/remove-bg-cli",
 	"author": "Kaleido AI GmbH",
-	"email" : "[email protected]",
+	"email": "[email protected]",
 	"private": true,
-	"version": "2.0.0",
+	"version": "2.0.1",
 	"engines": {
 		"node": "14.18.1"
 	},
@@ -16,6 +16,7 @@
 	},
 	"dependencies": {
 		"ag-psd": "^14.3.2",
+		"better-queue": "^3.8.10",
 		"canvas": "^2.8.0",
 		"canvas-exif-orientation": "^0.4.0",
 		"cli-progress": "^3.9.1",