From ff67b9896362f60589ff95b06aa9d1505254d406 Mon Sep 17 00:00:00 2001 From: Hoang Date: Tue, 21 Sep 2021 02:26:08 +0700 Subject: [PATCH] [Fix] bug timeout (p-limit & error handler) --- config.js | 3 ++- package-lock.json | 15 ++++++++++++- package.json | 1 + scripts/download_album.js | 34 ++++++++++++++--------------- scripts/download_wall_media.js | 29 ++++++++++++------------ scripts/utils.js | 40 +++++++++++++++++++--------------- 6 files changed, 72 insertions(+), 50 deletions(-) diff --git a/config.js b/config.js index 12f1c51..2de8519 100644 --- a/config.js +++ b/config.js @@ -1,6 +1,7 @@ // you can modify all the variables below export const ACCESS_TOKEN = "YOUR_ACCESS_TOKEN"; -export const WAIT_BEFORE_NEXT_FETCH = 0; // thời gian chờ (ms) trước mỗi lần fetch mới liên tiếp +export const NUMBER_OF_DOWNLOAD_THREADS = 20; // số lượng luồng tải ảnh/video cùng 1 lúc (càng lớn tải càng nhanh, nhưng dễ nghẽn mạng) +export const WAIT_BEFORE_NEXT_FETCH = 0; // thời gian chờ (ms) trước mỗi lần fetch tiếp theo export const ID_LINK_SEPERATOR = ";"; export const PHOTO_FILE_FORMAT = "png"; // OR jpg export const VIDEO_FILE_FORMAT = "mp4"; // OR wav ? diff --git a/package-lock.json b/package-lock.json index 3526c31..752ed58 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "FBAlbumDownloader", + "name": "FBMediaDownloader", "version": "1.0.0", "lockfileVersion": 1, "requires": true, @@ -239,6 +239,14 @@ "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, + "p-limit": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", + "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", + "requires": { + "yocto-queue": "^1.0.0" + } + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -361,6 +369,11 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.1.1.tgz", "integrity": "sha512-Czi3fG883e96T4DLEPRvufrF2ydhOOW1+1a6c3gNjH2aIh50DNFBdfwh2AKoOf1rXvpvavAoA11Qdq9+BKjE0Q==" + }, + "yocto-queue": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==" } } } diff --git a/package.json b/package.json index 91f9251..6813e53 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "license": "ISC", "dependencies": { "node-fetch": "^3.0.0", + "p-limit": "^4.0.0", "request": "^2.88.2" } } diff --git a/scripts/download_album.js b/scripts/download_album.js index 133323c..3d02f43 100644 --- a/scripts/download_album.js +++ b/scripts/download_album.js @@ -11,6 +11,7 @@ import { createIfNotExistDir, deleteFile, downloadFileSync, + limit, myFetch, saveToFile, sleep, @@ -25,6 +26,7 @@ const fetchAlbumPhotosFromCursor = async ({ albumId, cursor }) => { if (cursor) url += `&after=${cursor}`; const json = await myFetch(url); + if (!json) return null; // return imgData + next cursor return { @@ -47,9 +49,7 @@ const fetchAlbumPhotos = async ({ let allImgsData = []; while (hasNextCursor && currentPage <= pageLimit) { - console.log( - `Fetching page: ${currentPage}, pageSize: 100...` - ); + console.log(`Fetching page: ${currentPage}, pageSize: 100...`); const data = await fetchAlbumPhotosFromCursor({ albumId, @@ -144,23 +144,23 @@ export const downloadAlbumPhoto = async (albumId) => { const savePath = `${dir}/${photo_id}.${PHOTO_FILE_FORMAT}`; promises.push( - downloadFileSync({ - uri: photo_url, - filename: savePath, - successCallback: () => { - console.log(`> Saved ${savePath}`); - }, - failedCallback: (e) => { - console.log(`ERROR while save image ${savePath}`, e.toString()); - }, - }) + limit(() => + downloadFileSync({ + uri: photo_url, + filename: savePath, + successCallback: () => { + console.log(`> Saved ${savePath}`); + }, + failedCallback: (e) => { + console.log(`ERROR while save image ${savePath}`, e.toString()); + }, + }) + ) ); } - try { - await Promise.allSettled(promises); - console.log(`> Saved ${promises.length} images.`); - } catch (e) {} + await Promise.allSettled(promises); + console.log(`> Saved ${promises.length} images.`); }, }); }; diff --git a/scripts/download_wall_media.js b/scripts/download_wall_media.js index 29572bd..2ed2f73 100644 --- a/scripts/download_wall_media.js +++ b/scripts/download_wall_media.js @@ -11,6 +11,7 @@ import { createIfNotExistDir, deleteFile, downloadFileSync, + limit, myFetch, saveToFile, } from "./utils.js"; @@ -216,23 +217,23 @@ export const downloadWallMedia = async ({ const savePath = `${dir}/${media_id}.${file_format}`; promises.push( - downloadFileSync({ - uri: media_url, - filename: savePath, - successCallback: () => { - console.log(`> Saved ${savePath}`); - }, - failedCallback: (e) => { - console.log(`ERROR while save media ${savePath}`, e.toString()); - }, - }) + limit(() => + downloadFileSync({ + uri: media_url, + filename: savePath, + successCallback: () => { + console.log(`> Saved ${savePath}`); + }, + failedCallback: (e) => { + console.log(`ERROR while save media ${savePath}`, e.toString()); + }, + }) + ) ); } - try { - await Promise.allSettled(promises); - console.log(`> Saved ${promises.length} media.`); - } catch (e) {} + await Promise.allSettled(promises); + console.log(`> Saved ${promises.length} media.`); }, }); }; diff --git a/scripts/utils.js b/scripts/utils.js index 2ca3275..01aad9f 100644 --- a/scripts/utils.js +++ b/scripts/utils.js @@ -1,6 +1,10 @@ import fs from "fs"; import fetch from "node-fetch"; import request from "request"; +import pLimit from "p-limit"; +import { NUMBER_OF_DOWNLOAD_THREADS } from "../config.js"; + +export const limit = pLimit(NUMBER_OF_DOWNLOAD_THREADS); export const myFetch = async (_url) => { try { @@ -73,23 +77,25 @@ export const downloadFileSync = async function ({ failedCallback = () => {}, }) { await new Promise((resolve, reject) => { - try { - request.head(uri, function (err, res, body) { - if (err) { - failedCallback(err); - reject(err); - } else { - request({ uri, gzip: true }) - .pipe(fs.createWriteStream(filename, { flags: "w+" })) - .on("close", () => { - successCallback(); - resolve(); - }); - } - }); - } catch (e) { - reject(e); - } + request.head(uri, function (err, res, body) { + if (err) { + failedCallback(err); + reject(err); + } else { + // https://github.com/request/request/issues/636#issuecomment-23030577 + request + .get({ uri, gzip: true, timeout: 5000 }) + .on("error", function (e) { + console.log("ERROR", e.toString()); + reject(e); + }) + .on("close", () => { + successCallback(); + resolve(); + }) + .pipe(fs.createWriteStream(filename, { flags: "w+" })); + } + }); }); };