[Fix] bug timeout (p-limit & error handler)

HoangTran0410 · Sep 20, 2021 · ff67b98 · ff67b98
1 parent 09c8cc8
commit ff67b98
Show file tree

Hide file tree

Showing 6 changed files with 72 additions and 50 deletions.
diff --git a/config.js b/config.js
@@ -1,6 +1,7 @@
 // you can modify all the variables below
 export const ACCESS_TOKEN = "YOUR_ACCESS_TOKEN";
-export const WAIT_BEFORE_NEXT_FETCH = 0; // thời gian chờ (ms) trước mỗi lần fetch mới liên tiếp
+export const NUMBER_OF_DOWNLOAD_THREADS = 20; // số lượng luồng tải ảnh/video cùng 1 lúc (càng lớn tải càng nhanh, nhưng dễ nghẽn mạng)
+export const WAIT_BEFORE_NEXT_FETCH = 0; // thời gian chờ (ms) trước mỗi lần fetch tiếp theo
 export const ID_LINK_SEPERATOR = ";";
 export const PHOTO_FILE_FORMAT = "png"; // OR jpg
 export const VIDEO_FILE_FORMAT = "mp4"; // OR wav ?

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -12,6 +12,7 @@
   "license": "ISC",
   "dependencies": {
     "node-fetch": "^3.0.0",
+    "p-limit": "^4.0.0",
     "request": "^2.88.2"
   }
 }
diff --git a/scripts/download_album.js b/scripts/download_album.js
@@ -11,6 +11,7 @@ import {
   createIfNotExistDir,
   deleteFile,
   downloadFileSync,
+  limit,
   myFetch,
   saveToFile,
   sleep,
@@ -25,6 +26,7 @@ const fetchAlbumPhotosFromCursor = async ({ albumId, cursor }) => {
   if (cursor) url += `&after=${cursor}`;
 
   const json = await myFetch(url);
+  if (!json) return null;
 
   // return imgData + next cursor
   return {
@@ -47,9 +49,7 @@ const fetchAlbumPhotos = async ({
   let allImgsData = [];
 
   while (hasNextCursor && currentPage <= pageLimit) {
-    console.log(
-      `Fetching page: ${currentPage}, pageSize: 100...`
-    );
+    console.log(`Fetching page: ${currentPage}, pageSize: 100...`);
 
     const data = await fetchAlbumPhotosFromCursor({
       albumId,
@@ -144,23 +144,23 @@ export const downloadAlbumPhoto = async (albumId) => {
 
         const savePath = `${dir}/${photo_id}.${PHOTO_FILE_FORMAT}`;
         promises.push(
-          downloadFileSync({
-            uri: photo_url,
-            filename: savePath,
-            successCallback: () => {
-              console.log(`> Saved ${savePath}`);
-            },
-            failedCallback: (e) => {
-              console.log(`ERROR while save image ${savePath}`, e.toString());
-            },
-          })
+          limit(() =>
+            downloadFileSync({
+              uri: photo_url,
+              filename: savePath,
+              successCallback: () => {
+                console.log(`> Saved ${savePath}`);
+              },
+              failedCallback: (e) => {
+                console.log(`ERROR while save image ${savePath}`, e.toString());
+              },
+            })
+          )
         );
       }
 
-      try {
-        await Promise.allSettled(promises);
-        console.log(`> Saved ${promises.length} images.`);
-      } catch (e) {}
+      await Promise.allSettled(promises);
+      console.log(`> Saved ${promises.length} images.`);
     },
   });
 };
diff --git a/scripts/download_wall_media.js b/scripts/download_wall_media.js
@@ -11,6 +11,7 @@ import {
   createIfNotExistDir,
   deleteFile,
   downloadFileSync,
+  limit,
   myFetch,
   saveToFile,
 } from "./utils.js";
@@ -216,23 +217,23 @@ export const downloadWallMedia = async ({
         const savePath = `${dir}/${media_id}.${file_format}`;
 
         promises.push(
-          downloadFileSync({
-            uri: media_url,
-            filename: savePath,
-            successCallback: () => {
-              console.log(`> Saved ${savePath}`);
-            },
-            failedCallback: (e) => {
-              console.log(`ERROR while save media ${savePath}`, e.toString());
-            },
-          })
+          limit(() =>
+            downloadFileSync({
+              uri: media_url,
+              filename: savePath,
+              successCallback: () => {
+                console.log(`> Saved ${savePath}`);
+              },
+              failedCallback: (e) => {
+                console.log(`ERROR while save media ${savePath}`, e.toString());
+              },
+            })
+          )
         );
       }
 
-      try {
-        await Promise.allSettled(promises);
-        console.log(`> Saved ${promises.length} media.`);
-      } catch (e) {}
+      await Promise.allSettled(promises);
+      console.log(`> Saved ${promises.length} media.`);
     },
   });
 };
diff --git a/scripts/utils.js b/scripts/utils.js
@@ -1,6 +1,10 @@
 import fs from "fs";
 import fetch from "node-fetch";
 import request from "request";
+import pLimit from "p-limit";
+import { NUMBER_OF_DOWNLOAD_THREADS } from "../config.js";
+
+export const limit = pLimit(NUMBER_OF_DOWNLOAD_THREADS);
 
 export const myFetch = async (_url) => {
   try {
@@ -73,23 +77,25 @@ export const downloadFileSync = async function ({
   failedCallback = () => {},
 }) {
   await new Promise((resolve, reject) => {
-    try {
-      request.head(uri, function (err, res, body) {
-        if (err) {
-          failedCallback(err);
-          reject(err);
-        } else {
-          request({ uri, gzip: true })
-            .pipe(fs.createWriteStream(filename, { flags: "w+" }))
-            .on("close", () => {
-              successCallback();
-              resolve();
-            });
-        }
-      });
-    } catch (e) {
-      reject(e);
-    }
+    request.head(uri, function (err, res, body) {
+      if (err) {
+        failedCallback(err);
+        reject(err);
+      } else {
+        // https://github.com/request/request/issues/636#issuecomment-23030577
+        request
+          .get({ uri, gzip: true, timeout: 5000 })
+          .on("error", function (e) {
+            console.log("ERROR", e.toString());
+            reject(e);
+          })
+          .on("close", () => {
+            successCallback();
+            resolve();
+          })
+          .pipe(fs.createWriteStream(filename, { flags: "w+" }));
+      }
+    });
   });
 };