Added maxConcurrency config to set maximum concurrent parallel requests.

README.md

@@ -89,6 +89,8 @@ type Config = {
   maxFileSize?: number;
   /** Optional maximum number tokens to include in the output file */
   maxTokens?: number;
+   /** Optional - Maximum concurent parellel requets at a time */
+  maxConcurrency?: number;
 };
 ```
 

config.ts

@@ -25,5 +25,5 @@ export const defaultConfig: Config = {
   url: "https://www.builder.io/c/docs/developers",
   match: "https://www.builder.io/c/docs/**",
   maxPagesToCrawl: 50,
-  outputFileName: "output.json",
+  maxConcurrency: 1,
 };

src/config.ts

@@ -71,6 +71,13 @@ export const configSchema = z.object({
    * @example 5000
    */
   maxTokens: z.number().int().positive().optional(),
+    /** 
+     * **Optional:** 
+     * maxConcurrency 
+     * description: ( 0 = Unlimited, Doesn't stop until cancelled, undefined = max parellel requests possible ) 
+     * @default 1
+     * */
+    maxConcurrency: z.number().int().nonnegative().optional(),
 });
 
 export type Config = z.infer<typeof configSchema>;

src/core.ts

@@ -109,11 +109,7 @@ export async function crawl(config: Config) {
           globs:
             typeof config.match === "string" ? [config.match] : config.match,
         });
-      },
-      // Comment this option to scrape the full website.
-      maxRequestsPerCrawl: config.maxPagesToCrawl,
-      // Uncomment this option to see the browser window.
-      // headless: false,
+      maxConcurrency: config.maxConcurrency || 1 , // Set the max concurrency
       preNavigationHooks: [
         // Abort requests for certain resource types
         async ({ page, log }) => {