diff --git a/README.md b/README.md index 0111b472..8b08f997 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,8 @@ type Config = { maxFileSize?: number; /** Optional maximum number tokens to include in the output file */ maxTokens?: number; + /** Optional - Maximum concurent parellel requets at a time */ + maxConcurrency?: number; }; ``` diff --git a/config.ts b/config.ts index 07a3bcf6..dd658a6b 100644 --- a/config.ts +++ b/config.ts @@ -25,5 +25,5 @@ export const defaultConfig: Config = { url: "https://www.builder.io/c/docs/developers", match: "https://www.builder.io/c/docs/**", maxPagesToCrawl: 50, - outputFileName: "output.json", + maxConcurrency: 1, }; diff --git a/src/config.ts b/src/config.ts index 7ad3844e..848dcbeb 100644 --- a/src/config.ts +++ b/src/config.ts @@ -71,6 +71,13 @@ export const configSchema = z.object({ * @example 5000 */ maxTokens: z.number().int().positive().optional(), + /** + * **Optional:** + * maxConcurrency + * description: ( 0 = Unlimited, Doesn't stop until cancelled, undefined = max parellel requests possible ) + * @default 1 + * */ + maxConcurrency: z.number().int().nonnegative().optional(), }); export type Config = z.infer; diff --git a/src/core.ts b/src/core.ts index be010c6a..5eccb720 100644 --- a/src/core.ts +++ b/src/core.ts @@ -109,11 +109,7 @@ export async function crawl(config: Config) { globs: typeof config.match === "string" ? [config.match] : config.match, }); - }, - // Comment this option to scrape the full website. - maxRequestsPerCrawl: config.maxPagesToCrawl, - // Uncomment this option to see the browser window. - // headless: false, + maxConcurrency: config.maxConcurrency || 1 , // Set the max concurrency preNavigationHooks: [ // Abort requests for certain resource types async ({ page, log }) => {