Update to core.ts for maxPagesToCrawl

BuilderIO · Dec 4, 2023 · a6b4b1f · a6b4b1f
1 parent ac0ac25
commit a6b4b1f
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/core.ts b/src/core.ts
@@ -110,6 +110,7 @@ export async function crawl(config: Config) {
             typeof config.match === "string" ? [config.match] : config.match,
         });
       maxConcurrency: config.maxConcurrency || 1 , // Set the max concurrency
+      maxRequestsPerCrawl: config.maxPagesToCrawl, // Set the max pages to crawl or set to 0 to scrape the full website.          
       preNavigationHooks: [
         // Abort requests for certain resource types
         async ({ page, log }) => {