feat: Customizing selector can sometimes cause web crawlers to fail

BuilderIO · Feb 29, 2024 · b7ce8d1 · b7ce8d1
1 parent 4f799c5
commit b7ce8d1
Showing 1 changed file with 16 additions and 22 deletions.
diff --git a/src/core.ts b/src/core.ts
@@ -65,34 +65,28 @@ export async function crawl(config: Config) {
           );
 
           // Use custom handling for XPath selector
-          let html: string | null = null; // 初始化为null或空字符串""
-          if (config.selector) {
-            if (config.selector.startsWith("/")) {
-              await waitForXPath(
-                page,
-                config.selector,
-                config.waitForSelectorTimeout ?? 1000,
-              );
-              html = await getPageHtml(page, config.selector);
-            } else {
-              try {
-                // 尝试等待CSS选择器，捕获可能的异常
-                await page.waitForSelector(config.selector, {
-                  timeout: config.waitForSelectorTimeout ?? 1000,
-                });
-                html = await getPageHtml(page, config.selector);
-              } catch (error) {
-                // 如果CSS选择器等待失败，则输出日志并等待<body>
-                console.log(
-                  `CSS Selector "${config.selector}" not found. Waiting for <body> instead.`,
+          let effectiveSelector = config.selector;
+          if (effectiveSelector) {
+            try {
+              if (effectiveSelector.startsWith("/")) {
+                await waitForXPath(
+                  page,
+                  effectiveSelector,
+                  config.waitForSelectorTimeout ?? 1000,
                 );
-                await page.waitForSelector("body", {
+              } else {
+                await page.waitForSelector(effectiveSelector, {
                   timeout: config.waitForSelectorTimeout ?? 1000,
                 });
-                html = await getPageHtml(page, "body");
               }
+            } catch (error) {
+              console.log(
+                `Selector "${config.selector}" not found. Defaulting to <body>.`,
+              );
+              effectiveSelector = undefined;
             }
           }
+          const html = await getPageHtml(page, effectiveSelector);
           // Save results as JSON to ./storage/datasets/default
           await pushData({ title, url: request.loadedUrl, html });