Infrastructure: Include user-agent in link-checker to avoid errors wh…

…en servers require it (#2932)
w3c · Feb 13, 2024 · e032502 · e032502
1 parent 5446c67
commit e032502
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/scripts/link-checker.js b/scripts/link-checker.js
@@ -135,7 +135,14 @@ async function checkLinks() {
 
         const getPageData = async () => {
           try {
-            const response = await fetch(externalPageLink);
+            const response = await fetch(externalPageLink, {
+              headers: {
+                // Spoof a normal looking User-Agent to keep the servers happy
+                // See https://github.com/JustinBeckwith/linkinator/blob/main/src/index.ts
+                'User-Agent':
+                  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36',
+              },
+            });
             const text = await response.text();
             const html = HTMLParser.parse(text);
             const ids = html