Add script for checking lit.dev redirects (#468)

Custom script for checking lit.dev redirects.

It would be nice if we could use the 3rd party link checker we already have for this somehow, but it doesn't support checking for anchors (see stevenvachon/broken-link-checker#108 -- understandable since it would require DOM parsing) which is one of the main failure cases.

Fixes #467 (since we shouldn't need comments if we have the redirects checked in CI).

As part of this, I created a new lit-dev-tools-esm package.

The existing lit-dev-tools package is currently CommonJS, because mostly it is used for Eleventy plugins, and Eleventy doesn't support ES modules (11ty/eleventy#836).

We want ES modules for this new redirect checker script, because it needs to import some ES modules, and that is difficult to do with TypeScript, because TypeScript doesn't allow emitting an actual import statement, which is how CommonJS -> ESM interop works (microsoft/TypeScript#43329).

We also can't really have a mix of CommonJS and ESM in the same package, because the {"type": "module"} field has to be set to one or the other in the package.json. We could use .mjs extensions, but TypeScript won't emit those.

So the simplest solution seems to be to just have two packages.

.github/workflows/build.yml

@@ -22,6 +22,9 @@ jobs:
       - name: Build
         run: npm run build
 
+      - name: Check for broken redirects
+        run: npm run test:links:redirects
+
       - name: Check for broken links (internal)
         run: npm run test:links:internal
 

package.json

@@ -11,7 +11,8 @@
     "format": "prettier \"**/*.{ts,js,json,html,css,md}\" --write",
     "nuke": "rm -rf node_modules package-lock.json packages/*/node_modules packages/*/package-lock.json && npm install && npx lerna bootstrap",
     "test": "npm run test:links",
-    "test:links": "npm run test:links:internal && npm run test:links:external",
+    "test:links": "npm run test:links:redirects && npm run test:links:internal && npm run test:links:external",
+    "test:links:redirects": "node packages/lit-dev-tools-esm/lib/check-redirects.js",
     "test:links:internal": "run-p -r start check-links:internal",
     "test:links:external": "run-p -r start check-links:external",
     "check-links:internal": "wait-on tcp:8080 && blc http://localhost:8080 --recursive --exclude-external --ordered",

packages/lit-dev-tools-esm/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "lit-dev-tools-esm",
+  "private": true,
+  "version": "0.0.0",
+  "description": "Misc tools for lit.dev (ES Modules)",
+  "author": "Google LLC",
+  "license": "BSD-3-Clause",
+  "type": "module",
+  "scripts": {
+    "build": "npm run build:ts",
+    "build:ts": "../../node_modules/.bin/tsc",
+    "format": "../../node_modules/.bin/prettier \"**/*.{ts,js,json,html,css,md}\" --write"
+  },
+  "dependencies": {
+    "@types/ansi-escape-sequences": "^4.0.0",
+    "ansi-escape-sequences": "^6.2.0",
+    "lit-dev-server": "^0.0.0",
+    "node-fetch": "^3.0.0"
+  }
+}

packages/lit-dev-tools-esm/src/check-redirects.ts

@@ -0,0 +1,124 @@
+/**
+ * @license
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+import * as pathLib from 'path';
+import * as fs from 'fs/promises';
+import ansi from 'ansi-escape-sequences';
+import fetch from 'node-fetch';
+import {pageRedirects} from 'lit-dev-server/redirects.js';
+import {fileURLToPath} from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = pathLib.dirname(__filename);
+
+const {red, green, yellow, bold, reset} = ansi.style;
+
+const OK = Symbol();
+type ErrorMessage = string;
+
+const isAbsoluteUrl = (str: string) => {
+  try {
+    new URL(str);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+const trimTrailingSlash = (str: string) =>
+  str.endsWith('/') ? str.slice(0, str.length - 1) : str;
+
+const siteOutputDir = pathLib.resolve(
+  __dirname,
+  '../',
+  '../',
+  'lit-dev-content',
+  '_site'
+);
+
+const checkRedirect = async (
+  redirect: string
+): Promise<ErrorMessage | typeof OK> => {
+  if (isAbsoluteUrl(redirect)) {
+    // Remote URLs.
+    let res;
+    try {
+      res = await fetch(redirect);
+    } catch (e) {
+      return `Fetch error: ${(e as Error).message}`;
+    }
+    if (res.status !== 200) {
+      return `HTTP ${res.status} error`;
+    }
+  } else {
+    // Local paths. A bit hacky, but since we know how Eleventy works, we don't
+    // need to actually run the server, we can just look directly in the built
+    // HTML output directory.
+    const {pathname, hash} = new URL(redirect, 'http://lit.dev');
+    const diskPath = pathLib.relative(
+      process.cwd(),
+      pathLib.join(siteOutputDir, trimTrailingSlash(pathname), 'index.html')
+    );
+    let data;
+    try {
+      data = await fs.readFile(diskPath, {encoding: 'utf8'});
+    } catch {
+      return `Could not find file matching path ${pathname}
+Searched for file ${diskPath}`;
+    }
+    if (hash) {
+      // Another hack. Just do a regexp search for e.g. id="somesection" instead
+      // of DOM parsing. Should be good enough, especially given how regular our
+      // Markdown generated HTML is.
+      const idAttrRegExp = new RegExp(`\\sid=["']?${hash.slice(1)}["']?[\\s>]`);
+      if (data.match(idAttrRegExp) === null) {
+        return `Could not find section matching hash ${hash}.
+Searched in file ${diskPath}`;
+      }
+    }
+  }
+  return OK;
+};
+
+const checkAllRedirects = async () => {
+  console.log('==========================');
+  console.log('Checking lit.dev redirects');
+  console.log('==========================');
+  console.log();
+
+  let fail = false;
+  const promises = [];
+  for (const [from, to] of pageRedirects) {
+    promises.push(
+      (async () => {
+        const result = await checkRedirect(to);
+        if (result === OK) {
+          console.log(`${bold + green}OK${reset} ${from} -> ${to}`);
+        } else {
+          console.log();
+          console.log(
+            `${bold + red}BROKEN REDIRECT${reset} ${from} -> ${
+              yellow + to + reset
+            }`
+          );
+          console.log(result);
+          console.log();
+          fail = true;
+        }
+      })()
+    );
+  }
+  await Promise.all(promises);
+  console.log();
+  if (fail) {
+    console.log('Redirects were broken!');
+    process.exit(1);
+  } else {
+    console.error('All redirects OK!');
+  }
+};
+
+checkAllRedirects();

packages/lit-dev-tools-esm/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "compilerOptions": {
+    "target": "es2020",
+    "module": "esnext",
+    "moduleResolution": "node",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "outDir": "./lib",
+    "rootDir": "./src",
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "esModuleInterop": true,
+    "experimentalDecorators": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": []
+}

packages/lit-dev-tools/package.json

@@ -2,7 +2,7 @@
   "name": "lit-dev-tools",
   "private": true,
   "version": "0.0.0",
-  "description": "Misc tools for lit.dev",
+  "description": "Misc tools for lit.dev (CommonJS)",
   "author": "Google LLC",
   "license": "BSD-3-Clause",
   "scripts": {
@@ -12,8 +12,10 @@
     "test": "npm run build && uvu ./lib \".spec.js$\""
   },
   "dependencies": {
+    "@types/jsdom": "^16.2.13",
     "@types/markdown-it": "^12.0.1",
     "@types/source-map": "^0.5.7",
+    "@types/strip-comments": "^2.0.1",
     "@web/dev-server": "^0.1.6",
     "@web/dev-server-core": "^0.3.5",
     "fast-glob": "^3.2.5",
@@ -27,8 +29,6 @@
     "strip-comments": "^2.0.1",
     "striptags": "^3.2.0",
     "typedoc": "^0.20.30",
-    "@types/jsdom": "^16.2.13",
-    "@types/strip-comments": "^2.0.1",
     "uvu": "^0.5.1"
   }
 }