From aca82f6ab61cfcb2d9010ab1b1fd601189c320c2 Mon Sep 17 00:00:00 2001 From: Alexander Marks Date: Wed, 1 Sep 2021 12:43:05 -0700 Subject: [PATCH] Add script for checking lit.dev redirects (#468) Custom script for checking lit.dev redirects. It would be nice if we could use the 3rd party link checker we already have for this somehow, but it doesn't support checking for anchors (see stevenvachon/broken-link-checker#108 -- understandable since it would require DOM parsing) which is one of the main failure cases. Fixes #467 (since we shouldn't need comments if we have the redirects checked in CI). As part of this, I created a new lit-dev-tools-esm package. The existing lit-dev-tools package is currently CommonJS, because mostly it is used for Eleventy plugins, and Eleventy doesn't support ES modules (11ty/eleventy#836). We want ES modules for this new redirect checker script, because it needs to import some ES modules, and that is difficult to do with TypeScript, because TypeScript doesn't allow emitting an actual import statement, which is how CommonJS -> ESM interop works (microsoft/TypeScript#43329). We also can't really have a mix of CommonJS and ESM in the same package, because the {"type": "module"} field has to be set to one or the other in the package.json. We could use .mjs extensions, but TypeScript won't emit those. So the simplest solution seems to be to just have two packages. --- .github/workflows/build.yml | 3 + package.json | 3 +- packages/lit-dev-tools-esm/package-lock.json | 142 ++++++++++++++++++ packages/lit-dev-tools-esm/package.json | 20 +++ .../lit-dev-tools-esm/src/check-redirects.ts | 124 +++++++++++++++ packages/lit-dev-tools-esm/tsconfig.json | 22 +++ packages/lit-dev-tools/package.json | 6 +- 7 files changed, 316 insertions(+), 4 deletions(-) create mode 100644 packages/lit-dev-tools-esm/package-lock.json create mode 100644 packages/lit-dev-tools-esm/package.json create mode 100644 packages/lit-dev-tools-esm/src/check-redirects.ts create mode 100644 packages/lit-dev-tools-esm/tsconfig.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e4ce7de9..1555fa43c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,9 @@ jobs: - name: Build run: npm run build + - name: Check for broken redirects + run: npm run test:links:redirects + - name: Check for broken links (internal) run: npm run test:links:internal diff --git a/package.json b/package.json index 1e93fcfb2..b9c7952d2 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "format": "prettier \"**/*.{ts,js,json,html,css,md}\" --write", "nuke": "rm -rf node_modules package-lock.json packages/*/node_modules packages/*/package-lock.json && npm install && npx lerna bootstrap", "test": "npm run test:links", - "test:links": "npm run test:links:internal && npm run test:links:external", + "test:links": "npm run test:links:redirects && npm run test:links:internal && npm run test:links:external", + "test:links:redirects": "node packages/lit-dev-tools-esm/lib/check-redirects.js", "test:links:internal": "run-p -r start check-links:internal", "test:links:external": "run-p -r start check-links:external", "check-links:internal": "wait-on tcp:8080 && blc http://localhost:8080 --recursive --exclude-external --ordered", diff --git a/packages/lit-dev-tools-esm/package-lock.json b/packages/lit-dev-tools-esm/package-lock.json new file mode 100644 index 000000000..f33037e4f --- /dev/null +++ b/packages/lit-dev-tools-esm/package-lock.json @@ -0,0 +1,142 @@ +{ + "name": "lit-dev-tools-esm", + "version": "0.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "lit-dev-tools-esm", + "version": "0.0.0", + "license": "BSD-3-Clause", + "dependencies": { + "@types/ansi-escape-sequences": "^4.0.0", + "ansi-escape-sequences": "^6.2.0", + "node-fetch": "^3.0.0" + } + }, + "node_modules/@types/ansi-escape-sequences": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/ansi-escape-sequences/-/ansi-escape-sequences-4.0.0.tgz", + "integrity": "sha512-y9KJUf19SBowoaqhWdQNnErxFMNsKbuair2i3SGv5Su1ExLPAJz37iPXLHKIFQGYkHGxsSe45rNt8ZekXxJwUw==" + }, + "node_modules/ansi-escape-sequences": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/ansi-escape-sequences/-/ansi-escape-sequences-6.2.0.tgz", + "integrity": "sha512-tDwbanmlgu4wVCNM75YvwugiDn7iZtCewniuxZgLBbdVy/li7ufZhNpqPR7ZJgJVI2TzRcElDKBNlLaI+RSzbQ==", + "dependencies": { + "array-back": "^6.2.0" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/array-back": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.0.tgz", + "integrity": "sha512-mixVv03GOOn/ubHE4STQ+uevX42ETdk0JoMVEjNkSOCT7WgERh7C8/+NyhWYNpE3BN69pxFyJIBcF7CxWz/+4A==", + "engines": { + "node": ">=12.17" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz", + "integrity": "sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/fetch-blob": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.1.2.tgz", + "integrity": "sha512-hunJbvy/6OLjCD0uuhLdp0mMPzP/yd2ssd1t2FCJsaA7wkWhpbp9xfuNVpv7Ll4jFhzp6T4LAupSiV9uOeg0VQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/node-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.0.0.tgz", + "integrity": "sha512-bKMI+C7/T/SPU1lKnbQbwxptpCrG9ashG+VkytmXCPZyuM9jB6VU+hY0oi4lC8LxTtAeWdckNCTa3nrGsAdA3Q==", + "dependencies": { + "data-uri-to-buffer": "^3.0.1", + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.1.0.tgz", + "integrity": "sha512-wO9r1YnYe7kFBLHyyVEhV1H8VRWoNiNnuP+v/HUUmSTaRF8F93Kmd3JMrETx0f11GXxRek6OcL2QtjFIdc5WYw==", + "engines": { + "node": ">= 8" + } + } + }, + "dependencies": { + "@types/ansi-escape-sequences": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/ansi-escape-sequences/-/ansi-escape-sequences-4.0.0.tgz", + "integrity": "sha512-y9KJUf19SBowoaqhWdQNnErxFMNsKbuair2i3SGv5Su1ExLPAJz37iPXLHKIFQGYkHGxsSe45rNt8ZekXxJwUw==" + }, + "ansi-escape-sequences": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/ansi-escape-sequences/-/ansi-escape-sequences-6.2.0.tgz", + "integrity": "sha512-tDwbanmlgu4wVCNM75YvwugiDn7iZtCewniuxZgLBbdVy/li7ufZhNpqPR7ZJgJVI2TzRcElDKBNlLaI+RSzbQ==", + "requires": { + "array-back": "^6.2.0" + } + }, + "array-back": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.0.tgz", + "integrity": "sha512-mixVv03GOOn/ubHE4STQ+uevX42ETdk0JoMVEjNkSOCT7WgERh7C8/+NyhWYNpE3BN69pxFyJIBcF7CxWz/+4A==" + }, + "data-uri-to-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz", + "integrity": "sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og==" + }, + "fetch-blob": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.1.2.tgz", + "integrity": "sha512-hunJbvy/6OLjCD0uuhLdp0mMPzP/yd2ssd1t2FCJsaA7wkWhpbp9xfuNVpv7Ll4jFhzp6T4LAupSiV9uOeg0VQ==", + "requires": { + "web-streams-polyfill": "^3.0.3" + } + }, + "node-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.0.0.tgz", + "integrity": "sha512-bKMI+C7/T/SPU1lKnbQbwxptpCrG9ashG+VkytmXCPZyuM9jB6VU+hY0oi4lC8LxTtAeWdckNCTa3nrGsAdA3Q==", + "requires": { + "data-uri-to-buffer": "^3.0.1", + "fetch-blob": "^3.1.2" + } + }, + "web-streams-polyfill": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.1.0.tgz", + "integrity": "sha512-wO9r1YnYe7kFBLHyyVEhV1H8VRWoNiNnuP+v/HUUmSTaRF8F93Kmd3JMrETx0f11GXxRek6OcL2QtjFIdc5WYw==" + } + } +} diff --git a/packages/lit-dev-tools-esm/package.json b/packages/lit-dev-tools-esm/package.json new file mode 100644 index 000000000..9b5a1336e --- /dev/null +++ b/packages/lit-dev-tools-esm/package.json @@ -0,0 +1,20 @@ +{ + "name": "lit-dev-tools-esm", + "private": true, + "version": "0.0.0", + "description": "Misc tools for lit.dev (ES Modules)", + "author": "Google LLC", + "license": "BSD-3-Clause", + "type": "module", + "scripts": { + "build": "npm run build:ts", + "build:ts": "../../node_modules/.bin/tsc", + "format": "../../node_modules/.bin/prettier \"**/*.{ts,js,json,html,css,md}\" --write" + }, + "dependencies": { + "@types/ansi-escape-sequences": "^4.0.0", + "ansi-escape-sequences": "^6.2.0", + "lit-dev-server": "^0.0.0", + "node-fetch": "^3.0.0" + } +} diff --git a/packages/lit-dev-tools-esm/src/check-redirects.ts b/packages/lit-dev-tools-esm/src/check-redirects.ts new file mode 100644 index 000000000..b67eddfdf --- /dev/null +++ b/packages/lit-dev-tools-esm/src/check-redirects.ts @@ -0,0 +1,124 @@ +/** + * @license + * Copyright 2021 Google LLC + * SPDX-License-Identifier: BSD-3-Clause + */ + +import * as pathLib from 'path'; +import * as fs from 'fs/promises'; +import ansi from 'ansi-escape-sequences'; +import fetch from 'node-fetch'; +import {pageRedirects} from 'lit-dev-server/redirects.js'; +import {fileURLToPath} from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = pathLib.dirname(__filename); + +const {red, green, yellow, bold, reset} = ansi.style; + +const OK = Symbol(); +type ErrorMessage = string; + +const isAbsoluteUrl = (str: string) => { + try { + new URL(str); + return true; + } catch { + return false; + } +}; + +const trimTrailingSlash = (str: string) => + str.endsWith('/') ? str.slice(0, str.length - 1) : str; + +const siteOutputDir = pathLib.resolve( + __dirname, + '../', + '../', + 'lit-dev-content', + '_site' +); + +const checkRedirect = async ( + redirect: string +): Promise => { + if (isAbsoluteUrl(redirect)) { + // Remote URLs. + let res; + try { + res = await fetch(redirect); + } catch (e) { + return `Fetch error: ${(e as Error).message}`; + } + if (res.status !== 200) { + return `HTTP ${res.status} error`; + } + } else { + // Local paths. A bit hacky, but since we know how Eleventy works, we don't + // need to actually run the server, we can just look directly in the built + // HTML output directory. + const {pathname, hash} = new URL(redirect, 'http://lit.dev'); + const diskPath = pathLib.relative( + process.cwd(), + pathLib.join(siteOutputDir, trimTrailingSlash(pathname), 'index.html') + ); + let data; + try { + data = await fs.readFile(diskPath, {encoding: 'utf8'}); + } catch { + return `Could not find file matching path ${pathname} +Searched for file ${diskPath}`; + } + if (hash) { + // Another hack. Just do a regexp search for e.g. id="somesection" instead + // of DOM parsing. Should be good enough, especially given how regular our + // Markdown generated HTML is. + const idAttrRegExp = new RegExp(`\\sid=["']?${hash.slice(1)}["']?[\\s>]`); + if (data.match(idAttrRegExp) === null) { + return `Could not find section matching hash ${hash}. +Searched in file ${diskPath}`; + } + } + } + return OK; +}; + +const checkAllRedirects = async () => { + console.log('=========================='); + console.log('Checking lit.dev redirects'); + console.log('=========================='); + console.log(); + + let fail = false; + const promises = []; + for (const [from, to] of pageRedirects) { + promises.push( + (async () => { + const result = await checkRedirect(to); + if (result === OK) { + console.log(`${bold + green}OK${reset} ${from} -> ${to}`); + } else { + console.log(); + console.log( + `${bold + red}BROKEN REDIRECT${reset} ${from} -> ${ + yellow + to + reset + }` + ); + console.log(result); + console.log(); + fail = true; + } + })() + ); + } + await Promise.all(promises); + console.log(); + if (fail) { + console.log('Redirects were broken!'); + process.exit(1); + } else { + console.error('All redirects OK!'); + } +}; + +checkAllRedirects(); diff --git a/packages/lit-dev-tools-esm/tsconfig.json b/packages/lit-dev-tools-esm/tsconfig.json new file mode 100644 index 000000000..428eb597d --- /dev/null +++ b/packages/lit-dev-tools-esm/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "es2020", + "module": "esnext", + "moduleResolution": "node", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./lib", + "rootDir": "./src", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*.ts"], + "exclude": [] +} diff --git a/packages/lit-dev-tools/package.json b/packages/lit-dev-tools/package.json index f4f50baa9..d7fe912a4 100644 --- a/packages/lit-dev-tools/package.json +++ b/packages/lit-dev-tools/package.json @@ -2,7 +2,7 @@ "name": "lit-dev-tools", "private": true, "version": "0.0.0", - "description": "Misc tools for lit.dev", + "description": "Misc tools for lit.dev (CommonJS)", "author": "Google LLC", "license": "BSD-3-Clause", "scripts": { @@ -12,8 +12,10 @@ "test": "npm run build && uvu ./lib \".spec.js$\"" }, "dependencies": { + "@types/jsdom": "^16.2.13", "@types/markdown-it": "^12.0.1", "@types/source-map": "^0.5.7", + "@types/strip-comments": "^2.0.1", "@web/dev-server": "^0.1.6", "@web/dev-server-core": "^0.3.5", "fast-glob": "^3.2.5", @@ -27,8 +29,6 @@ "strip-comments": "^2.0.1", "striptags": "^3.2.0", "typedoc": "^0.20.30", - "@types/jsdom": "^16.2.13", - "@types/strip-comments": "^2.0.1", "uvu": "^0.5.1" } }