diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e4ce7de9..1555fa43c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,9 @@ jobs: - name: Build run: npm run build + - name: Check for broken redirects + run: npm run test:links:redirects + - name: Check for broken links (internal) run: npm run test:links:internal diff --git a/package.json b/package.json index 1e93fcfb2..b9c7952d2 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "format": "prettier \"**/*.{ts,js,json,html,css,md}\" --write", "nuke": "rm -rf node_modules package-lock.json packages/*/node_modules packages/*/package-lock.json && npm install && npx lerna bootstrap", "test": "npm run test:links", - "test:links": "npm run test:links:internal && npm run test:links:external", + "test:links": "npm run test:links:redirects && npm run test:links:internal && npm run test:links:external", + "test:links:redirects": "node packages/lit-dev-tools-esm/lib/check-redirects.js", "test:links:internal": "run-p -r start check-links:internal", "test:links:external": "run-p -r start check-links:external", "check-links:internal": "wait-on tcp:8080 && blc http://localhost:8080 --recursive --exclude-external --ordered", diff --git a/packages/lit-dev-tools-esm/package-lock.json b/packages/lit-dev-tools-esm/package-lock.json new file mode 100644 index 000000000..f33037e4f --- /dev/null +++ b/packages/lit-dev-tools-esm/package-lock.json @@ -0,0 +1,142 @@ +{ + "name": "lit-dev-tools-esm", + "version": "0.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "lit-dev-tools-esm", + "version": "0.0.0", + "license": "BSD-3-Clause", + "dependencies": { + "@types/ansi-escape-sequences": "^4.0.0", + "ansi-escape-sequences": "^6.2.0", + "node-fetch": "^3.0.0" + } + }, + "node_modules/@types/ansi-escape-sequences": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/ansi-escape-sequences/-/ansi-escape-sequences-4.0.0.tgz", + "integrity": "sha512-y9KJUf19SBowoaqhWdQNnErxFMNsKbuair2i3SGv5Su1ExLPAJz37iPXLHKIFQGYkHGxsSe45rNt8ZekXxJwUw==" + }, + "node_modules/ansi-escape-sequences": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/ansi-escape-sequences/-/ansi-escape-sequences-6.2.0.tgz", + "integrity": "sha512-tDwbanmlgu4wVCNM75YvwugiDn7iZtCewniuxZgLBbdVy/li7ufZhNpqPR7ZJgJVI2TzRcElDKBNlLaI+RSzbQ==", + "dependencies": { + "array-back": "^6.2.0" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/array-back": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.0.tgz", + "integrity": "sha512-mixVv03GOOn/ubHE4STQ+uevX42ETdk0JoMVEjNkSOCT7WgERh7C8/+NyhWYNpE3BN69pxFyJIBcF7CxWz/+4A==", + "engines": { + "node": ">=12.17" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz", + "integrity": "sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/fetch-blob": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.1.2.tgz", + "integrity": "sha512-hunJbvy/6OLjCD0uuhLdp0mMPzP/yd2ssd1t2FCJsaA7wkWhpbp9xfuNVpv7Ll4jFhzp6T4LAupSiV9uOeg0VQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/node-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.0.0.tgz", + "integrity": "sha512-bKMI+C7/T/SPU1lKnbQbwxptpCrG9ashG+VkytmXCPZyuM9jB6VU+hY0oi4lC8LxTtAeWdckNCTa3nrGsAdA3Q==", + "dependencies": { + "data-uri-to-buffer": "^3.0.1", + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.1.0.tgz", + "integrity": "sha512-wO9r1YnYe7kFBLHyyVEhV1H8VRWoNiNnuP+v/HUUmSTaRF8F93Kmd3JMrETx0f11GXxRek6OcL2QtjFIdc5WYw==", + "engines": { + "node": ">= 8" + } + } + }, + "dependencies": { + "@types/ansi-escape-sequences": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/ansi-escape-sequences/-/ansi-escape-sequences-4.0.0.tgz", + "integrity": "sha512-y9KJUf19SBowoaqhWdQNnErxFMNsKbuair2i3SGv5Su1ExLPAJz37iPXLHKIFQGYkHGxsSe45rNt8ZekXxJwUw==" + }, + "ansi-escape-sequences": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/ansi-escape-sequences/-/ansi-escape-sequences-6.2.0.tgz", + "integrity": "sha512-tDwbanmlgu4wVCNM75YvwugiDn7iZtCewniuxZgLBbdVy/li7ufZhNpqPR7ZJgJVI2TzRcElDKBNlLaI+RSzbQ==", + "requires": { + "array-back": "^6.2.0" + } + }, + "array-back": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.0.tgz", + "integrity": "sha512-mixVv03GOOn/ubHE4STQ+uevX42ETdk0JoMVEjNkSOCT7WgERh7C8/+NyhWYNpE3BN69pxFyJIBcF7CxWz/+4A==" + }, + "data-uri-to-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz", + "integrity": "sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og==" + }, + "fetch-blob": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.1.2.tgz", + "integrity": "sha512-hunJbvy/6OLjCD0uuhLdp0mMPzP/yd2ssd1t2FCJsaA7wkWhpbp9xfuNVpv7Ll4jFhzp6T4LAupSiV9uOeg0VQ==", + "requires": { + "web-streams-polyfill": "^3.0.3" + } + }, + "node-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.0.0.tgz", + "integrity": "sha512-bKMI+C7/T/SPU1lKnbQbwxptpCrG9ashG+VkytmXCPZyuM9jB6VU+hY0oi4lC8LxTtAeWdckNCTa3nrGsAdA3Q==", + "requires": { + "data-uri-to-buffer": "^3.0.1", + "fetch-blob": "^3.1.2" + } + }, + "web-streams-polyfill": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.1.0.tgz", + "integrity": "sha512-wO9r1YnYe7kFBLHyyVEhV1H8VRWoNiNnuP+v/HUUmSTaRF8F93Kmd3JMrETx0f11GXxRek6OcL2QtjFIdc5WYw==" + } + } +} diff --git a/packages/lit-dev-tools-esm/package.json b/packages/lit-dev-tools-esm/package.json new file mode 100644 index 000000000..9b5a1336e --- /dev/null +++ b/packages/lit-dev-tools-esm/package.json @@ -0,0 +1,20 @@ +{ + "name": "lit-dev-tools-esm", + "private": true, + "version": "0.0.0", + "description": "Misc tools for lit.dev (ES Modules)", + "author": "Google LLC", + "license": "BSD-3-Clause", + "type": "module", + "scripts": { + "build": "npm run build:ts", + "build:ts": "../../node_modules/.bin/tsc", + "format": "../../node_modules/.bin/prettier \"**/*.{ts,js,json,html,css,md}\" --write" + }, + "dependencies": { + "@types/ansi-escape-sequences": "^4.0.0", + "ansi-escape-sequences": "^6.2.0", + "lit-dev-server": "^0.0.0", + "node-fetch": "^3.0.0" + } +} diff --git a/packages/lit-dev-tools-esm/src/check-redirects.ts b/packages/lit-dev-tools-esm/src/check-redirects.ts new file mode 100644 index 000000000..b67eddfdf --- /dev/null +++ b/packages/lit-dev-tools-esm/src/check-redirects.ts @@ -0,0 +1,124 @@ +/** + * @license + * Copyright 2021 Google LLC + * SPDX-License-Identifier: BSD-3-Clause + */ + +import * as pathLib from 'path'; +import * as fs from 'fs/promises'; +import ansi from 'ansi-escape-sequences'; +import fetch from 'node-fetch'; +import {pageRedirects} from 'lit-dev-server/redirects.js'; +import {fileURLToPath} from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = pathLib.dirname(__filename); + +const {red, green, yellow, bold, reset} = ansi.style; + +const OK = Symbol(); +type ErrorMessage = string; + +const isAbsoluteUrl = (str: string) => { + try { + new URL(str); + return true; + } catch { + return false; + } +}; + +const trimTrailingSlash = (str: string) => + str.endsWith('/') ? str.slice(0, str.length - 1) : str; + +const siteOutputDir = pathLib.resolve( + __dirname, + '../', + '../', + 'lit-dev-content', + '_site' +); + +const checkRedirect = async ( + redirect: string +): Promise => { + if (isAbsoluteUrl(redirect)) { + // Remote URLs. + let res; + try { + res = await fetch(redirect); + } catch (e) { + return `Fetch error: ${(e as Error).message}`; + } + if (res.status !== 200) { + return `HTTP ${res.status} error`; + } + } else { + // Local paths. A bit hacky, but since we know how Eleventy works, we don't + // need to actually run the server, we can just look directly in the built + // HTML output directory. + const {pathname, hash} = new URL(redirect, 'http://lit.dev'); + const diskPath = pathLib.relative( + process.cwd(), + pathLib.join(siteOutputDir, trimTrailingSlash(pathname), 'index.html') + ); + let data; + try { + data = await fs.readFile(diskPath, {encoding: 'utf8'}); + } catch { + return `Could not find file matching path ${pathname} +Searched for file ${diskPath}`; + } + if (hash) { + // Another hack. Just do a regexp search for e.g. id="somesection" instead + // of DOM parsing. Should be good enough, especially given how regular our + // Markdown generated HTML is. + const idAttrRegExp = new RegExp(`\\sid=["']?${hash.slice(1)}["']?[\\s>]`); + if (data.match(idAttrRegExp) === null) { + return `Could not find section matching hash ${hash}. +Searched in file ${diskPath}`; + } + } + } + return OK; +}; + +const checkAllRedirects = async () => { + console.log('=========================='); + console.log('Checking lit.dev redirects'); + console.log('=========================='); + console.log(); + + let fail = false; + const promises = []; + for (const [from, to] of pageRedirects) { + promises.push( + (async () => { + const result = await checkRedirect(to); + if (result === OK) { + console.log(`${bold + green}OK${reset} ${from} -> ${to}`); + } else { + console.log(); + console.log( + `${bold + red}BROKEN REDIRECT${reset} ${from} -> ${ + yellow + to + reset + }` + ); + console.log(result); + console.log(); + fail = true; + } + })() + ); + } + await Promise.all(promises); + console.log(); + if (fail) { + console.log('Redirects were broken!'); + process.exit(1); + } else { + console.error('All redirects OK!'); + } +}; + +checkAllRedirects(); diff --git a/packages/lit-dev-tools-esm/tsconfig.json b/packages/lit-dev-tools-esm/tsconfig.json new file mode 100644 index 000000000..428eb597d --- /dev/null +++ b/packages/lit-dev-tools-esm/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "es2020", + "module": "esnext", + "moduleResolution": "node", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./lib", + "rootDir": "./src", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*.ts"], + "exclude": [] +} diff --git a/packages/lit-dev-tools/package.json b/packages/lit-dev-tools/package.json index f4f50baa9..d7fe912a4 100644 --- a/packages/lit-dev-tools/package.json +++ b/packages/lit-dev-tools/package.json @@ -2,7 +2,7 @@ "name": "lit-dev-tools", "private": true, "version": "0.0.0", - "description": "Misc tools for lit.dev", + "description": "Misc tools for lit.dev (CommonJS)", "author": "Google LLC", "license": "BSD-3-Clause", "scripts": { @@ -12,8 +12,10 @@ "test": "npm run build && uvu ./lib \".spec.js$\"" }, "dependencies": { + "@types/jsdom": "^16.2.13", "@types/markdown-it": "^12.0.1", "@types/source-map": "^0.5.7", + "@types/strip-comments": "^2.0.1", "@web/dev-server": "^0.1.6", "@web/dev-server-core": "^0.3.5", "fast-glob": "^3.2.5", @@ -27,8 +29,6 @@ "strip-comments": "^2.0.1", "striptags": "^3.2.0", "typedoc": "^0.20.30", - "@types/jsdom": "^16.2.13", - "@types/strip-comments": "^2.0.1", "uvu": "^0.5.1" } }