Added regex test coverage

.github/workflows/test.yml

@@ -57,3 +57,16 @@ jobs:
           node-version: 14.x
       - run: npm ci
       - run: npm run lint:ci
+
+  coverage:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - run: npm ci
+      - run: npm run regex-coverage

package.json

@@ -14,6 +14,7 @@
 		"lint": "eslint . --cache",
 		"lint:fix": "npm run lint -- --fix",
 		"lint:ci": "eslint . --max-warnings 0",
+		"regex-coverage": "mocha tests/coverage.js",
 		"test:aliases": "mocha tests/aliases-test.js",
 		"test:core": "mocha tests/core/**/*.js",
 		"test:dependencies": "mocha tests/dependencies-test.js",

tests/coverage.js

@@ -0,0 +1,260 @@
+'use strict';
+
+const TestDiscovery = require('./helper/test-discovery');
+const TestCase = require('./helper/test-case');
+const PrismLoader = require('./helper/prism-loader');
+const { BFS, BFSPathToPrismTokenPath } = require('./helper/util');
+const { assert } = require('chai');
+const components = require('../components.json');
+const ALL_LANGUAGES = [...Object.keys(components.languages).filter(k => k !== 'meta')];
+
+
+describe('Pattern test coverage', function () {
+	/**
+	 * @type {Map<string, PatternData>}
+	 * @typedef PatternData
+	 * @property {RegExp} pattern
+	 * @property {string} language
+	 * @property {Set<string>} from
+	 * @property {RegExpExecArray[]} matches
+	 */
+	const patterns = new Map();
+
+	/**
+	 * @param {string | string[]} languages
+	 * @returns {import("./helper/prism-loader").Prism}
+	 */
+	function createInstance(languages) {
+		const Prism = PrismLoader.createInstance(languages);
+
+		BFS(Prism.languages, (path, object) => {
+			const { key, value } = path[path.length - 1];
+			const tokenPath = BFSPathToPrismTokenPath(path);
+
+			if (Object.prototype.toString.call(value) == '[object RegExp]') {
+				const regex = makeGlobal(value);
+				object[key] = regex;
+
+				const patternKey = String(regex);
+				let data = patterns.get(patternKey);
+				if (!data) {
+					data = {
+						pattern: regex,
+						language: path[1].key,
+						from: new Set([tokenPath]),
+						matches: []
+					};
+					patterns.set(patternKey, data);
+				} else {
+					data.from.add(tokenPath);
+				}
+
+				regex.exec = string => {
+					let match = RegExp.prototype.exec.call(regex, string);
+					if (match) {
+						data.matches.push(match);
+					}
+					return match;
+				};
+			}
+		});
+
+		return Prism;
+	}
+
+	describe('Register all patterns', function () {
+		it('all', function () {
+			this.slow(10 * 1000);
+			// This will cause ALL regexes of Prism to be registered in the patterns map.
+			// (Languages that don't have any tests can't be caught otherwise.)
+			createInstance(ALL_LANGUAGES);
+		});
+	});
+
+	describe('Run all language tests', function () {
+		// define tests for all tests in all languages in the test suite
+		for (const [languageIdentifier, files] of TestDiscovery.loadAllTests()) {
+			it(languageIdentifier, function () {
+				this.timeout(10 * 1000);
+
+				for (const filePath of files) {
+					try {
+						TestCase.run({
+							languageIdentifier,
+							filePath,
+							updateMode: 'none',
+							createInstance
+						});
+					} catch (error) {
+						// we don't case about whether the test succeeds,
+						// we just want to gather usage data
+					}
+				}
+			});
+		}
+	});
+
+	describe('Coverage', function () {
+		for (const language of ALL_LANGUAGES) {
+			describe(language, function () {
+				it(`- should cover all patterns`, function () {
+					const untested = getAllOf(language).filter(d => d.matches.length === 0);
+					if (untested.length === 0) {
+						return;
+					}
+
+					const problems = untested.map(data => {
+						return formatProblem(data, [
+							'This pattern is completely untested. Add test files that match this pattern.'
+						]);
+					});
+
+					assert.fail([
+						`${problems.length} pattern(s) are untested:\n`
+						+ 'You can learn more about writing tests at https://prismjs.com/test-suite.html#writing-tests',
+						...problems
+					].join('\n\n'));
+				});
+
+				it(`- should exhaustively cover all keywords in keyword lists`, function () {
+					const problems = [];
+
+					for (const data of getAllOf(language)) {
+						if (data.matches.length === 0) {
+							// don't report the same pattern twice
+							continue;
+						}
+
+						const keywords = getKeywordList(data.pattern);
+						if (!keywords) {
+							continue;
+						}
+						const keywordCount = keywords.size;
+
+						data.matches.forEach(([m]) => {
+							if (data.pattern.ignoreCase) {
+								m = m.toUpperCase();
+							}
+							keywords.delete(m);
+						});
+
+						if (keywords.size > 0) {
+							problems.push(formatProblem(data, [
+								`Add test files to test all keywords. The following keywords (${keywords.size}/${keywordCount}) are untested:`,
+								...[...keywords].map(k => `    ${k}`)
+							]));
+						}
+					}
+
+					if (problems.length === 0) {
+						return;
+					}
+
+					assert.fail([
+						`${problems.length} keyword list(s) are not exhaustively tested:\n`
+						+ 'You can learn more about writing tests at https://prismjs.com/test-suite.html#writing-tests',
+						...problems
+					].join('\n\n'));
+				});
+			});
+		}
+	});
+
+	/**
+	 * @param {string} language
+	 * @returns {PatternData[]}
+	 */
+	function getAllOf(language) {
+		return [...patterns.values()].filter(d => d.language === language);
+	}
+
+	/**
+	 * @param {string} string
+	 * @param {number} maxLength
+	 * @returns {string}
+	 */
+	function short(string, maxLength) {
+		if (string.length > maxLength) {
+			return string.slice(0, maxLength - 1) + '…';
+		} else {
+			return string;
+		}
+	}
+
+	/**
+	 * If the given pattern string describes a keyword list, all keyword will be returned. Otherwise, `null` will be
+	 * returned.
+	 *
+	 * @param {RegExp} pattern
+	 * @returns {Set<string> | null}
+	 */
+	function getKeywordList(pattern) {
+		// Right now, only keyword lists of the form /\b(?:foo|bar)\b/ are supported.
+		// In the future, we might want to convert these regexes to NFAs and iterate all words to cover more complex
+		// keyword lists and even operator and punctuation lists.
+
+		let source = pattern.source.replace(/^\\b|\\b$/g, '');
+		if (source.startsWith('(?:') && source.endsWith(')')) {
+			source = source.slice('(?:'.length, source.length - ')'.length);
+		}
+
+		if (/^\w+(?:\|\w+)*$/.test(source)) {
+			if (pattern.ignoreCase) {
+				source = source.toUpperCase();
+			}
+			return new Set(source.split(/\|/g));
+		} else {
+			return null;
+		}
+	}
+
+	/**
+	 * @param {Iterable<string>} occurrences
+	 * @returns {{ origin: string; otherOccurrences: string[] }}
+	 */
+	function splitOccurrences(occurrences) {
+		const all = [...occurrences];
+		return {
+			origin: all[0],
+			otherOccurrences: all.slice(1),
+		};
+	}
+
+	/**
+	 * @param {PatternData} data
+	 * @param {string[]} messageLines
+	 * @returns {string}
+	 */
+	function formatProblem(data, messageLines) {
+		const { origin, otherOccurrences } = splitOccurrences(data.from);
+
+		const lines = [
+			`${origin}:`,
+			short(String(data.pattern), 100),
+			'',
+			...messageLines,
+		];
+
+		if (otherOccurrences.length) {
+			lines.push(
+				'',
+				'Other occurrences of this pattern:',
+				...otherOccurrences.map(o => `- ${o}`)
+			);
+		}
+
+		return lines.join('\n    ');
+	}
+});
+
+/**
+ * @param {RegExp} regex
+ * @returns {RegExp}
+ */
+function makeGlobal(regex) {
+	if (regex.global) {
+		return regex;
+	} else {
+		return RegExp(regex.source, regex.flags + 'g');
+	}
+}

tests/helper/test-case.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const fs = require('fs');
+const path = require('path');
 const { assert } = require('chai');
 const Prettier = require('prettier');
 const PrismLoader = require('./prism-loader');
@@ -11,6 +12,12 @@ const TokenStreamTransformer = require('./token-stream-transformer');
  * @typedef {import("../../components/prism-core.js")} Prism
  */
 
+/**
+ * @param {string[]} languages
+ * @returns {Prism}
+ */
+const defaultCreateInstance = (languages) => PrismLoader.createInstance(languages);
+
 /**
  * Handles parsing and printing of a test case file.
  *
@@ -297,6 +304,29 @@ class HighlightHTMLRunner {
 module.exports = {
 	TestCaseFile,
 
+	/**
+	 * Runs the given test file and asserts the result.
+	 *
+	 * This function will determine what kind of test files the given file is and call the appropriate method to run the
+	 * test.
+	 *
+	 * @param {RunOptions} options
+	 * @returns {void}
+	 *
+	 * @typedef RunOptions
+	 * @property {string} languageIdentifier
+	 * @property {string} filePath
+	 * @property {"none" | "insert" | "update"} updateMode
+	 * @property {(languages: string[]) => Prism} [createInstance]
+	 */
+	run(options) {
+		if (path.extname(options.filePath) === '.test') {
+			this.runTestCase(options.languageIdentifier, options.filePath, options.updateMode, options.createInstance);
+		} else {
+			this.runTestsWithHooks(options.languageIdentifier, require(options.filePath), options.createInstance);
+		}
+	},
+
 	/**
 	 * Runs the given test case file and asserts the result
 	 *
@@ -312,27 +342,31 @@ module.exports = {
 	 * @param {string} languageIdentifier
 	 * @param {string} filePath
 	 * @param {"none" | "insert" | "update"} updateMode
+	 * @param {(languages: string[]) => Prism} [createInstance]
 	 */
-	runTestCase(languageIdentifier, filePath, updateMode) {
+	runTestCase(languageIdentifier, filePath, updateMode, createInstance = defaultCreateInstance) {
+		let runner;
 		if (/\.html\.test$/i.test(filePath)) {
-			this.runTestCaseWithRunner(languageIdentifier, filePath, updateMode, new HighlightHTMLRunner());
+			runner = new HighlightHTMLRunner();
 		} else {
-			this.runTestCaseWithRunner(languageIdentifier, filePath, updateMode, new TokenizeJSONRunner());
+			runner = new TokenizeJSONRunner();
 		}
+		this.runTestCaseWithRunner(languageIdentifier, filePath, updateMode, runner, createInstance);
 	},
 
 	/**
 	 * @param {string} languageIdentifier
 	 * @param {string} filePath
 	 * @param {"none" | "insert" | "update"} updateMode
 	 * @param {Runner<T>} runner
+	 * @param {(languages: string[]) => Prism} createInstance
 	 * @template T
 	 */
-	runTestCaseWithRunner(languageIdentifier, filePath, updateMode, runner) {
+	runTestCaseWithRunner(languageIdentifier, filePath, updateMode, runner, createInstance) {
 		const testCase = TestCaseFile.readFromFile(filePath);
 		const usedLanguages = this.parseLanguageNames(languageIdentifier);
 
-		const Prism = PrismLoader.createInstance(usedLanguages.languages);
+		const Prism = createInstance(usedLanguages.languages);
 
 		// the first language is the main language to highlight
 		const actualValue = runner.run(Prism, testCase.code, usedLanguages.mainLanguage);