Improve Recovery of Unterminated Regular Expressions

src/compiler/parser.ts

@@ -62,7 +62,6 @@ import {
     DeleteExpression,
     Diagnostic,
     DiagnosticArguments,
-    DiagnosticCategory,
     DiagnosticMessage,
     Diagnostics,
     DiagnosticWithDetachedLocation,
@@ -2144,11 +2143,7 @@ namespace Parser {
         // Don't report another error if it would just be at the same position as the last error.
         const lastError = lastOrUndefined(parseDiagnostics);
         let result: DiagnosticWithDetachedLocation | undefined;
-        if (message.category === DiagnosticCategory.Message && lastError && start === lastError.start && length === lastError.length) {
-            result = createDetachedDiagnostic(fileName, sourceText, start, length, message, ...args);
-            addRelatedInfo(lastError, result);
-        }
-        else if (!lastError || start !== lastError.start) {
+        if (!lastError || start !== lastError.start) {
             result = createDetachedDiagnostic(fileName, sourceText, start, length, message, ...args);
             parseDiagnostics.push(result);
         }

src/compiler/scanner.ts

@@ -21,6 +21,7 @@ import {
     KeywordSyntaxKind,
     LanguageFeatureMinimumTarget,
     LanguageVariant,
+    lastOrUndefined,
     LineAndCharacter,
     MapLike,
     parsePseudoBigInt,
@@ -2424,7 +2425,8 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
     function reScanSlashToken(reportErrors?: boolean): SyntaxKind {
         if (token === SyntaxKind.SlashToken || token === SyntaxKind.SlashEqualsToken) {
             // Quickly get to the end of regex such that we know the flags
-            let p = tokenStart + 1;
+            const startOfRegExpBody = tokenStart + 1;
+            pos = startOfRegExpBody;
             let inEscape = false;
             // Although nested character classes are allowed in Unicode Sets mode,
             // an unescaped slash is nevertheless invalid even in a character class in Unicode mode.
@@ -2436,16 +2438,14 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
             while (true) {
                 // If we reach the end of a file, or hit a newline, then this is an unterminated
                 // regex.  Report error and return what we have so far.
-                if (p >= end) {
+                if (pos >= end) {
                     tokenFlags |= TokenFlags.Unterminated;
-                    error(Diagnostics.Unterminated_regular_expression_literal);
                     break;
                 }
 
-                const ch = charCodeUnchecked(p);
+                const ch = charCodeUnchecked(pos);
                 if (isLineBreak(ch)) {
                     tokenFlags |= TokenFlags.Unterminated;
-                    error(Diagnostics.Unterminated_regular_expression_literal);
                     break;
                 }
 
@@ -2457,7 +2457,6 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
                 else if (ch === CharacterCodes.slash && !inCharacterClass) {
                     // A slash within a character class is permissible,
                     // but in general it signals the end of the regexp literal.
-                    p++;
                     break;
                 }
                 else if (ch === CharacterCodes.openBracket) {
@@ -2469,55 +2468,92 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
                 else if (ch === CharacterCodes.closeBracket) {
                     inCharacterClass = false;
                 }
-                p++;
+                pos++;
             }
-            const isUnterminated = !!(tokenFlags & TokenFlags.Unterminated);
-            const endOfBody = p - (isUnterminated ? 0 : 1);
-            let regExpFlags = RegularExpressionFlags.None;
-            while (p < end) {
-                const ch = charCodeUnchecked(p);
-                if (!isIdentifierPart(ch, languageVersion)) {
-                    break;
-                }
-                if (reportErrors) {
-                    const flag = characterToRegularExpressionFlag(String.fromCharCode(ch));
-                    if (flag === undefined) {
-                        error(Diagnostics.Unknown_regular_expression_flag, p, 1);
+            const endOfRegExpBody = pos;
+            if (tokenFlags & TokenFlags.Unterminated) {
+                // Search for the nearest unbalanced bracket for better recovery. Since the expression is
+                // invalid anyways, we take nested square brackets into consideration for the best guess.
+                pos = startOfRegExpBody;
+                inEscape = false;
+                let characterClassDepth = 0;
+                const bracketStack: CharacterCodes[] = [];
+                while (pos < endOfRegExpBody) {
+                    const ch = charCodeUnchecked(pos);
+                    if (inEscape) {
+                        inEscape = false;
                     }
-                    else if (regExpFlags & flag) {
-                        error(Diagnostics.Duplicate_regular_expression_flag, p, 1);
+                    else if (ch === CharacterCodes.backslash) {
+                        inEscape = true;
                     }
-                    else if (((regExpFlags | flag) & RegularExpressionFlags.UnicodeMode) === RegularExpressionFlags.UnicodeMode) {
-                        error(Diagnostics.The_Unicode_u_flag_and_the_Unicode_Sets_v_flag_cannot_be_set_simultaneously, p, 1);
+                    else if (ch === CharacterCodes.openBracket) {
+                        characterClassDepth++;
                     }
-                    else {
-                        regExpFlags |= flag;
-                        checkRegularExpressionFlagAvailable(flag, p);
-                    }
-                }
-                p++;
-            }
-            pos = p;
-            if (reportErrors) {
-                const saveTokenStart = tokenStart;
-                const saveTokenFlags = tokenFlags;
-                const savePos = pos;
-                const saveEnd = end;
-                pos = tokenStart + 1;
-                end = endOfBody;
-                scanRegularExpressionWorker(regExpFlags, isUnterminated, /*annexB*/ true);
-                tokenStart = saveTokenStart;
-                tokenFlags = saveTokenFlags;
-                pos = savePos;
-                end = saveEnd;
+                    else if (ch === CharacterCodes.closeBracket && characterClassDepth) {
+                        characterClassDepth--;
+                    }
+                    else if (!characterClassDepth) {
+                        if (ch === CharacterCodes.openParen) {
+                            bracketStack.push(CharacterCodes.closeParen);
+                        }
+                        else if (ch === CharacterCodes.openBrace) {
+                            bracketStack.push(CharacterCodes.closeBrace);
+                        }
+                        else if (ch === lastOrUndefined(bracketStack)) {
+                            bracketStack.pop();
+                        }
+                        else if (ch === CharacterCodes.closeParen || ch === CharacterCodes.closeBracket || ch === CharacterCodes.closeBrace) {
+                            // We encountered an unbalanced bracket outside a character class. Treat this position as the end of regex.
+                            break;
+                        }
+                    }
+                    pos++;
+                }
+                // Whitespaces and semicolons at the end are not likely to be part of the regex
+                while (isWhiteSpaceLike(charCodeChecked(pos - 1)) || charCodeChecked(pos - 1) === CharacterCodes.semicolon) pos--;
+                error(Diagnostics.Unterminated_regular_expression_literal, tokenStart, pos - tokenStart);
+            }
+            else {
+                // Consume the slash character
+                pos++;
+                let regExpFlags = RegularExpressionFlags.None;
+                while (pos < end) {
+                    const ch = codePointUnchecked(pos);
+                    if (!isIdentifierPart(ch, languageVersion)) {
+                        break;
+                    }
+                    const size = charSize(ch);
+                    if (reportErrors) {
+                        const flag = characterToRegularExpressionFlag(utf16EncodeAsString(ch));
+                        if (flag === undefined) {
+                            error(Diagnostics.Unknown_regular_expression_flag, pos, size);
+                        }
+                        else if (regExpFlags & flag) {
+                            error(Diagnostics.Duplicate_regular_expression_flag, pos, size);
+                        }
+                        else if (((regExpFlags | flag) & RegularExpressionFlags.UnicodeMode) === RegularExpressionFlags.UnicodeMode) {
+                            error(Diagnostics.The_Unicode_u_flag_and_the_Unicode_Sets_v_flag_cannot_be_set_simultaneously, pos, size);
+                        }
+                        else {
+                            regExpFlags |= flag;
+                            checkRegularExpressionFlagAvailability(flag, size);
+                        }
+                    }
+                    pos += size;
+                }
+                if (reportErrors) {
+                    scanRange(startOfRegExpBody, endOfRegExpBody - startOfRegExpBody, () => {
+                        scanRegularExpressionWorker(regExpFlags, /*annexB*/ true);
+                    });
+                }
             }
             tokenValue = text.substring(tokenStart, pos);
             token = SyntaxKind.RegularExpressionLiteral;
         }
         return token;
     }
 
-    function scanRegularExpressionWorker(regExpFlags: RegularExpressionFlags, isUnterminated: boolean, annexB: boolean) {
+    function scanRegularExpressionWorker(regExpFlags: RegularExpressionFlags, annexB: boolean) {
         // Why var? It avoids TDZ checks in the runtime which can be costly.
         // See: https://github.com/microsoft/TypeScript/issues/52924
         /* eslint-disable no-var */
@@ -2535,7 +2571,7 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
         /** @see {scanClassSetExpression} */
         var mayContainStrings = false;
 
-        /** The number of numeric (anonymous) capturing groups defined in the regex. */
+        /** The number of all (named and unnamed) capturing groups defined in the regex. */
         var numberOfCapturingGroups = 0;
         /** All named capturing groups defined in the regex. */
         var groupSpecifiers: Set<string> | undefined;
@@ -2740,10 +2776,6 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
                     // falls through
                     case CharacterCodes.closeBracket:
                     case CharacterCodes.closeBrace:
-                        if (isUnterminated && !isInGroup) {
-                            // Assume what starting from the character to be outside of the regex
-                            return;
-                        }
                         if (unicodeMode || ch === CharacterCodes.closeParen) {
                             error(Diagnostics.Unexpected_0_Did_you_mean_to_escape_it_with_backslash, pos, 1, String.fromCharCode(ch));
                         }
@@ -2763,25 +2795,26 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
 
         function scanPatternModifiers(currFlags: RegularExpressionFlags): RegularExpressionFlags {
             while (true) {
-                const ch = charCodeChecked(pos);
+                const ch = codePointChecked(pos);
                 if (ch === CharacterCodes.EOF || !isIdentifierPart(ch, languageVersion)) {
                     break;
                 }
-                const flag = characterToRegularExpressionFlag(String.fromCharCode(ch));
+                const size = charSize(ch);
+                const flag = characterToRegularExpressionFlag(utf16EncodeAsString(ch));
                 if (flag === undefined) {
-                    error(Diagnostics.Unknown_regular_expression_flag, pos, 1);
+                    error(Diagnostics.Unknown_regular_expression_flag, pos, size);
                 }
                 else if (currFlags & flag) {
-                    error(Diagnostics.Duplicate_regular_expression_flag, pos, 1);
+                    error(Diagnostics.Duplicate_regular_expression_flag, pos, size);
                 }
                 else if (!(flag & RegularExpressionFlags.Modifiers)) {
-                    error(Diagnostics.This_regular_expression_flag_cannot_be_toggled_within_a_subpattern, pos, 1);
+                    error(Diagnostics.This_regular_expression_flag_cannot_be_toggled_within_a_subpattern, pos, size);
                 }
                 else {
                     currFlags |= flag;
-                    checkRegularExpressionFlagAvailable(flag, pos);
+                    checkRegularExpressionFlagAvailability(flag, size);
                 }
-                pos++;
+                pos += size;
             }
             return currFlags;
         }
@@ -3494,10 +3527,10 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
         });
     }
 
-    function checkRegularExpressionFlagAvailable(flag: RegularExpressionFlags, pos: number) {
+    function checkRegularExpressionFlagAvailability(flag: RegularExpressionFlags, size: number) {
         const availableFrom = regExpFlagToFirstAvailableLanguageVersion.get(flag) as ScriptTarget | undefined;
         if (availableFrom && languageVersion < availableFrom) {
-            error(Diagnostics.This_regular_expression_flag_is_only_available_when_targeting_0_or_later, pos, 1, getNameOfScriptTarget(availableFrom));
+            error(Diagnostics.This_regular_expression_flag_is_only_available_when_targeting_0_or_later, pos, size, getNameOfScriptTarget(availableFrom));
         }
     }
 

src/testRunner/tests.ts

@@ -47,6 +47,7 @@ export * from "./unittests/paths.js";
 export * from "./unittests/printer.js";
 export * from "./unittests/programApi.js";
 export * from "./unittests/publicApi.js";
+export * from "./unittests/regExpParserRecovery.js";
 export * from "./unittests/reuseProgramStructure.js";
 export * from "./unittests/semver.js";
 export * from "./unittests/services/cancellableLanguageServiceOperations.js";

src/testRunner/unittests/incrementalParser.ts

@@ -160,7 +160,7 @@ describe("unittests:: incrementalParser::", () => {
         const oldText = ts.ScriptSnapshot.fromString(source);
         const newTextAndChange = withInsert(oldText, semicolonIndex, "/");
 
-        compareTrees(oldText, newTextAndChange.text, newTextAndChange.textChangeRange, 0);
+        compareTrees(oldText, newTextAndChange.text, newTextAndChange.textChangeRange, 4);
     });
 
     it("Regular expression 2", () => {

src/testRunner/unittests/regExpParserRecovery.ts

@@ -0,0 +1,81 @@
+import * as ts from "../_namespaces/ts.js";
+
+describe("unittests:: regExpParserRecovery", () => {
+    const testCases = [
+        "/",
+        "/[]",
+        "/{}",
+        "/()",
+        "/foo",
+        "/foo[]",
+        "/foo{}",
+        "/foo()",
+        "/[]foo",
+        "/{}foo",
+        "/()foo",
+        "/{[]}",
+        "/([])",
+        "/[)}({]",
+        "/({[)}]})",
+        "/\\[",
+        "/\\{",
+        "/\\(",
+        "/[\\[]",
+        "/(\\[)",
+        "/{\\[}",
+        "/[\\(]",
+        "/(\\()",
+        "/{\\(}",
+        "/[\\{]",
+        "/(\\{)",
+        "/{\\{}",
+        "/\\{(\\[\\([{])",
+        "/\\]",
+        "/\\}",
+        "/\\)",
+        "/[\\]]",
+        "/(\\])",
+        "/{\\]}",
+        "/[\\)]",
+        "/(\\))",
+        "/{\\)}",
+        "/[\\}]",
+        "/(\\})",
+        "/{\\}}",
+        "/({[\\]})]})",
+    ];
+    const whiteSpaceSequences = [
+        "",
+        "  ",
+        "\t\v\r\n",
+        "\u3000\u2028",
+    ];
+    it("stops parsing unterminated regexes at correct position", () => {
+        ts.forEach(testCases, testCase => {
+            ts.forEach(whiteSpaceSequences, whiteSpaces => {
+                const testCaseWithWhiteSpaces = testCase + whiteSpaces;
+                const sources = [
+                    `const regex = ${testCaseWithWhiteSpaces};`,
+                    `(${testCaseWithWhiteSpaces});`,
+                    `([${testCaseWithWhiteSpaces}]);`,
+                    `({prop: ${testCaseWithWhiteSpaces}});`,
+                    `({prop: ([(${testCaseWithWhiteSpaces})])});`,
+                    `({[(${testCaseWithWhiteSpaces}).source]: 42});`,
+                ];
+                ts.forEach(sources, source => {
+                    const { parseDiagnostics } = ts.createLanguageServiceSourceFile(
+                        /*fileName*/ "",
+                        ts.ScriptSnapshot.fromString(source),
+                        ts.ScriptTarget.Latest,
+                        /*version*/ "0",
+                        /*setNodeParents*/ false,
+                    );
+                    const diagnostic = ts.find(parseDiagnostics, ({ code }) => code === ts.Diagnostics.Unterminated_regular_expression_literal.code);
+                    assert(diagnostic, "There should be an 'Unterminated regular expression literal.' error");
+                    assert.equal(diagnostic.start, source.indexOf("/"), "Diagnostic should start at where the regex starts");
+                    assert.equal(diagnostic.length, testCase.length, "Diagnostic should end at where the regex ends");
+                });
+            });
+        });
+    });
+});

tests/baselines/reference/parser645086_1.errors.txt

@@ -1,13 +1,10 @@
 parser645086_1.ts(1,13): error TS1005: ',' expected.
 parser645086_1.ts(1,14): error TS1134: Variable declaration expected.
-parser645086_1.ts(1,15): error TS1161: Unterminated regular expression literal.
 
 
-==== parser645086_1.ts (3 errors) ====
+==== parser645086_1.ts (2 errors) ====
     var v = /[]/]/
                 ~
 !!! error TS1005: ',' expected.
                  ~
-!!! error TS1134: Variable declaration expected.
-
-!!! error TS1161: Unterminated regular expression literal.
+!!! error TS1134: Variable declaration expected.

tests/baselines/reference/parser645086_2.errors.txt

@@ -1,13 +1,10 @@
 parser645086_2.ts(1,14): error TS1005: ',' expected.
 parser645086_2.ts(1,15): error TS1134: Variable declaration expected.
-parser645086_2.ts(1,16): error TS1161: Unterminated regular expression literal.
 
 
-==== parser645086_2.ts (3 errors) ====
+==== parser645086_2.ts (2 errors) ====
     var v = /[^]/]/
                  ~
 !!! error TS1005: ',' expected.
                   ~
-!!! error TS1134: Variable declaration expected.
-
-!!! error TS1161: Unterminated regular expression literal.
+!!! error TS1134: Variable declaration expected.