From 64a638882562d9af7a4ce5c1dd18f1671a442358 Mon Sep 17 00:00:00 2001 From: Evan Wallace Date: Thu, 9 Feb 2023 17:11:23 -0500 Subject: [PATCH] fix #2201: update instantiation expression parsing --- CHANGELOG.md | 13 ++ internal/js_parser/ts_parser.go | 202 ++++++++++++++++++++------- internal/js_parser/ts_parser_test.go | 53 ++++--- 3 files changed, 198 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b5b4ba0f53..c27dfab3e92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## Unreleased +* Change esbuild's parsing of TypeScript instantiation expressions to match TypeScript 4.8+ ([#2201](https://github.com/evanw/esbuild/issues/2201)) + + This release updates esbuild's implementation of instantiation expression erasure to match [microsoft/TypeScript#49353](https://github.com/microsoft/TypeScript/pull/49353). The new rules are as follows (copied from TypeScript's PR description): + + > When a potential type argument list is followed by + > + > * a line break, + > * an `(` token, + > * a template literal string, or + > * any token except `<` or `>` that isn't the start of an expression, + > + > we consider that construct to be a type argument list. Otherwise we consider the construct to be a `<` relational expression followed by a `>` relational expression. + * Ignore `sideEffects: false` for imported CSS files ([#1370](https://github.com/evanw/esbuild/issues/1370), [#1458](https://github.com/evanw/esbuild/pull/1458), [#2905](https://github.com/evanw/esbuild/issues/2905)) This release ignores the `sideEffects` annotation in `package.json` for CSS files that are imported into JS files using esbuild's `css` loader. This means that these CSS files are no longer be tree-shaken. diff --git a/internal/js_parser/ts_parser.go b/internal/js_parser/ts_parser.go index 61999a5ac01..64e95636e05 100644 --- a/internal/js_parser/ts_parser.go +++ b/internal/js_parser/ts_parser.go @@ -806,7 +806,7 @@ func (p *parser) trySkipTypeScriptTypeArgumentsWithBacktracking() bool { if p.skipTypeScriptTypeArguments(false /* isInsideJSXElement */) { // Check the token after the type argument list and backtrack if it's invalid - if !p.canFollowTypeArgumentsInExpression() { + if !p.tsCanFollowTypeArgumentsInExpression() { p.lexer.Unexpected() } } @@ -944,22 +944,14 @@ func (p *parser) isTSArrowFnJSX() (isTSArrowFn bool) { return } -func (p *parser) nextTokenIsOpenParenOrLessThanOrDot() (result bool) { - oldLexer := p.lexer - p.lexer.Next() - - result = p.lexer.Token == js_lexer.TOpenParen || - p.lexer.Token == js_lexer.TLessThan || - p.lexer.Token == js_lexer.TDot - - // Restore the lexer - p.lexer = oldLexer - return -} - // This function is taken from the official TypeScript compiler source code: // https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts -func (p *parser) canFollowTypeArgumentsInExpression() bool { +// +// This function is pretty inefficient as written, and could be collapsed into +// a single switch statement. But that would make it harder to keep this in +// sync with the TypeScript compiler's source code, so we keep doing it the +// slow way. +func (p *parser) tsCanFollowTypeArgumentsInExpression() bool { switch p.lexer.Token { case // These tokens can follow a type argument list in a call expression. @@ -968,9 +960,81 @@ func (p *parser) canFollowTypeArgumentsInExpression() bool { js_lexer.TTemplateHead: // foo `...${100}...` return true - // Consider something a type argument list only if the following token can't start an expression. + // A type argument list followed by `<` never makes sense, and a type argument list followed + // by `>` is ambiguous with a (re-scanned) `>>` operator, so we disqualify both. Also, in + // this context, `+` and `-` are unary operators, not binary operators. + case js_lexer.TLessThan, + js_lexer.TGreaterThan, + js_lexer.TPlus, + js_lexer.TMinus, + // TypeScript always sees "TGreaterThan" instead of these tokens since + // their scanner works a little differently than our lexer. So since + // "TGreaterThan" is forbidden above, we also forbid these too. + js_lexer.TGreaterThanEquals, + js_lexer.TGreaterThanGreaterThan, + js_lexer.TGreaterThanGreaterThanEquals, + js_lexer.TGreaterThanGreaterThanGreaterThan, + js_lexer.TGreaterThanGreaterThanGreaterThanEquals: + return false + } + + // We favor the type argument list interpretation when it is immediately followed by + // a line break, a binary operator, or something that can't start an expression. + return p.lexer.HasNewlineBefore || p.tsIsBinaryOperator() || !p.tsIsStartOfExpression() +} + +// This function is taken from the official TypeScript compiler source code: +// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts +func (p *parser) tsIsBinaryOperator() bool { + switch p.lexer.Token { + case js_lexer.TIn: + return p.allowIn + + case + js_lexer.TQuestionQuestion, + js_lexer.TBarBar, + js_lexer.TAmpersandAmpersand, + js_lexer.TBar, + js_lexer.TCaret, + js_lexer.TAmpersand, + js_lexer.TEqualsEquals, + js_lexer.TExclamationEquals, + js_lexer.TEqualsEqualsEquals, + js_lexer.TExclamationEqualsEquals, + js_lexer.TLessThan, + js_lexer.TGreaterThan, + js_lexer.TLessThanEquals, + js_lexer.TGreaterThanEquals, + js_lexer.TInstanceof, + js_lexer.TLessThanLessThan, + js_lexer.TGreaterThanGreaterThan, + js_lexer.TGreaterThanGreaterThanGreaterThan, + js_lexer.TPlus, + js_lexer.TMinus, + js_lexer.TAsterisk, + js_lexer.TSlash, + js_lexer.TPercent, + js_lexer.TAsteriskAsterisk: + return true + + case js_lexer.TIdentifier: + if p.lexer.IsContextualKeyword("as") || p.lexer.IsContextualKeyword("satisfies") { + return true + } + } + + return false +} + +// This function is taken from the official TypeScript compiler source code: +// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts +func (p *parser) tsIsStartOfExpression() bool { + if p.tsIsStartOfLeftHandSideExpression() { + return true + } + + switch p.lexer.Token { case - // From "isStartOfExpression()" js_lexer.TPlus, js_lexer.TMinus, js_lexer.TTilde, @@ -981,8 +1045,35 @@ func (p *parser) canFollowTypeArgumentsInExpression() bool { js_lexer.TPlusPlus, js_lexer.TMinusMinus, js_lexer.TLessThan, + js_lexer.TPrivateIdentifier, + js_lexer.TAt: + return true - // From "isStartOfLeftHandSideExpression()" + default: + if p.lexer.Token == js_lexer.TIdentifier && (p.lexer.Identifier.String == "await" || p.lexer.Identifier.String == "yield") { + // Yield/await always starts an expression. Either it is an identifier (in which case + // it is definitely an expression). Or it's a keyword (either because we're in + // a generator or async function, or in strict mode (or both)) and it started a yield or await expression. + return true + } + + // Error tolerance. If we see the start of some binary operator, we consider + // that the start of an expression. That way we'll parse out a missing identifier, + // give a good message about an identifier being missing, and then consume the + // rest of the binary expression. + if p.tsIsBinaryOperator() { + return true + } + + return p.tsIsIdentifier() + } +} + +// This function is taken from the official TypeScript compiler source code: +// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts +func (p *parser) tsIsStartOfLeftHandSideExpression() bool { + switch p.lexer.Token { + case js_lexer.TThis, js_lexer.TSuper, js_lexer.TNull, @@ -991,6 +1082,9 @@ func (p *parser) canFollowTypeArgumentsInExpression() bool { js_lexer.TNumericLiteral, js_lexer.TBigIntegerLiteral, js_lexer.TStringLiteral, + js_lexer.TNoSubstitutionTemplateLiteral, + js_lexer.TTemplateHead, + js_lexer.TOpenParen, js_lexer.TOpenBracket, js_lexer.TOpenBrace, js_lexer.TFunction, @@ -998,46 +1092,52 @@ func (p *parser) canFollowTypeArgumentsInExpression() bool { js_lexer.TNew, js_lexer.TSlash, js_lexer.TSlashEquals, - js_lexer.TIdentifier, + js_lexer.TIdentifier: + return true - // From "isBinaryOperator()" - js_lexer.TQuestionQuestion, - js_lexer.TBarBar, - js_lexer.TAmpersandAmpersand, - js_lexer.TBar, - js_lexer.TCaret, - js_lexer.TAmpersand, - js_lexer.TEqualsEquals, - js_lexer.TExclamationEquals, - js_lexer.TEqualsEqualsEquals, - js_lexer.TExclamationEqualsEquals, - js_lexer.TGreaterThan, - js_lexer.TLessThanEquals, - js_lexer.TGreaterThanEquals, - js_lexer.TInstanceof, - js_lexer.TLessThanLessThan, - js_lexer.TGreaterThanGreaterThan, - js_lexer.TGreaterThanGreaterThanGreaterThan, - js_lexer.TAsterisk, - js_lexer.TPercent, - js_lexer.TAsteriskAsterisk, + case js_lexer.TImport: + return p.tsLookAheadNextTokenIsOpenParenOrLessThanOrDot() - // TypeScript always sees "TGreaterThan" instead of these tokens since - // their scanner works a little differently than our lexer. So since - // "TGreaterThan" is forbidden above, we also forbid these too. - js_lexer.TGreaterThanGreaterThanEquals, - js_lexer.TGreaterThanGreaterThanGreaterThanEquals: - return false + default: + return p.tsIsIdentifier() + } +} - case js_lexer.TIn: - return !p.allowIn +// This function is taken from the official TypeScript compiler source code: +// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts +func (p *parser) tsLookAheadNextTokenIsOpenParenOrLessThanOrDot() (result bool) { + oldLexer := p.lexer + p.lexer.Next() - case js_lexer.TImport: - return !p.nextTokenIsOpenParenOrLessThanOrDot() + result = p.lexer.Token == js_lexer.TOpenParen || + p.lexer.Token == js_lexer.TLessThan || + p.lexer.Token == js_lexer.TDot + + // Restore the lexer + p.lexer = oldLexer + return +} + +// This function is taken from the official TypeScript compiler source code: +// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts +func (p *parser) tsIsIdentifier() bool { + if p.lexer.Token == js_lexer.TIdentifier { + // If we have a 'yield' keyword, and we're in the [yield] context, then 'yield' is + // considered a keyword and is not an identifier. + if p.fnOrArrowDataParse.yield != allowIdent && p.lexer.Identifier.String == "yield" { + return false + } + + // If we have a 'await' keyword, and we're in the [Await] context, then 'await' is + // considered a keyword and is not an identifier. + if p.fnOrArrowDataParse.await != allowIdent && p.lexer.Identifier.String == "await" { + return false + } - default: return true } + + return false } func (p *parser) skipTypeScriptInterfaceStmt(opts parseStmtOpts) { diff --git a/internal/js_parser/ts_parser_test.go b/internal/js_parser/ts_parser_test.go index 2c56ec9762a..d394d907973 100644 --- a/internal/js_parser/ts_parser_test.go +++ b/internal/js_parser/ts_parser_test.go @@ -2014,7 +2014,7 @@ func TestTSInstantiationExpression(t *testing.T) { // Function call expectPrintedTS(t, "const x1 = f\n(true);", "const x1 = f(true);\n") // Relational expression - expectPrintedTS(t, "const x1 = f\ntrue;", "const x1 = f < true > true;\n") + expectPrintedTS(t, "const x1 = f\ntrue;", "const x1 = f;\ntrue;\n") // Instantiation expression expectPrintedTS(t, "const x1 = f;\n(true);", "const x1 = f;\ntrue;\n") @@ -2034,8 +2034,6 @@ func TestTSInstantiationExpression(t *testing.T) { expectPrintedTS(t, "type T21 = typeof Array; f();", "f();\n") expectPrintedTS(t, "type T22 = typeof Array; f();", "f();\n") - // This behavior matches TypeScript 4.7.0 nightly (specifically "typescript@4.7.0-dev.20220421") - // after various fixes from Microsoft that landed after the TypeScript 4.7.0 beta expectPrintedTS(t, "f, g;", "f, g;\n") expectPrintedTS(t, "f<() => T>;", "f;\n") expectPrintedTS(t, "f.x<() => T>;", "f.x;\n") @@ -2058,11 +2056,14 @@ func TestTSInstantiationExpression(t *testing.T) { expectPrintedTS(t, "{ f }", "{\n f;\n}\n") expectPrintedTS(t, "f + g;", "f < x > +g;\n") expectPrintedTS(t, "f - g;", "f < x > -g;\n") - expectParseErrorTS(t, "f * g;", ": ERROR: Unexpected \"*\"\n") - expectParseErrorTS(t, "f == g;", ": ERROR: Unexpected \"==\"\n") - expectParseErrorTS(t, "f ?? g;", ": ERROR: Unexpected \"??\"\n") - expectParseErrorTS(t, "f in g;", ": ERROR: Unexpected \"in\"\n") - expectParseErrorTS(t, "f instanceof g;", ": ERROR: Unexpected \"instanceof\"\n") + expectPrintedTS(t, "f * g;", "f * g;\n") + expectPrintedTS(t, "f *= g;", "f *= g;\n") + expectPrintedTS(t, "f == g;", "f == g;\n") + expectPrintedTS(t, "f ?? g;", "f ?? g;\n") + expectPrintedTS(t, "f in g;", "f in g;\n") + expectPrintedTS(t, "f instanceof g;", "f instanceof g;\n") + expectPrintedTS(t, "f as g;", "f;\n") + expectPrintedTS(t, "f satisfies g;", "f;\n") expectParseErrorTS(t, "const a8 = f;", ": ERROR: Unexpected \";\"\n") expectParseErrorTS(t, "const b1 = f?.;", ": ERROR: Expected \"(\" but found \";\"\n") @@ -2082,20 +2083,26 @@ func TestTSInstantiationExpression(t *testing.T) { expectParseErrorTSX(t, "type x = typeof y\n\nz", ": ERROR: Unexpected end of file before a closing \"number\" tag\n: NOTE: The opening \"number\" tag is here:\n") // See: https://github.com/microsoft/TypeScript/issues/48654 - expectPrintedTS(t, "x\ny", "x < true > y;\n") + expectPrintedTS(t, "x y", "x < true > y;\n") + expectPrintedTS(t, "x\ny", "x;\ny;\n") expectPrintedTS(t, "x\nif (y) {}", "x;\nif (y) {\n}\n") expectPrintedTS(t, "x\nimport 'y'", "x;\nimport \"y\";\n") - expectPrintedTS(t, "x\nimport('y')", "x < true > import(\"y\");\n") - expectPrintedTS(t, "x\nimport.meta", "x < true > import.meta;\n") - expectPrintedTS(t, "new x\ny", "new x() < number > y;\n") + expectPrintedTS(t, "x\nimport('y')", "x;\nimport(\"y\");\n") + expectPrintedTS(t, "x\nimport.meta", "x;\nimport.meta;\n") + expectPrintedTS(t, "x import('y')", "x < true > import(\"y\");\n") + expectPrintedTS(t, "x import.meta", "x < true > import.meta;\n") + expectPrintedTS(t, "new x y", "new x() < number > y;\n") + expectPrintedTS(t, "new x\ny", "new x();\ny;\n") expectPrintedTS(t, "new x\nif (y) {}", "new x();\nif (y) {\n}\n") expectPrintedTS(t, "new x\nimport 'y'", "new x();\nimport \"y\";\n") - expectPrintedTS(t, "new x\nimport('y')", "new x() < true > import(\"y\");\n") - expectPrintedTS(t, "new x\nimport.meta", "new x() < true > import.meta;\n") + expectPrintedTS(t, "new x\nimport('y')", "new x();\nimport(\"y\");\n") + expectPrintedTS(t, "new x\nimport.meta", "new x();\nimport.meta;\n") + expectPrintedTS(t, "new x import('y')", "new x() < true > import(\"y\");\n") + expectPrintedTS(t, "new x import.meta", "new x() < true > import.meta;\n") // See: https://github.com/microsoft/TypeScript/issues/48759 - expectParseErrorTS(t, "x\nimport('y')", ": ERROR: Expected \"(\" but found \"<\"\n") - expectParseErrorTS(t, "new x\nimport('y')", ": ERROR: Expected \"(\" but found \"<\"\n") + expectParseErrorTS(t, "x\nimport('y')", ": ERROR: Unexpected \"<\"\n") + expectParseErrorTS(t, "new x\nimport('y')", ": ERROR: Unexpected \"<\"\n") // See: https://github.com/evanw/esbuild/issues/2201 expectParseErrorTS(t, "return Array < ;", ": ERROR: Unexpected \";\"\n") @@ -2115,12 +2122,20 @@ func TestTSInstantiationExpression(t *testing.T) { expectPrintedTS(t, "return Array < Array < number >> +1;", "return Array < Array < number >> 1;\n") expectPrintedTS(t, "return Array < Array < number >> (1);", "return Array(1);\n") expectPrintedTS(t, "return Array < Array < number > > (1);", "return Array(1);\n") - expectParseErrorTS(t, "return Array < number > in x;", ": ERROR: Unexpected \"in\"\n") - expectParseErrorTS(t, "return Array < Array < number >> in x;", ": ERROR: Unexpected \"in\"\n") - expectParseErrorTS(t, "return Array < Array < number > > in x;", ": ERROR: Unexpected \">\"\n") + expectPrintedTS(t, "return Array < number > in x;", "return Array in x;\n") + expectPrintedTS(t, "return Array < Array < number >> in x;", "return Array in x;\n") + expectPrintedTS(t, "return Array < Array < number > > in x;", "return Array in x;\n") expectPrintedTS(t, "for (var x = Array < number > in y) ;", "x = Array;\nfor (var x in y)\n ;\n") expectPrintedTS(t, "for (var x = Array < Array < number >> in y) ;", "x = Array;\nfor (var x in y)\n ;\n") expectPrintedTS(t, "for (var x = Array < Array < number > > in y) ;", "x = Array;\nfor (var x in y)\n ;\n") + + // See: https://github.com/microsoft/TypeScript/pull/49353 + expectPrintedTS(t, "F<{}> 0", "F < {} > 0;\n") + expectPrintedTS(t, "F<{}> class F {}", "F < {} > class F {\n};\n") + expectPrintedTS(t, "f<{}> function f() {}", "f < {} > function f() {\n};\n") + expectPrintedTS(t, "F<{}>\n0", "F;\n0;\n") + expectPrintedTS(t, "F<{}>\nclass F {}", "F;\nclass F {\n}\n") + expectPrintedTS(t, "f<{}>\nfunction f() {}", "f;\nfunction f() {\n}\n") } func TestTSExponentiation(t *testing.T) {