fix #2201: update instantiation expression parsing

evanw · Feb 9, 2023 · 64a6388 · 64a6388
1 parent 88e17d8
commit 64a6388
Show file tree

Hide file tree

Showing 3 changed files with 198 additions and 70 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,19 @@
 
 ## Unreleased
 
+* Change esbuild's parsing of TypeScript instantiation expressions to match TypeScript 4.8+ ([#2201](https://github.com/evanw/esbuild/issues/2201))
+
+    This release updates esbuild's implementation of instantiation expression erasure to match [microsoft/TypeScript#49353](https://github.com/microsoft/TypeScript/pull/49353). The new rules are as follows (copied from TypeScript's PR description):
+
+    > When a potential type argument list is followed by
+    >
+    > * a line break,
+    > * an `(` token,
+    > * a template literal string, or
+    > * any token except `<` or `>` that isn't the start of an expression,
+    >
+    > we consider that construct to be a type argument list. Otherwise we consider the construct to be a `<` relational expression followed by a `>` relational expression.
+
 * Ignore `sideEffects: false` for imported CSS files ([#1370](https://github.com/evanw/esbuild/issues/1370), [#1458](https://github.com/evanw/esbuild/pull/1458), [#2905](https://github.com/evanw/esbuild/issues/2905))
 
     This release ignores the `sideEffects` annotation in `package.json` for CSS files that are imported into JS files using esbuild's `css` loader. This means that these CSS files are no longer be tree-shaken.

diff --git a/internal/js_parser/ts_parser.go b/internal/js_parser/ts_parser.go
@@ -806,7 +806,7 @@ func (p *parser) trySkipTypeScriptTypeArgumentsWithBacktracking() bool {
 
 	if p.skipTypeScriptTypeArguments(false /* isInsideJSXElement */) {
 		// Check the token after the type argument list and backtrack if it's invalid
-		if !p.canFollowTypeArgumentsInExpression() {
+		if !p.tsCanFollowTypeArgumentsInExpression() {
 			p.lexer.Unexpected()
 		}
 	}
@@ -944,22 +944,14 @@ func (p *parser) isTSArrowFnJSX() (isTSArrowFn bool) {
 	return
 }
 
-func (p *parser) nextTokenIsOpenParenOrLessThanOrDot() (result bool) {
-	oldLexer := p.lexer
-	p.lexer.Next()
-
-	result = p.lexer.Token == js_lexer.TOpenParen ||
-		p.lexer.Token == js_lexer.TLessThan ||
-		p.lexer.Token == js_lexer.TDot
-
-	// Restore the lexer
-	p.lexer = oldLexer
-	return
-}
-
 // This function is taken from the official TypeScript compiler source code:
 // https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts
-func (p *parser) canFollowTypeArgumentsInExpression() bool {
+//
+// This function is pretty inefficient as written, and could be collapsed into
+// a single switch statement. But that would make it harder to keep this in
+// sync with the TypeScript compiler's source code, so we keep doing it the
+// slow way.
+func (p *parser) tsCanFollowTypeArgumentsInExpression() bool {
 	switch p.lexer.Token {
 	case
 		// These tokens can follow a type argument list in a call expression.
@@ -968,9 +960,81 @@ func (p *parser) canFollowTypeArgumentsInExpression() bool {
 		js_lexer.TTemplateHead:                  // foo<T> `...${100}...`
 		return true
 
-	// Consider something a type argument list only if the following token can't start an expression.
+	// A type argument list followed by `<` never makes sense, and a type argument list followed
+	// by `>` is ambiguous with a (re-scanned) `>>` operator, so we disqualify both. Also, in
+	// this context, `+` and `-` are unary operators, not binary operators.
+	case js_lexer.TLessThan,
+		js_lexer.TGreaterThan,
+		js_lexer.TPlus,
+		js_lexer.TMinus,
+		// TypeScript always sees "TGreaterThan" instead of these tokens since
+		// their scanner works a little differently than our lexer. So since
+		// "TGreaterThan" is forbidden above, we also forbid these too.
+		js_lexer.TGreaterThanEquals,
+		js_lexer.TGreaterThanGreaterThan,
+		js_lexer.TGreaterThanGreaterThanEquals,
+		js_lexer.TGreaterThanGreaterThanGreaterThan,
+		js_lexer.TGreaterThanGreaterThanGreaterThanEquals:
+		return false
+	}
+
+	// We favor the type argument list interpretation when it is immediately followed by
+	// a line break, a binary operator, or something that can't start an expression.
+	return p.lexer.HasNewlineBefore || p.tsIsBinaryOperator() || !p.tsIsStartOfExpression()
+}
+
+// This function is taken from the official TypeScript compiler source code:
+// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts
+func (p *parser) tsIsBinaryOperator() bool {
+	switch p.lexer.Token {
+	case js_lexer.TIn:
+		return p.allowIn
+
+	case
+		js_lexer.TQuestionQuestion,
+		js_lexer.TBarBar,
+		js_lexer.TAmpersandAmpersand,
+		js_lexer.TBar,
+		js_lexer.TCaret,
+		js_lexer.TAmpersand,
+		js_lexer.TEqualsEquals,
+		js_lexer.TExclamationEquals,
+		js_lexer.TEqualsEqualsEquals,
+		js_lexer.TExclamationEqualsEquals,
+		js_lexer.TLessThan,
+		js_lexer.TGreaterThan,
+		js_lexer.TLessThanEquals,
+		js_lexer.TGreaterThanEquals,
+		js_lexer.TInstanceof,
+		js_lexer.TLessThanLessThan,
+		js_lexer.TGreaterThanGreaterThan,
+		js_lexer.TGreaterThanGreaterThanGreaterThan,
+		js_lexer.TPlus,
+		js_lexer.TMinus,
+		js_lexer.TAsterisk,
+		js_lexer.TSlash,
+		js_lexer.TPercent,
+		js_lexer.TAsteriskAsterisk:
+		return true
+
+	case js_lexer.TIdentifier:
+		if p.lexer.IsContextualKeyword("as") || p.lexer.IsContextualKeyword("satisfies") {
+			return true
+		}
+	}
+
+	return false
+}
+
+// This function is taken from the official TypeScript compiler source code:
+// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts
+func (p *parser) tsIsStartOfExpression() bool {
+	if p.tsIsStartOfLeftHandSideExpression() {
+		return true
+	}
+
+	switch p.lexer.Token {
 	case
-		// From "isStartOfExpression()"
 		js_lexer.TPlus,
 		js_lexer.TMinus,
 		js_lexer.TTilde,
@@ -981,8 +1045,35 @@ func (p *parser) canFollowTypeArgumentsInExpression() bool {
 		js_lexer.TPlusPlus,
 		js_lexer.TMinusMinus,
 		js_lexer.TLessThan,
+		js_lexer.TPrivateIdentifier,
+		js_lexer.TAt:
+		return true
 
-		// From "isStartOfLeftHandSideExpression()"
+	default:
+		if p.lexer.Token == js_lexer.TIdentifier && (p.lexer.Identifier.String == "await" || p.lexer.Identifier.String == "yield") {
+			// Yield/await always starts an expression.  Either it is an identifier (in which case
+			// it is definitely an expression).  Or it's a keyword (either because we're in
+			// a generator or async function, or in strict mode (or both)) and it started a yield or await expression.
+			return true
+		}
+
+		// Error tolerance.  If we see the start of some binary operator, we consider
+		// that the start of an expression.  That way we'll parse out a missing identifier,
+		// give a good message about an identifier being missing, and then consume the
+		// rest of the binary expression.
+		if p.tsIsBinaryOperator() {
+			return true
+		}
+
+		return p.tsIsIdentifier()
+	}
+}
+
+// This function is taken from the official TypeScript compiler source code:
+// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts
+func (p *parser) tsIsStartOfLeftHandSideExpression() bool {
+	switch p.lexer.Token {
+	case
 		js_lexer.TThis,
 		js_lexer.TSuper,
 		js_lexer.TNull,
@@ -991,53 +1082,62 @@ func (p *parser) canFollowTypeArgumentsInExpression() bool {
 		js_lexer.TNumericLiteral,
 		js_lexer.TBigIntegerLiteral,
 		js_lexer.TStringLiteral,
+		js_lexer.TNoSubstitutionTemplateLiteral,
+		js_lexer.TTemplateHead,
+		js_lexer.TOpenParen,
 		js_lexer.TOpenBracket,
 		js_lexer.TOpenBrace,
 		js_lexer.TFunction,
 		js_lexer.TClass,
 		js_lexer.TNew,
 		js_lexer.TSlash,
 		js_lexer.TSlashEquals,
-		js_lexer.TIdentifier,
+		js_lexer.TIdentifier:
+		return true
 
-		// From "isBinaryOperator()"
-		js_lexer.TQuestionQuestion,
-		js_lexer.TBarBar,
-		js_lexer.TAmpersandAmpersand,
-		js_lexer.TBar,
-		js_lexer.TCaret,
-		js_lexer.TAmpersand,
-		js_lexer.TEqualsEquals,
-		js_lexer.TExclamationEquals,
-		js_lexer.TEqualsEqualsEquals,
-		js_lexer.TExclamationEqualsEquals,
-		js_lexer.TGreaterThan,
-		js_lexer.TLessThanEquals,
-		js_lexer.TGreaterThanEquals,
-		js_lexer.TInstanceof,
-		js_lexer.TLessThanLessThan,
-		js_lexer.TGreaterThanGreaterThan,
-		js_lexer.TGreaterThanGreaterThanGreaterThan,
-		js_lexer.TAsterisk,
-		js_lexer.TPercent,
-		js_lexer.TAsteriskAsterisk,
+	case js_lexer.TImport:
+		return p.tsLookAheadNextTokenIsOpenParenOrLessThanOrDot()
 
-		// TypeScript always sees "TGreaterThan" instead of these tokens since
-		// their scanner works a little differently than our lexer. So since
-		// "TGreaterThan" is forbidden above, we also forbid these too.
-		js_lexer.TGreaterThanGreaterThanEquals,
-		js_lexer.TGreaterThanGreaterThanGreaterThanEquals:
-		return false
+	default:
+		return p.tsIsIdentifier()
+	}
+}
 
-	case js_lexer.TIn:
-		return !p.allowIn
+// This function is taken from the official TypeScript compiler source code:
+// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts
+func (p *parser) tsLookAheadNextTokenIsOpenParenOrLessThanOrDot() (result bool) {
+	oldLexer := p.lexer
+	p.lexer.Next()
 
-	case js_lexer.TImport:
-		return !p.nextTokenIsOpenParenOrLessThanOrDot()
+	result = p.lexer.Token == js_lexer.TOpenParen ||
+		p.lexer.Token == js_lexer.TLessThan ||
+		p.lexer.Token == js_lexer.TDot
+
+	// Restore the lexer
+	p.lexer = oldLexer
+	return
+}
+
+// This function is taken from the official TypeScript compiler source code:
+// https://github.com/microsoft/TypeScript/blob/master/src/compiler/parser.ts
+func (p *parser) tsIsIdentifier() bool {
+	if p.lexer.Token == js_lexer.TIdentifier {
+		// If we have a 'yield' keyword, and we're in the [yield] context, then 'yield' is
+		// considered a keyword and is not an identifier.
+		if p.fnOrArrowDataParse.yield != allowIdent && p.lexer.Identifier.String == "yield" {
+			return false
+		}
+
+		// If we have a 'await' keyword, and we're in the [Await] context, then 'await' is
+		// considered a keyword and is not an identifier.
+		if p.fnOrArrowDataParse.await != allowIdent && p.lexer.Identifier.String == "await" {
+			return false
+		}
 
-	default:
 		return true
 	}
+
+	return false
 }
 
 func (p *parser) skipTypeScriptInterfaceStmt(opts parseStmtOpts) {

diff --git a/internal/js_parser/ts_parser_test.go b/internal/js_parser/ts_parser_test.go
@@ -2014,7 +2014,7 @@ func TestTSInstantiationExpression(t *testing.T) {
 	// Function call
 	expectPrintedTS(t, "const x1 = f<true>\n(true);", "const x1 = f(true);\n")
 	// Relational expression
-	expectPrintedTS(t, "const x1 = f<true>\ntrue;", "const x1 = f < true > true;\n")
+	expectPrintedTS(t, "const x1 = f<true>\ntrue;", "const x1 = f;\ntrue;\n")
 	// Instantiation expression
 	expectPrintedTS(t, "const x1 = f<true>;\n(true);", "const x1 = f;\ntrue;\n")
 
@@ -2034,8 +2034,6 @@ func TestTSInstantiationExpression(t *testing.T) {
 	expectPrintedTS(t, "type T21 = typeof Array<string>; f();", "f();\n")
 	expectPrintedTS(t, "type T22 = typeof Array<string, number>; f();", "f();\n")
 
-	// This behavior matches TypeScript 4.7.0 nightly (specifically "[email protected]")
-	// after various fixes from Microsoft that landed after the TypeScript 4.7.0 beta
 	expectPrintedTS(t, "f<x>, g<y>;", "f, g;\n")
 	expectPrintedTS(t, "f<<T>() => T>;", "f;\n")
 	expectPrintedTS(t, "f.x<<T>() => T>;", "f.x;\n")
@@ -2058,11 +2056,14 @@ func TestTSInstantiationExpression(t *testing.T) {
 	expectPrintedTS(t, "{ f<x> }", "{\n  f;\n}\n")
 	expectPrintedTS(t, "f<x> + g<y>;", "f < x > +g;\n")
 	expectPrintedTS(t, "f<x> - g<y>;", "f < x > -g;\n")
-	expectParseErrorTS(t, "f<x> * g<y>;", "<stdin>: ERROR: Unexpected \"*\"\n")
-	expectParseErrorTS(t, "f<x> == g<y>;", "<stdin>: ERROR: Unexpected \"==\"\n")
-	expectParseErrorTS(t, "f<x> ?? g<y>;", "<stdin>: ERROR: Unexpected \"??\"\n")
-	expectParseErrorTS(t, "f<x> in g<y>;", "<stdin>: ERROR: Unexpected \"in\"\n")
-	expectParseErrorTS(t, "f<x> instanceof g<y>;", "<stdin>: ERROR: Unexpected \"instanceof\"\n")
+	expectPrintedTS(t, "f<x> * g<y>;", "f * g;\n")
+	expectPrintedTS(t, "f<x> *= g<y>;", "f *= g;\n")
+	expectPrintedTS(t, "f<x> == g<y>;", "f == g;\n")
+	expectPrintedTS(t, "f<x> ?? g<y>;", "f ?? g;\n")
+	expectPrintedTS(t, "f<x> in g<y>;", "f in g;\n")
+	expectPrintedTS(t, "f<x> instanceof g<y>;", "f instanceof g;\n")
+	expectPrintedTS(t, "f<x> as g<y>;", "f;\n")
+	expectPrintedTS(t, "f<x> satisfies g<y>;", "f;\n")
 
 	expectParseErrorTS(t, "const a8 = f<number><number>;", "<stdin>: ERROR: Unexpected \";\"\n")
 	expectParseErrorTS(t, "const b1 = f?.<number>;", "<stdin>: ERROR: Expected \"(\" but found \";\"\n")
@@ -2082,20 +2083,26 @@ func TestTSInstantiationExpression(t *testing.T) {
 	expectParseErrorTSX(t, "type x = typeof y\n<number>\nz", "<stdin>: ERROR: Unexpected end of file before a closing \"number\" tag\n<stdin>: NOTE: The opening \"number\" tag is here:\n")
 
 	// See: https://github.com/microsoft/TypeScript/issues/48654
-	expectPrintedTS(t, "x<true>\ny", "x < true > y;\n")
+	expectPrintedTS(t, "x<true> y", "x < true > y;\n")
+	expectPrintedTS(t, "x<true>\ny", "x;\ny;\n")
 	expectPrintedTS(t, "x<true>\nif (y) {}", "x;\nif (y) {\n}\n")
 	expectPrintedTS(t, "x<true>\nimport 'y'", "x;\nimport \"y\";\n")
-	expectPrintedTS(t, "x<true>\nimport('y')", "x < true > import(\"y\");\n")
-	expectPrintedTS(t, "x<true>\nimport.meta", "x < true > import.meta;\n")
-	expectPrintedTS(t, "new x<number>\ny", "new x() < number > y;\n")
+	expectPrintedTS(t, "x<true>\nimport('y')", "x;\nimport(\"y\");\n")
+	expectPrintedTS(t, "x<true>\nimport.meta", "x;\nimport.meta;\n")
+	expectPrintedTS(t, "x<true> import('y')", "x < true > import(\"y\");\n")
+	expectPrintedTS(t, "x<true> import.meta", "x < true > import.meta;\n")
+	expectPrintedTS(t, "new x<number> y", "new x() < number > y;\n")
+	expectPrintedTS(t, "new x<number>\ny", "new x();\ny;\n")
 	expectPrintedTS(t, "new x<number>\nif (y) {}", "new x();\nif (y) {\n}\n")
 	expectPrintedTS(t, "new x<true>\nimport 'y'", "new x();\nimport \"y\";\n")
-	expectPrintedTS(t, "new x<true>\nimport('y')", "new x() < true > import(\"y\");\n")
-	expectPrintedTS(t, "new x<true>\nimport.meta", "new x() < true > import.meta;\n")
+	expectPrintedTS(t, "new x<true>\nimport('y')", "new x();\nimport(\"y\");\n")
+	expectPrintedTS(t, "new x<true>\nimport.meta", "new x();\nimport.meta;\n")
+	expectPrintedTS(t, "new x<true> import('y')", "new x() < true > import(\"y\");\n")
+	expectPrintedTS(t, "new x<true> import.meta", "new x() < true > import.meta;\n")
 
 	// See: https://github.com/microsoft/TypeScript/issues/48759
-	expectParseErrorTS(t, "x<true>\nimport<T>('y')", "<stdin>: ERROR: Expected \"(\" but found \"<\"\n")
-	expectParseErrorTS(t, "new x<true>\nimport<T>('y')", "<stdin>: ERROR: Expected \"(\" but found \"<\"\n")
+	expectParseErrorTS(t, "x<true>\nimport<T>('y')", "<stdin>: ERROR: Unexpected \"<\"\n")
+	expectParseErrorTS(t, "new x<true>\nimport<T>('y')", "<stdin>: ERROR: Unexpected \"<\"\n")
 
 	// See: https://github.com/evanw/esbuild/issues/2201
 	expectParseErrorTS(t, "return Array < ;", "<stdin>: ERROR: Unexpected \";\"\n")
@@ -2115,12 +2122,20 @@ func TestTSInstantiationExpression(t *testing.T) {
 	expectPrintedTS(t, "return Array < Array < number >> +1;", "return Array < Array < number >> 1;\n")
 	expectPrintedTS(t, "return Array < Array < number >> (1);", "return Array(1);\n")
 	expectPrintedTS(t, "return Array < Array < number > > (1);", "return Array(1);\n")
-	expectParseErrorTS(t, "return Array < number > in x;", "<stdin>: ERROR: Unexpected \"in\"\n")
-	expectParseErrorTS(t, "return Array < Array < number >> in x;", "<stdin>: ERROR: Unexpected \"in\"\n")
-	expectParseErrorTS(t, "return Array < Array < number > > in x;", "<stdin>: ERROR: Unexpected \">\"\n")
+	expectPrintedTS(t, "return Array < number > in x;", "return Array in x;\n")
+	expectPrintedTS(t, "return Array < Array < number >> in x;", "return Array in x;\n")
+	expectPrintedTS(t, "return Array < Array < number > > in x;", "return Array in x;\n")
 	expectPrintedTS(t, "for (var x = Array < number > in y) ;", "x = Array;\nfor (var x in y)\n  ;\n")
 	expectPrintedTS(t, "for (var x = Array < Array < number >> in y) ;", "x = Array;\nfor (var x in y)\n  ;\n")
 	expectPrintedTS(t, "for (var x = Array < Array < number > > in y) ;", "x = Array;\nfor (var x in y)\n  ;\n")
+
+	// See: https://github.com/microsoft/TypeScript/pull/49353
+	expectPrintedTS(t, "F<{}> 0", "F < {} > 0;\n")
+	expectPrintedTS(t, "F<{}> class F<T> {}", "F < {} > class F {\n};\n")
+	expectPrintedTS(t, "f<{}> function f<T>() {}", "f < {} > function f() {\n};\n")
+	expectPrintedTS(t, "F<{}>\n0", "F;\n0;\n")
+	expectPrintedTS(t, "F<{}>\nclass F<T> {}", "F;\nclass F {\n}\n")
+	expectPrintedTS(t, "f<{}>\nfunction f<T>() {}", "f;\nfunction f() {\n}\n")
 }
 
 func TestTSExponentiation(t *testing.T) {