From 7bd35f776783475f2e47812d45db6a40fa3b4b0c Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 19 Jan 2015 18:07:54 -0800 Subject: [PATCH 1/7] Rudimentary, but imperfect, lexical classification for templates. --- src/services/services.ts | 92 +++++++++++++++++++++++++++++++++++----- 1 file changed, 82 insertions(+), 10 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 665a06ac9b59d..5ba9589ab4aff 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1451,6 +1451,9 @@ module ts { InMultiLineCommentTrivia, InSingleQuoteStringLiteral, InDoubleQuoteStringLiteral, + InTemplateHeadLiteral, // this could also be a NoSubstitutionTemplateLiteral + InTemplateMiddleLiteral, //this could also be a TemplateTail + InTemplateSubstitutionPosition, } export enum TokenClass { @@ -5671,12 +5674,12 @@ module ts { // if there are more cases we want the classifier to be better at. return true; } - - // 'classifyKeywordsInGenerics' should be 'true' when a syntactic classifier is not present. - function getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult { + + function getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult { var offset = 0; var token = SyntaxKind.Unknown; var lastNonTriviaToken = SyntaxKind.Unknown; + var templateStack: SyntaxKind[]; // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. @@ -5696,6 +5699,21 @@ module ts { text = "/*\n" + text; offset = 3; break; + case EndOfLineState.InTemplateHeadLiteral: + if (syntacticClassifierAbsent) { + text = "`\n" + text; + offset = 2; + } + break; + case EndOfLineState.InTemplateMiddleLiteral: + if (syntacticClassifierAbsent) { + text = "${\n" + text; + offset = 3; + } + // fallthrough + case EndOfLineState.InTemplateSubstitutionPosition: + templateStack = [SyntaxKind.TemplateHead]; + break; } scanner.setText(text); @@ -5757,16 +5775,54 @@ module ts { angleBracketStack--; } else if (token === SyntaxKind.AnyKeyword || - token === SyntaxKind.StringKeyword || - token === SyntaxKind.NumberKeyword || - token === SyntaxKind.BooleanKeyword) { - if (angleBracketStack > 0 && !classifyKeywordsInGenerics) { + token === SyntaxKind.StringKeyword || + token === SyntaxKind.NumberKeyword || + token === SyntaxKind.BooleanKeyword) { + if (angleBracketStack > 0 && !syntacticClassifierAbsent) { // If it looks like we're could be in something generic, don't classify this // as a keyword. We may just get overwritten by the syntactic classifier, // causing a noisy experience for the user. token = SyntaxKind.Identifier; } } + else if (token === SyntaxKind.TemplateHead && syntacticClassifierAbsent) { + if (!templateStack) { + templateStack = [token]; + } + else { + templateStack.push(token); + } + } + else if (token === SyntaxKind.OpenBraceToken && syntacticClassifierAbsent) { + // If we don't have anything on the template stack, + // then we aren't trying to keep track of a previously scanned template head. + if (templateStack && templateStack.length > 0) { + templateStack.push(token); + } + } + else if (token === SyntaxKind.CloseBraceToken && syntacticClassifierAbsent) { + // If we don't have anything on the template stack, + // then we aren't trying to keep track of a previously scanned template head. + if (templateStack && templateStack.length > 0) { + var lastTemplateStackToken = lastOrUndefined(templateStack); + + if (lastTemplateStackToken === SyntaxKind.TemplateHead) { + token = scanner.reScanTemplateToken(); + + // Only pop on a TemplateTail; a TemplateMiddle indicates there is more for us. + if (token === SyntaxKind.TemplateTail) { + templateStack.pop(); + } + else { + Debug.assert(token === SyntaxKind.TemplateMiddle, "Should have been a template middle. Was " + token); + } + } + else { + Debug.assert(token === SyntaxKind.CloseBraceToken, "Should have been an open brace. Was: " + token); + templateStack.pop(); + } + } + } lastNonTriviaToken = token; } @@ -5781,8 +5837,7 @@ module ts { var start = scanner.getTokenPos(); var end = scanner.getTextPos(); - // add the token - addResult(end - start, classFromKind(token)); + addResult(end - start, classFromKind(token, syntacticClassifierAbsent)); if (end >= text.length) { if (token === SyntaxKind.StringLiteral) { @@ -5811,6 +5866,19 @@ module ts { result.finalLexState = EndOfLineState.InMultiLineCommentTrivia; } } + else if (isTemplateLiteralKind(token) && syntacticClassifierAbsent) { + if (scanner.isUnterminated()) { + if (token === SyntaxKind.TemplateMiddle) { + result.finalLexState = EndOfLineState.InTemplateMiddleLiteral; + } + else { + result.finalLexState = EndOfLineState.InTemplateHeadLiteral; + } + } + } + else if (templateStack && templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) { + result.finalLexState = EndOfLineState.InTemplateSubstitutionPosition; + } } } @@ -5888,7 +5956,7 @@ module ts { return token >= SyntaxKind.FirstKeyword && token <= SyntaxKind.LastKeyword; } - function classFromKind(token: SyntaxKind) { + function classFromKind(token: SyntaxKind, syntacticClassifierAbsent?: boolean) { if (isKeyword(token)) { return TokenClass.Keyword; } @@ -5913,6 +5981,10 @@ module ts { return TokenClass.Whitespace; case SyntaxKind.Identifier: default: + // Only give a classification if nothing will more accurately classify. + if (syntacticClassifierAbsent && isTemplateLiteralKind(token)) { + return TokenClass.StringLiteral; // should make a TemplateLiteral + } return TokenClass.Identifier; } } From 796eeb54978c793134717ec327b138e780fe9f0d Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 20 Jan 2015 12:42:03 -0800 Subject: [PATCH 2/7] Fixed issue with the kinds we check. --- src/services/services.ts | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 5ba9589ab4aff..fa707a2edd5e2 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1451,8 +1451,8 @@ module ts { InMultiLineCommentTrivia, InSingleQuoteStringLiteral, InDoubleQuoteStringLiteral, - InTemplateHeadLiteral, // this could also be a NoSubstitutionTemplateLiteral - InTemplateMiddleLiteral, //this could also be a TemplateTail + InTemplateHeadOrNoSubstitutionTemplate, + InTemplateMiddleOrTail, InTemplateSubstitutionPosition, } @@ -5699,20 +5699,22 @@ module ts { text = "/*\n" + text; offset = 3; break; - case EndOfLineState.InTemplateHeadLiteral: + case EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate: if (syntacticClassifierAbsent) { text = "`\n" + text; offset = 2; } break; - case EndOfLineState.InTemplateMiddleLiteral: + case EndOfLineState.InTemplateMiddleOrTail: if (syntacticClassifierAbsent) { - text = "${\n" + text; - offset = 3; + text = "}\n" + text; + offset = 2; } // fallthrough case EndOfLineState.InTemplateSubstitutionPosition: - templateStack = [SyntaxKind.TemplateHead]; + if (syntacticClassifierAbsent) { + templateStack = [SyntaxKind.TemplateHead]; + } break; } @@ -5868,11 +5870,14 @@ module ts { } else if (isTemplateLiteralKind(token) && syntacticClassifierAbsent) { if (scanner.isUnterminated()) { - if (token === SyntaxKind.TemplateMiddle) { - result.finalLexState = EndOfLineState.InTemplateMiddleLiteral; + if (token === SyntaxKind.TemplateTail) { + result.finalLexState = EndOfLineState.InTemplateMiddleOrTail; + } + else if (token === SyntaxKind.NoSubstitutionTemplateLiteral) { + result.finalLexState = EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate; } else { - result.finalLexState = EndOfLineState.InTemplateHeadLiteral; + Debug.fail("Only 'NoSubstitutionTemplateLiteral's and 'TemplateTail's can be unterminated; got SyntaxKind #" + token); } } } From a6e94fd73a2e710d427c25ecd040f14a250cd690 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 12:06:01 -0800 Subject: [PATCH 3/7] Addressed CR feedback. --- src/services/services.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index fa707a2edd5e2..01bf08718bb4b 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5641,6 +5641,10 @@ module ts { noRegexTable[SyntaxKind.TrueKeyword] = true; noRegexTable[SyntaxKind.FalseKeyword] = true; + // Just a stack of TemplateHeads and OpenCurlyBraces, used + // to perform rudimentary classification on templates. + var templateStack: SyntaxKind[] = []; + function isAccessibilityModifier(kind: SyntaxKind) { switch (kind) { case SyntaxKind.PublicKeyword: @@ -5679,7 +5683,6 @@ module ts { var offset = 0; var token = SyntaxKind.Unknown; var lastNonTriviaToken = SyntaxKind.Unknown; - var templateStack: SyntaxKind[]; // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. @@ -5746,6 +5749,11 @@ module ts { // work well enough in practice. var angleBracketStack = 0; + // Empty out the template stack for reuse. + while (templateStack.length > 0) { + templateStack.pop(); + } + do { token = scanner.scan(); @@ -5788,24 +5796,19 @@ module ts { } } else if (token === SyntaxKind.TemplateHead && syntacticClassifierAbsent) { - if (!templateStack) { - templateStack = [token]; - } - else { - templateStack.push(token); - } + templateStack.push(token); } else if (token === SyntaxKind.OpenBraceToken && syntacticClassifierAbsent) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. - if (templateStack && templateStack.length > 0) { + if (templateStack.length > 0) { templateStack.push(token); } } else if (token === SyntaxKind.CloseBraceToken && syntacticClassifierAbsent) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. - if (templateStack && templateStack.length > 0) { + if (templateStack.length > 0) { var lastTemplateStackToken = lastOrUndefined(templateStack); if (lastTemplateStackToken === SyntaxKind.TemplateHead) { @@ -5881,7 +5884,7 @@ module ts { } } } - else if (templateStack && templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) { + else if (templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) { result.finalLexState = EndOfLineState.InTemplateSubstitutionPosition; } } From a7f35c5ea93e238bfe5d97402c1116512428e810 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 12:37:03 -0800 Subject: [PATCH 4/7] classifyKeywordsInGenerics -> syntacticClassifierAbsent --- src/services/services.ts | 2 +- src/services/shims.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 01bf08718bb4b..19d4445f66994 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1479,7 +1479,7 @@ module ts { } export interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; } export interface DocumentRegistry { diff --git a/src/services/shims.ts b/src/services/shims.ts index e1eb388321a4e..c76cd5d0d0ac7 100644 --- a/src/services/shims.ts +++ b/src/services/shims.ts @@ -161,7 +161,7 @@ module ts { } export interface ClassifierShim extends Shim { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): string; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): string; } export interface CoreServicesShim extends Shim { From 570a36bb8fb92fddba82419967ac8fdf9ec01d05 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 15:41:01 -0800 Subject: [PATCH 5/7] Added tests, fixed order of emptying templateStack, unconditionally perform template classification. --- src/services/services.ts | 43 +++--- .../cases/unittests/services/colorization.ts | 143 ++++++++++++++++-- 2 files changed, 146 insertions(+), 40 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 19d4445f66994..cd60dc714d8e2 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5684,6 +5684,11 @@ module ts { var token = SyntaxKind.Unknown; var lastNonTriviaToken = SyntaxKind.Unknown; + // Empty out the template stack for reuse. + while (templateStack.length > 0) { + templateStack.pop(); + } + // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. // @@ -5703,21 +5708,15 @@ module ts { offset = 3; break; case EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate: - if (syntacticClassifierAbsent) { - text = "`\n" + text; - offset = 2; - } + text = "`\n" + text; + offset = 2; break; case EndOfLineState.InTemplateMiddleOrTail: - if (syntacticClassifierAbsent) { - text = "}\n" + text; - offset = 2; - } + text = "}\n" + text; + offset = 2; // fallthrough case EndOfLineState.InTemplateSubstitutionPosition: - if (syntacticClassifierAbsent) { - templateStack = [SyntaxKind.TemplateHead]; - } + templateStack.push(SyntaxKind.TemplateHead); break; } @@ -5749,11 +5748,6 @@ module ts { // work well enough in practice. var angleBracketStack = 0; - // Empty out the template stack for reuse. - while (templateStack.length > 0) { - templateStack.pop(); - } - do { token = scanner.scan(); @@ -5795,17 +5789,17 @@ module ts { token = SyntaxKind.Identifier; } } - else if (token === SyntaxKind.TemplateHead && syntacticClassifierAbsent) { + else if (token === SyntaxKind.TemplateHead) { templateStack.push(token); } - else if (token === SyntaxKind.OpenBraceToken && syntacticClassifierAbsent) { + else if (token === SyntaxKind.OpenBraceToken) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. if (templateStack.length > 0) { templateStack.push(token); } } - else if (token === SyntaxKind.CloseBraceToken && syntacticClassifierAbsent) { + else if (token === SyntaxKind.CloseBraceToken) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. if (templateStack.length > 0) { @@ -5842,7 +5836,7 @@ module ts { var start = scanner.getTokenPos(); var end = scanner.getTextPos(); - addResult(end - start, classFromKind(token, syntacticClassifierAbsent)); + addResult(end - start, classFromKind(token)); if (end >= text.length) { if (token === SyntaxKind.StringLiteral) { @@ -5871,7 +5865,7 @@ module ts { result.finalLexState = EndOfLineState.InMultiLineCommentTrivia; } } - else if (isTemplateLiteralKind(token) && syntacticClassifierAbsent) { + else if (isTemplateLiteralKind(token)) { if (scanner.isUnterminated()) { if (token === SyntaxKind.TemplateTail) { result.finalLexState = EndOfLineState.InTemplateMiddleOrTail; @@ -5964,7 +5958,7 @@ module ts { return token >= SyntaxKind.FirstKeyword && token <= SyntaxKind.LastKeyword; } - function classFromKind(token: SyntaxKind, syntacticClassifierAbsent?: boolean) { + function classFromKind(token: SyntaxKind) { if (isKeyword(token)) { return TokenClass.Keyword; } @@ -5989,9 +5983,8 @@ module ts { return TokenClass.Whitespace; case SyntaxKind.Identifier: default: - // Only give a classification if nothing will more accurately classify. - if (syntacticClassifierAbsent && isTemplateLiteralKind(token)) { - return TokenClass.StringLiteral; // should make a TemplateLiteral + if (isTemplateLiteralKind(token)) { + return TokenClass.StringLiteral; // maybe make a TemplateLiteral } return TokenClass.Identifier; } diff --git a/tests/cases/unittests/services/colorization.ts b/tests/cases/unittests/services/colorization.ts index b29f119f944b2..741773fe1237a 100644 --- a/tests/cases/unittests/services/colorization.ts +++ b/tests/cases/unittests/services/colorization.ts @@ -2,9 +2,9 @@ /// interface Classification { - position: number; length: number; class: ts.TokenClass; + position: number; } interface ClassiferResult { @@ -15,6 +15,7 @@ interface ClassiferResult { interface ClassificationEntry { value: any; class: ts.TokenClass; + position?: number; } describe('Colorization', function () { @@ -60,16 +61,23 @@ describe('Colorization', function () { return undefined; } - function punctuation(text: string) { return { value: text, class: ts.TokenClass.Punctuation }; } - function keyword(text: string) { return { value: text, class: ts.TokenClass.Keyword }; } - function operator(text: string) { return { value: text, class: ts.TokenClass.Operator }; } - function comment(text: string) { return { value: text, class: ts.TokenClass.Comment }; } - function whitespace(text: string) { return { value: text, class: ts.TokenClass.Whitespace }; } - function identifier(text: string) { return { value: text, class: ts.TokenClass.Identifier }; } - function numberLiteral(text: string) { return { value: text, class: ts.TokenClass.NumberLiteral }; } - function stringLiteral(text: string) { return { value: text, class: ts.TokenClass.StringLiteral }; } - function regExpLiteral(text: string) { return { value: text, class: ts.TokenClass.RegExpLiteral }; } - function finalEndOfLineState(value: number) { return { value: value, class: undefined }; } + function punctuation(text: string, position?: number) { return createClassification(text, ts.TokenClass.Punctuation, position); } + function keyword(text: string, position?: number) { return createClassification(text, ts.TokenClass.Keyword, position); } + function operator(text: string, position?: number) { return createClassification(text, ts.TokenClass.Operator, position); } + function comment(text: string, position?: number) { return createClassification(text, ts.TokenClass.Comment, position); } + function whitespace(text: string, position?: number) { return createClassification(text, ts.TokenClass.Whitespace, position); } + function identifier(text: string, position?: number) { return createClassification(text, ts.TokenClass.Identifier, position); } + function numberLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.NumberLiteral, position); } + function stringLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.StringLiteral, position); } + function regExpLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.RegExpLiteral, position); } + function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, class: undefined, position: 0 }; } + function createClassification(text: string, tokenClass: ts.TokenClass, position?: number): ClassificationEntry { + return { + value: text, + class: tokenClass, + position: position, + }; + } function test(text: string, initialEndOfLineState: ts.EndOfLineState, ...expectedEntries: ClassificationEntry[]): void { var result = getClassifications(text, initialEndOfLineState); @@ -81,18 +89,23 @@ describe('Colorization', function () { assert.equal(result.finalEndOfLineState, expectedEntry.value, "final endOfLineState does not match expected."); } else { - var actualEntryPosition = text.indexOf(expectedEntry.value); + var actualEntryPosition = expectedEntry.position !== undefined ? expectedEntry.position : text.indexOf(expectedEntry.value); assert(actualEntryPosition >= 0, "token: '" + expectedEntry.value + "' does not exit in text: '" + text + "'."); var actualEntry = getEntryAtPosistion(result, actualEntryPosition); - assert(actualEntry, "Could not find classification entry for '" + expectedEntry.value + "' at position: " + actualEntryPosition); - assert.equal(actualEntry.class, expectedEntry.class, "Classification class does not match expected. Expected: " + ts.TokenClass[expectedEntry.class] + ", Actual: " + ts.TokenClass[actualEntry.class]); - assert.equal(actualEntry.length, expectedEntry.value.length, "Classification length does not match expected. Expected: " + ts.TokenClass[expectedEntry.value.length] + ", Actual: " + ts.TokenClass[actualEntry.length]); + assert(actualEntry, "Could not find classification entry for '" + expectedEntry.value + "' at position: " + actualEntryPosition + "\n\n" + JSON.stringify(result)); + assert.equal(actualEntry.class, expectedEntry.class, + "Classification class does not match expected. Expected: " + ts.TokenClass[expectedEntry.class] + " - '" + expectedEntry.value + "', Actual: " + ts.TokenClass[actualEntry.class] + " - '" + getActualText(text, actualEntry) + "\n\n" + JSON.stringify(result)); + assert.equal(actualEntry.length, expectedEntry.value.length, "Classification length does not match expected. Expected: " + expectedEntry.value.length + " - '" + expectedEntry.value + "', Actual: " + actualEntry.length + " - '" + getActualText(text, actualEntry) + "'\n\n" + JSON.stringify(result)); } } } + function getActualText(sourceText: string, classification: Classification): string { + return sourceText.substr(classification.position, classification.length); + } + describe("test getClassifications", function () { it("Returns correct token classes", function () { test("var x: string = \"foo\"; //Hello", @@ -291,6 +304,106 @@ describe('Colorization', function () { finalEndOfLineState(ts.EndOfLineState.Start)); }); + it("classifies a single line no substitution template string correctly", () => { + test("`number number public string`", + ts.EndOfLineState.Start, + stringLiteral("`number number public string`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies substitution parts of a template string correctly", () => { + test("`number '${ 1 + 1 }' string '${ 'hello' }'`", + ts.EndOfLineState.Start, + stringLiteral("`number '${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}' string '${"), + stringLiteral("'hello'"), + stringLiteral("}'`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies an unterminated no substitution template string correctly", () => { + test("`hello world", + ts.EndOfLineState.Start, + stringLiteral("`hello world"), + finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); + }); + it("classifies the entire line of an unterminated multiline no-substitution/head template", () => { + test("...", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("..."), + finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); + }); + it("classifies the entire line of an unterminated multiline template middle/end",() => { + test("...", + ts.EndOfLineState.InTemplateMiddleOrTail, + stringLiteral("..."), + finalEndOfLineState(ts.EndOfLineState.InTemplateMiddleOrTail)); + }); + it("classifies a termination of a multiline template head", () => { + test("...${", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("...${"), + finalEndOfLineState(ts.EndOfLineState.InTemplateSubstitutionPosition)); + }); + it("classifies the termination of a multiline no substitution template", () => { + test("...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies the substitution parts and middle/tail of a multiline template string", () => { + test("${ 1 + 1 }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies a template middle and propagates the end of line state",() => { + test("${ 1 + 1 }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies substitution expressions with curly braces appropriately", () => { + var pos = 0; + var lastLength = 0; + + test("...${ () => { } } ${ { x: `1` } }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral(track("...${"), pos), + punctuation(track(" ", "("), pos), + punctuation(track(")"), pos), + punctuation(track(" ", "=>"), pos), + punctuation(track(" ", "{"), pos), + punctuation(track(" ", "}"), pos), + stringLiteral(track(" ", "} ${"), pos), + punctuation(track(" ", "{"), pos), + identifier(track(" ", "x"), pos), + punctuation(track(":"), pos), + stringLiteral(track(" ", "`1`"), pos), + punctuation(track(" ", "}"), pos), + stringLiteral(track(" ", "}...`"), pos), + finalEndOfLineState(ts.EndOfLineState.Start)); + + // Adjusts 'pos' by accounting for the length of each portion of the string, + // but only return the last given string + function track(...vals: string[]): string { + for (var i = 0, n = vals.length; i < n; i++) { + pos += lastLength; + lastLength = vals[i].length; + } + return ts.lastOrUndefined(vals); + } + }); + it("classifies partially written generics correctly.", function () { test("Foo Date: Mon, 26 Jan 2015 15:43:26 -0800 Subject: [PATCH 6/7] Fixed assertion --- src/services/services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/services.ts b/src/services/services.ts index cd60dc714d8e2..f86c883d50b15 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5817,7 +5817,7 @@ module ts { } } else { - Debug.assert(token === SyntaxKind.CloseBraceToken, "Should have been an open brace. Was: " + token); + Debug.assert(lastTemplateStackToken === SyntaxKind.OpenBraceToken, "Should have been an open brace. Was: " + token); templateStack.pop(); } } From 31b0f66a063afa0ed3b02f35d9fc1ccf1f0c2ed4 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 16:06:56 -0800 Subject: [PATCH 7/7] Added comment for 'syntacticClassifierAbsent'. --- src/services/services.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/services/services.ts b/src/services/services.ts index f86c883d50b15..3b6e667c85631 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5679,6 +5679,8 @@ module ts { return true; } + // If there is a syntactic classifier ('syntacticClassifierAbsent' is false), + // we will be more conservative in order to avoid conflicting with the syntactic classifier. function getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult { var offset = 0; var token = SyntaxKind.Unknown;