Lexical colorization/classification for template strings.

src/services/services.ts

@@ -1143,6 +1143,9 @@ module ts {
         InMultiLineCommentTrivia,
         InSingleQuoteStringLiteral,
         InDoubleQuoteStringLiteral,
+        InTemplateHeadOrNoSubstitutionTemplate,
+        InTemplateMiddleOrTail,
+        InTemplateSubstitutionPosition,
     }
 
     export enum TokenClass {
@@ -1168,7 +1171,7 @@ module ts {
     }
 
     export interface Classifier {
-        getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult;
+        getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult;
     }
 
     /**
@@ -5617,6 +5620,10 @@ module ts {
         noRegexTable[SyntaxKind.TrueKeyword] = true;
         noRegexTable[SyntaxKind.FalseKeyword] = true;
 
+        // Just a stack of TemplateHeads and OpenCurlyBraces, used
+        // to perform rudimentary classification on templates.
+        var templateStack: SyntaxKind[] = [];
+
         function isAccessibilityModifier(kind: SyntaxKind) {
             switch (kind) {
                 case SyntaxKind.PublicKeyword:
@@ -5650,13 +5657,19 @@ module ts {
             // if there are more cases we want the classifier to be better at.
             return true;
         }
-
-        // 'classifyKeywordsInGenerics' should be 'true' when a syntactic classifier is not present.
-        function getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult {
+
+        // If there is a syntactic classifier ('syntacticClassifierAbsent' is false),
+        // we will be more conservative in order to avoid conflicting with the syntactic classifier.
+        function getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult {
             var offset = 0;
             var token = SyntaxKind.Unknown;
             var lastNonTriviaToken = SyntaxKind.Unknown;
 
+            // Empty out the template stack for reuse.
+            while (templateStack.length > 0) {
+                templateStack.pop();
+            }
+
             // If we're in a string literal, then prepend: "\
             // (and a newline).  That way when we lex we'll think we're still in a string literal.
             //
@@ -5675,6 +5688,17 @@ module ts {
                     text = "/*\n" + text;
                     offset = 3;
                     break;
+                case EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate:
+                    text = "`\n" + text;
+                    offset = 2;
+                    break;
+                case EndOfLineState.InTemplateMiddleOrTail:
+                    text = "}\n" + text;
+                    offset = 2;
+                    // fallthrough
+                case EndOfLineState.InTemplateSubstitutionPosition:
+                    templateStack.push(SyntaxKind.TemplateHead);
+                    break;
             }
 
             scanner.setText(text);
@@ -5739,12 +5763,45 @@ module ts {
                              token === SyntaxKind.StringKeyword ||
                              token === SyntaxKind.NumberKeyword ||
                              token === SyntaxKind.BooleanKeyword) {
-                             if (angleBracketStack > 0 && !classifyKeywordsInGenerics) {
-                                 // If it looks like we're could be in something generic, don't classify this 
-                                 // as a keyword.  We may just get overwritten by the syntactic classifier,
-                                 // causing a noisy experience for the user.
-                                 token = SyntaxKind.Identifier;
-                             }
+                        if (angleBracketStack > 0 && !syntacticClassifierAbsent) {
+                            // If it looks like we're could be in something generic, don't classify this 
+                            // as a keyword.  We may just get overwritten by the syntactic classifier,
+                            // causing a noisy experience for the user.
+                            token = SyntaxKind.Identifier;
+                        }
+                    }
+                    else if (token === SyntaxKind.TemplateHead) {
+                        templateStack.push(token);
+                    }
+                    else if (token === SyntaxKind.OpenBraceToken) {
+                        // If we don't have anything on the template stack,
+                        // then we aren't trying to keep track of a previously scanned template head.
+                        if (templateStack.length > 0) {
+                            templateStack.push(token);
+                        }
+                    }
+                    else if (token === SyntaxKind.CloseBraceToken) {
+                        // If we don't have anything on the template stack,
+                        // then we aren't trying to keep track of a previously scanned template head.
+                        if (templateStack.length > 0) {
+                            var lastTemplateStackToken = lastOrUndefined(templateStack);
+
+                            if (lastTemplateStackToken === SyntaxKind.TemplateHead) {
+                                token = scanner.reScanTemplateToken();
+
+                                // Only pop on a TemplateTail; a TemplateMiddle indicates there is more for us.
+                                if (token === SyntaxKind.TemplateTail) {
+                                    templateStack.pop();
+                                }
+                                else {
+                                    Debug.assert(token === SyntaxKind.TemplateMiddle, "Should have been a template middle. Was " + token);
+                                }
+                            }
+                            else {
+                                Debug.assert(lastTemplateStackToken === SyntaxKind.OpenBraceToken, "Should have been an open brace. Was: " + token);
+                                templateStack.pop();
+                            }
+                        }
                     }
 
                     lastNonTriviaToken = token;
@@ -5789,6 +5846,22 @@ module ts {
                             result.finalLexState = EndOfLineState.InMultiLineCommentTrivia;
                         }
                     }
+                    else if (isTemplateLiteralKind(token)) {
+                        if (scanner.isUnterminated()) {
+                            if (token === SyntaxKind.TemplateTail) {
+                                result.finalLexState = EndOfLineState.InTemplateMiddleOrTail;
+                            }
+                            else if (token === SyntaxKind.NoSubstitutionTemplateLiteral) {
+                                result.finalLexState = EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate;
+                            }
+                            else {
+                                Debug.fail("Only 'NoSubstitutionTemplateLiteral's and 'TemplateTail's can be unterminated; got SyntaxKind #" + token);
+                            }
+                        }
+                    }
+                    else if (templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) {
+                        result.finalLexState = EndOfLineState.InTemplateSubstitutionPosition;
+                    }
                 }
             }
 
@@ -5892,6 +5965,9 @@ module ts {
                     return TokenClass.Whitespace;
                 case SyntaxKind.Identifier:
                 default:
+                    if (isTemplateLiteralKind(token)) {
+                        return TokenClass.StringLiteral; // maybe make a TemplateLiteral
+                    }
                     return TokenClass.Identifier;
             }
         }

src/services/shims.ts

@@ -165,7 +165,7 @@ module ts {
     }
 
     export interface ClassifierShim extends Shim {
-        getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): string;
+        getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): string;
     }
 
     export interface CoreServicesShim extends Shim {

tests/baselines/reference/APISample_compile.js

@@ -1741,6 +1741,9 @@ declare module "typescript" {
         InMultiLineCommentTrivia = 1,
         InSingleQuoteStringLiteral = 2,
         InDoubleQuoteStringLiteral = 3,
+        InTemplateHeadOrNoSubstitutionTemplate = 4,
+        InTemplateMiddleOrTail = 5,
+        InTemplateSubstitutionPosition = 6,
     }
     enum TokenClass {
         Punctuation = 0,
@@ -1762,7 +1765,7 @@ declare module "typescript" {
         classification: TokenClass;
     }
     interface Classifier {
-        getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult;
+        getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult;
     }
     /**
       * The document registry represents a store of SourceFile objects that can be shared between

tests/baselines/reference/APISample_compile.types

@@ -5539,6 +5539,15 @@ declare module "typescript" {
 
         InDoubleQuoteStringLiteral = 3,
 >InDoubleQuoteStringLiteral : EndOfLineState
+
+        InTemplateHeadOrNoSubstitutionTemplate = 4,
+>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState
+
+        InTemplateMiddleOrTail = 5,
+>InTemplateMiddleOrTail : EndOfLineState
+
+        InTemplateSubstitutionPosition = 6,
+>InTemplateSubstitutionPosition : EndOfLineState
     }
     enum TokenClass {
 >TokenClass : TokenClass
@@ -5594,12 +5603,12 @@ declare module "typescript" {
     interface Classifier {
 >Classifier : Classifier
 
-        getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult;
->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult
+        getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult;
+>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult
 >text : string
 >lexState : EndOfLineState
 >EndOfLineState : EndOfLineState
->classifyKeywordsInGenerics : boolean
+>syntacticClassifierAbsent : boolean
 >ClassificationResult : ClassificationResult
     }
     /**

tests/baselines/reference/APISample_linter.js

@@ -1772,6 +1772,9 @@ declare module "typescript" {
         InMultiLineCommentTrivia = 1,
         InSingleQuoteStringLiteral = 2,
         InDoubleQuoteStringLiteral = 3,
+        InTemplateHeadOrNoSubstitutionTemplate = 4,
+        InTemplateMiddleOrTail = 5,
+        InTemplateSubstitutionPosition = 6,
     }
     enum TokenClass {
         Punctuation = 0,
@@ -1793,7 +1796,7 @@ declare module "typescript" {
         classification: TokenClass;
     }
     interface Classifier {
-        getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult;
+        getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult;
     }
     /**
       * The document registry represents a store of SourceFile objects that can be shared between

tests/baselines/reference/APISample_linter.types

@@ -5683,6 +5683,15 @@ declare module "typescript" {
 
         InDoubleQuoteStringLiteral = 3,
 >InDoubleQuoteStringLiteral : EndOfLineState
+
+        InTemplateHeadOrNoSubstitutionTemplate = 4,
+>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState
+
+        InTemplateMiddleOrTail = 5,
+>InTemplateMiddleOrTail : EndOfLineState
+
+        InTemplateSubstitutionPosition = 6,
+>InTemplateSubstitutionPosition : EndOfLineState
     }
     enum TokenClass {
 >TokenClass : TokenClass
@@ -5738,12 +5747,12 @@ declare module "typescript" {
     interface Classifier {
 >Classifier : Classifier
 
-        getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult;
->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult
+        getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult;
+>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult
 >text : string
 >lexState : EndOfLineState
 >EndOfLineState : EndOfLineState
->classifyKeywordsInGenerics : boolean
+>syntacticClassifierAbsent : boolean
 >ClassificationResult : ClassificationResult
     }
     /**

tests/baselines/reference/APISample_transform.js

@@ -1773,6 +1773,9 @@ declare module "typescript" {
         InMultiLineCommentTrivia = 1,
         InSingleQuoteStringLiteral = 2,
         InDoubleQuoteStringLiteral = 3,
+        InTemplateHeadOrNoSubstitutionTemplate = 4,
+        InTemplateMiddleOrTail = 5,
+        InTemplateSubstitutionPosition = 6,
     }
     enum TokenClass {
         Punctuation = 0,
@@ -1794,7 +1797,7 @@ declare module "typescript" {
         classification: TokenClass;
     }
     interface Classifier {
-        getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult;
+        getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult;
     }
     /**
       * The document registry represents a store of SourceFile objects that can be shared between

tests/baselines/reference/APISample_transform.types

@@ -5635,6 +5635,15 @@ declare module "typescript" {
 
         InDoubleQuoteStringLiteral = 3,
 >InDoubleQuoteStringLiteral : EndOfLineState
+
+        InTemplateHeadOrNoSubstitutionTemplate = 4,
+>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState
+
+        InTemplateMiddleOrTail = 5,
+>InTemplateMiddleOrTail : EndOfLineState
+
+        InTemplateSubstitutionPosition = 6,
+>InTemplateSubstitutionPosition : EndOfLineState
     }
     enum TokenClass {
 >TokenClass : TokenClass
@@ -5690,12 +5699,12 @@ declare module "typescript" {
     interface Classifier {
 >Classifier : Classifier
 
-        getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult;
->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult
+        getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult;
+>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult
 >text : string
 >lexState : EndOfLineState
 >EndOfLineState : EndOfLineState
->classifyKeywordsInGenerics : boolean
+>syntacticClassifierAbsent : boolean
 >ClassificationResult : ClassificationResult
     }
     /**

tests/baselines/reference/APISample_watcher.js

@@ -1810,6 +1810,9 @@ declare module "typescript" {
         InMultiLineCommentTrivia = 1,
         InSingleQuoteStringLiteral = 2,
         InDoubleQuoteStringLiteral = 3,
+        InTemplateHeadOrNoSubstitutionTemplate = 4,
+        InTemplateMiddleOrTail = 5,
+        InTemplateSubstitutionPosition = 6,
     }
     enum TokenClass {
         Punctuation = 0,
@@ -1831,7 +1834,7 @@ declare module "typescript" {
         classification: TokenClass;
     }
     interface Classifier {
-        getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult;
+        getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult;
     }
     /**
       * The document registry represents a store of SourceFile objects that can be shared between

tests/baselines/reference/APISample_watcher.types

@@ -5808,6 +5808,15 @@ declare module "typescript" {
 
         InDoubleQuoteStringLiteral = 3,
 >InDoubleQuoteStringLiteral : EndOfLineState
+
+        InTemplateHeadOrNoSubstitutionTemplate = 4,
+>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState
+
+        InTemplateMiddleOrTail = 5,
+>InTemplateMiddleOrTail : EndOfLineState
+
+        InTemplateSubstitutionPosition = 6,
+>InTemplateSubstitutionPosition : EndOfLineState
     }
     enum TokenClass {
 >TokenClass : TokenClass
@@ -5863,12 +5872,12 @@ declare module "typescript" {
     interface Classifier {
 >Classifier : Classifier
 
-        getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult;
->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult
+        getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult;
+>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult
 >text : string
 >lexState : EndOfLineState
 >EndOfLineState : EndOfLineState
->classifyKeywordsInGenerics : boolean
+>syntacticClassifierAbsent : boolean
 >ClassificationResult : ClassificationResult
     }
     /**