Handle XML doc comments rather than delegating to text.xml (so we can…

… write tests!) and add regression test for issue dotnet#706
DustinCampbell · Jan 10, 2017 · e467be7 · e467be7
1 parent 39db070
commit e467be7
Show file tree

Hide file tree

Showing 3 changed files with 379 additions and 3 deletions.
diff --git a/syntaxes/csharp.tmLanguage.yml b/syntaxes/csharp.tmLanguage.yml
@@ -13,7 +13,8 @@ uuid: f7de61e2-bdde-4e2a-a139-8221b179584e
 # * compound assignement
 # * verbatim identifiers
 # * hexadecimal and unicode character escape sequences
-# * unsafe code: fixed, sizeof, unsafe blocks
+# * unsafe code: fixed, sizeof, unsafe blocks, pointer member access
+# * multi-line XML doc comments
 
 patterns:
 - include: '#preprocessor'
@@ -2233,9 +2234,118 @@ repository:
           '0': { name: punctuation.definition.comment.cs }
         end: (?=$)
         patterns:
-        - include: 'text.xml'
+        - include: '#xml-doc-comment'
       - name: comment.line.double-slash.cs
         begin: (?<!/)//(?!/)
         beginCaptures:
           '0': { name: punctuation.definition.comment.cs }
         end: (?=$)
+
+  xml-doc-comment:
+    patterns:
+    - include: '#xml-comment'
+    - include: '#xml-character-entity'
+    - include: '#xml-cdata'
+    - include: '#xml-tag'
+
+  xml-tag:
+    name: meta.tag.cs
+    begin: |-
+      (?x)
+      (</?)
+      (
+        (?:
+          ([-_[:alnum:]]+)
+          (:)
+        )?
+        ([-_[:alnum:]]+)
+      )
+    beginCaptures:
+      '1': { name: punctuation.definition.tag.cs }
+      '2': { name: entity.name.tag.cs }
+      '3': { name: entity.name.tag.namespace.cs }
+      '4': { name: punctuation.separator.colon.cs }
+      '5': { name: entity.name.tag.localname.cs }
+    end: (/?>)
+    endCaptures:
+      '1': { name: punctuation.definition.tag.cs }
+    patterns:
+    - include: '#xml-attribute'
+
+  xml-attribute:
+    patterns:
+    - match: |-
+        (?x)
+        (?:^|\s+)
+        (
+          (?:
+            ([-_[:alnum:]]+)
+            (:)
+          )?
+          ([-_[:alnum:]]+)
+        )
+        (=)
+      captures:
+        '1': { name: entity.other.attribute-name.cs }
+        '2': { name: entity.other.attribute-name.namespace.cs }
+        '3': { name: punctuation.separator.colon.cs }
+        '4': { name: entity.other.attribute-name.localname.cs }
+        '5': { name: punctuation.separator.equals.cs }
+    - include: '#xml-string'
+
+  xml-cdata:
+    name: string.unquoted.cdata.cs
+    begin: <!\[CDATA\[
+    beginCaptures:
+      '0': { name: punctuation.definition.string.begin.cs }
+    end: \]\]>
+    endCaptures:
+      '0': { name: punctuation.definition.string.end.cs }
+
+  xml-string:
+    patterns:
+    - name: string.quoted.single.cs
+      begin: \'
+      beginCaptures:
+        '0': { name: punctuation.definition.string.begin.cs }
+      end: \'
+      endCaptures:
+        '0': { name: punctuation.definition.string.end.cs }
+      patterns:
+      - include: '#xml-character-entity'
+    - name: string.quoted.double.cs
+      begin: \"
+      beginCaptures:
+        '0': { name: punctuation.definition.string.begin.cs }
+      end: \"
+      endCaptures:
+        '0': { name: punctuation.definition.string.end.cs }
+      patterns:
+      - include: '#xml-character-entity'
+
+  xml-character-entity:
+    patterns:
+    - name: constant.character.entity.cs
+      match: |-
+        (?x)
+        (&)
+        (
+          (?:[[:alpha:]:_][[:alnum:]:_.-]*)|
+          (?:\#[[:digit:]]+)|
+          (?:\#x[[:xdigit:]]+)
+        )
+        (;)
+      captures:
+        '1': { name: punctuation.definition.constant.cs }
+        '3': { name: punctuation.definition.constant.cs }
+    - name: invalid.illegal.bad-ampersand.cs
+      match: '&'
+
+  xml-comment:
+    name: comment.block.cs
+    begin: <!--
+    beginCaptures:
+      '0': { name: punctuation.definition.comment.cs }
+    end: -->
+    endCaptures:
+      '0': { name: punctuation.definition.comment.cs }
diff --git a/test/syntaxes/utils/tokenize.ts b/test/syntaxes/utils/tokenize.ts
@@ -7,7 +7,7 @@ import { ITokenizeLineResult, Registry, StackElement } from 'vscode-textmate';
 
 const registry = new Registry();
 const grammar = registry.loadGrammarFromPathSync('syntaxes/csharp.tmLanguage');
-const excludedTypes = ['source.cs', 'meta.interpolation.cs', 'meta.preprocessor.cs', 'meta.type.parameters.cs']
+const excludedTypes = ['source.cs', 'meta.interpolation.cs', 'meta.preprocessor.cs', 'meta.tag.cs', 'meta.type.parameters.cs']
 
 export function tokenize(input: string | Input, excludeTypes: boolean = true): Token[] {
     if (typeof input === "string") {
@@ -444,6 +444,61 @@ export namespace Token {
         export const ReadWrite = (text: string) => createToken(text, 'variable.other.readwrite.cs');
     }
 
+    export namespace XmlDocComments {
+        export namespace Attribute {
+            export const Name = (text: string) => createToken(text, 'entity.other.attribute-name.localname.cs');
+        }
+
+        export namespace CData {
+            export const Begin = createToken('<![CDATA[', 'punctuation.definition.string.begin.cs');
+            export const End = createToken(']]>', 'punctuation.definition.string.end.cs');
+            export const Text = (text: string) => createToken(text, 'string.unquoted.cdata.cs');
+        }
+
+        export namespace CharacterEntity {
+            export const Begin = createToken('&', 'punctuation.definition.constant.cs');
+            export const End = createToken(';', 'punctuation.definition.constant.cs');
+            export const Text = (text: string) => createToken(text, 'constant.character.entity.cs');
+        }
+
+        export namespace Comment {
+            export const Begin = createToken('<!--', 'punctuation.definition.comment.cs')
+            export const End = createToken('-->', 'punctuation.definition.comment.cs')
+            export const Text = (text: string) => createToken(text, 'comment.block.cs')
+        }
+
+        export namespace Tag {
+            // punctuation
+            export const StartTagBegin = createToken('<', 'punctuation.definition.tag.cs');
+            export const StartTagEnd = createToken('>', 'punctuation.definition.tag.cs');
+            export const EndTagBegin = createToken('</', 'punctuation.definition.tag.cs');
+            export const EndTagEnd = createToken('>', 'punctuation.definition.tag.cs');
+            export const EmptyTagBegin = createToken('<', 'punctuation.definition.tag.cs');
+            export const EmptyTagEnd = createToken('/>', 'punctuation.definition.tag.cs');
+
+            export const Name = (text: string) => createToken(text, 'entity.name.tag.localname.cs');
+        }
+
+        export namespace String {
+            export namespace DoubleQuoted {
+                export const Begin = createToken('"', 'punctuation.definition.string.begin.cs');
+                export const End = createToken('"', 'punctuation.definition.string.end.cs');
+                export const Text = (text: string) => createToken(text, 'string.quoted.double.cs');
+            }
+
+            export namespace SingleQuoted {
+                export const Begin = createToken('\'', 'punctuation.definition.string.begin.cs');
+                export const End = createToken('\'', 'punctuation.definition.string.end.cs');
+                export const Text = (text: string) => createToken(text, 'string.quoted.single.cs');
+            }
+        }
+
+        export const Begin = createToken('///', 'punctuation.definition.comment.cs');
+        export const Colon = createToken(':', 'punctuation.separator.colon.cs');
+        export const Equals = createToken('=', 'punctuation.separator.equals.cs');
+        export const Text = (text: string) => createToken(text, 'comment.block.documentation.cs');
+    }
+
     export const IllegalNewLine = (text: string) => createToken(text, 'invalid.illegal.newline.cs');
     export const PreprocessorMessage = (text: string) => createToken(text, 'string.unquoted.preprocessor.message.cs');
     export const Type = (text: string) => createToken(text, 'storage.type.cs');