Added tests for syntax highlighting

dotnet · Sep 2, 2016 · 559c606 · 559c606
1 parent 3f50222
commit 559c606
Show file tree

Hide file tree

Showing 9 changed files with 874 additions and 14 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,12 +1,17 @@
-sudo: false
 language: node_js
 
 node_js:
 - "5.1"
 
-os:
-  -osx
-  -linux
+env:
+  - CXX=g++-4.8
+
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - g++-4.8
 
 install:
 - npm install

diff --git a/gulpfile.js b/gulpfile.js
@@ -42,7 +42,7 @@ function installOmnisharp(omnisharps) {
 
         return download.go(omni.flavor, omni.platform, log);
     });
-    
+
     return Promise.all(promises);
 }
 
@@ -53,7 +53,7 @@ function cleanOmnisharp() {
 gulp.task('omnisharp:clean', () => {
     return cleanOmnisharp();
 });
- 
+
 gulp.task('omnisharp:install', ['omnisharp:clean'], () => {
     const flavor = gulpUtil.env.flavor || Flavor.CoreCLR;
     const platform = gulpUtil.env.platform || platform.getCurrentPlatform();
@@ -157,7 +157,7 @@ gulp.task('package:offline', ['clean'], () => {
 
 /// Test Task
 gulp.task('test', () => {
-    gulp.src('out/test/*.tests.js')
+    gulp.src('out/test/**/*.tests.js')
         .pipe(mocha({ui: "tdd"}))
         .once('error', () => {
             process.exit(1);

diff --git a/package.json b/package.json
@@ -21,7 +21,7 @@
   "scripts": {
     "compile": "node ./node_modules/vscode/bin/compile -p ./",
     "watch": "node ./node_modules/vscode/bin/compile -watch -p ./",
-    "test": "mocha --timeout 15000 -u tdd ./out/test/*.tests.js",
+    "test": "mocha --timeout 15000 -u tdd ./out/test/**/*.tests.js",
     "postinstall": "node ./node_modules/vscode/bin/install"
   },
   "dependencies": {
@@ -45,7 +45,9 @@
     "tslint-microsoft-contrib": "^2.0.0",
     "typescript": "^1.7.3",
     "vscode": "^0.11.13",
-    "vsce": "^1.7.0"
+    "vsce": "^1.7.0",
+    "chai": "^3.5.0",
+    "vscode-textmate": "^2.1.1"
   },
   "engines": {
     "vscode": "^1.3.0"
@@ -149,7 +151,7 @@
       {
         "command": "csharp.listProcess",
         "title": "List process for attach",
-        "category": "CSharp" 
+        "category": "CSharp"
       }
     ],
     "keybindings": [
@@ -494,7 +496,7 @@
                         "additionalProperties": { "type": "string" },
                         "description": "Environment variables passed to the pipe program.",
                         "default": { }
-                      }                      
+                      }
                     }
                   },
                   "linux": {
@@ -545,18 +547,18 @@
                 "default": "The process name to attach to. If this is used, 'processId' should not be used."
               },
               "processId": {
-                "anyOf": [ 
+                "anyOf": [
                     {
                       "type": "string",
                       "description": "The process id to attach to. Use \"${command.pickProcesss}\" to get a list of running processes to attach to. If 'processId' used, 'processName' should not be used.",
                       "default": "${command.pickProcess}"
-                    }, 
+                    },
                     {
                       "type": "integer",
                       "description": "The process id to attach to. Use \"${command.pickProcesss}\" to get a list of running processes to attach to. If 'processId' used, 'processName' should not be used.",
                       "default": 0
                     }
-                ]                
+                ]
               },
               "sourceFileMap": {
                 "type": "object",

diff --git a/test/syntaxes/class.tests.ts b/test/syntaxes/class.tests.ts
@@ -0,0 +1,79 @@
+import { should } from 'chai';
+import { Tokens, Token } from './utils/tokenizer';
+import { TokenizerUtil } from'./utils/tokenizerUtil';
+
+describe("Grammar", function() {
+    before(function() {
+        should();
+    });
+
+    describe("Class", function() {
+        it("has a class keyword, a name and optional storage modifiers", function() {
+
+const input = `
+namespace TestNamespace
+{
+    public             class PublicClass { }
+
+                       class DefaultClass { }
+
+    internal           class InternalClass { }
+
+              static   class DefaultStaticClass { }
+
+    public    static   class PublicStaticClass { }
+
+              sealed   class DefaultSealedClass { }
+
+    public    sealed   class PublicSealedClass { }
+
+    public    abstract class PublicAbstractClass { }
+
+              abstract class DefaultAbstractClass { }
+}`;
+            let tokens: Token[] = TokenizerUtil.tokenize(input);
+
+            tokens.should.contain(Tokens.StorageModifierKeyword("public", 4, 5));
+            tokens.should.contain(Tokens.ClassKeyword("class", 4, 24));
+            tokens.should.contain(Tokens.ClassIdentifier("PublicClass", 4, 30));
+
+            tokens.should.contain(Tokens.ClassKeyword("class", 6, 24));
+            tokens.should.contain(Tokens.ClassIdentifier("DefaultClass", 6, 30));
+
+            tokens.should.contain(Tokens.StorageModifierKeyword("internal", 8, 5));
+            tokens.should.contain(Tokens.ClassKeyword("class", 8, 24));
+            tokens.should.contain(Tokens.ClassIdentifier("InternalClass", 8, 30));
+
+            tokens.should.contain(Tokens.StorageModifierKeyword("static", 10, 15));
+            tokens.should.contain(Tokens.ClassKeyword("class", 10, 24));
+            tokens.should.contain(Tokens.ClassIdentifier("DefaultStaticClass", 10, 30));
+
+            tokens.should.contain(Tokens.StorageModifierKeyword("public", 12, 5));
+            tokens.should.contain(Tokens.StorageModifierKeyword("static", 12, 15));
+            tokens.should.contain(Tokens.ClassKeyword("class", 12, 24));
+            tokens.should.contain(Tokens.ClassIdentifier("PublicStaticClass", 12, 30));
+
+            tokens.should.contain(Tokens.StorageModifierKeyword("sealed", 14, 15));
+            tokens.should.contain(Tokens.ClassKeyword("class", 14, 24));
+            tokens.should.contain(Tokens.ClassIdentifier("DefaultSealedClass", 14, 30));
+
+            tokens.should.contain(Tokens.StorageModifierKeyword("public", 16, 5));
+            tokens.should.contain(Tokens.StorageModifierKeyword("sealed", 16, 15));
+            tokens.should.contain(Tokens.ClassKeyword("class", 16, 24));
+            tokens.should.contain(Tokens.ClassIdentifier("PublicSealedClass", 16, 30));
+
+            tokens.should.contain(Tokens.StorageModifierKeyword("public", 18, 5));
+            tokens.should.contain(Tokens.StorageModifierKeyword("abstract", 18, 15));
+            tokens.should.contain(Tokens.ClassKeyword("class", 18, 24));
+            tokens.should.contain(Tokens.ClassIdentifier("PublicAbstractClass", 18, 30));
+
+            tokens.should.contain(Tokens.StorageModifierKeyword("abstract", 20, 15));
+            tokens.should.contain(Tokens.ClassKeyword("class", 20, 24));
+            tokens.should.contain(Tokens.ClassIdentifier("DefaultAbstractClass", 20, 30));
+
+        });
+
+    });
+});
+
+
diff --git a/test/syntaxes/namespace.tests.ts b/test/syntaxes/namespace.tests.ts
@@ -0,0 +1,78 @@
+import { should } from 'chai';
+import { Tokens, Token } from './utils/tokenizer';
+import { TokenizerUtil } from'./utils/tokenizerUtil';
+
+describe("Grammar", function() {
+    before(function () {
+        should();
+    });
+
+    describe("Namespace", function() {
+        it("has a namespace keyword and a name", function() {
+
+const input = `
+namespace TestNamespace
+{
+}`;
+            let tokens: Token[] = TokenizerUtil.tokenize(input);
+
+            tokens.should.contain(Tokens.NamespaceKeyword("namespace", 2, 1));
+            tokens.should.contain(Tokens.NamespaceIdentifier("TestNamespace", 2, 11));
+        });
+
+        it("can be nested", function() {
+
+const input = `
+namespace TestNamespace
+{
+    namespace NestedNamespace {
+
+    }
+}`;
+            let tokens: Token[] = TokenizerUtil.tokenize(input);
+
+            tokens.should.contain(Tokens.NamespaceKeyword("namespace", 2, 1));
+            tokens.should.contain(Tokens.NamespaceIdentifier("TestNamespace", 2, 11));
+
+            tokens.should.contain(Tokens.NamespaceKeyword("namespace", 4, 5));
+            tokens.should.contain(Tokens.NamespaceIdentifier("NestedNamespace", 4, 15));
+        });
+
+        it("can contain using statements", function() {
+
+const input = `
+using UsineOne;
+using one = UsineOne.Something;
+
+namespace TestNamespace
+{
+    using UsingTwo;
+    using two = UsineOne.Something;
+
+    namespace NestedNamespace
+    {
+        using UsingThree;
+        using three = UsineOne.Something;
+    }
+}`;
+            let tokens: Token[] = TokenizerUtil.tokenize(input);
+
+            tokens.should.contain(Tokens.UsingKeyword("using", 2, 1));
+            tokens.should.contain(Tokens.UsingKeyword("using", 3, 1));
+
+            tokens.should.contain(Tokens.NamespaceKeyword("namespace", 5, 1));
+            tokens.should.contain(Tokens.NamespaceIdentifier("TestNamespace", 5, 11));
+
+            tokens.should.contain(Tokens.UsingKeyword("using", 7, 5));
+            tokens.should.contain(Tokens.UsingKeyword("using", 8, 5));
+
+            tokens.should.contain(Tokens.NamespaceKeyword("namespace", 10, 5));
+            tokens.should.contain(Tokens.NamespaceIdentifier("NestedNamespace", 10, 15));
+
+            tokens.should.contain(Tokens.UsingKeyword("using", 12, 9));
+            tokens.should.contain(Tokens.UsingKeyword("using", 12, 9));
+        });
+    });
+});
+
+
diff --git a/test/syntaxes/utils/tokenizer.ts b/test/syntaxes/utils/tokenizer.ts
@@ -0,0 +1,77 @@
+import {ITokenizeLineResult, Registry, IGrammar, StackElement} from 'vscode-textmate';
+
+export class Tokenizer
+{
+    private _grammar : IGrammar;
+
+    constructor(grammarFilePath: string) {
+        this._grammar = new Registry().loadGrammarFromPathSync(grammarFilePath);
+    }
+
+    public tokenize(input: string): Token[] {
+        let tokens: Token[] = [];
+
+        // ensure consistent line-endings irrelevant of OS
+        input = input.replace("\r\n","\n");
+
+        let previousStack : StackElement = null;
+
+        const lines: string[] = input.split("\n");
+
+        for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
+            const line = lines[lineIndex];
+
+            let result: ITokenizeLineResult = this._grammar.tokenizeLine(line, previousStack);
+            previousStack = result.ruleStack;
+
+            for (const token of result.tokens) {
+                const text = line.substring(token.startIndex, token.endIndex);
+                const type : string = token.scopes[token.scopes.length - 1];
+                tokens.push(new Token(text, type, lineIndex+1, token.startIndex + 1));
+            }
+        }
+
+        return tokens;
+    }
+}
+
+export class Token {
+    constructor(text: string, type: string, line?: number, column?: number) {
+        this.text = text;
+        this.type = type;
+        this.column = column;
+        this.line = line;
+    }
+
+    public text: string;
+    public type: string;
+    public line: number;
+    public column: number;
+}
+
+export namespace Tokens {
+
+    function createToken(text: string, type: string, line?: number, column?: number) : Token {
+        return new Token(text, type, line, column);
+    }
+
+    export const NamespaceKeyword = (text: string, line?: number, column?: number) =>
+        createToken(text, "keyword.other.namespace.cs", line, column);
+
+    export const NamespaceIdentifier = (text: string, line?: number, column?: number) =>
+        createToken(text, "entity.name.type.namespace.cs", line, column);
+
+    export const UsingKeyword = (text: string, line?: number, column?: number) =>
+        createToken(text, "keyword.other.using.cs", line, column);
+
+    export const ClassKeyword = (text: string, line?: number, column?: number) =>
+        createToken(text, "storage.modifier.cs", line, column);
+
+    export const ClassIdentifier = (text: string, line?: number, column?: number) =>
+        createToken(text, "entity.name.type.class.cs", line, column);
+
+    export const StorageModifierKeyword = (text: string, line?: number, column?: number) =>
+        createToken(text, "storage.modifier.cs", line, column);
+
+}
+
diff --git a/test/syntaxes/utils/tokenizerUtil.ts b/test/syntaxes/utils/tokenizerUtil.ts
@@ -0,0 +1,10 @@
+import { Tokenizer, Token } from './tokenizer';
+
+export class TokenizerUtil
+{
+    private static _tokenizer: Tokenizer = new Tokenizer("syntaxes/csharp.json");
+
+    public static tokenize(input: string): Token[] {
+        return TokenizerUtil._tokenizer.tokenize(input);
+    }
+}