Global markdown mode remake based on Scopes

demo/kitchen-sink/demo.js

@@ -130,6 +130,7 @@ function loadLanguageProvider(editor) {
             }
         }, true);
         function showOccurrenceMarkers(session, positions) {
+            if (!session.state) session.state = {}
             if (!session.state.occurrenceMarkers) {
                 session.state.occurrenceMarkers = new MarkerGroup(session);
             }
@@ -179,7 +180,7 @@ function loadLanguageProvider(editor) {
             let docPos = e.getDocumentPosition();
 
             languageProvider.doHover(session, docPos, function(hover) {
-                var errorMarker = session.state?.diagnosticMarkers.getMarkerAtPosition(docPos);
+                var errorMarker = session.state?.diagnosticMarkers?.getMarkerAtPosition(docPos);
 
                 if (!errorMarker && !hover?.content) return;
 

demo/kitchen-sink/token_tooltip.js

@@ -58,13 +58,23 @@ class TokenTooltip extends Tooltip {
             return;
         }
 
-        var tokenText = token.type;
-        if (token.state)
-            tokenText += "|" + token.state;
-        if (token.merge)
-            tokenText += "\n  merge";
-        if (token.stateTransitions)
-            tokenText += "\n  " + token.stateTransitions.join("\n  ");
+        var tokenText = "";
+        var scope = token.type;
+        if (scope.name !== undefined) {
+            do {
+                tokenText += scope.name + "\n"
+                if (!scope.parent)
+                    tokenText += "\ntoken count:" + count(scope);
+            } while(scope = scope.parent)
+        } else {
+            tokenText += token.type;
+            if (token.state)
+                tokenText += "|" + token.state;
+            if (token.merge)
+                tokenText += "\n  merge";
+            if (token.stateTransitions)
+                tokenText += "\n  " + token.stateTransitions.join("\n  ");
+        }
 
         if (this.tokenText != tokenText) {
             this.setText(tokenText);
@@ -121,4 +131,16 @@ class TokenTooltip extends Tooltip {
 
 }
 
+function count(root) {
+    return Object.keys(root.children).reduce(function (n, key) {
+        return n + count(root.children[key]);
+    }, 1);
+}
+
+function count(root) {
+    return Object.keys(root.children).reduce(function (n, key) {
+        return n + count(root.children[key]);
+    }, 1);
+}
+
 exports.TokenTooltip = TokenTooltip;

src/background_tokenizer.js

@@ -192,9 +192,12 @@ class BackgroundTokenizer {
         var state = this.states[row - 1];
         // @ts-expect-error TODO: potential wrong argument
         var data = this.tokenizer.getLineTokens(line, state, row);
+        var lastToken = data.tokens[data.tokens.length - 1];
+        var newState = (lastToken !== undefined && lastToken.type !== undefined && lastToken.type.parent !== undefined)
+            ? lastToken.type.parent : data.state;
 
-        if (this.states[row] + "" !== data.state + "") {
-            this.states[row] = data.state;
+        if (this.states[row] !== newState) {
+            this.states[row] = newState;
             this.lines[row + 1] = null;
             if (this.currentLine > row + 1)
                 this.currentLine = row + 1;

src/edit_session/bracket_match.js

@@ -178,7 +178,7 @@ function BracketMatch() {
             // whose type matches typeRe
             do {
                 token = iterator.stepBackward();
-            } while (token && !typeRe.test(token.type));
+            } while (token && !typeRe.test(token.type.toString()));
 
             if (token == null)
                 break;
@@ -245,7 +245,7 @@ function BracketMatch() {
             // whose type matches typeRe
             do {
                 token = iterator.stepForward();
-            } while (token && !typeRe.test(token.type));
+            } while (token && !typeRe.test(token.type.toString()));
 
             if (token == null)
                 break;

src/edit_session/folding.js

@@ -708,9 +708,9 @@ function Folding() {
         var token = iterator.getCurrentToken();
         var type = token && token.type;
         if (token && /^comment|string/.test(type)) {
-            type = type.match(/comment|string/)[0];
+            type = /comment|string/.exec(type)[0];
             if (type == "comment")
-                type += "|doc-start|\\.doc";
+                type += "|doc-start|\\.doc|empty";
             var re = new RegExp(type);
             var range = new Range();
             if (dir != 1) {

src/ext/beautify.js

@@ -147,13 +147,13 @@ exports.beautify = function(session) {
                 breakBefore = true;
 
                 // trim value if not in a comment or string
-                if (!is(token, "comment") && !token.type.match(/^(comment|string)$/))
+                if (!is(token, "comment") && !/^(comment|string)$/.test(token.type))
                    value = value.trimLeft();
             }
 
             if (value) {
                 // whitespace
-                if (token.type === "keyword" && value.match(/^(if|else|elseif|for|foreach|while|switch)$/)) {
+                if (token.type === "keyword" && /^(if|else|elseif|for|foreach|while|switch)$/.test(value)) {
                     parents[depth] = value;
 
                     trimNext();

src/layer/text.js

@@ -413,10 +413,10 @@ class Text {
 
         valueFragment.appendChild(this.dom.createTextNode(i ? value.slice(i) : value, this.element));
 
-        if (!isTextToken(token.type)) {
-            var classes = "ace_" + token.type.replace(/\./g, " ace_");
+        if (!isTextToken(token.type.toString())) {
+            var classes = "ace_" + token.type.toString().replace(/\./g, " ace_");
             var span = this.dom.createElement("span");
-            if (token.type == "fold"){
+            if (token.type.toString() === "fold") {
                 span.style.width = (token.value.length * this.config.characterWidth) + "px";
                 span.setAttribute("title", nls("inline-fold.closed.title", "Unfold code"));
             }

src/mode/_test/highlight_rules_test.js

@@ -50,11 +50,11 @@ function checkModes() {
         testComments(m.lineCommentStart, testLineComment, tokenizer, modeName);
         testComments(m.blockComment, testBlockComment, tokenizer, modeName);
         testBrackets(m, modeName);
-        
+
         if (m.snippetFileId)
             snippets[m.snippetFileId] = modeName;
     });
-    
+
     jsFileList(cwd + "../../snippets").forEach(function(snippetFileName) {
         if (/\.snippets$/.test(snippetFileName)) return;
         if (!snippets["ace/snippets/" + snippetFileName])
@@ -65,7 +65,7 @@ function checkModes() {
         console.error("Snippet files missing", snippets);
         throw new Error("Snippet files missing");
     }
-    
+
     function testNextState(tokenizer, modeName) {
         let keys = Object.keys(tokenizer.states);
         for (let i = 0; i < keys.length; i++) {
@@ -97,24 +97,28 @@ function checkModes() {
             }
         }
     }
-    
+
     function testBlockComment(tokenizer, blockComment, modeName) {
         if (blockComment.lineStartOnly)
-            return; // TODO test 
+            return; // TODO test
         var str = blockComment.start + " " + blockComment.end;
         str = blockComment.start + str;
         if (blockComment.nestable)
-            str += blockComment.end;     
+            str += blockComment.end;
         var data = tokenizer.getLineTokens(str, "start");
+
+        var type = data.tokens.length > 0 ? data.tokens[data.tokens.length - 1].type : "";
+        var state = typeof type === "string" ? data.state : type.parent;
+
         var isBroken = data.tokens.some(function(t) { return !/comment/.test(t.type); });
         if (isBroken) {
             die("broken blockComment in " + modeName, data);
         }
-        if (!/start/.test(data.state)) {
+        if (!/start/.test(state)) {
             die("broken state after blockComment in " + modeName, data);
         }
     }
-    
+
     function testLineComment(tokenizer, commentStart, modeName) {
         var tokens = tokenizer.getLineTokens(commentStart + " ", "start").tokens;
         if (!/comment/.test(tokens[0].type)) {
@@ -124,11 +128,11 @@ function checkModes() {
 
     function testBrackets(mode, modeName) {
         if (/^(forth|mask)$/.test(modeName)) return;
-        
+
         var session = new EditSession("{ foo[ bar(baz) ] }", mode);
-        
-        var isInvalid = session.getTokens(0).some(function(t) { 
-            return /invalid|illegal|string/.test(t.type); 
+
+        var isInvalid = session.getTokens(0).some(function(t) {
+            return /invalid|illegal|string/.test(t.type);
         });
         if (isInvalid) return;
 
@@ -141,7 +145,7 @@ function checkModes() {
         position = session.findMatchingBracket({row:0, column:11});
         if (!position || position.column != 14)
             die("Matching bracket not found in " + modeName);
-        
+
         if (mode.$behaviour) {
             session.setValue("");
             editor.setSession(session);
@@ -161,7 +165,7 @@ function checkModes() {
 }
 
 function generateTestData(names, force) {
-    var docRoot = root + "/demo/kitchen-sink/docs";
+    var docRoot = root + "/ace/demo/kitchen-sink/docs";
     var docs = fs.readdirSync(docRoot);
     var specialDocs = fs.readdirSync(cwd);
     var modes = modeList();
@@ -183,7 +187,7 @@ function generateTestData(names, force) {
 
         if (names && names.length && names.indexOf(modeName) == -1)
             return;
-        
+
         var outputPath = cwd + "tokens_" + modeName + ".json";
         try {
             var oldOutput = require(outputPath);
@@ -197,7 +201,7 @@ function generateTestData(names, force) {
                 }).join("");
             }).join("\n");
         }
-        
+
         var filePath = "text_" + modeName + ".txt";
         if (specialDocs.indexOf(filePath) !== -1) {
             filePath = cwd + filePath;
@@ -206,7 +210,7 @@ function generateTestData(names, force) {
             // oldText = "";
         }
         var text = oldText ||fs.readFileSync(filePath, "utf8");
-        
+
         try {
             var Mode = require("../" + modeName).Mode;
         } catch(e) {
@@ -217,26 +221,33 @@ function generateTestData(names, force) {
         var tokenizer = new Mode().getTokenizer();
 
         var state = "start";
-        var data = text.split(/\r\n|\r|\n/).map(function(line) {
+        var data = text.split(/\r\n|\r|\n/).map(function (line) {
             var data = tokenizer.getLineTokens(line, state);
+
+            var type = data.tokens.length > 0 ? data.tokens[data.tokens.length - 1].type : "";
             var tmp = [];
-            tmp.push(JSON.stringify(data.state));
+            var stateString;
+            if (/^[\x00]/.test(line)) {
+                stateString = "start";
+            }
+            else {
+                stateString = typeof type === "string" ? data.state : type.getAllScopeNames();
+            }
+            tmp.push(JSON.stringify(stateString));
             var tokenizedLine = "";
-            data.tokens.forEach(function(x) {
+            data.tokens.forEach(function (x) {
                 tokenizedLine += x.value;
-                tmp.push(JSON.stringify([x.type, x.value]));
+                tmp.push(JSON.stringify([x.type.toString(), x.value]));
             });
-            if (tokenizedLine != line)
-                tmp.push(JSON.stringify(line));
-            state = data.state;
+            if (tokenizedLine !== line) tmp.push(JSON.stringify(line));
+            state = typeof type === "string" ? data.state : type.parent;
             return tmp.join(",\n  ");
         });
-        
+
         var jsonStr = "[[\n   " + data.join("\n],[\n   ") + "\n]]";
-
-        if (oldOutput && JSON.stringify(JSON.parse(jsonStr)) == JSON.stringify(oldOutput))
-            return;
-
+
+        if (oldOutput && JSON.stringify(JSON.parse(jsonStr)) == JSON.stringify(oldOutput)) return;
+
         fs.writeFileSync(outputPath, jsonStr, "utf8");
     });
 }
@@ -280,18 +291,19 @@ function testMode(modeName, i) {
         var tokens = tokenizer.getLineTokens(line, state);
         var values = tokens.tokens.map(function(x) {return x.value;});
         var types = tokens.tokens.map(function(x) {return x.type;});
-
+        var type = tokens.tokens.length > 0 ? tokens.tokens[tokens.tokens.length - 1].type: "";
+        var stateString = /^[\x00]/.test(line) ? "start" : typeof type === "string" ? tokens.state : type.getAllScopeNames();
         var err = testEqual([
-            JSON.stringify(lineData.state), JSON.stringify(tokens.state),
+            JSON.stringify(lineData.state), JSON.stringify(stateString),
             lineData.types, types,
             lineData.values, values]);
-        
+
         if (err) {
             console.log(line);
             throw "error";
         }
 
-        state = tokens.state;
+        state = typeof type === "string" ? tokens.state : type.parent;
     });
 }
 function testEqual(a) {