Scopes tokenizer: new implementation

ace-internal.d.ts

@@ -1217,7 +1217,7 @@ export namespace Ace {
         trim?: boolean,
         firstLineNumber?: number,
         showGutter?: boolean
-    }
+}
 }
 
 
@@ -1554,6 +1554,27 @@ declare module "./src/mouse/default_gutter_handler" {
     }
 }
 
+declare module "./src/scope" {
+
+    export interface Scope extends String {
+        name: string;
+        children: { [name: string]: Scope };
+        parent?: Scope;
+        data: any;
+
+        get(name: any, extraId: string): Scope;
+
+        find(states): Scope | undefined;
+
+        hasParent(states): boolean;
+
+        count(): number;
+
+        getAllScopeNames(): string[];
+
+        toStack(): any[];
+    }
+}
 declare module "./src/lib/keys" {
     export function keyCodeToString(keyCode: number): string;
 }

demo/kitchen-sink/token_tooltip.js

@@ -58,13 +58,25 @@ class TokenTooltip extends Tooltip {
             return;
         }
 
-        var tokenText = token.type;
-        if (token.state)
-            tokenText += "|" + token.state;
-        if (token.merge)
-            tokenText += "\n  merge";
-        if (token.stateTransitions)
-            tokenText += "\n  " + token.stateTransitions.join("\n  ");
+        var tokenText = "";
+        var scope = token.type;
+        if (scope.name !== undefined) {
+            let firstScope = true;
+            do {
+                tokenText += firstScope ? scope.name + "\n" :"  " + scope.name + "\n";
+                firstScope = false;
+                if (!scope.parent)
+                    tokenText += "\ntoken count:" + count(scope);
+            } while(scope = scope.parent)
+        } else {
+            tokenText += token.type;
+            if (token.state)
+                tokenText += "|" + token.state;
+            if (token.merge)
+                tokenText += "\n  merge";
+            if (token.stateTransitions)
+                tokenText += "\n  " + token.stateTransitions.join("\n  ");
+        }
 
         if (this.tokenText != tokenText) {
             this.setText(tokenText);
@@ -121,4 +133,10 @@ class TokenTooltip extends Tooltip {
 
 }
 
+function count(root) {
+    return Object.keys(root.children).reduce(function (n, key) {
+        return n + count(root.children[key]);
+    }, 1);
+}
+
 exports.TokenTooltip = TokenTooltip;

src/autocomplete/inline_test.js

@@ -344,7 +344,7 @@ module.exports = {
         // Text to the right of the cursor should be tokenized normally again.
         var tokens = editor.session.getTokens(2);
         assert.strictEqual(tokens[0].value, "f hi I should be hidden");
-        assert.strictEqual(tokens[0].type, "text");
+        assert.equal(tokens[0].type, "text");
 
         done();
     },
@@ -375,7 +375,7 @@ module.exports = {
         // Text to the right of the cursor should be tokenized normally again.
         var tokens = editor.session.getTokens(2);
         assert.strictEqual(tokens[0].value, "fhi I should be hidden");
-        assert.strictEqual(tokens[0].type, "text");
+        assert.equal(tokens[0].type, "text");
 
         done();
     },

src/background_tokenizer.js

@@ -193,8 +193,8 @@ class BackgroundTokenizer {
         var state = this.states[row - 1];
         // @ts-expect-error TODO: potential wrong argument
         var data = this.tokenizer.getLineTokens(line, state, row);
-
-        if (this.states[row] + "" !== data.state + "") {
+        
+        if (this.states[row] !== data.state) {
             this.states[row] = data.state;
             this.lines[row + 1] = null;
             if (this.currentLine > row + 1)

src/edit_session/folding.js

@@ -710,7 +710,7 @@ function Folding() {
         if (token && /^comment|string/.test(type)) {
             type = type.match(/comment|string/)[0];
             if (type == "comment")
-                type += "|doc-start|\\.doc";
+                type += "|doc-start|\\.doc|empty";
             var re = new RegExp(type);
             var range = new Range();
             if (dir != 1) {

src/ext/beautify.js

@@ -127,7 +127,7 @@ exports.beautify = function(session) {
             }
 
             // line break before }
-            if (!inTag && !rowsToAdd && token.type === "paren.rparen" && token.value.substr(0, 1) === "}") {
+            if (!inTag && !rowsToAdd && token.type == "paren.rparen" && token.value.substr(0, 1) === "}") {
                 rowsToAdd++;
             }
 
@@ -153,7 +153,7 @@ exports.beautify = function(session) {
 
             if (value) {
                 // whitespace
-                if (token.type === "keyword" && value.match(/^(if|else|elseif|for|foreach|while|switch)$/)) {
+                if (token.type == "keyword" && value.match(/^(if|else|elseif|for|foreach|while|switch)$/)) {
                     parents[depth] = value;
 
                     trimNext();
@@ -167,7 +167,7 @@ exports.beautify = function(session) {
                         }
                     }
                 // trim value after opening paren
-                } else if (token.type === "paren.lparen") {
+                } else if (token.type == "paren.lparen") {
                     trimNext();
 
                     // whitespace after {
@@ -194,7 +194,7 @@ exports.beautify = function(session) {
                         }
                     }
                 // remove space before closing paren
-                } else if (token.type === "paren.rparen") {
+                } else if (token.type == "paren.rparen") {
                     unindent = 1;
 
                     // ensure curly brace is preceeded by whitespace
@@ -232,21 +232,21 @@ exports.beautify = function(session) {
 
                     trimLine();
                 // add spaces around conditional operators
-                } else if ((token.type === "keyword.operator" || token.type === "keyword") && value.match(/^(=|==|===|!=|!==|&&|\|\||and|or|xor|\+=|.=|>|>=|<|<=|=>)$/)) {
+                } else if ((token.type == "keyword.operator" || token.type == "keyword") && value.match(/^(=|==|===|!=|!==|&&|\|\||and|or|xor|\+=|.=|>|>=|<|<=|=>)$/)) {
                     trimCode();
                     trimNext();
                     spaceBefore = true;
                     spaceAfter = true;
                 // remove space before semicolon
-                } else if (token.type === "punctuation.operator" && value === ';') {
+                } else if (token.type == "punctuation.operator" && value === ';') {
                     trimCode();
                     trimNext();
                     spaceAfter = true;
 
                     if (inCSS)
                         rowsToAdd++;
                 // space after colon or comma
-                } else if (token.type === "punctuation.operator" && value.match(/^(:|,)$/)) {
+                } else if (token.type == "punctuation.operator" && value.match(/^(:|,)$/)) {
                     trimCode();
                     trimNext();
 
@@ -258,7 +258,7 @@ exports.beautify = function(session) {
                         breakBefore = false;
                     }
                 // ensure space before php closing tag
-                } else if (token.type === "support.php_tag" && value === "?>" && !breakBefore) {
+                } else if (token.type == "support.php_tag" && value === "?>" && !breakBefore) {
                     trimCode();
                     spaceBefore = true;
                 // remove excess space before HTML attribute
@@ -273,7 +273,7 @@ exports.beautify = function(session) {
                     trimLine();
                     if(value === "/>")
                         spaceBefore = true;
-                } else if (token.type === "keyword" && value.match(/^(case|default)$/)) {
+                } else if (token.type == "keyword" && value.match(/^(case|default)$/)) {
                     if (caseBody)
                         unindent = 1;
                 }
@@ -306,33 +306,33 @@ exports.beautify = function(session) {
                         code += tabString;
                 }
 
-                if (token.type === "keyword" && value.match(/^(case|default)$/)) {
+                if (token.type == "keyword" && value.match(/^(case|default)$/)) {
                     if (caseBody === false) {
                         parents[depth] = value;
                         depth++;
                         caseBody = true;
                     }
-                } else if (token.type === "keyword" && value.match(/^(break)$/)) {
+                } else if (token.type == "keyword" && value.match(/^(break)$/)) {
                     if(parents[depth-1] && parents[depth-1].match(/^(case|default)$/)) {
                         depth--;
                         caseBody = false;
                     }
                 }
 
                 // indent one line after if or else
-                if (token.type === "paren.lparen") {
+                if (token.type == "paren.lparen") {
                     roundDepth += (value.match(/\(/g) || []).length;
                     curlyDepth += (value.match(/\{/g) || []).length;
                     depth += value.length;
                 }
 
-                if (token.type === "keyword" && value.match(/^(if|else|elseif|for|while)$/)) {
+                if (token.type == "keyword" && value.match(/^(if|else|elseif|for|while)$/)) {
                     indentNextLine = true;
                     roundDepth = 0;
                 } else if (!roundDepth && value.trim() && token.type !== "comment")
                     indentNextLine = false;
 
-                if (token.type === "paren.rparen") {
+                if (token.type == "paren.rparen") {
                     roundDepth -= (value.match(/\)/g) || []).length;
                     curlyDepth -= (value.match(/\}/g) || []).length;
 

src/ext/simple_tokenizer_test.js

@@ -69,3 +69,7 @@ module.exports = {
     }
 
 };
+
+if (typeof module !== "undefined" && module === require.main) {
+    require("asyncjs").test.testcase(module.exports).exec();
+}

src/layer/text_util.js

@@ -2,5 +2,5 @@
 const textTokens = new Set(["text", "rparen", "lparen"]);
 
 exports.isTextToken = function(tokenType) {
-    return textTokens.has(tokenType);
+    return textTokens.has(tokenType.toString());
 };

src/mode/_test/highlight_rules_test.js

@@ -104,9 +104,9 @@ function checkModes() {
         var str = blockComment.start + " " + blockComment.end;
         str = blockComment.start + str;
         if (blockComment.nestable)
-            str += blockComment.end;     
+            str += blockComment.end;
         var data = tokenizer.getLineTokens(str, "start");
-        var isBroken = data.tokens.some(function(t) { return !/comment/.test(t.type); });
+        var isBroken = data.tokens.some(function(t) { return !/comment|empty/.test(t.type); });
         if (isBroken) {
             die("broken blockComment in " + modeName, data);
         }
@@ -224,7 +224,9 @@ function generateTestData(names, force) {
             var tokenizedLine = "";
             data.tokens.forEach(function(x) {
                 tokenizedLine += x.value;
-                tmp.push(JSON.stringify([x.type, x.value]));
+                if (x.type != "empty") {
+                    tmp.push(JSON.stringify([x.type, x.value]));
+                }
             });
             if (tokenizedLine != line)
                 tmp.push(JSON.stringify(line));
@@ -278,6 +280,7 @@ function testMode(modeName, i) {
             line = lineData.values.join("");
 
         var tokens = tokenizer.getLineTokens(line, state);
+        tokens.tokens = tokens.tokens.filter((el) => el.type != "empty");
         var values = tokens.tokens.map(function(x) {return x.value;});
         var types = tokens.tokens.map(function(x) {return x.type;});
 

src/mode/folding/mixed.js

@@ -13,7 +13,7 @@ oop.inherits(FoldMode, BaseFoldMode);
 
 
     this.$getMode = function(state) {
-        if (typeof state != "string") 
+        if (Array.isArray(state)) 
             state = state[0];
         for (var key in this.subModes) {
             if (state.indexOf(key) === 0)

src/mode/lua_highlight_rules.js

@@ -7,13 +7,13 @@ var LuaHighlightRules = function() {
 
     var keywords = (
         "break|do|else|elseif|end|for|function|if|in|local|repeat|"+
-         "return|then|until|while|or|and|not"
+        "return|then|until|while|or|and|not"
     );
 
     var builtinConstants = ("true|false|nil|_G|_VERSION");
 
     var functions = (
-      // builtinFunctions
+        // builtinFunctions
         "string|xpcall|package|tostring|print|os|unpack|require|"+
         "getfenv|setmetatable|next|assert|tonumber|io|rawequal|"+
         "collectgarbage|getmetatable|module|rawset|math|debug|"+
@@ -34,9 +34,9 @@ var LuaHighlightRules = function() {
         "setupvalue|getlocal|getregistry|getfenv|setn|insert|getn|"+
         "foreachi|maxn|foreach|concat|sort|remove|resume|yield|"+
         "status|wrap|create|running|"+
-      // metatableMethods
+        // metatableMethods
         "__add|__sub|__mod|__unm|__concat|__lt|__index|__call|__gc|__metatable|"+
-         "__mul|__div|__pow|__len|__eq|__le|__newindex|__tostring|__mode|__tonumber"
+        "__mul|__div|__pow|__len|__eq|__le|__newindex|__tostring|__mode|__tonumber"
     );
 
     var stdLibaries = ("string|package|os|io|math|debug|table|coroutine");
@@ -64,27 +64,22 @@ var LuaHighlightRules = function() {
     this.$rules = {
         "start" : [{
             stateName: "bracketedComment",
-            onMatch : function(value, currentState, stack){
-                stack.unshift(this.next, value.length - 2, currentState);
-                return "comment";
+            onMatch2 : function(value, scope){
+                return scope.get(this.next, value.length - 2).get("comment");
             },
             regex : /\-\-\[=*\[/,
             next  : [
                 {
-                    onMatch : function(value, currentState, stack) {
-                        if (value.length == stack[1]) {
-                            stack.shift();
-                            stack.shift();
-                            this.next = stack.shift();
+                    onMatch2 : function(value, scope) {
+                        if (scope == "bracketedComment" && value.length == scope.data) {
+                            return scope.parent.get("comment");
                         } else {
-                            this.next = "";
+                            return scope.get("comment");
                         }
-                        return "comment";
                     },
                     regex : /\]=*\]/,
-                    next  : "start"
                 }, {
-                    defaultToken: "comment.body"
+                    defaultToken : "comment.body"
                 }
             ]
         },
@@ -95,26 +90,21 @@ var LuaHighlightRules = function() {
         },
         {
             stateName: "bracketedString",
-            onMatch : function(value, currentState, stack){
-                stack.unshift(this.next, value.length, currentState);
-                return "string.start";
+            onMatch2 : function(value, scope){
+                return scope.get(this.next, value.length - 2).get("string.start");
             },
             regex : /\[=*\[/,
             next  : [
                 {
-                    onMatch : function(value, currentState, stack) {
-                        if (value.length == stack[1]) {
-                            stack.shift();
-                            stack.shift();
-                            this.next = stack.shift();
+                    onMatch2 : function(value, scope) {
+                        if (scope == "bracketedString" && value.length == scope.data) {
+                            return scope.parent.get("string.end");
                         } else {
-                            this.next = "";
+                            return scope.get("string.end");
                         }
-                        return "string.end";
                     },
-                    
+
                     regex : /\]=*\]/,
-                    next  : "start"
                 }, {
                     defaultToken : "string"
                 }
@@ -149,7 +139,7 @@ var LuaHighlightRules = function() {
             regex : "\\s+|\\w+"
         } ]
     };
-    
+
     this.normalizeRules();
 };