tweak mode_creator

demo/kitchen-sink/util.js

@@ -40,6 +40,10 @@ var Renderer = require("ace/virtual_renderer").VirtualRenderer;
 var Editor = require("ace/editor").Editor;
 var MultiSelect = require("ace/multi_select").MultiSelect;
 
+exports.createEditor = function(el) {
+    return new Editor(new Renderer(el));
+}
+
 exports.createSplitEditor = function(el) {
     if (typeof(el) == "string")
         el = document.getElementById(el);
@@ -54,8 +58,8 @@ exports.createSplitEditor = function(el) {
     el.style.position = "relative";
     var split = {$container: el};
 
-    split.editor0 = split[0] = new Editor(new Renderer(e0, require("ace/theme/textmate")));
+    split.editor0 = split[0] = new Editor(new Renderer(e0));
-    split.editor1 = split[1] = new Editor(new Renderer(e1, require("ace/theme/textmate")));
+    split.editor1 = split[1] = new Editor(new Renderer(e1));
     split.splitter = s;
 
     MultiSelect(split.editor0);

lib/ace/tokenizer_dev.js

@@ -30,13 +30,15 @@
 
 define(function(require, exports, module) {
 
+// tokenizing lines longer than this makes editor very slow
+var MAX_TOKEN_COUNT = 1000;
 /*
  * version of Tokenizer with additional logging
  * and infinite loop checks
  * can be used for developing/testing new modes
  **/
-var Tokenizer = function(rules, flag) {
+
-    flag = flag ? "g" + flag : "g";
+var Tokenizer = function(rules) {
     this.states = rules;
 
     this.regExps = {};
@@ -46,13 +48,17 @@ var Tokenizer = function(rules, flag) {
         var ruleRegExps = [];
         var matchTotal = 0;
         var mapping = this.matchMappings[key] = {defaultToken: "default.text"};
+        var flag = "g";
 
         for (var i = 0; i < state.length; i++) {
             var rule = state[i];
-            if (rule.defaultToken) {
+            if (rule.defaultToken)
                 mapping.defaultToken = rule.defaultToken;
+            if (rule.caseInsensitive)
+                flag = "gi";
+            if (rule.regex == null)
                 continue;
-            }
+            
             if (rule.regex instanceof RegExp)
                 rule.regex = rule.regex.toString().slice(1, -1);
 
@@ -241,6 +247,12 @@ var Tokenizer = function(rules, flag) {
                 break;
 
             lastIndex = index;
+
+            if (tokens.length > MAX_TOKEN_COUNT) {
+                token.value += line.substr(lastIndex);
+                currentState = "start"
+                break;
+            }
         }
 
         if (token.type)

tool/mode_creator.html

@@ -21,6 +21,34 @@
         .separator-h {
             padding: 0 20px;
         }
+        #closeBtn {
+            background: rgba(245, 146, 146, 0.5);
+            border: 1px solid #F48A8A;
+            border-radius: 50%;
+            padding: 7px;
+            position: absolute;
+            right: -8px;
+            top: -8px;
+            z-index: 1000;
+        }
+        #closeBtn:hover {
+            background: rgba(245, 146, 146, 0.9);
+        }
+        #console{
+            /border: 1px solid lightblue;
+            bottom: 0;
+            height: 80px;
+            margin: 0 4%;
+            position: absolute;
+            width: 92%;
+            z-index: 1000;
+            box-shadow: 0 0 1px 2px gray
+        }
+        #consoleEditor{
+            height: 100%;
+            position: relative;
+            width: 100%;
+        }
     </style>
     <link href="../doc/site/images/favicon.ico" rel="icon" type="image/x-icon">
 </head>
@@ -29,10 +57,13 @@
 <div id="header">
     <label for="modeEl">mode</label>
     <select id="modeEl" size="1"></select>
+    <input type="button" value="&#10227;" title="sync" id="syncToMode"></select>
     <span id="tablist"></span>
     <span class="separator-h"></span>
     <label for="autorunEl">live preview</label>
     <input type="checkbox" label="autorun" id="autorunEl" checked>
+    <span class="separator-h"></span>
+    <input type="button" value="measure speed" id="perfTest"></select>
 
 
     <div style='float:right'>
@@ -44,6 +75,10 @@
     </div>
 </div>
 <div id="editor"></div>
+<div id="console" style="display:none">
+    <span id="closeBtn" onclick="this.parentNode.style.display='none'"></span>
+    <div id="consoleEditor"></div>
+</div>
 
 
 <script type="text/javascript">

tool/mode_creator.js

@@ -14,6 +14,7 @@ var EditSession = require("ace/edit_session").EditSession;
 var UndoManager = require("ace/undomanager").UndoManager;
 
 var DebugTokenizer = require("ace/tokenizer_dev").Tokenizer;
+var Tokenizer = require("ace/tokenizer").Tokenizer;
 
 // createEditor
 var splitEditor = window.splitEditor = util.createSplitEditor("editor");
@@ -74,6 +75,33 @@ util.bindDropdown(modeEl, function(value) {
     });
 });
 
+document.getElementById("syncToMode").onclick = function() {
+    docEl.value = modelist.modesByName[modeEl.value].desc;
+    docEl.onchange();
+    run();
+}
+document.getElementById("perfTest").onclick = function() {
+    var lines = editor2.session.doc.getAllLines()
+    if (!lines.length)
+        return
+    while (lines.length < 1000) {
+        lines = lines.concat(lines)
+    }
+
+    var tk = new Tokenizer(currentRules);
+    var testPerf = function(lines, tk){
+        var state = "start"
+        for (var i=0, l = lines.length; i <l; i++) {
+            state = tk.getLineTokens(lines[i], state).state
+        }
+    }
+
+    var t = performance.now();
+    testPerf(lines, tk);
+    t = t - performance.now(t);
+    log("tokenized " + lines.length + " lines in " + t + " ms");
+}
+
 util.fillDropdown("themeEl", {
     bright: [
         "chrome", "clouds", "crimson_editor", "dawn", "dreamweaver", "eclipse", "github",
@@ -118,18 +146,26 @@ function run() {
     var path = "ace/mode/new";
     var deps = getDeps(src, path);
     src = src.replace("define(", 'define("' + path +'", ["require","exports","module",' + deps +'],');
-    src += ';require(["ace/mode/new"], continueRun, function(e){console.log(e);window.require.undef("ace/mode/new")})';
+    src += ';require(["ace/mode/new"], function(e) {\
+        try{continueRun(e)}catch(e){log(e)}\
+    }, function(e){\
+        log(e);\
+        window.require.undef("ace/mode/new")\
+    });';
     try {
         eval(src);
+        hideLog()
     } catch(e) {
-        console.log(e);
+        log(e);
     }
 }
+var currentRules
 var continueRun = function(rules) {
     rules = rules[Object.keys(rules)[0]];
+    currentRules = new rules().getRules()
     var Tokenizer = DebugTokenizer;
 
-    var tk = new Tokenizer(new rules().getRules());
+    var tk = new Tokenizer(currentRules);
     editor2.session.$mode.$tokenizer = tk;
     editor2.session.bgTokenizer.setTokenizer(tk);
     editor2.renderer.updateText();
@@ -137,5 +173,23 @@ var continueRun = function(rules) {
 
 editor1.commands.bindKey("ctrl-Return", run);
 
+var logEditor
+function log(e) {
+    console.log(e)
+    if (!logEditor) {
+        logEditor = util.createEditor(document.getElementById("consoleEditor"));
+        logEditor.session.setMode("ace/mode/javascript")
+        logEditor.session.setUseWorker(false)
+    }
+    logEditor.container.parentNode.style.display = '';
+    logEditor.resize()
+    logEditor.navigateFileEnd(e);
+    logEditor.insert(e + "\n");
+}
+function hideLog() {
+    if (logEditor)
+        logEditor.container.parentNode.style.display = 'none';
+}
+
 });