highlight js identifiers with unicode characters.

fix #148

lib/ace/edit_session.js

@@ -384,9 +384,6 @@ var EditSession = function(text, mode) {
         }
     };
 
-    this.tokenRe = /^[\w\d]+/g;
-    this.nonTokenRe = /^(?:[^\w\d]|[\u3040-\u309F]|[\u30A0-\u30FF]|[\u4E00-\u9FFF\uF900-\uFAFF\u3400-\u4DBF])+/g;
-
     this.getWordRange = function(row, column) {
         var line = this.getLine(row);
 
@@ -477,6 +474,9 @@ var EditSession = function(text, mode) {
 
         this.bgTokenizer.setDocument(this.getDocument());
         this.bgTokenizer.start(0);
+        
+        this.tokenRe = mode.tokenRe;
+        this.nonTokenRe = mode.nonTokenRe;
 
         this._dispatchEvent("changeMode");
     };

lib/ace/mode/javascript_highlight_rules.js

@@ -40,6 +40,7 @@ define(function(require, exports, module) {
 
 var oop = require("pilot/oop");
 var lang = require("pilot/lang");
+var regexp = require("ace/regexp");
 var DocCommentHighlightRules = require("ace/mode/doc_comment_highlight_rules").DocCommentHighlightRules;
 var TextHighlightRules = require("ace/mode/text_highlight_rules").TextHighlightRules;
 
@@ -60,6 +61,13 @@ var JavaScriptHighlightRules = function() {
         "public|interface|package|protected|static").split("|")
     );
 
+    // TODO: Unicode escape sequences
+    var identifierRe = "[" + regexp.unicode.L + "\\$_][" 
+        + regexp.unicode.L
+        + regexp.unicode.Mn + regexp.unicode.Mc
+        + regexp.unicode.Nd
+        + regexp.unicode.Pc + "\\$_]*\\b";
+        
     // regexp must not have capturing parentheses. Use (?:) instead.
     // regexps are ordered -> the first match is used
 
@@ -115,9 +123,7 @@ var JavaScriptHighlightRules = function() {
                     else
                         return "identifier";
                 },
-                // TODO: Unicode escape sequences
-                // TODO: Unicode identifiers
-                regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
+                regex : identifierRe
             }, {
                 token : "keyword.operator",
                 regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)"

lib/ace/mode/javascript_tokenizer_test.js

@@ -107,6 +107,11 @@ module.exports = {
     "test tokenize regular expressions": function() {
         var tokens = this.tokenizer.getLineTokens("a/b/c", "start").tokens;
         assert.equal(5, tokens.length);
+    },
+    
+    "test tokenize identifier with umlauts": function() {
+        var tokens = this.tokenizer.getLineTokens("füße", "start").tokens;
+        assert.equal(1, tokens.length);
     }
 };
 

lib/ace/mode/text.js

@@ -43,6 +43,7 @@ define(function(require, exports, module) {
 var Tokenizer = require("ace/tokenizer").Tokenizer;
 var TextHighlightRules = require("ace/mode/text_highlight_rules").TextHighlightRules;
 var Behaviour = require("ace/mode/behaviour").Behaviour;
+var regexp = require("ace/regexp");
 
 var Mode = function() {
     this.$tokenizer = new Tokenizer(new TextHighlightRules().getRules());
@@ -51,6 +52,20 @@ var Mode = function() {
 
 (function() {
 
+    this.tokenRe = new RegExp("^["
+        + regexp.unicode.L
+        + regexp.unicode.Mn + regexp.unicode.Mc
+        + regexp.unicode.Nd
+        + regexp.unicode.Pc + "\\$_]+", "g"
+    );
+    
+    this.nonTokenRe = new RegExp("^(?:[^"
+        + regexp.unicode.L
+        + regexp.unicode.Mn + regexp.unicode.Mc
+        + regexp.unicode.Nd
+        + regexp.unicode.Pc + "\\$_]|\s])+", "g"
+    );
+
     this.getTokenizer = function() {
         return this.$tokenizer;
     };

lib/ace/selection_test.js

@@ -144,8 +144,12 @@ module.exports = {
     },
 
     "test: moveCursor word left" : function() {
-        var session = new EditSession( ["ab",
-                                         " Juhu Kinners (abc, 12)", " cde"].join("\n"));
+        var session = new EditSession([
+            "ab",
+            " Juhu Kinners (abc, 12)",
+            " cde"
+        ].join("\n"));
+        
         var selection = session.getSelection();
 
         selection.moveCursorDown();
@@ -183,6 +187,18 @@ module.exports = {
         selection.moveCursorWordLeft();
         assert.position(selection.getCursor(), 0, 2);
     },
+        
+    "test: moveCursor word left" : function() {
+        var session = new EditSession(" Fuß Füße");
+
+        var selection = session.getSelection();
+        selection.moveCursorTo(0, 9)
+        selection.moveCursorWordLeft();
+        assert.position(selection.getCursor(), 0, 5);
+        
+        selection.moveCursorWordLeft();
+        assert.position(selection.getCursor(), 0, 4);
+    },
 
     "test: select word left if cursor in word" : function() {
         var session = new EditSession("Juhu Kinners");