Merge pull request #1415 from ajaxorg/highlighting/bugfix

Highlighting bugfix

demo/kitchen-sink/doclist.js

@@ -85,6 +85,7 @@ var docs = {
     "docs/haml.haml": "Haml",
     "docs/Haxe.hx": "haXe",
     "docs/html.html": "HTML",
+    "docs/html_ruby.erb": "HTML (Ruby)",
     "docs/jade.jade": "Jade",
     "docs/java.java": "Java",
     "docs/jsp.jsp": "JSP",

demo/kitchen-sink/docs/css.css

@@ -1,6 +1,6 @@
 .text-layer {
     font-family: Monaco, "Courier New", monospace;
-    font-size: 12px;
+    font-size: 12pX;
     cursor: text;
 }
 

demo/kitchen-sink/docs/html_ruby.erb

@@ -0,0 +1,25 @@
+<h1>Listing Books</h1>
+ 
+<table>
+  <tr>
+    <th>Title</th>
+    <th>Summary</th>
+    <th></th>
+    <th></th>
+    <th></th>
+  </tr>
+ 
+<% @books.each do |book| %>
+  <tr>
+    <td><%= book.title %></td>
+    <td><%= book.content %></td>
+    <td><%= link_to 'Show', book %></td>
+    <td><%= link_to 'Edit', edit_book_path(book) %></td>
+    <td><%= link_to 'Remove', book, :confirm => 'Are you sure?', :method => :delete %></td>
+  </tr>
+<% end %>
+</table>
+ 
+<br />
+ 
+<%= link_to 'New book', new_book_path %>

lib/ace/ext/modelist.js

@@ -68,6 +68,7 @@ var supportedModes = {
     Haskell:     ["hs"],
     haXe:        ["hx"],
     HTML:        ["htm|html|xhtml"],
+    HTML_Ruby:   ["erb|rhtml|html.erb"],
     Ini:         ["Ini|conf"],
     Jade:        ["jade"],
     Java:        ["java"],
@@ -133,7 +134,8 @@ var nameOverrides = {
     CSharp: "C#",
     golang: "Go",
     C_Cpp: "C/C++",
-    coffee: "CoffeeScript"
+    coffee: "CoffeeScript",
+    HTML_Ruby: "HTML (Ruby)"
 };
 var modesByName = {};
 for (var name in supportedModes) {

lib/ace/mode/_test/tokens_css.json

@@ -18,7 +18,7 @@
   ["support.type","font-size"],
   ["text",": "],
   ["constant.numeric","12"],
-  ["keyword","px"],
+  ["keyword","pX"],
   ["text",";"]
 ],[
    "ruleset",

lib/ace/mode/_test/tokens_python.json

@@ -29,7 +29,7 @@
   ["constant.numeric","1"],
   ["text",":"]
 ],[
-   "qstring",
+   "qstring3",
   ["text","    "],
   ["keyword","print"],
   ["text"," "],
@@ -129,7 +129,11 @@
   ["text","        "],
   ["keyword","print"],
   ["text"," "],
-  ["string","'%i\\260F = %i\\260C'"],
+  ["string","'%i"],
+  ["constant.language.escape","\\260"],
+  ["string","F = %i"],
+  ["constant.language.escape","\\260"],
+  ["string","C'"],
   ["text"," "],
   ["keyword.operator","%"],
   ["text"," "],

lib/ace/mode/behaviour/html.js

@@ -59,6 +59,9 @@ var HtmlBehaviour = function () {
             var position = editor.getCursorPosition();
             var iterator = new TokenIterator(session, position.row, position.column);
             var token = iterator.getCurrentToken();
+
+            if (hasType(token, 'string') && iterator.getCurrentTokenColumn() + token.value.length > position.column)
+                return;
             var atCursor = false;
             if (!token || !hasType(token, 'meta.tag') && !(hasType(token, 'text') && token.value.match('/'))){
                 do {

lib/ace/mode/html_highlight_rules.js

@@ -86,7 +86,7 @@ var HtmlHighlightRules = function() {
             next : "style"
         }, {
             token : "meta.tag", // opening tag
-            regex : "<\\/?",
+            regex : "<\\/?(?=\\S)",
             next : "tag"
         }, {
             token : "text",

lib/ace/mode/html_ruby.js

@@ -0,0 +1,61 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2012, Ajax.org B.V.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+
+define(function(require, exports, module) {
+"use strict";
+
+var oop = require("../lib/oop");
+var Tokenizer = require("../tokenizer").Tokenizer;
+var HtmlRubyHighlightRules = require("./html_ruby_highlight_rules").HtmlRubyHighlightRules;
+var HtmlMode = require("./html").Mode;
+var JavaScriptMode = require("./javascript").Mode;
+var CssMode = require("./css").Mode;
+var RubyMode = require("./ruby").Mode;
+
+var Mode = function() {
+    HtmlMode.call(this);   
+    var highlighter = new HtmlRubyHighlightRules();
+    this.$tokenizer = new Tokenizer(highlighter.getRules());    
+    this.$embeds = highlighter.getEmbeds();
+    this.createModeDelegates({
+        "js-": JavaScriptMode,
+        "css-": CssMode,
+        "ruby-": RubyMode
+    });
+};
+oop.inherits(Mode, HtmlMode);
+
+(function() {
+
+}).call(Mode.prototype);
+
+exports.Mode = Mode;
+});

lib/ace/mode/html_ruby_highlight_rules.js

@@ -0,0 +1,74 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Distributed under the BSD license:
+ *
+ * Copyright (c) 2012, Ajax.org B.V.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Ajax.org B.V. nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+define(function(require, exports, module) {
+    "use strict";
+
+    var oop = require("../lib/oop");
+    var HtmlHighlightRules = require("./html_highlight_rules").HtmlHighlightRules;
+    var RubyHighlightRules = require("./ruby_highlight_rules").RubyHighlightRules;
+
+    var HtmlRubyHighlightRules = function() {
+        HtmlHighlightRules.call(this);
+        
+        for (var i in this.$rules) {
+            this.$rules[i].unshift({
+                regex: "<%%|%%>",
+                token: "constant.language.escape"
+            }, {
+                token : "comment.start.erb", 
+                regex : "<%#",
+                push  : [{regex: "%>", next: "pop"}]
+            }, {
+                token : "support.ruby_tag",
+                regex : "<%+(?!>)[-=]?",
+                push  : "ruby-start"
+            });
+        }
+        
+        this.embedRules(RubyHighlightRules, "ruby-");
+        
+        this.$rules["ruby-start"].unshift({
+            token : "support.ruby_tag",
+            regex : "%>",
+            next  : "pop"
+        }, {
+            token: "comment",
+            regex: /#(?:[^%]|%[^>])*/
+        });
+
+        this.normalizeRules();
+    };
+
+
+    oop.inherits(HtmlRubyHighlightRules, HtmlHighlightRules);
+
+    exports.HtmlRubyHighlightRules = HtmlRubyHighlightRules;
+});

lib/ace/mode/python_highlight_rules.js

@@ -84,30 +84,28 @@ var PythonHighlightRules = function() {
     var exponentFloat = "(?:(?:" + pointFloat + "|" +  intPart + ")" + exponent + ")";
     var floatNumber = "(?:" + exponentFloat + "|" + pointFloat + ")";
 
+    var stringEscape =  "\\\\(x[0-9A-Fa-f]{2}|[0-7]{3}|[\\\\abfnrtv'\"]|U[0-9A-Fa-f]{8}|u[0-9A-Fa-f]{4})";
+
     this.$rules = {
         "start" : [ {
             token : "comment",
             regex : "#.*$"
-        }, {
-            token : "string",           // """ string
-            regex : strPre + '"{3}(?:[^\\\\]|\\\\.)*?"{3}'
         }, {
             token : "string",           // multi line """ string start
-            regex : strPre + '"{3}.*$',
+            regex : strPre + '"{3}',
-            next : "qqstring"
+            next : "qqstring3"
         }, {
             token : "string",           // " string
-            regex : strPre + '"(?:[^\\\\]|\\\\.)*?"'
+            regex : strPre + '"(?=.)',
-        }, {
+            next : "qqstring"
-            token : "string",           // ''' string
-            regex : strPre + "'{3}(?:[^\\\\]|\\\\.)*?'{3}"
         }, {
             token : "string",           // multi line ''' string start
-            regex : strPre + "'{3}.*$",
+            regex : strPre + "'{3}",
-            next : "qstring"
+            next : "qstring3"
         }, {
             token : "string",           // ' string
-            regex : strPre + "'(?:[^\\\\]|\\\\.)*?'"
+            regex : strPre + "'(?=.)",
+            next : "qstring"
         }, {
             token : "constant.numeric", // imaginary
             regex : "(?:" + floatNumber + "|\\d+)[jJ]\\b"
@@ -136,22 +134,54 @@ var PythonHighlightRules = function() {
             token : "text",
             regex : "\\s+"
         } ],
-        "qqstring" : [ {
+        "qqstring3" : [ {
+            token : "constant.language.escape",
+            regex : stringEscape
+        }, {
             token : "string", // multi line """ string end
-            regex : '(?:[^\\\\]|\\\\.)*?"{3}',
+            regex : '"{3}',
             next : "start"
         }, {
-            token : "string",
+            defaultToken : "string"
-            regex : '.+'
         } ],
-        "qstring" : [ {
+        "qstring3" : [ {
+            token : "constant.language.escape",
+            regex : stringEscape
+        }, {
             token : "string",  // multi line ''' string end
-            regex : "(?:[^\\\\]|\\\\.)*?'{3}",
+            regex : "'{3}",
             next : "start"
+        }, {
+            defaultToken : "string"
+        } ],
+        "qqstring" : [{
+            token : "constant.language.escape",
+            regex : stringEscape
         }, {
             token : "string",
-            regex : '.+'
+            regex : "\\\\$",
-        } ]
+            next  : "qqstring"
+        }, {
+            token : "string",
+            regex : '"|$',
+            next  : "start"
+        }, {
+            defaultToken: "string"
+        }],
+        "qstring" : [{
+            token : "constant.language.escape",
+            regex : stringEscape
+        }, {
+            token : "string",
+            regex : "\\\\$",
+            next  : "qstring"
+        }, {
+            token : "string",
+            regex : "'|$",
+            next  : "start"
+        }, {
+            defaultToken: "string"
+        }]
     };
 };
 

lib/ace/mode/r.js

@@ -36,7 +36,9 @@ define(function(require, exports, module) {
 
    (function()
    {
-      this.tokenRe = new RegExp("^["
+      this.lineCommentStart = "#";
+      // todo import codeModel from RStudio
+      /*this.tokenRe = new RegExp("^["
           + unicode.packages.L
           + unicode.packages.Mn + unicode.packages.Mc
           + unicode.packages.Nd
@@ -126,7 +128,7 @@ define(function(require, exports, module) {
             }
          }
          return false;
-      };
+      };*/
    }).call(Mode.prototype);
    exports.Mode = Mode;
 });

lib/ace/mode/rhtml.js

@@ -56,10 +56,10 @@ oop.inherits(Mode, HtmlMode);
       return this.$session.getState(position.row).match(/^r-/) ? 'R' : 'HTML';
    };
 
-   this.getNextLineIndent = function(state, line, tab, tabSize, row)
+   /* this.getNextLineIndent = function(state, line, tab, tabSize, row)
    {
       return this.codeModel.getNextLineIndent(row, line, state, tab, tabSize);
-   };
+   }; */
 
 }).call(Mode.prototype);
 

lib/ace/mode/text_highlight_rules.js

@@ -56,7 +56,11 @@ var TextHighlightRules = function() {
             for (var i = 0; i < state.length; i++) {
                 var rule = state[i];
                 if (rule.next) {
+                    if (typeof rule.next != "string")
+                        rule.nextState = prefix + rule.nextState;
+                    else
                         rule.next = prefix + rule.next;
+
                 }
             }
             this.$rules[prefix + key] = state;

lib/ace/mode/vbscript_highlight_rules.js

@@ -181,7 +181,7 @@ var VBScriptHighlightRules = function() {
             token: [
                 "support.type.vb.asp"
             ],
-            regex: "(?:\\b(vbtrue|fvbalse|vbcr|vbcrlf|vbformfeed|vblf|vbnewline|vbnullchar|vbnullstring|int32|vbtab|vbverticaltab|vbbinarycompare|vbtextcomparevbsunday|vbmonday|vbtuesday|vbwednesday|vbthursday|vbfriday|vbsaturday|vbusesystemdayofweek|vbfirstjan1|vbfirstfourdays|vbfirstfullweek|vbgeneraldate|vblongdate|vbshortdate|vblongtime|vbshorttime|vbobjecterror|vbEmpty|vbNull|vbInteger|vbLong|vbSingle|vbDouble|vbCurrency|vbDate|vbString|vbObject|vbError|vbBoolean|vbVariant|vbDataObject|vbDecimal|vbByte|vbArray)\\b)"
+            regex: "(?:\\b(vbtrue|vbfalse|vbcr|vbcrlf|vbformfeed|vblf|vbnewline|vbnullchar|vbnullstring|int32|vbtab|vbverticaltab|vbbinarycompare|vbtextcomparevbsunday|vbmonday|vbtuesday|vbwednesday|vbthursday|vbfriday|vbsaturday|vbusesystemdayofweek|vbfirstjan1|vbfirstfourdays|vbfirstfullweek|vbgeneraldate|vblongdate|vbshortdate|vblongtime|vbshorttime|vbobjecterror|vbEmpty|vbNull|vbInteger|vbLong|vbSingle|vbDouble|vbCurrency|vbDate|vbString|vbObject|vbError|vbBoolean|vbVariant|vbDataObject|vbDecimal|vbByte|vbArray)\\b)"
         },
         {
             token: [

lib/ace/tokenizer.js

@@ -58,6 +58,7 @@ var Tokenizer = function(rules) {
         var mapping = this.matchMappings[key] = {defaultToken: "text"};
         var flag = "g";
 
+        var splitterRurles = [];
         for (var i = 0; i < state.length; i++) {
             var rule = state[i];
             if (rule.defaultToken)
@@ -104,7 +105,7 @@ var Tokenizer = function(rules) {
                     adjustedregex = this.removeCapturingGroups(rule.regex);
                 }
                 if (!rule.splitRegex && typeof rule.token != "string")
-                    rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
+                    splitterRurles.push(rule); // flag will be known only at the very end
             }
 
             mapping[matchTotal] = i;
@@ -118,6 +119,10 @@ var Tokenizer = function(rules) {
             rule.__proto__ = null;
         }
         
+        splitterRurles.forEach(function(rule) {
+            rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
+        }, this);
+
         this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag);
     }
 };
@@ -146,6 +151,8 @@ var Tokenizer = function(rules) {
         if (!str)
             return [];
         var values = this.splitRegex.exec(str);
+        if (!values)
+            return "text";
         var tokens = [];
         var types = this.tokenArray;
         for (var i = 0, l = types.length; i < l; i++) {