jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

don't merge tokens (Includes coding style cleanups)

Browse source 
fix #355
This commit is contained in:
Fabian Jakobs 2011-08-01 15:01:50 +02:00
commit 1c5fb5b75a
28 changed files with 835 additions and 883 deletions
Download patch file Download diff file Expand all files Collapse all files

14
index.html
View file

@ -31,7 +31,6 @@
<option value="java">Java</option> <option value="java">Java</option>
<option value="csharp">C#</option> <option value="csharp">C#</option>
<option value="c_cpp">C++</option> <option value="c_cpp">C++</option>
<option value="svg">SVG</option>
<option value="clojure">Clojure</option> <option value="clojure">Clojure</option>
<option value="ocaml">OCaml</option> <option value="ocaml">OCaml</option>
<option value="textile">Textile</option> <option value="textile">Textile</option>
@ -221,9 +220,7 @@
for (var i=0; i<items.length; i++) { for (var i=0; i<items.length; i++) {
alert(items[i] + "juhu"); alert(items[i] + "juhu");
} // Real Tab. } // Real Tab.
} }</script>
// A magic character: >> ぁ <<
</script>
<script type="text/editor" id="csstext">.text-layer { <script type="text/editor" id="csstext">.text-layer {
font-family: Monaco, "Courier New", monospace; font-family: Monaco, "Courier New", monospace;
@ -302,8 +299,7 @@ for i in sys.argv[1:]:
print repr(i), "not a numeric value" print repr(i), "not a numeric value"
else: else:
celsius=(fahrenheit-32)*5.0/9.0 celsius=(fahrenheit-32)*5.0/9.0
print '%i\260F = %i\260C' % (int(fahrenheit), int(celsius+.5)) print '%i\260F = %i\260C' % (int(fahrenheit), int(celsius+.5))</script>
</script>
<script type="text/editor" id="phptext"><?php <script type="text/editor" id="phptext"><?php
@ -674,8 +670,7 @@ th.join()
assert counter.get() == 12 assert counter.get() == 12
</script> </script>
<script type="text/editor" id="scala"> <script type="text/editor" id="scala">//http://www.scala-lang.org/node/227
//http://www.scala-lang.org/node/227
/* Defines a new method 'sort' for array objects */ /* Defines a new method 'sort' for array objects */
object implicits extends Application { object implicits extends Application {
implicit def arrayWrapper[A : ClassManifest](x: Array[A]) = implicit def arrayWrapper[A : ClassManifest](x: Array[A]) =
@ -686,8 +681,7 @@ object implicits extends Application {
} }
val x = Array(2, 3, 1, 4) val x = Array(2, 3, 1, 4)
println("x = "+ x.sort((x: Int, y: Int) => x < y)) println("x = "+ x.sort((x: Int, y: Int) => x < y))
} }</script>
</script>
<input id="cockpitInput" type="text"/> <input id="cockpitInput" type="text"/>
<script type="text/javascript"> <script type="text/javascript">

32
lib/ace/layer/text.js
View file

@ -384,17 +384,10 @@ var Text = function(parentEl) {
var screenColumn = 0; var screenColumn = 0;
var self = this; var self = this;
function addToken(token, value) { if (!splits || splits.length == 0)
screenColumn = self.$renderToken(
stringBuilder, screenColumn, token, value
);
}
if (!splits || splits.length == 0) {
splitChars = Number.MAX_VALUE; splitChars = Number.MAX_VALUE;
} else { else
splitChars = splits[0]; splitChars = splits[0];
}
if (!onlyContents) { if (!onlyContents) {
stringBuilder.push("<div class='ace_line' style='height:", stringBuilder.push("<div class='ace_line' style='height:",
@ -408,11 +401,17 @@ var Text = function(parentEl) {
var value = token.value; var value = token.value;
if (chars + value.length < splitChars) { if (chars + value.length < splitChars) {
addToken(token, value); screenColumn = self.$renderToken(
stringBuilder, screenColumn, token, value
);
chars += value.length; chars += value.length;
} else { }
else {
while (chars + value.length >= splitChars) { while (chars + value.length >= splitChars) {
addToken(token, value.substring(0, splitChars - chars)); screenColumn = self.$renderToken(
stringBuilder, screenColumn,
token, value.substring(0, splitChars - chars)
);
value = value.substring(splitChars - chars); value = value.substring(splitChars - chars);
chars = splitChars; chars = splitChars;
@ -430,17 +429,18 @@ var Text = function(parentEl) {
} }
if (value.length != 0) { if (value.length != 0) {
chars += value.length; chars += value.length;
addToken(token, value); screenColumn = self.$renderToken(
stringBuilder, screenColumn, token, value
);
} }
} }
} }
if (this.showInvisibles) { if (this.showInvisibles) {
if (lastRow !== this.session.getLength() - 1) { if (lastRow !== this.session.getLength() - 1)
stringBuilder.push("<span class='ace_invisible'>" + this.EOL_CHAR + "</span>"); stringBuilder.push("<span class='ace_invisible'>" + this.EOL_CHAR + "</span>");
} else { else
stringBuilder.push("<span class='ace_invisible'>" + this.EOF_CHAR + "</span>"); stringBuilder.push("<span class='ace_invisible'>" + this.EOF_CHAR + "</span>");
}
} }
stringBuilder.push("</div>"); stringBuilder.push("</div>");
}; };

162
lib/ace/mode/c_cpp_highlight_rules.js
View file

@ -67,96 +67,102 @@ var c_cppHighlightRules = function() {
this.$rules = { this.$rules = {
"start" : [ "start" : [
{ {
token : "comment", token : "comment",
regex : "\\/\\/.*$" regex : "\\/\\/.*$"
}, },
new DocCommentHighlightRules().getStartRule("doc-start"), new DocCommentHighlightRules().getStartRule("doc-start"),
{ {
token : "comment", // multi line comment token : "comment", // multi line comment
regex : "\\/\\*", merge : true,
next : "comment" regex : "\\/\\*",
}, { next : "comment"
token : "string", // single line }, {
regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' token : "string", // single line
}, { regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
token : "string", // multi line string start }, {
regex : '["].*\\\\$', token : "string", // multi line string start
next : "qqstring" merge : true,
}, { regex : '["].*\\\\$',
token : "string", // single line next : "qqstring"
regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']" }, {
}, { token : "string", // single line
token : "string", // multi line string start regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
regex : "['].*\\\\$", }, {
next : "qstring" token : "string", // multi line string start
}, { merge : true,
token : "constant.numeric", // hex regex : "['].*\\\\$",
regex : "0[xX][0-9a-fA-F]+\\b" next : "qstring"
}, { }, {
token : "constant.numeric", // float token : "constant.numeric", // hex
regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b" regex : "0[xX][0-9a-fA-F]+\\b"
}, { }, {
token : "constant.numeric", // float
regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
}, {
token : "constant", // <CONSTANT> token : "constant", // <CONSTANT>
regex : "<[a-zA-Z0-9.]+>" regex : "<[a-zA-Z0-9.]+>"
}, { }, {
token : "keyword", // pre-compiler directivs token : "keyword", // pre-compiler directivs
regex : "(?:#include|#pragma|#line|#define|#undef|#ifdef|#else|#elif|#endif|#ifndef)" regex : "(?:#include|#pragma|#line|#define|#undef|#ifdef|#else|#elif|#endif|#ifndef)"
}, { }, {
token : function(value) { token : function(value) {
if (value == "this") if (value == "this")
return "variable.language"; return "variable.language";
else if (keywords.hasOwnProperty(value)) else if (keywords.hasOwnProperty(value))
return "keyword"; return "keyword";
else if (buildinConstants.hasOwnProperty(value)) else if (buildinConstants.hasOwnProperty(value))
return "constant.language"; return "constant.language";
else else
return "identifier"; return "identifier";
}, },
regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b" regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
}, { }, {
token : "keyword.operator", token : "keyword.operator",
regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|==|=|!=|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|new|delete|typeof|void)" regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|==|=|!=|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|new|delete|typeof|void)"
}, { }, {
token : "lparen", token : "lparen",
regex : "[[({]" regex : "[[({]"
}, { }, {
token : "rparen", token : "rparen",
regex : "[\\])}]" regex : "[\\])}]"
}, { }, {
token : "text", token : "text",
regex : "\\s+" regex : "\\s+"
} }
], ],
"comment" : [ "comment" : [
{ {
token : "comment", // closing comment token : "comment", // closing comment
regex : ".*?\\*\\/", regex : ".*?\\*\\/",
next : "start" next : "start"
}, { }, {
token : "comment", // comment spanning whole line token : "comment", // comment spanning whole line
regex : ".+" merge : true,
} regex : ".+"
}
], ],
"qqstring" : [ "qqstring" : [
{ {
token : "string", token : "string",
regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"', regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
regex : '.+' merge : true,
} regex : '.+'
}
], ],
"qstring" : [ "qstring" : [
{ {
token : "string", token : "string",
regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'", regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
regex : '.+' merge : true,
} regex : '.+'
}
] ]
}; };

256
lib/ace/mode/clojure_highlight_rules.js
View file

@ -49,78 +49,78 @@ var ClojureHighlightRules = function() {
var builtinFunctions = lang.arrayToMap( var builtinFunctions = lang.arrayToMap(
('* *1 *2 *3 *agent* *allow-unresolved-vars* *assert* *clojure-version* ' + ('* *1 *2 *3 *agent* *allow-unresolved-vars* *assert* *clojure-version* ' +
'*command-line-args* *compile-files* *compile-path* *e *err* *file* ' + '*command-line-args* *compile-files* *compile-path* *e *err* *file* ' +
'*flush-on-newline* *in* *macro-meta* *math-context* *ns* *out* ' + '*flush-on-newline* *in* *macro-meta* *math-context* *ns* *out* ' +
'*print-dup* *print-length* *print-level* *print-meta* *print-readably* ' + '*print-dup* *print-length* *print-level* *print-meta* *print-readably* ' +
'*read-eval* *source-path* *use-context-classloader* ' + '*read-eval* *source-path* *use-context-classloader* ' +
'*warn-on-reflection* + - -> -&gt; ->> -&gt;&gt; .. / < &lt; <= &lt;= = ' + '*warn-on-reflection* + - -> -&gt; ->> -&gt;&gt; .. / < &lt; <= &lt;= = ' +
'== > &gt; >= &gt;= accessor aclone ' + '== > &gt; >= &gt;= accessor aclone ' +
'add-classpath add-watch agent agent-errors aget alength alias all-ns ' + 'add-classpath add-watch agent agent-errors aget alength alias all-ns ' +
'alter alter-meta! alter-var-root amap ancestors and apply areduce ' + 'alter alter-meta! alter-var-root amap ancestors and apply areduce ' +
'array-map aset aset-boolean aset-byte aset-char aset-double aset-float ' + 'array-map aset aset-boolean aset-byte aset-char aset-double aset-float ' +
'aset-int aset-long aset-short assert assoc assoc! assoc-in associative? ' + 'aset-int aset-long aset-short assert assoc assoc! assoc-in associative? ' +
'atom await await-for await1 bases bean bigdec bigint binding bit-and ' + 'atom await await-for await1 bases bean bigdec bigint binding bit-and ' +
'bit-and-not bit-clear bit-flip bit-not bit-or bit-set bit-shift-left ' + 'bit-and-not bit-clear bit-flip bit-not bit-or bit-set bit-shift-left ' +
'bit-shift-right bit-test bit-xor boolean boolean-array booleans ' + 'bit-shift-right bit-test bit-xor boolean boolean-array booleans ' +
'bound-fn bound-fn* butlast byte byte-array bytes cast char char-array ' + 'bound-fn bound-fn* butlast byte byte-array bytes cast char char-array ' +
'char-escape-string char-name-string char? chars chunk chunk-append ' + 'char-escape-string char-name-string char? chars chunk chunk-append ' +
'chunk-buffer chunk-cons chunk-first chunk-next chunk-rest chunked-seq? ' + 'chunk-buffer chunk-cons chunk-first chunk-next chunk-rest chunked-seq? ' +
'class class? clear-agent-errors clojure-version coll? comment commute ' + 'class class? clear-agent-errors clojure-version coll? comment commute ' +
'comp comparator compare compare-and-set! compile complement concat cond ' + 'comp comparator compare compare-and-set! compile complement concat cond ' +
'condp conj conj! cons constantly construct-proxy contains? count ' + 'condp conj conj! cons constantly construct-proxy contains? count ' +
'counted? create-ns create-struct cycle dec decimal? declare definline ' + 'counted? create-ns create-struct cycle dec decimal? declare definline ' +
'defmacro defmethod defmulti defn defn- defonce defstruct delay delay? ' + 'defmacro defmethod defmulti defn defn- defonce defstruct delay delay? ' +
'deliver deref derive descendants destructure disj disj! dissoc dissoc! ' + 'deliver deref derive descendants destructure disj disj! dissoc dissoc! ' +
'distinct distinct? doall doc dorun doseq dosync dotimes doto double ' + 'distinct distinct? doall doc dorun doseq dosync dotimes doto double ' +
'double-array doubles drop drop-last drop-while empty empty? ensure ' + 'double-array doubles drop drop-last drop-while empty empty? ensure ' +
'enumeration-seq eval even? every? false? ffirst file-seq filter find ' + 'enumeration-seq eval even? every? false? ffirst file-seq filter find ' +
'find-doc find-ns find-var first float float-array float? floats flush ' + 'find-doc find-ns find-var first float float-array float? floats flush ' +
'fn fn? fnext for force format future future-call future-cancel ' + 'fn fn? fnext for force format future future-call future-cancel ' +
'future-cancelled? future-done? future? gen-class gen-interface gensym ' + 'future-cancelled? future-done? future? gen-class gen-interface gensym ' +
'get get-in get-method get-proxy-class get-thread-bindings get-validator ' + 'get get-in get-method get-proxy-class get-thread-bindings get-validator ' +
'hash hash-map hash-set identical? identity if-let if-not ifn? import ' + 'hash hash-map hash-set identical? identity if-let if-not ifn? import ' +
'in-ns inc init-proxy instance? int int-array integer? interleave intern ' + 'in-ns inc init-proxy instance? int int-array integer? interleave intern ' +
'interpose into into-array ints io! isa? iterate iterator-seq juxt key ' + 'interpose into into-array ints io! isa? iterate iterator-seq juxt key ' +
'keys keyword keyword? last lazy-cat lazy-seq let letfn line-seq list ' + 'keys keyword keyword? last lazy-cat lazy-seq let letfn line-seq list ' +
'list* list? load load-file load-reader load-string loaded-libs locking ' + 'list* list? load load-file load-reader load-string loaded-libs locking ' +
'long long-array longs loop macroexpand macroexpand-1 make-array ' + 'long long-array longs loop macroexpand macroexpand-1 make-array ' +
'make-hierarchy map map? mapcat max max-key memfn memoize merge ' + 'make-hierarchy map map? mapcat max max-key memfn memoize merge ' +
'merge-with meta method-sig methods min min-key mod name namespace neg? ' + 'merge-with meta method-sig methods min min-key mod name namespace neg? ' +
'newline next nfirst nil? nnext not not-any? not-empty not-every? not= ' + 'newline next nfirst nil? nnext not not-any? not-empty not-every? not= ' +
'ns ns-aliases ns-imports ns-interns ns-map ns-name ns-publics ' + 'ns ns-aliases ns-imports ns-interns ns-map ns-name ns-publics ' +
'ns-refers ns-resolve ns-unalias ns-unmap nth nthnext num number? odd? ' + 'ns-refers ns-resolve ns-unalias ns-unmap nth nthnext num number? odd? ' +
'or parents partial partition pcalls peek persistent! pmap pop pop! ' + 'or parents partial partition pcalls peek persistent! pmap pop pop! ' +
'pop-thread-bindings pos? pr pr-str prefer-method prefers ' + 'pop-thread-bindings pos? pr pr-str prefer-method prefers ' +
'primitives-classnames print print-ctor print-doc print-dup print-method ' + 'primitives-classnames print print-ctor print-doc print-dup print-method ' +
'print-namespace-doc print-simple print-special-doc print-str printf ' + 'print-namespace-doc print-simple print-special-doc print-str printf ' +
'println println-str prn prn-str promise proxy proxy-call-with-super ' + 'println println-str prn prn-str promise proxy proxy-call-with-super ' +
'proxy-mappings proxy-name proxy-super push-thread-bindings pvalues quot ' + 'proxy-mappings proxy-name proxy-super push-thread-bindings pvalues quot ' +
'rand rand-int range ratio? rational? rationalize re-find re-groups ' + 'rand rand-int range ratio? rational? rationalize re-find re-groups ' +
're-matcher re-matches re-pattern re-seq read read-line read-string ' + 're-matcher re-matches re-pattern re-seq read read-line read-string ' +
'reduce ref ref-history-count ref-max-history ref-min-history ref-set ' + 'reduce ref ref-history-count ref-max-history ref-min-history ref-set ' +
'refer refer-clojure release-pending-sends rem remove remove-method ' + 'refer refer-clojure release-pending-sends rem remove remove-method ' +
'remove-ns remove-watch repeat repeatedly replace replicate require ' + 'remove-ns remove-watch repeat repeatedly replace replicate require ' +
'reset! reset-meta! resolve rest resultset-seq reverse reversible? rseq ' + 'reset! reset-meta! resolve rest resultset-seq reverse reversible? rseq ' +
'rsubseq second select-keys send send-off seq seq? seque sequence ' + 'rsubseq second select-keys send send-off seq seq? seque sequence ' +
'sequential? set set-validator! set? short short-array shorts ' + 'sequential? set set-validator! set? short short-array shorts ' +
'shutdown-agents slurp some sort sort-by sorted-map sorted-map-by ' + 'shutdown-agents slurp some sort sort-by sorted-map sorted-map-by ' +
'sorted-set sorted-set-by sorted? special-form-anchor special-symbol? ' + 'sorted-set sorted-set-by sorted? special-form-anchor special-symbol? ' +
'split-at split-with str stream? string? struct struct-map subs subseq ' + 'split-at split-with str stream? string? struct struct-map subs subseq ' +
'subvec supers swap! symbol symbol? sync syntax-symbol-anchor take ' + 'subvec supers swap! symbol symbol? sync syntax-symbol-anchor take ' +
'take-last take-nth take-while test the-ns time to-array to-array-2d ' + 'take-last take-nth take-while test the-ns time to-array to-array-2d ' +
'trampoline transient tree-seq true? type unchecked-add unchecked-dec ' + 'trampoline transient tree-seq true? type unchecked-add unchecked-dec ' +
'unchecked-divide unchecked-inc unchecked-multiply unchecked-negate ' + 'unchecked-divide unchecked-inc unchecked-multiply unchecked-negate ' +
'unchecked-remainder unchecked-subtract underive unquote ' + 'unchecked-remainder unchecked-subtract underive unquote ' +
'unquote-splicing update-in update-proxy use val vals var-get var-set ' + 'unquote-splicing update-in update-proxy use val vals var-get var-set ' +
'var? vary-meta vec vector vector? when when-first when-let when-not ' + 'var? vary-meta vec vector vector? when when-first when-let when-not ' +
'while with-bindings with-bindings* with-in-str with-loading-context ' + 'while with-bindings with-bindings* with-in-str with-loading-context ' +
'with-local-vars with-meta with-open with-out-str with-precision xml-seq ' + 'with-local-vars with-meta with-open with-out-str with-precision xml-seq ' +
'zero? zipmap ').split(" ") 'zero? zipmap ').split(" ")
); );
var keywords = lang.arrayToMap( var keywords = lang.arrayToMap(
('def do fn if let loop monitor-enter monitor-exit new quote recur set! ' + ('def do fn if let loop monitor-enter monitor-exit new quote recur set! ' +
'throw try var').split(" ") 'throw try var').split(" ")
); );
var buildinConstants = lang.arrayToMap( var buildinConstants = lang.arrayToMap(
@ -133,98 +133,82 @@ var ClojureHighlightRules = function() {
this.$rules = { this.$rules = {
"start" : [ "start" : [
{
{ token : "comment",
token : "comment", regex : ";.*$"
regex : ";.*$" }, {
}, {
token : "comment", // multi line comment token : "comment", // multi line comment
regex : "^\=begin$", regex : "^\=begin$",
next : "comment" next : "comment"
}, }, {
{ token : "keyword", //parens
token : "keyword", //parens regex : "[\\(|\\)]"
regex : "[\\(|\\)]" }, {
}, token : "keyword", //lists
{ regex : "[\\'\\(]"
token : "keyword", //lists }, {
regex : "[\\'\\(]" token : "keyword", //vectors
}, regex : "[\\[|\\]]"
{ }, {
token : "keyword", //vectors token : "keyword", //sets and maps
regex : "[\\[|\\]]" regex : "[\\{|\\}|\\#\\{|\\#\\}]"
}, }, {
{
token : "keyword", //sets and maps
regex : "[\\{|\\}|\\#\\{|\\#\\}]"
},
{
token : "keyword", // ampersands token : "keyword", // ampersands
regex : '[\\&]' regex : '[\\&]'
}, }, {
{
token : "keyword", // metadata token : "keyword", // metadata
regex : '[\\#\\^\\{]' regex : '[\\#\\^\\{]'
}, }, {
{
token : "keyword", // anonymous fn syntactic sugar token : "keyword", // anonymous fn syntactic sugar
regex : '[\\%]' regex : '[\\%]'
}, }, {
{
token : "keyword", // deref reader macro token : "keyword", // deref reader macro
regex : '[@]' regex : '[@]'
}, }, {
{ token : "constant.numeric", // hex
token : "constant.numeric", // hex regex : "0[xX][0-9a-fA-F]+\\b"
regex : "0[xX][0-9a-fA-F]+\\b" }, {
}, token : "constant.numeric", // float
{ regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
token : "constant.numeric", // float }, {
regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b" token : "constant.language",
}, regex : '[!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+||=|!=|<=|>=|<>|<|>|!|&&]'
{ }, {
token : "constant.language", token : function(value) {
regex : '[!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+||=|!=|<=|>=|<>|<|>|!|&&]' if (keywords.hasOwnProperty(value))
}, return "keyword";
else if (buildinConstants.hasOwnProperty(value))
{ return "constant.language";
token : function(value) {
if (keywords.hasOwnProperty(value))
return "keyword";
else if (buildinConstants.hasOwnProperty(value))
return "constant.language";
else if (builtinFunctions.hasOwnProperty(value)) else if (builtinFunctions.hasOwnProperty(value))
return "support.function"; return "support.function";
else else
return "identifier"; return "identifier";
}, },
// TODO: Unicode escape sequences // TODO: Unicode escape sequences
// TODO: Unicode identifiers // TODO: Unicode identifiers
regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b" regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
}, }, {
{
token : "string", // single line token : "string", // single line
regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
}, }, {
{
token : "string", // symbol token : "string", // symbol
regex : "[:](?:[a-zA-Z]|\d)+" regex : "[:](?:[a-zA-Z]|\d)+"
}, }, {
{ token : "string.regexp", //Regular Expressions
token : "string.regexp", //Regular Expressions regex : '/#"(?:\.|(\\\")|[^\""\n])*"/g'
regex : '/#"(?:\.|(\\\")|[^\""\n])*"/g' }
}
], ],
"comment" : [ "comment" : [
{ {
token : "comment", // closing comment token : "comment", // closing comment
regex : "^\=end$", regex : "^\=end$",
next : "start" next : "start"
}, { }, {
token : "comment", // comment spanning whole line token : "comment", // comment spanning whole line
regex : ".+" merge : true,
} regex : ".+"
}
] ]
}; };
}; };

292
lib/ace/mode/coffee_highlight_rules.js
View file

@ -37,146 +37,158 @@
define(function(require, exports, module) { define(function(require, exports, module) {
require("pilot/oop").inherits( require("pilot/oop").inherits(CoffeeHighlightRules,
CoffeeHighlightRules, require("ace/mode/text_highlight_rules").TextHighlightRules);
require("ace/mode/text_highlight_rules").TextHighlightRules);
function CoffeeHighlightRules() { function CoffeeHighlightRules() {
var identifier = "[$A-Za-z_\\x7f-\\uffff][$\\w\\x7f-\\uffff]*" var identifier = "[$A-Za-z_\\x7f-\\uffff][$\\w\\x7f-\\uffff]*";
, keywordend = "(?![$\\w]|\\s*:)" var keywordend = "(?![$\\w]|\\s*:)";
, stringfill = {token: "string", regex: ".+"} var stringfill = {
; token : "string",
this.$rules = merge : true,
{ start: regex : ".+"
[ { token: "identifier" };
, regex: "(?:@|(?:\\.|::)\\s*)" + identifier
} this.$rules = {
, { token: "keyword" start : [
, regex: "(?:t(?:h(?:is|row|en)|ry|ypeof)|s(?:uper|witch)|return|b(?:reak|y)|c(?:ontinue|atch|lass)|i(?:n(?:stanceof)?|s(?:nt)?|f)|e(?:lse|xtends)|f(?:or (?:own)?|inally|unction)|wh(?:ile|en)|n(?:ew|ot?)|d(?:e(?:lete|bugger)|o)|loop|o(?:ff?|[rn])|un(?:less|til)|and|yes)" + keywordend {
} token : "identifier",
, { token: "constant.language" regex : "(?:@|(?:\\.|::)\\s*)" + identifier
, regex: "(?:true|false|null|undefined)" + keywordend }, {
} token : "keyword",
, { token: "invalid.illegal" regex : "(?:t(?:h(?:is|row|en)|ry|ypeof)|s(?:uper|witch)|return|b(?:reak|y)|c(?:ontinue|atch|lass)|i(?:n(?:stanceof)?|s(?:nt)?|f)|e(?:lse|xtends)|f(?:or (?:own)?|inally|unction)|wh(?:ile|en)|n(?:ew|ot?)|d(?:e(?:lete|bugger)|o)|loop|o(?:ff?|[rn])|un(?:less|til)|and|yes)"
, regex: "(?:c(?:ase|onst)|default|function|v(?:ar|oid)|with|e(?:num|xport)|i(?:mplements|nterface)|let|p(?:ackage|r(?:ivate|otected)|ublic)|static|yield|__(?:hasProp|extends|slice|bind|indexOf))" + keywordend + keywordend
} }, {
, { token: "language.support.class" token : "constant.language",
, regex: "(?:Array|Boolean|Date|Function|Number|Object|R(?:e(?:gExp|ferenceError)|angeError)|S(?:tring|yntaxError)|E(?:rror|valError)|TypeError|URIError)" + keywordend regex : "(?:true|false|null|undefined)" + keywordend
} }, {
, { token: "language.support.function" token : "invalid.illegal",
, regex: "(?:Math|JSON|is(?:NaN|Finite)|parse(?:Int|Float)|encodeURI(?:Component)?|decodeURI(?:Component)?)" + keywordend regex : "(?:c(?:ase|onst)|default|function|v(?:ar|oid)|with|e(?:num|xport)|i(?:mplements|nterface)|let|p(?:ackage|r(?:ivate|otected)|ublic)|static|yield|__(?:hasProp|extends|slice|bind|indexOf))"
} + keywordend
, { token: "identifier" }, {
, regex: identifier token : "language.support.class",
} regex : "(?:Array|Boolean|Date|Function|Number|Object|R(?:e(?:gExp|ferenceError)|angeError)|S(?:tring|yntaxError)|E(?:rror|valError)|TypeError|URIError)"
, { token: "constant.numeric" + keywordend
, regex: "(?:0x[\\da-fA-F]+|(?:\\d+(?:\\.\\d+)?|\\.\\d+)(?:[eE][+-]?\\d+)?)" }, {
} token : "language.support.function",
, { token: "string" regex : "(?:Math|JSON|is(?:NaN|Finite)|parse(?:Int|Float)|encodeURI(?:Component)?|decodeURI(?:Component)?)"
, regex: "'''" + keywordend
, next : "qdoc" }, {
} token : "identifier",
, { token: "string" regex : identifier
, regex: '"""' }, {
, next : "qqdoc" token : "constant.numeric",
} regex : "(?:0x[\\da-fA-F]+|(?:\\d+(?:\\.\\d+)?|\\.\\d+)(?:[eE][+-]?\\d+)?)"
, { token: "string" }, {
, regex: "'" token : "string",
, next : "qstring" merge : true,
} regex : "'''",
, { token: "string" next : "qdoc"
, regex: '"' }, {
, next : "qqstring" token : "string",
} merge : true,
, { token: "string" regex : '"""',
, regex: "`" next : "qqdoc"
, next : "js" }, {
} token : "string",
, { token: "string.regex" merge : true,
, regex: "///" regex : "'",
, next : "heregex" next : "qstring"
} }, {
, { token: "string.regex" token : "string",
, regex: "/(?!\\s)[^[/\\n\\\\]*(?: (?:\\\\.|\\[[^\\]\\n\\\\]*(?:\\\\.[^\\]\\n\\\\]*)*\\])[^[/\\n\\\\]*)*/[imgy]{0,4}(?!\\w)" merge : true,
} regex : '"',
, { token: "comment" next : "qqstring"
, regex: "###(?!#)" }, {
, next : "comment" token : "string",
} merge : true,
, { token: "comment" regex : "`",
, regex: "#.*" next : "js"
} }, {
, { token: "lparen" token : "string.regex",
, regex: "[({[]" merge : true,
} regex : "///",
, { token: "rparen" next : "heregex"
, regex: "[\\]})]" }, {
} token : "string.regex",
, { token: "keyword.operator" regex : "/(?!\\s)[^[/\\n\\\\]*(?: (?:\\\\.|\\[[^\\]\\n\\\\]*(?:\\\\.[^\\]\\n\\\\]*)*\\])[^[/\\n\\\\]*)*/[imgy]{0,4}(?!\\w)"
, regex: "\\S+" }, {
} token : "comment",
, { token: "text" merge : true,
, regex: "\\s+" regex : "###(?!#)",
} next : "comment"
] }, {
, qdoc: token : "comment",
[ { token: "string" regex : "#.*"
, regex: ".*?'''" }, {
, next : "start" token : "lparen",
} regex : "[({[]"
, stringfill }, {
] token : "rparen",
, qqdoc: regex : "[\\]})]"
[ { token: "string" }, {
, regex: '.*?"""' token : "keyword.operator",
, next : "start" regex : "\\S+"
} }, {
, stringfill token : "text",
] regex : "\\s+"
, qstring: }],
[ { token: "string"
, regex: "[^\\\\']*(?:\\\\.[^\\\\']*)*'" qdoc : [{
, next : "start" token : "string",
} regex : ".*?'''",
, stringfill next : "start"
] }, stringfill],
, qqstring:
[ { token: "string" qqdoc : [{
, regex: '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"' token : "string",
, next : "start" regex : '.*?"""',
} next : "start"
, stringfill }, stringfill],
]
, js: qstring : [{
[ { token: "string" token : "string",
, regex: "[^\\\\`]*(?:\\\\.[^\\\\`]*)*`" regex : "[^\\\\']*(?:\\\\.[^\\\\']*)*'",
, next : "start" next : "start"
} }, stringfill],
, stringfill
] qqstring : [{
, heregex: token : "string",
[ { token: "string.regex" regex : '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"',
, regex: '.*?///[imgy]{0,4}' next : "start"
, next : "start" }, stringfill],
}
, { token: "comment.regex" js : [{
, regex: "\\s+(?:#.*)?" token : "string",
} merge : true,
, { token: "string.regex" regex : "[^\\\\`]*(?:\\\\.[^\\\\`]*)*`",
, regex: "\\S+" next : "start"
} }, stringfill],
]
, comment: heregex : [{
[ { token: "comment" token : "string.regex",
, regex: '.*?###' regex : '.*?///[imgy]{0,4}',
, next : "start" next : "start"
} }, {
, { token: "comment" token : "comment.regex",
, regex: ".+" regex : "\\s+(?:#.*)?"
} }, {
] token : "string.regex",
}; merge : true,
} regex : "\\S+"
}],
comment : [{
token : "comment",
regex : '.*?###',
next : "start"
}, {
token : "comment",
merge : true,
regex : ".+"
}]
};
}
exports.CoffeeHighlightRules = CoffeeHighlightRules; exports.CoffeeHighlightRules = CoffeeHighlightRules;
}); });

140
lib/ace/mode/csharp_highlight_rules.js
View file

@ -8,7 +8,7 @@ var TextHighlightRules = require("ace/mode/text_highlight_rules").TextHighlightR
var CSharpHighlightRules = function() { var CSharpHighlightRules = function() {
var keywords = lang.arrayToMap( var keywords = lang.arrayToMap(
("abstract|event|new|struct|as|explicit|null|switch|base|extern|object|this|bool|false|operator|throw|break|finally|out|true|byte|fixed|override|try|case|float|params|typeof|catch|for|private|uint|char|foreach|protected|ulong|checked|goto|public|unchecked|class|if|readonly|unsafe|const|implicit|ref|ushort|continue|in|return|using|decimal|int|sbyte|virtual|default|interface|sealed|volatile|delegate|internal|short|void|do|is|sizeof|while|double|lock|stackalloc|else|long|static|enum|namespace|string|var|dynamic").split("|") ("abstract|event|new|struct|as|explicit|null|switch|base|extern|object|this|bool|false|operator|throw|break|finally|out|true|byte|fixed|override|try|case|float|params|typeof|catch|for|private|uint|char|foreach|protected|ulong|checked|goto|public|unchecked|class|if|readonly|unsafe|const|implicit|ref|ushort|continue|in|return|using|decimal|int|sbyte|virtual|default|interface|sealed|volatile|delegate|internal|short|void|do|is|sizeof|while|double|lock|stackalloc|else|long|static|enum|namespace|string|var|dynamic").split("|")
); );
var buildinConstants = lang.arrayToMap( var buildinConstants = lang.arrayToMap(
@ -21,94 +21,72 @@ var CSharpHighlightRules = function() {
this.$rules = { this.$rules = {
"start" : [ "start" : [
{ {
token : "comment", token : "comment",
regex : "\\/\\/.*$" regex : "\\/\\/.*$"
}, },
new DocCommentHighlightRules().getStartRule("doc-start"), new DocCommentHighlightRules().getStartRule("doc-start"),
{ {
token : "comment", // multi line comment token : "comment", // multi line comment
regex : "\\/\\*", regex : "\\/\\*",
merge : true,
next : "comment" next : "comment"
}, { }, {
token : "comment", // multi line comment token : "string.regexp",
regex : "\\/\\*\\*", regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)"
next : "comment" }, {
}, { token : "string", // single line
token : "string.regexp", regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)" }, {
}, { token : "string", // single line
token : "string", // single line regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' }, {
}, { token : "constant.numeric", // hex
token : "string", // single line regex : "0[xX][0-9a-fA-F]+\\b"
regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']" }, {
}, { token : "constant.numeric", // float
token : "constant.numeric", // hex regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
regex : "0[xX][0-9a-fA-F]+\\b" }, {
}, { token : "constant.language.boolean",
token : "constant.numeric", // float regex : "(?:true|false)\\b"
regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b" }, {
}, { token : function(value) {
token : "constant.language.boolean", if (value == "this")
regex : "(?:true|false)\\b" return "variable.language";
}, { else if (keywords.hasOwnProperty(value))
token : function(value) { return "keyword";
if (value == "this") else if (buildinConstants.hasOwnProperty(value))
return "variable.language"; return "constant.language";
else if (keywords.hasOwnProperty(value)) else
return "keyword"; return "identifier";
else if (buildinConstants.hasOwnProperty(value)) },
return "constant.language"; // TODO: Unicode escape sequences
else // TODO: Unicode identifiers
return "identifier"; regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
}, }, {
// TODO: Unicode escape sequences token : "keyword.operator",
// TODO: Unicode identifiers regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)"
regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b" }, {
}, { token : "lparen",
token : "keyword.operator", regex : "[[({]"
regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)" }, {
}, { token : "rparen",
token : "lparen", regex : "[\\])}]"
regex : "[[({]" }, {
}, { token : "text",
token : "rparen", regex : "\\s+"
regex : "[\\])}]" }
}, {
token : "text",
regex : "\\s+"
}
], ],
"comment" : [ "comment" : [
{
token : "comment", // closing comment
regex : ".*?\\*\\/",
next : "start"
}, {
token : "comment", // comment spanning whole line
regex : ".+"
}
],
"qqstring" : [
{ {
token : "string", token : "comment", // closing comment
regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"', regex : ".*?\\*\\/",
next : "start" next : "start"
}, { }, {
token : "string", token : "comment", // comment spanning whole line
regex : '.+' merge : true,
} regex : ".+"
], }
"qstring" : [
{
token : "string",
regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
next : "start"
}, {
token : "string",
regex : '.+'
}
] ]
}; };

60
lib/ace/mode/css_highlight_rules.js
View file

@ -125,6 +125,7 @@ var CssHighlightRules = function() {
var base_ruleset = [ var base_ruleset = [
{ {
token : "comment", // multi line comment token : "comment", // multi line comment
merge : true,
regex : "\\/\\*", regex : "\\/\\*",
next : "ruleset_comment" next : "ruleset_comment"
},{ },{
@ -212,66 +213,56 @@ var CssHighlightRules = function() {
} }
]; ];
var ruleset = lang.copyArray( base_ruleset ); var ruleset = lang.copyArray(base_ruleset);
ruleset.unshift( ruleset.unshift({
{
token : "rparen", token : "rparen",
regex : "\}", regex : "\\}",
next: "start" next: "start"
} });
);
var media_ruleset = lang.copyArray( base_ruleset ); var media_ruleset = lang.copyArray( base_ruleset );
media_ruleset.unshift( media_ruleset.unshift({
{
token : "rparen", token : "rparen",
regex : "\}", regex : "\\}",
next: "media" next: "media"
} });
);
var base_comment = [ var base_comment = [{
{
token : "comment", // comment spanning whole line token : "comment", // comment spanning whole line
merge : true,
regex : ".+" regex : ".+"
} }];
];
var comment = lang.copyArray( base_comment ); var comment = lang.copyArray(base_comment);
comment.unshift( comment.unshift({
{
token : "comment", // closing comment token : "comment", // closing comment
regex : ".*?\\*\\/", regex : ".*?\\*\\/",
next : "start" next : "start"
} });
);
var media_comment = lang.copyArray( base_comment ); var media_comment = lang.copyArray(base_comment);
media_comment.unshift( media_comment.unshift({
{
token : "comment", // closing comment token : "comment", // closing comment
regex : ".*?\\*\\/", regex : ".*?\\*\\/",
next : "media" next : "media"
} });
);
var ruleset_comment = lang.copyArray( base_comment ); var ruleset_comment = lang.copyArray(base_comment);
ruleset_comment.unshift( ruleset_comment.unshift({
{
token : "comment", // closing comment token : "comment", // closing comment
regex : ".*?\\*\\/", regex : ".*?\\*\\/",
next : "ruleset" next : "ruleset"
} });
);
this.$rules = { this.$rules = {
"start" : [ { "start" : [{
token : "comment", // multi line comment token : "comment", // multi line comment
merge : true,
regex : "\\/\\*", regex : "\\/\\*",
next : "comment" next : "comment"
}, { }, {
token: "lparen", token: "lparen",
regex: "{", regex: "\\{",
next: "ruleset" next: "ruleset"
}, { }, {
token: "string", token: "string",
@ -293,15 +284,16 @@ var CssHighlightRules = function() {
"media" : [ { "media" : [ {
token : "comment", // multi line comment token : "comment", // multi line comment
merge : true,
regex : "\\/\\*", regex : "\\/\\*",
next : "media_comment" next : "media_comment"
}, { }, {
token: "lparen", token: "lparen",
regex: "{", regex: "\\{",
next: "media_ruleset" next: "media_ruleset"
},{ },{
token: "string", token: "string",
regex: "}", regex: "\\}",
next: "start" next: "start"
},{ },{
token: "keyword", token: "keyword",

3
lib/ace/mode/css_test.js
View file

@ -46,6 +46,9 @@ var CssMode = require("ace/mode/css").Mode;
var assert = require("ace/test/assertions"); var assert = require("ace/test/assertions");
module.exports = { module.exports = {
name: "CSS",
setUp : function() { setUp : function() {
this.mode = new CssMode(); this.mode = new CssMode();
}, },

3
lib/ace/mode/css_tokenizer_test.js
View file

@ -45,6 +45,9 @@ var CssMode = require("ace/mode/css").Mode;
var assert = require("ace/test/assertions"); var assert = require("ace/test/assertions");
module.exports = { module.exports = {
name: "CSS Tokenizer",
setUp : function() { setUp : function() {
this.tokenizer = new CssMode().getTokenizer(); this.tokenizer = new CssMode().getTokenizer();
}, },

8
lib/ace/mode/doc_comment_highlight_rules.js
View file

@ -48,15 +48,19 @@ var DocCommentHighlightRules = function() {
regex : "@[\\w\\d_]+" // TODO: fix email addresses regex : "@[\\w\\d_]+" // TODO: fix email addresses
}, { }, {
token : "comment.doc", token : "comment.doc",
merge : true,
regex : "\\s+" regex : "\\s+"
}, { }, {
token : "comment.doc", token : "comment.doc",
merge : true,
regex : "TODO" regex : "TODO"
}, { }, {
token : "comment.doc", token : "comment.doc",
merge : true,
regex : "[^@\\*]+" regex : "[^@\\*]+"
}, { }, {
token : "comment.doc", token : "comment.doc",
merge : true,
regex : "." regex : "."
}] }]
}; };
@ -69,14 +73,16 @@ oop.inherits(DocCommentHighlightRules, TextHighlightRules);
this.getStartRule = function(start) { this.getStartRule = function(start) {
return { return {
token : "comment.doc", // doc comment token : "comment.doc", // doc comment
merge : true,
regex : "\\/\\*(?=\\*)", regex : "\\/\\*(?=\\*)",
next: start next : start
}; };
}; };
this.getEndRule = function (start) { this.getEndRule = function (start) {
return { return {
token : "comment.doc", // closing comment token : "comment.doc", // closing comment
merge : true,
regex : "\\*\\/", regex : "\\*\\/",
next : start next : start
}; };

158
lib/ace/mode/groovy_highlight_rules.js
View file

@ -8,16 +8,16 @@ var TextHighlightRules = require("ace/mode/text_highlight_rules").TextHighlightR
var GroovyHighlightRules = function() { var GroovyHighlightRules = function() {
var keywords = lang.arrayToMap( var keywords = lang.arrayToMap(
("assert|with|abstract|continue|for|new|switch|" + ("assert|with|abstract|continue|for|new|switch|" +
"assert|default|goto|package|synchronized|" + "assert|default|goto|package|synchronized|" +
"boolean|do|if|private|this|" + "boolean|do|if|private|this|" +
"break|double|implements|protected|throw|" + "break|double|implements|protected|throw|" +
"byte|else|import|public|throws|" + "byte|else|import|public|throws|" +
"case|enum|instanceof|return|transient|" + "case|enum|instanceof|return|transient|" +
"catch|extends|int|short|try|" + "catch|extends|int|short|try|" +
"char|final|interface|static|void|" + "char|final|interface|static|void|" +
"class|finally|long|strictfp|volatile|" + "class|finally|long|strictfp|volatile|" +
"def|float|native|super|while").split("|") "def|float|native|super|while").split("|")
); );
var buildinConstants = lang.arrayToMap( var buildinConstants = lang.arrayToMap(
@ -59,98 +59,76 @@ var GroovyHighlightRules = function() {
this.$rules = { this.$rules = {
"start" : [ "start" : [
{ {
token : "comment", token : "comment",
regex : "\\/\\/.*$" regex : "\\/\\/.*$"
}, },
new DocCommentHighlightRules().getStartRule("doc-start"), new DocCommentHighlightRules().getStartRule("doc-start"),
{ {
token : "comment", // multi line comment token : "comment", // multi line comment
merge : true,
regex : "\\/\\*", regex : "\\/\\*",
next : "comment" next : "comment"
}, { }, {
token : "comment", // multi line comment token : "string.regexp",
regex : "\\/\\*\\*", regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)"
next : "comment" }, {
}, { token : "string", // single line
token : "string.regexp", regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)" }, {
}, { token : "string", // single line
token : "string", // single line regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' }, {
}, { token : "constant.numeric", // hex
token : "string", // single line regex : "0[xX][0-9a-fA-F]+\\b"
regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']" }, {
}, { token : "constant.numeric", // float
token : "constant.numeric", // hex regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
regex : "0[xX][0-9a-fA-F]+\\b" }, {
}, { token : "constant.language.boolean",
token : "constant.numeric", // float regex : "(?:true|false)\\b"
regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b" }, {
}, { token : function(value) {
token : "constant.language.boolean", if (value == "this")
regex : "(?:true|false)\\b" return "variable.language";
}, { else if (keywords.hasOwnProperty(value))
token : function(value) { return "keyword";
if (value == "this")
return "variable.language";
else if (keywords.hasOwnProperty(value))
return "keyword";
else if (langClasses.hasOwnProperty(value)) else if (langClasses.hasOwnProperty(value))
return "support.function"; return "support.function";
else if (importClasses.hasOwnProperty(value)) else if (importClasses.hasOwnProperty(value))
return "support.function"; return "support.function";
else if (buildinConstants.hasOwnProperty(value)) else if (buildinConstants.hasOwnProperty(value))
return "constant.language"; return "constant.language";
else else
return "identifier"; return "identifier";
}, },
// TODO: Unicode escape sequences // TODO: Unicode escape sequences
// TODO: Unicode identifiers // TODO: Unicode identifiers
regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b" regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
}, { }, {
token : "keyword.operator", token : "keyword.operator",
regex : "\\?:|\\?\\.|\\*\\.|<=>|=~|==~|\\.@|\\*\\.@|\\.&|as|in|is|!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)" regex : "\\?:|\\?\\.|\\*\\.|<=>|=~|==~|\\.@|\\*\\.@|\\.&|as|in|is|!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)"
}, { }, {
token : "lparen", token : "lparen",
regex : "[[({]" regex : "[[({]"
}, { }, {
token : "rparen", token : "rparen",
regex : "[\\])}]" regex : "[\\])}]"
}, { }, {
token : "text", token : "text",
regex : "\\s+" regex : "\\s+"
} }
], ],
"comment" : [ "comment" : [
{
token : "comment", // closing comment
regex : ".*?\\*\\/",
next : "start"
}, {
token : "comment", // comment spanning whole line
regex : ".+"
}
],
"qqstring" : [
{ {
token : "string", token : "comment", // closing comment
regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"', regex : ".*?\\*\\/",
next : "start" next : "start"
}, { }, {
token : "string", token : "comment", // comment spanning whole line
regex : '.+' merge : true,
} regex : ".+"
], }
"qstring" : [
{
token : "string",
regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
next : "start"
}, {
token : "string",
regex : '.+'
}
] ]
}; };

11
lib/ace/mode/html_highlight_rules.js
View file

@ -54,6 +54,7 @@ var HtmlHighlightRules = function() {
regex : '".*?"' regex : '".*?"'
}, { }, {
token : "string", // multi line string start token : "string", // multi line string start
merge : true,
regex : '["].*$', regex : '["].*$',
next : state + "-qqstring" next : state + "-qqstring"
}, { }, {
@ -61,6 +62,7 @@ var HtmlHighlightRules = function() {
regex : "'.*?'" regex : "'.*?'"
}, { }, {
token : "string", // multi line string start token : "string", // multi line string start
merge : true,
regex : "['].*$", regex : "['].*$",
next : state + "-qstring" next : state + "-qstring"
}] }]
@ -69,10 +71,12 @@ var HtmlHighlightRules = function() {
function multiLineString(quote, state) { function multiLineString(quote, state) {
return [{ return [{
token : "string", token : "string",
merge : true,
regex : ".*" + quote, regex : ".*" + quote,
next : state next : state
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
}] }]
} }
@ -80,6 +84,7 @@ var HtmlHighlightRules = function() {
this.$rules = { this.$rules = {
start : [ { start : [ {
token : "text", token : "text",
merge : true,
regex : "<\\!\\[CDATA\\[", regex : "<\\!\\[CDATA\\[",
next : "cdata" next : "cdata"
}, { }, {
@ -87,6 +92,7 @@ var HtmlHighlightRules = function() {
regex : "<\\?.*?\\?>" regex : "<\\?.*?\\?>"
}, { }, {
token : "comment", token : "comment",
merge : true,
regex : "<\\!--", regex : "<\\!--",
next : "comment" next : "comment"
}, { }, {
@ -158,9 +164,11 @@ var HtmlHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "text", token : "text",
merge : true,
regex : "\\s+" regex : "\\s+"
}, { }, {
token : "text", token : "text",
merge : true,
regex : ".+" regex : ".+"
} ], } ],
@ -170,6 +178,7 @@ var HtmlHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "comment", token : "comment",
merge : true,
regex : ".+" regex : ".+"
} ] } ]
}; };
@ -194,4 +203,4 @@ var HtmlHighlightRules = function() {
oop.inherits(HtmlHighlightRules, TextHighlightRules); oop.inherits(HtmlHighlightRules, TextHighlightRules);
exports.HtmlHighlightRules = HtmlHighlightRules; exports.HtmlHighlightRules = HtmlHighlightRules;
}); });

158
lib/ace/mode/java_highlight_rules.js
View file

@ -9,16 +9,16 @@ var JavaHighlightRules = function() {
// taken from http://download.oracle.com/javase/tutorial/java/nutsandbolts/_keywords.html // taken from http://download.oracle.com/javase/tutorial/java/nutsandbolts/_keywords.html
var keywords = lang.arrayToMap( var keywords = lang.arrayToMap(
("abstract|continue|for|new|switch|" + ("abstract|continue|for|new|switch|" +
"assert|default|goto|package|synchronized|" + "assert|default|goto|package|synchronized|" +
"boolean|do|if|private|this|" + "boolean|do|if|private|this|" +
"break|double|implements|protected|throw|" + "break|double|implements|protected|throw|" +
"byte|else|import|public|throws|" + "byte|else|import|public|throws|" +
"case|enum|instanceof|return|transient|" + "case|enum|instanceof|return|transient|" +
"catch|extends|int|short|try|" + "catch|extends|int|short|try|" +
"char|final|interface|static|void|" + "char|final|interface|static|void|" +
"class|finally|long|strictfp|volatile|" + "class|finally|long|strictfp|volatile|" +
"const|float|native|super|while").split("|") "const|float|native|super|while").split("|")
); );
var buildinConstants = lang.arrayToMap( var buildinConstants = lang.arrayToMap(
@ -60,98 +60,76 @@ var JavaHighlightRules = function() {
this.$rules = { this.$rules = {
"start" : [ "start" : [
{ {
token : "comment", token : "comment",
regex : "\\/\\/.*$" regex : "\\/\\/.*$"
}, },
new DocCommentHighlightRules().getStartRule("doc-start"), new DocCommentHighlightRules().getStartRule("doc-start"),
{ {
token : "comment", // multi line comment token : "comment", // multi line comment
merge : true,
regex : "\\/\\*", regex : "\\/\\*",
next : "comment" next : "comment"
}, { }, {
token : "comment", // multi line comment token : "string.regexp",
regex : "\\/\\*\\*", regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)"
next : "comment" }, {
}, { token : "string", // single line
token : "string.regexp", regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)" }, {
}, { token : "string", // single line
token : "string", // single line regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' }, {
}, { token : "constant.numeric", // hex
token : "string", // single line regex : "0[xX][0-9a-fA-F]+\\b"
regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']" }, {
}, { token : "constant.numeric", // float
token : "constant.numeric", // hex regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
regex : "0[xX][0-9a-fA-F]+\\b" }, {
}, { token : "constant.language.boolean",
token : "constant.numeric", // float regex : "(?:true|false)\\b"
regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b" }, {
}, { token : function(value) {
token : "constant.language.boolean", if (value == "this")
regex : "(?:true|false)\\b" return "variable.language";
}, { else if (keywords.hasOwnProperty(value))
token : function(value) { return "keyword";
if (value == "this")
return "variable.language";
else if (keywords.hasOwnProperty(value))
return "keyword";
else if (langClasses.hasOwnProperty(value)) else if (langClasses.hasOwnProperty(value))
return "support.function"; return "support.function";
else if (importClasses.hasOwnProperty(value)) else if (importClasses.hasOwnProperty(value))
return "support.function"; return "support.function";
else if (buildinConstants.hasOwnProperty(value)) else if (buildinConstants.hasOwnProperty(value))
return "constant.language"; return "constant.language";
else else
return "identifier"; return "identifier";
}, },
// TODO: Unicode escape sequences // TODO: Unicode escape sequences
// TODO: Unicode identifiers // TODO: Unicode identifiers
regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b" regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
}, { }, {
token : "keyword.operator", token : "keyword.operator",
regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)" regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)"
}, { }, {
token : "lparen", token : "lparen",
regex : "[[({]" regex : "[[({]"
}, { }, {
token : "rparen", token : "rparen",
regex : "[\\])}]" regex : "[\\])}]"
}, { }, {
token : "text", token : "text",
regex : "\\s+" regex : "\\s+"
} }
], ],
"comment" : [ "comment" : [
{
token : "comment", // closing comment
regex : ".*?\\*\\/",
next : "start"
}, {
token : "comment", // comment spanning whole line
regex : ".+"
}
],
"qqstring" : [
{ {
token : "string", token : "comment", // closing comment
regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"', regex : ".*?\\*\\/",
next : "start" next : "start"
}, { }, {
token : "string", token : "comment", // comment spanning whole line
regex : '.+' merge : true,
} regex : ".+"
], }
"qstring" : [
{
token : "string",
regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
next : "start"
}, {
token : "string",
regex : '.+'
}
] ]
}; };

8
lib/ace/mode/javascript_highlight_rules.js
View file

@ -80,6 +80,7 @@ var JavaScriptHighlightRules = function() {
new DocCommentHighlightRules().getStartRule("doc-start"), new DocCommentHighlightRules().getStartRule("doc-start"),
{ {
token : "comment", // multi line comment token : "comment", // multi line comment
merge : true,
regex : "\\/\\*", regex : "\\/\\*",
next : "comment" next : "comment"
}, { }, {
@ -87,6 +88,7 @@ var JavaScriptHighlightRules = function() {
regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
}, { }, {
token : "string", // multi line string start token : "string", // multi line string start
merge : true,
regex : '["].*\\\\$', regex : '["].*\\\\$',
next : "qqstring" next : "qqstring"
}, { }, {
@ -94,6 +96,7 @@ var JavaScriptHighlightRules = function() {
regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']" regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
}, { }, {
token : "string", // multi line string start token : "string", // multi line string start
merge : true,
regex : "['].*\\\\$", regex : "['].*\\\\$",
next : "qstring" next : "qstring"
}, { }, {
@ -157,7 +160,7 @@ var JavaScriptHighlightRules = function() {
token : "text", token : "text",
regex : "\\s+" regex : "\\s+"
}, { }, {
// immediately return to the start mode without mathcing // immediately return to the start mode without matching
// anything // anything
token: "empty", token: "empty",
regex: "", regex: "",
@ -171,6 +174,7 @@ var JavaScriptHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "comment", // comment spanning whole line token : "comment", // comment spanning whole line
merge : true,
regex : ".+" regex : ".+"
} }
], ],
@ -181,6 +185,7 @@ var JavaScriptHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
} }
], ],
@ -191,6 +196,7 @@ var JavaScriptHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
} }
] ]

13
lib/ace/mode/javascript_tokenizer_test.js
View file

@ -45,6 +45,9 @@ var JavaScriptMode = require("ace/mode/javascript").Mode;
var assert = require("ace/test/assertions"); var assert = require("ace/test/assertions");
module.exports = { module.exports = {
name: "JavaScript Tokenizer",
setUp : function() { setUp : function() {
this.tokenizer = new JavaScriptMode().getTokenizer(); this.tokenizer = new JavaScriptMode().getTokenizer();
}, },
@ -91,10 +94,14 @@ module.exports = {
var tokens = this.tokenizer.getLineTokens(line, "start").tokens; var tokens = this.tokenizer.getLineTokens(line, "start").tokens;
assert.equal(3, tokens.length); assert.equal(7, tokens.length);
assert.equal("lparen", tokens[0].type); assert.equal("lparen", tokens[0].type);
assert.equal("text", tokens[1].type); assert.equal("lparen", tokens[1].type);
assert.equal("rparen", tokens[2].type); assert.equal("lparen", tokens[2].type);
assert.equal("text", tokens[3].type);
assert.equal("rparen", tokens[4].type);
assert.equal("rparen", tokens[5].type);
assert.equal("rparen", tokens[6].type);
}, },
"test for last rule in ruleset to catch capturing group bugs" : function() { "test for last rule in ruleset to catch capturing group bugs" : function() {

9
lib/ace/mode/ocaml_highlight_rules.js
View file

@ -237,9 +237,8 @@ var OcamlHighlightRules = function() {
"Int32|Int64|LargeFile|Lazy|Lexing|List|ListLabels|Make|Map|Marshal|" + "Int32|Int64|LargeFile|Lazy|Lexing|List|ListLabels|Make|Map|Marshal|" +
"MoreLabels|Mutex|Nativeint|Num|Obj|Oo|Parsing|Pervasives|Printexc|" + "MoreLabels|Mutex|Nativeint|Num|Obj|Oo|Parsing|Pervasives|Printexc|" +
"Printf|Queue|Random|Scanf|Scanning|Set|Sort|Stack|State|StdLabels|Str|" + "Printf|Queue|Random|Scanf|Scanning|Set|Sort|Stack|State|StdLabels|Str|" +
"Stream|String|StringLabels|Sys|Thread|ThreadUnix|Tk|Unix|UnixLabels|Weak") "Stream|String|StringLabels|Sys|Thread|ThreadUnix|Tk|Unix|UnixLabels|Weak"
).split("|"));
.split("|"));
var decimalInteger = "(?:(?:[1-9]\\d*)|(?:0))"; var decimalInteger = "(?:(?:[1-9]\\d*)|(?:0))";
var octInteger = "(?:0[oO]?[0-7]+)"; var octInteger = "(?:0[oO]?[0-7]+)";
@ -262,6 +261,7 @@ var OcamlHighlightRules = function() {
}, },
{ {
token : "comment", token : "comment",
merge : true,
regex : '\\(\\*.*', regex : '\\(\\*.*',
next : "comment" next : "comment"
}, },
@ -275,6 +275,7 @@ var OcamlHighlightRules = function() {
}, },
{ {
token : "string", // " string token : "string", // " string
merge : true,
regex : '"', regex : '"',
next : "qstring" next : "qstring"
}, },
@ -328,6 +329,7 @@ var OcamlHighlightRules = function() {
}, },
{ {
token : "comment", // comment spanning whole line token : "comment", // comment spanning whole line
merge : true,
regex : ".+" regex : ".+"
} }
], ],
@ -339,6 +341,7 @@ var OcamlHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
} }
] ]

4
lib/ace/mode/perl_highlight_rules.js
View file

@ -93,6 +93,7 @@ var PerlHighlightRules = function() {
regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
}, { }, {
token : "string", // multi line string start token : "string", // multi line string start
merge : true,
regex : '["].*\\\\$', regex : '["].*\\\\$',
next : "qqstring" next : "qqstring"
}, { }, {
@ -100,6 +101,7 @@ var PerlHighlightRules = function() {
regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']" regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
}, { }, {
token : "string", // multi line string start token : "string", // multi line string start
merge : true,
regex : "['].*\\\\$", regex : "['].*\\\\$",
next : "qstring" next : "qstring"
}, { }, {
@ -141,6 +143,7 @@ var PerlHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
} }
], ],
@ -151,6 +154,7 @@ var PerlHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
} }
] ]

6
lib/ace/mode/php_highlight_rules.js
View file

@ -921,6 +921,7 @@ var PhpHighlightRules = function() {
new DocCommentHighlightRules().getStartRule("doc-start"), new DocCommentHighlightRules().getStartRule("doc-start"),
{ {
token : "comment", // multi line comment token : "comment", // multi line comment
merge : true,
regex : "\\/\\*", regex : "\\/\\*",
next : "comment" next : "comment"
}, { }, {
@ -931,6 +932,7 @@ var PhpHighlightRules = function() {
regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
}, { }, {
token : "string", // multi line string start token : "string", // multi line string start
merge : true,
regex : '["].*\\\\$', regex : '["].*\\\\$',
next : "qqstring" next : "qqstring"
}, { }, {
@ -938,6 +940,7 @@ var PhpHighlightRules = function() {
regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']" regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
}, { }, {
token : "string", // multi line string start token : "string", // multi line string start
merge : true,
regex : "['].*\\\\$", regex : "['].*\\\\$",
next : "qstring" next : "qstring"
}, { }, {
@ -1018,6 +1021,7 @@ var PhpHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "comment", // comment spanning whole line token : "comment", // comment spanning whole line
merge : true,
regex : ".+" regex : ".+"
} }
], ],
@ -1028,6 +1032,7 @@ var PhpHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
} }
], ],
@ -1038,6 +1043,7 @@ var PhpHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
} }
] ]

6
lib/ace/mode/python_highlight_rules.js
View file

@ -96,6 +96,7 @@ var PythonHighlightRules = function() {
regex : strPre + '"{3}(?:[^\\\\]|\\\\.)*?"{3}' regex : strPre + '"{3}(?:[^\\\\]|\\\\.)*?"{3}'
}, { }, {
token : "string", // multi line """ string start token : "string", // multi line """ string start
merge : true,
regex : strPre + '"{3}.*$', regex : strPre + '"{3}.*$',
next : "qqstring" next : "qqstring"
}, { }, {
@ -106,6 +107,7 @@ var PythonHighlightRules = function() {
regex : strPre + "'{3}(?:[^\\\\]|\\\\.)*?'{3}" regex : strPre + "'{3}(?:[^\\\\]|\\\\.)*?'{3}"
}, { }, {
token : "string", // multi line ''' string start token : "string", // multi line ''' string start
merge : true,
regex : strPre + "'{3}.*$", regex : strPre + "'{3}.*$",
next : "qstring" next : "qstring"
}, { }, {
@ -158,14 +160,16 @@ var PythonHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
} ], } ],
"qstring" : [ { "qstring" : [ {
token : "string", // multi line ''' string end token : "string", // multi line ''' string end
regex : "(?:[^\\\\]|\\\\.)*?'{3}", regex : "(?:[^\\\\]|\\\\.)*?'{3}",
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
} ] } ]
}; };

112
lib/ace/mode/ruby_highlight_rules.js
View file

@ -98,89 +98,91 @@ var RubyHighlightRules = function() {
this.$rules = { this.$rules = {
"start" : [ "start" : [
{ {
token : "comment", token : "comment",
regex : "#.*$" regex : "#.*$"
}, { }, {
token : "comment", // multi line comment token : "comment", // multi line comment
merge : true,
regex : "^\=begin$", regex : "^\=begin$",
next : "comment" next : "comment"
}, { }, {
token : "string.regexp", token : "string.regexp",
regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)" regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)"
}, { }, {
token : "string", // single line token : "string", // single line
regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
}, { }, {
token : "string", // single line token : "string", // single line
regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']" regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
}, { }, {
token : "string", // backtick string token : "string", // backtick string
regex : "[`](?:(?:\\\\.)|(?:[^'\\\\]))*?[`]" regex : "[`](?:(?:\\\\.)|(?:[^'\\\\]))*?[`]"
}, { }, {
token : "text", // namespaces aren't symbols token : "text", // namespaces aren't symbols
regex : "::" regex : "::"
}, { }, {
token : "variable.instancce", // instance variable token : "variable.instancce", // instance variable
regex : "@{1,2}(?:[a-zA-Z_]|\d)+" regex : "@{1,2}(?:[a-zA-Z_]|\d)+"
}, { }, {
token : "variable.class", // class name token : "variable.class", // class name
regex : "[A-Z](?:[a-zA-Z_]|\d)+" regex : "[A-Z](?:[a-zA-Z_]|\d)+"
}, { }, {
token : "string", // symbol token : "string", // symbol
regex : "[:](?:[A-Za-z_]|[@$](?=[a-zA-Z0-9_]))[a-zA-Z0-9_]*[!=?]?" regex : "[:](?:[A-Za-z_]|[@$](?=[a-zA-Z0-9_]))[a-zA-Z0-9_]*[!=?]?"
}, { }, {
token : "constant.numeric", // hex token : "constant.numeric", // hex
regex : "0[xX][0-9a-fA-F](?:[0-9a-fA-F]|_(?=[0-9a-fA-F]))*\\b" regex : "0[xX][0-9a-fA-F](?:[0-9a-fA-F]|_(?=[0-9a-fA-F]))*\\b"
}, { }, {
token : "constant.numeric", // float token : "constant.numeric", // float
regex : "[+-]?\\d(?:\\d|_(?=\\d))*(?:(?:\\.\\d(?:\\d|_(?=\\d))*)?(?:[eE][+-]?\\d+)?)?\\b" regex : "[+-]?\\d(?:\\d|_(?=\\d))*(?:(?:\\.\\d(?:\\d|_(?=\\d))*)?(?:[eE][+-]?\\d+)?)?\\b"
}, { }, {
token : "constant.language.boolean", token : "constant.language.boolean",
regex : "(?:true|false)\\b" regex : "(?:true|false)\\b"
}, { }, {
token : function(value) { token : function(value) {
if (value == "self") if (value == "self")
return "variable.language"; return "variable.language";
else if (keywords.hasOwnProperty(value)) else if (keywords.hasOwnProperty(value))
return "keyword"; return "keyword";
else if (buildinConstants.hasOwnProperty(value)) else if (buildinConstants.hasOwnProperty(value))
return "constant.language"; return "constant.language";
else if (builtinVariables.hasOwnProperty(value)) else if (builtinVariables.hasOwnProperty(value))
return "variable.language"; return "variable.language";
else if (builtinFunctions.hasOwnProperty(value)) else if (builtinFunctions.hasOwnProperty(value))
return "support.function"; return "support.function";
else if (value == "debugger") else if (value == "debugger")
return "invalid.deprecated"; return "invalid.deprecated";
else else
return "identifier"; return "identifier";
}, },
// TODO: Unicode escape sequences // TODO: Unicode escape sequences
// TODO: Unicode identifiers // TODO: Unicode identifiers
regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b" regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
}, { }, {
token : "keyword.operator", token : "keyword.operator",
regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)" regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)"
}, { }, {
token : "lparen", token : "lparen",
regex : "[[({]" regex : "[[({]"
}, { }, {
token : "rparen", token : "rparen",
regex : "[\\])}]" regex : "[\\])}]"
}, { }, {
token : "text", token : "text",
regex : "\\s+" regex : "\\s+"
} }
], ],
"comment" : [ "comment" : [
{ {
token : "comment", // closing comment token : "comment", // closing comment
regex : "^\=end$", regex : "^\=end$",
next : "start" next : "start"
}, { }, {
token : "comment", // comment spanning whole line token : "comment", // comment spanning whole line
regex : ".+" merge : true,
} regex : ".+"
}
] ]
}; };
}; };

3
lib/ace/mode/ruby_tokenizer_test.js
View file

@ -46,6 +46,9 @@ var RubyMode = require("ace/mode/ruby").Mode;
var assert = require("ace/test/assertions"); var assert = require("ace/test/assertions");
module.exports = { module.exports = {
name: "Ruby Tokenizer",
setUp : function() { setUp : function() {
this.tokenizer = new RubyMode().getTokenizer(); this.tokenizer = new RubyMode().getTokenizer();
}, },

140
lib/ace/mode/scala_highlight_rules.js
View file

@ -9,7 +9,7 @@ var ScalaHighlightRules = function() {
// taken from http://download.oracle.com/javase/tutorial/java/nutsandbolts/_keywords.html // taken from http://download.oracle.com/javase/tutorial/java/nutsandbolts/_keywords.html
var keywords = lang.arrayToMap( var keywords = lang.arrayToMap(
( (
"case|default|do|else|for|if|match|while|throw|return|try|catch|finally|yield|" + "case|default|do|else|for|if|match|while|throw|return|try|catch|finally|yield|" +
"abstract|class|def|extends|final|forSome|implicit|implicits|import|lazy|new|object|" + "abstract|class|def|extends|final|forSome|implicit|implicits|import|lazy|new|object|" +
"override|package|private|protected|sealed|super|this|trait|type|val|var|with" "override|package|private|protected|sealed|super|this|trait|type|val|var|with"
@ -60,98 +60,76 @@ var ScalaHighlightRules = function() {
this.$rules = { this.$rules = {
"start" : [ "start" : [
{ {
token : "comment", token : "comment",
regex : "\\/\\/.*$" regex : "\\/\\/.*$"
}, },
new DocCommentHighlightRules().getStartRule("doc-start"), new DocCommentHighlightRules().getStartRule("doc-start"),
{ {
token : "comment", // multi line comment token : "comment", // multi line comment
merge : true,
regex : "\\/\\*", regex : "\\/\\*",
next : "comment" next : "comment"
}, { }, {
token : "comment", // multi line comment token : "string.regexp",
regex : "\\/\\*\\*", regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)"
next : "comment" }, {
}, { token : "string", // single line
token : "string.regexp", regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)" }, {
}, { token : "string", // single line
token : "string", // single line regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' }, {
}, { token : "constant.numeric", // hex
token : "string", // single line regex : "0[xX][0-9a-fA-F]+\\b"
regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']" }, {
}, { token : "constant.numeric", // float
token : "constant.numeric", // hex regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b"
regex : "0[xX][0-9a-fA-F]+\\b" }, {
}, { token : "constant.language.boolean",
token : "constant.numeric", // float regex : "(?:true|false)\\b"
regex : "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b" }, {
}, { token : function(value) {
token : "constant.language.boolean", if (value == "this")
regex : "(?:true|false)\\b" return "variable.language";
}, { else if (keywords.hasOwnProperty(value))
token : function(value) { return "keyword";
if (value == "this")
return "variable.language";
else if (keywords.hasOwnProperty(value))
return "keyword";
else if (langClasses.hasOwnProperty(value)) else if (langClasses.hasOwnProperty(value))
return "support.function"; return "support.function";
else if (importClasses.hasOwnProperty(value)) else if (importClasses.hasOwnProperty(value))
return "support.function"; return "support.function";
else if (buildinConstants.hasOwnProperty(value)) else if (buildinConstants.hasOwnProperty(value))
return "constant.language"; return "constant.language";
else else
return "identifier"; return "identifier";
}, },
// TODO: Unicode escape sequences // TODO: Unicode escape sequences
// TODO: Unicode identifiers // TODO: Unicode identifiers
regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b" regex : "[a-zA-Z_$][a-zA-Z0-9_$]*\\b"
}, { }, {
token : "keyword.operator", token : "keyword.operator",
regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)" regex : "!|\\$|%|&|\\*|\\-\\-|\\-|\\+\\+|\\+|~|===|==|=|!=|!==|<=|>=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?\\:|\\*=|%=|\\+=|\\-=|&=|\\^=|\\b(?:in|instanceof|new|delete|typeof|void)"
}, { }, {
token : "lparen", token : "lparen",
regex : "[[({]" regex : "[[({]"
}, { }, {
token : "rparen", token : "rparen",
regex : "[\\])}]" regex : "[\\])}]"
}, { }, {
token : "text", token : "text",
regex : "\\s+" regex : "\\s+"
} }
], ],
"comment" : [ "comment" : [
{
token : "comment", // closing comment
regex : ".*?\\*\\/",
next : "start"
}, {
token : "comment", // comment spanning whole line
regex : ".+"
}
],
"qqstring" : [
{ {
token : "string", token : "comment", // closing comment
regex : '(?:(?:\\\\.)|(?:[^"\\\\]))*?"', regex : ".*?\\*\\/",
next : "start" next : "start"
}, { }, {
token : "string", token : "comment", // comment spanning whole line
regex : '.+' merge : true,
} regex : ".+"
], }
"qstring" : [
{
token : "string",
regex : "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
next : "start"
}, {
token : "string",
regex : '.+'
}
] ]
}; };

61
lib/ace/mode/scss_highlight_rules.js
View file

@ -42,11 +42,11 @@ var lang = require("pilot/lang");
var TextHighlightRules = require("ace/mode/text_highlight_rules").TextHighlightRules; var TextHighlightRules = require("ace/mode/text_highlight_rules").TextHighlightRules;
var ScssHighlightRules = function() { var ScssHighlightRules = function() {
var properties = lang.arrayToMap( (function () { var properties = lang.arrayToMap( (function () {
var browserPrefix = ("-webkit-|-moz-|-o-|-ms-|-svg-|-pie-|-khtml-").split("|"); var browserPrefix = ("-webkit-|-moz-|-o-|-ms-|-svg-|-pie-|-khtml-").split("|");
var prefixProperties = ("appearance|background-clip|background-inline-policy|background-origin|" + var prefixProperties = ("appearance|background-clip|background-inline-policy|background-origin|" +
"background-size|binding|border-bottom-colors|border-left-colors|" + "background-size|binding|border-bottom-colors|border-left-colors|" +
"border-right-colors|border-top-colors|border-end|border-end-color|" + "border-right-colors|border-top-colors|border-end|border-end-color|" +
@ -97,20 +97,20 @@ var ScssHighlightRules = function() {
//All prefixProperties will get the browserPrefix in //All prefixProperties will get the browserPrefix in
//the begning by join the prefixProperties array with the value of browserPrefix //the begning by join the prefixProperties array with the value of browserPrefix
for (var i=0, ln=browserPrefix.length; i<ln; i++) { for (var i=0, ln=browserPrefix.length; i<ln; i++) {
Array.prototype.push.apply( Array.prototype.push.apply(
ret, ret,
(( browserPrefix[i] + prefixProperties.join("|" + browserPrefix[i]) ).split("|")) (( browserPrefix[i] + prefixProperties.join("|" + browserPrefix[i]) ).split("|"))
); );
} }
//Add also prefixProperties and properties without any browser prefix //Add also prefixProperties and properties without any browser prefix
Array.prototype.push.apply(ret, prefixProperties); Array.prototype.push.apply(ret, prefixProperties);
Array.prototype.push.apply(ret, properties); Array.prototype.push.apply(ret, properties);
return ret; return ret;
})() ); })() );
var functions = lang.arrayToMap( var functions = lang.arrayToMap(
@ -151,19 +151,19 @@ var ScssHighlightRules = function() {
); );
var keywords = lang.arrayToMap( var keywords = lang.arrayToMap(
("@mixin|@extend|@include|@import|@media|@debug|@warn|@if|@for|@each|@while|@else|@font-face|@-webkit-keyframes|if|and|!default|module|def|end|declare").split("|") ("@mixin|@extend|@include|@import|@media|@debug|@warn|@if|@for|@each|@while|@else|@font-face|@-webkit-keyframes|if|and|!default|module|def|end|declare").split("|")
) )
var tags = lang.arrayToMap( var tags = lang.arrayToMap(
("a|abbr|acronym|address|applet|area|article|aside|audio|b|base|basefont|bdo|" + ("a|abbr|acronym|address|applet|area|article|aside|audio|b|base|basefont|bdo|" +
"big|blockquote|body|br|button|canvas|caption|center|cite|code|col|colgroup|" + "big|blockquote|body|br|button|canvas|caption|center|cite|code|col|colgroup|" +
"command|datalist|dd|del|details|dfn|dir|div|dl|dt|em|embed|fieldset|" + "command|datalist|dd|del|details|dfn|dir|div|dl|dt|em|embed|fieldset|" +
"figcaption|figure|font|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|" + "figcaption|figure|font|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|" +
"header|hgroup|hr|html|i|iframe|img|input|ins|keygen|kbd|label|legend|li|" + "header|hgroup|hr|html|i|iframe|img|input|ins|keygen|kbd|label|legend|li|" +
"link|map|mark|menu|meta|meter|nav|noframes|noscript|object|ol|optgroup|" + "link|map|mark|menu|meta|meter|nav|noframes|noscript|object|ol|optgroup|" +
"option|output|p|param|pre|progress|q|rp|rt|ruby|s|samp|script|section|select|" + "option|output|p|param|pre|progress|q|rp|rt|ruby|s|samp|script|section|select|" +
"small|source|span|strike|strong|style|sub|summary|sup|table|tbody|td|" + "small|source|span|strike|strong|style|sub|summary|sup|table|tbody|td|" +
"textarea|tfoot|th|thead|time|title|tr|tt|u|ul|var|video|wbr|xmp").split("|") "textarea|tfoot|th|thead|time|title|tr|tt|u|ul|var|video|wbr|xmp").split("|")
); );
// regexp must not have capturing parentheses. Use (?:) instead. // regexp must not have capturing parentheses. Use (?:) instead.
@ -197,6 +197,7 @@ var ScssHighlightRules = function() {
}, },
{ {
token : "comment", // multi line comment token : "comment", // multi line comment
merge : true,
regex : "\\/\\*", regex : "\\/\\*",
next : "comment" next : "comment"
}, { }, {
@ -204,6 +205,7 @@ var ScssHighlightRules = function() {
regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' regex : '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
}, { }, {
token : "string", // multi line string start token : "string", // multi line string start
merge : true,
regex : '["].*\\\\$', regex : '["].*\\\\$',
next : "qqstring" next : "qqstring"
}, { }, {
@ -211,6 +213,7 @@ var ScssHighlightRules = function() {
regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']" regex : "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
}, { }, {
token : "string", // multi line string start token : "string", // multi line string start
merge : true,
regex : "['].*\\\\$", regex : "['].*\\\\$",
next : "qstring" next : "qstring"
}, { }, {
@ -272,8 +275,8 @@ var ScssHighlightRules = function() {
regex : numRe regex : numRe
}, { }, {
token : function(value) { token : function(value) {
if (properties.hasOwnProperty(value.toLowerCase())) if (properties.hasOwnProperty(value.toLowerCase()))
return "support.type"; return "support.type";
if (keywords.hasOwnProperty(value)) if (keywords.hasOwnProperty(value))
return "keyword"; return "keyword";
else if (constants.hasOwnProperty(value)) else if (constants.hasOwnProperty(value))
@ -283,9 +286,9 @@ var ScssHighlightRules = function() {
else if (colors.hasOwnProperty(value.toLowerCase())) else if (colors.hasOwnProperty(value.toLowerCase()))
return "support.constant.color"; return "support.constant.color";
else if (tags.hasOwnProperty(value.toLowerCase())) else if (tags.hasOwnProperty(value.toLowerCase()))
return "variable.language"; return "variable.language";
else else
return "text"; return "text";
}, },
regex : "\\-?[@a-zA-Z_][@a-zA-Z0-9_\\-]*" regex : "\\-?[@a-zA-Z_][@a-zA-Z0-9_\\-]*"
}, { }, {
@ -324,6 +327,7 @@ var ScssHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "comment", // comment spanning whole line token : "comment", // comment spanning whole line
merge : true,
regex : ".+" regex : ".+"
} }
], ],
@ -334,6 +338,7 @@ var ScssHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
} }
], ],
@ -344,11 +349,11 @@ var ScssHighlightRules = function() {
next : "start" next : "start"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
} }
] ]
}; };
}; };
oop.inherits(ScssHighlightRules, TextHighlightRules); oop.inherits(ScssHighlightRules, TextHighlightRules);

4
lib/ace/mode/text_highlight_rules.js
View file

@ -45,13 +45,13 @@ var TextHighlightRules = function() {
// regexps are ordered -> the first match is used // regexps are ordered -> the first match is used
this.$rules = { this.$rules = {
"start" : [ { "start" : [{
token : "empty_line", token : "empty_line",
regex : '^$' regex : '^$'
}, { }, {
token : "text", token : "text",
regex : ".+" regex : ".+"
} ] }]
}; };
}; };

18
lib/ace/mode/textile_highlight_rules.js
View file

@ -40,23 +40,7 @@ define(function(require, exports, module) {
var oop = require("pilot/oop"); var oop = require("pilot/oop");
var TextHighlightRules = require("ace/mode/text_highlight_rules").TextHighlightRules; var TextHighlightRules = require("ace/mode/text_highlight_rules").TextHighlightRules;
var TextileHighlightRules = function() var TextileHighlightRules = function() {
{
/*
var phraseModifiers = lang.arrayToMap(
("_|*|__|**|??|-|+|^|%|@").split("|")
);
var blockModifiers = lang.arrayToMap(
("h1|h2|h3|h4|h5|h6|bq|p|bc|pre").split("|")
);
*/
/*
var punctuation = lang.arrayToMap(
("-|--|(tm)|(r)|(c)").split("|")
);
*/
this.$rules = { this.$rules = {
"start" : [ "start" : [
{ {

20
lib/ace/mode/xml_highlight_rules.js
View file

@ -46,7 +46,7 @@ var XmlHighlightRules = function() {
// regexps are ordered -> the first match is used // regexps are ordered -> the first match is used
this.$rules = { this.$rules = {
start : [ { start : [{
token : "text", token : "text",
regex : "<\\!\\[CDATA\\[", regex : "<\\!\\[CDATA\\[",
next : "cdata" next : "cdata"
@ -55,6 +55,7 @@ var XmlHighlightRules = function() {
regex : "<\\?.*?\\?>" regex : "<\\?.*?\\?>"
}, { }, {
token : "comment", token : "comment",
merge : true,
regex : "<\\!--", regex : "<\\!--",
next : "comment" next : "comment"
}, { }, {
@ -67,9 +68,9 @@ var XmlHighlightRules = function() {
}, { }, {
token : "text", token : "text",
regex : "[^<]+" regex : "[^<]+"
} ], }],
tag : [ { tag : [{
token : "text", token : "text",
regex : ">", regex : ">",
next : "start" next : "start"
@ -84,6 +85,7 @@ var XmlHighlightRules = function() {
regex : '".*?"' regex : '".*?"'
}, { }, {
token : "string", // multi line string start token : "string", // multi line string start
merge : true,
regex : '["].*$', regex : '["].*$',
next : "qqstring" next : "qqstring"
}, { }, {
@ -91,6 +93,7 @@ var XmlHighlightRules = function() {
regex : "'.*?'" regex : "'.*?'"
}, { }, {
token : "string", // multi line string start token : "string", // multi line string start
merge : true,
regex : "['].*$", regex : "['].*$",
next : "qstring" next : "qstring"
}], }],
@ -101,6 +104,7 @@ var XmlHighlightRules = function() {
next : "tag" next : "tag"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
}], }],
@ -110,10 +114,11 @@ var XmlHighlightRules = function() {
next : "tag" next : "tag"
}, { }, {
token : "string", token : "string",
merge : true,
regex : '.+' regex : '.+'
}], }],
cdata : [ { cdata : [{
token : "text", token : "text",
regex : "\\]\\]>", regex : "\\]\\]>",
next : "start" next : "start"
@ -123,16 +128,17 @@ var XmlHighlightRules = function() {
}, { }, {
token : "text", token : "text",
regex : "(?:[^\\]]|\\](?!\\]>))+" regex : "(?:[^\\]]|\\](?!\\]>))+"
} ], }],
comment : [ { comment : [{
token : "comment", token : "comment",
regex : ".*?-->", regex : ".*?-->",
next : "start" next : "start"
}, { }, {
token : "comment", token : "comment",
merge : true,
regex : ".+" regex : ".+"
} ] }]
}; };
}; };

4
lib/ace/mode/xml_tokenizer_test.js
View file

@ -45,12 +45,14 @@ var XmlMode = require("ace/mode/xml").Mode;
var assert = require("ace/test/assertions"); var assert = require("ace/test/assertions");
module.exports = { module.exports = {
name: "XML Tokenizer",
setUp : function() { setUp : function() {
this.tokenizer = new XmlMode().getTokenizer(); this.tokenizer = new XmlMode().getTokenizer();
}, },
"test: tokenize1" : function() { "test: tokenize1" : function() {
var line = "<Juhu>//Juhu Kinners</Kinners>"; var line = "<Juhu>//Juhu Kinners</Kinners>";
var tokens = this.tokenizer.getLineTokens(line, "start").tokens; var tokens = this.tokenizer.getLineTokens(line, "start").tokens;

13
lib/ace/tokenizer.js
View file

@ -69,7 +69,6 @@ var Tokenizer = function(rules) {
} }
this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g"); this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", "g");
} }
}; };
@ -93,11 +92,12 @@ var Tokenizer = function(rules) {
while (match = re.exec(line)) { while (match = re.exec(line)) {
var type = "text"; var type = "text";
var rule = null;
var value = [match[0]]; var value = [match[0]];
for ( var i = 0; i < match.length-2; i++) { for (var i = 0; i < match.length-2; i++) {
if (match[i + 1] !== undefined) { if (match[i + 1] !== undefined) {
var rule = state[mapping[i].rule]; rule = state[mapping[i].rule];
if (mapping[i].len > 1) { if (mapping[i].len > 1) {
value = match.slice(i+2, i+1+mapping[i].len); value = match.slice(i+2, i+1+mapping[i].len);
@ -128,9 +128,10 @@ var Tokenizer = function(rules) {
value = [value.join("")]; value = [value.join("")];
type = [type]; type = [type];
} }
for (var i = 0; i < value.length; i++) { for (var i = 0; i < value.length; i++) {
if (token.type !== type[i]) { if ((!rule || rule.merge || type[i] === "text") && token.type === type[i]) {
token.value += value[i];
} else {
if (token.type) { if (token.type) {
tokens.push(token); tokens.push(token);
} }
@ -139,8 +140,6 @@ var Tokenizer = function(rules) {
type: type[i], type: type[i],
value: value[i] value: value[i]
} }
} else {
token.value += value[i];
} }
} }
} }