Merge pull request #1298 from ajaxorg/highlighting/logicblox

Highlighting logicblox and some cleanup
This commit is contained in:
Lennart Kats 2013-03-12 08:31:27 -07:00
commit f16d0dc1d3
19 changed files with 600 additions and 35 deletions

View file

@ -93,7 +93,9 @@ var docs = {
"docs/lisp.lisp": "Lisp",
"docs/lsl.lsl": "LSL",
"docs/scheme.scm": "Scheme",
"docs/livescript.ls": "LiveScript",
"docs/liquid.liquid": "Liquid",
"docs/logicql.logic": "LogicQL",
"docs/lua.lua": "Lua",
"docs/lucene.lucene": "Lucene",
"docs/luapage.lp": "LuaPage",

View file

@ -0,0 +1,245 @@
# Defines an editing mode for [Ace](http://ace.ajax.org).
#
# Open [test/ace.html](../test/ace.html) to test.
require, exports, module <-! define \ace/mode/ls
identifier = /(?![\d\s])[$\w\xAA-\uFFDC](?:(?!\s)[$\w\xAA-\uFFDC]|-[A-Za-z])*/$
exports.Mode = class LiveScriptMode extends require(\ace/mode/text)Mode
->
@$tokenizer =
new (require \ace/tokenizer)Tokenizer LiveScriptMode.Rules
if require \ace/mode/matching_brace_outdent
@$outdent = new that.MatchingBraceOutdent
indenter = // (?
: [({[=:]
| [-~]>
| \b (?: e(?:lse|xport) | d(?:o|efault) | t(?:ry|hen) | finally |
import (?:\s* all)? | const | var |
let | new | catch (?:\s* #identifier)? )
) \s* $ //
getNextLineIndent: (state, line, tab) ->
indent = @$getIndent line
{tokens} = @$tokenizer.getLineTokens line, state
unless tokens.length and tokens[*-1]type is \comment
indent += tab if state is \start and indenter.test line
indent
toggleCommentLines: (state, doc, startRow, endRow) ->
comment = /^(\s*)#/; range = new (require \ace/range)Range 0 0 0 0
for i from startRow to endRow
if out = comment.test line = doc.getLine i
then line.=replace comment, \$1
else line.=replace /^\s*/ \$&#
range.end.row = range.start.row = i
range.end.column = line.length + 1
doc.replace range, line
1 - out * 2
checkOutdent: (state, line, input) -> @$outdent?checkOutdent line, input
autoOutdent: (state, doc, row) -> @$outdent?autoOutdent doc, row
### Highlight Rules
keywordend = /(?![$\w]|-[A-Za-z]|\s*:(?![:=]))/$
stringfill = token: \string, regex: '.+'
LiveScriptMode.Rules =
start:
* token: \keyword
regex: //(?
:t(?:h(?:is|row|en)|ry|ypeof!?)
|c(?:on(?:tinue|st)|a(?:se|tch)|lass)
|i(?:n(?:stanceof)?|mp(?:ort(?:\s+all)?|lements)|[fs])
|d(?:e(?:fault|lete|bugger)|o)
|f(?:or(?:\s+own)?|inally|unction)
|s(?:uper|witch)
|e(?:lse|x(?:tends|port)|val)
|a(?:nd|rguments)
|n(?:ew|ot)
|un(?:less|til)
|w(?:hile|ith)
|o[fr]|return|break|let|var|loop
)//$ + keywordend
* token: \constant.language
regex: '(?:true|false|yes|no|on|off|null|void|undefined)' + keywordend
* token: \invalid.illegal
regex: '(?
:p(?:ackage|r(?:ivate|otected)|ublic)
|i(?:mplements|nterface)
|enum|static|yield
)' + keywordend
* token: \language.support.class
regex: '(?
:R(?:e(?:gExp|ferenceError)|angeError)
|S(?:tring|yntaxError)
|E(?:rror|valError)
|Array|Boolean|Date|Function|Number|Object|TypeError|URIError
)' + keywordend
* token: \language.support.function
regex: '(?
:is(?:NaN|Finite)
|parse(?:Int|Float)
|Math|JSON
|(?:en|de)codeURI(?:Component)?
)' + keywordend
* token: \variable.language
regex: '(?:t(?:hat|il|o)|f(?:rom|allthrough)|it|by|e)' + keywordend
* token: \identifier
regex: identifier + /\s*:(?![:=])/$
* token: \variable
regex: identifier
* token: \keyword.operator
regex: /(?:\.{3}|\s+\?)/$
* token: \keyword.variable
regex: /(?:@+|::|\.\.)/$
next : \key
* token: \keyword.operator
regex: /\.\s*/$
next : \key
* token: \string
regex: /\\\S[^\s,;)}\]]*/$
* token: \string.doc
regex: \'''
next : \qdoc
* token: \string.doc
regex: \"""
next : \qqdoc
* token: \string
regex: \'
next : \qstring
* token: \string
regex: \"
next : \qqstring
* token: \string
regex: \`
next : \js
* token: \string
regex: '<\\['
next : \words
* token: \string.regex
regex: \//
next : \heregex
* token: \comment.doc
regex: '/\\*'
next : \comment
* token: \comment
regex: '#.*'
* token: \string.regex
regex: //
/(?: [^ [ / \n \\ ]*
(?: (?: \\.
| \[ [^\]\n\\]* (?:\\.[^\]\n\\]*)* \]
) [^ [ / \n \\ ]*
)*
)/ [gimy$]{0,4}
//$
next : \key
* token: \constant.numeric
regex: '(?:0x[\\da-fA-F][\\da-fA-F_]*
|(?:[2-9]|[12]\\d|3[0-6])r[\\da-zA-Z][\\da-zA-Z_]*
|(?:\\d[\\d_]*(?:\\.\\d[\\d_]*)?|\\.\\d[\\d_]*)
(?:e[+-]?\\d[\\d_]*)?[\\w$]*)'
* token: \lparen
regex: '[({[]'
* token: \rparen
regex: '[)}\\]]'
next : \key
* token: \keyword.operator
regex: \\\S+
* token: \text
regex: \\\s+
heregex:
* token: \string.regex
regex: '.*?//[gimy$?]{0,4}'
next : \start
* token: \string.regex
regex: '\\s*#{'
* token: \comment.regex
regex: '\\s+(?:#.*)?'
* token: \string.regex
regex: '\\S+'
key:
* token: \keyword.operator
regex: '[.?@!]+'
* token: \identifier
regex: identifier
next : \start
* token: \text
regex: '.'
next : \start
comment:
* token: \comment.doc
regex: '.*?\\*/'
next : \start
* token: \comment.doc
regex: '.+'
qdoc:
token: \string
regex: ".*?'''"
next : \key
stringfill
qqdoc:
token: \string
regex: '.*?"""'
next : \key
stringfill
qstring:
token: \string
regex: /[^\\']*(?:\\.[^\\']*)*'/$
next : \key
stringfill
qqstring:
token: \string
regex: /[^\\"]*(?:\\.[^\\"]*)*"/$
next : \key
stringfill
js:
token: \string
regex: /[^\\`]*(?:\\.[^\\`]*)*`/$
next : \key
stringfill
words:
token: \string
regex: '.*?\\]>'
next : \key
stringfill

View file

@ -0,0 +1,16 @@
// ancestors
parentof("douglas", "john").
parentof("john", "bob").
parentof("bob", "ebbon").
parentof("douglas", "jane").
parentof("jane", "jan").
ancestorof(A, B) <- parentof(A, B).
ancestorof(A, C) <- ancestorof(A, B), parentof(B,C).
grandparentof(A, B) <- parentof(A, C), parentof(C, B).
cousins(A,B) <- grandparentof(C,A), grandparentof(C,B).
parentof[`arg](A, B) -> int[32](A), !string(B).

View file

@ -1,4 +1,8 @@
#!/usr/bin/perl
=begin
perl example code for Ace
=cut
use strict;
use warnings;
my $num_primes = 0;
@ -30,3 +34,4 @@ for my $p (0 .. ($num_primes-1))
print $primes[$p], ", ";
}
print "\n";

View file

@ -66,6 +66,8 @@ var modesByName = {
lisp: ["Lisp" , "lisp"],
scheme: ["Scheme" , "scm|rkt"],
liquid: ["Liquid" , "liquid"],
livescript: ["LiveScript" , "ls"],
logicql: ["LogicQL" , "logic"],
lua: ["Lua" , "lua"],
luapage: ["LuaPage" , "lp"], // http://keplerproject.github.com/cgilua/manual.html#templates
lucene: ["Lucene" , "lucene"],

View file

@ -369,8 +369,11 @@ var Editor = function(renderer, session) {
var pos = self.session.findMatchingBracket(self.getCursorPosition());
if (pos) {
var range = new Range(pos.row, pos.column, pos.row, pos.column+1);
self.session.$bracketHighlight = self.session.addMarker(range, "ace_bracket", "text");
} else if (self.session.$mode.getMatching) {
var range = self.session.$mode.getMatching(self.session);
}
if (range)
self.session.$bracketHighlight = self.session.addMarker(range, "ace_bracket", "text");
}, 50);
};

View file

@ -0,0 +1 @@
# comment

View file

@ -1,6 +1,17 @@
[[
"start",
["comment","#!/usr/bin/perl"]
],[
"block_comment",
["comment.doc","=begin"]
],[
"block_comment",
["comment.doc"," perl example code for Ace"]
],[
"start",
["comment.doc","=cut"]
],[
"start"
],[
"start",
["keyword","use"],
@ -211,4 +222,6 @@
["text",";"]
],[
"start"
],[
"start"
]]

View file

@ -35,7 +35,16 @@ var oop = require("../../lib/oop");
var Range = require("../../range").Range;
var BaseFoldMode = require("./fold_mode").FoldMode;
var FoldMode = exports.FoldMode = function() {};
var FoldMode = exports.FoldMode = function(commentRegex) {
if (commentRegex) {
this.foldingStartMarker = new RegExp(
this.foldingStartMarker.source.replace(/\|[^|]*?$/, "|" + commentRegex.start)
);
this.foldingStopMarker = new RegExp(
this.foldingStopMarker.source.replace(/\|[^|]*?$/, "|" + commentRegex.end)
);
}
};
oop.inherits(FoldMode, BaseFoldMode);
(function() {

View file

@ -60,7 +60,7 @@ oop.inherits(FoldMode, BaseFoldMode);
return "start";
} else if (match[2]) {
var type = session.bgTokenizer.getState(row) || "";
if (type.indexOf("comment") != -1 || type.indexOf("string") != -1)
if (type[0] == "bracketedComment" || type[0] == "bracketedString")
return "start";
} else {
return "start";
@ -75,7 +75,7 @@ oop.inherits(FoldMode, BaseFoldMode);
return "end";
} else if (match[0][0] === "]") {
var type = session.bgTokenizer.getState(row - 1) || "";
if (type.indexOf("comment") != -1 || type.indexOf("string") != -1)
if (type[0] == "bracketedComment" || type[0] == "bracketedString")
return "end";
} else
return "end";

123
lib/ace/mode/logicql.js Normal file
View file

@ -0,0 +1,123 @@
/* ***** BEGIN LICENSE BLOCK *****
* Distributed under the BSD license:
*
* Copyright (c) 2012, Ajax.org B.V.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Ajax.org B.V. nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ***** END LICENSE BLOCK ***** */
define(function(require, exports, module) {
"use strict";
var oop = require("../lib/oop");
var TextMode = require("./text").Mode;
var Tokenizer = require("../tokenizer").Tokenizer;
var LogicQLHighlightRules = require("./logicql_highlight_rules").LogicQLHighlightRules;
var FoldMode = require("./folding/coffee").FoldMode;
var TokenIterator = require("ace/token_iterator").TokenIterator;
var Range = require("ace/range").Range;
var Mode = function() {
var highlighter = new LogicQLHighlightRules();
this.foldingRules = new FoldMode();
this.$tokenizer = new Tokenizer(highlighter.getRules());
};
oop.inherits(Mode, TextMode);
(function() {
this.lineCommentStart = "//";
this.blockComment = {start: "/*", end: "*/"};
this.getNextLineIndent = function(state, line, tab) {
var indent = this.$getIndent(line);
var match = line.match();
if (/(-->|<--|<-|->)\s*$/.test(line))
indent += tab;
return indent;
};
this.checkOutdent = function(state, line, input) {
if (input !== "\n" && input !== "\r\n")
return false;
if (!/^\s+/.test(line))
return false;
return true;
};
this.autoOutdent = function(state, doc, row) {
var prevLine = doc.getLine(row);
var match = prevLine.match(/^\s+/);
var column = prevLine.lastIndexOf(".") + 1;
if (!match || !row || !column) return 0;
var line = doc.getLine(row + 1);
var startRange = this.getMatching(doc, {row: row, column: column});
if (!startRange || startRange.start.row == row) return 0;
column = match[0].length;
var indent = this.$getIndent(doc.getLine(startRange.start.row));
doc.replace(new Range(row + 1, 0, row + 1, column), indent);
};
this.getMatching = function(session, row, column) {
if (row == undefined)
row = session.selection.lead
if (typeof row == "object") {
column = row.column;
row = row.row;
}
var startToken = session.getTokenAt(row, column);
var KW_START = "keyword.start", KW_END = "keyword.end";
var tok;
if (!startToken)
return;
if (startToken.type == KW_START) {
var it = new TokenIterator(session, row, column);
it.step = it.stepForward;
} else if (startToken.type == KW_END) {
var it = new TokenIterator(session, row, column);
it.step = it.stepBackward;
} else
return;
while (tok = it.step()) {
if (tok.type == KW_START || tok.type == KW_END)
break;
}
if (!tok)
return;
var col = it.getCurrentTokenColumn();
var row = it.getCurrentTokenRow();
return new Range(row, col, row, col + tok.value.length);
};
// Extra logic goes here.
}).call(Mode.prototype);
exports.Mode = Mode;
});

View file

@ -0,0 +1,119 @@
/* ***** BEGIN LICENSE BLOCK *****
* Distributed under the BSD license:
*
* Copyright (c) 2012, Ajax.org B.V.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Ajax.org B.V. nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ***** END LICENSE BLOCK ***** */
/* THIS FILE WAS AUTOGENERATED FROM tool\LogicBlox.tmbundle\Syntaxes\LogicBlox.tmLanguage (UUID: 59bf5022-e261-453f-b1cb-9f9fa0712413) */
define(function(require, exports, module) {
"use strict";
var oop = require("../lib/oop");
var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
var LogicQLHighlightRules = function() {
// regexp must not have capturing parentheses. Use (?:) instead.
// regexps are ordered -> the first match is used
this.$rules = { start:
[ { token: 'comment.block',
regex: '/\\*',
push:
[ { token: 'comment.block', regex: '\\*/', next: 'pop' },
{ defaultToken: 'comment.block' } ],
//A block comment.
},
{ token: 'comment.single',
regex: '//.*',
//A single line comment.
},
{ token: 'constant.numeric',
regex: '\\d+(?:\\.\\d+)?(?:[eE][+-]?\\d+)?',
//An integer constant.
//Or a Real number.
},
{ token: 'string',
regex: '"',
push:
[ { token: 'string', regex: '"', next: 'pop' },
{ defaultToken: 'string' } ],
//Strings
},
{ token: 'constant.language',
regex: '\\b(true|false)\\b',
//Boolean values.
},
{ token: 'entity.name.type.logicblox',
regex: '`[a-zA-Z_:]+(\\d|\\a)*\\b',
//LogicBlox Symbol
},
{ token: 'keyword.start', regex: '->', comment: 'Constraint' },
{ token: 'keyword.start', regex: '-->', comment: 'Level 1 Constraint'},
{ token: 'keyword.start', regex: '<-', comment: 'Rule' },
{ token: 'keyword.start', regex: '<--', comment: 'Level 1 Rule' },
{ token: 'keyword.end', regex: '\\.', comment: 'Terminator' },
{ token: 'keyword.other', regex: '!', comment: 'Negation' },
{ token: 'keyword.other', regex: ',', comment: 'Conjunction' },
{ token: 'keyword.other', regex: ';', comment: 'Disjunction' },
{ token: 'keyword.operator', regex: '<=|>=|!=|<|>', comment: 'Equality'},
{ token: 'keyword.other', regex: '@', comment: 'Equality' },
{ token: 'keyword.operator', regex: '\\+|-|\\*|/', comment: 'Arithmetic operations'},
{ token: 'keyword', regex: '::', comment: 'Colon colon' },
{ token: 'support.function',
regex: '\\b(agg\\s*<<)',
push:
[ { include: '$self' },
{ token: 'support.function',
regex: '>>',
next: 'pop' } ],
//Aggregations
},
{ token: 'storage.modifier',
regex: '\\b(lang:[\\w:]*)',
//All the lang system predicates
},
{ token: [ 'storage.type', 'text' ],
regex: '(export|sealed|clauses|block|alias)\\s*\\((?=`)',
//Module keywords
},
{ token: 'entity.name',
regex: '[a-zA-Z_][a-zA-Z_0-9:]*(@prev|@init|@final)?(?=(\\(|\\[))',
//A predicate name.
},
{ token: 'variable.parameter',
regex: '([a-zA-Z][a-zA-Z_0-9]*|_)\\s*(?=(,|\\.|<-|->|\\)|\\]|=))',
//A variable to a functional predicate.
} ] }
this.normalizeRules();
};
oop.inherits(LogicQLHighlightRules, TextHighlightRules);
exports.LogicQLHighlightRules = LogicQLHighlightRules;
});

View file

@ -60,12 +60,7 @@ var MatchingBraceOutdent = function() {};
};
this.$getIndent = function(line) {
var match = line.match(/^(\s+)/);
if (match) {
return match[1];
}
return "";
return line.match(/^\s*/)[0];
};
}).call(MatchingBraceOutdent.prototype);

View file

@ -42,13 +42,18 @@ var CStyleFoldMode = require("./folding/cstyle").FoldMode;
var Mode = function() {
this.$tokenizer = new Tokenizer(new PerlHighlightRules().getRules());
this.$outdent = new MatchingBraceOutdent();
this.foldingRules = new CStyleFoldMode();
this.foldingRules = new CStyleFoldMode({start: "^=(begin|item)\\b", end: "^=(cut)\\b"});
};
oop.inherits(Mode, TextMode);
(function() {
this.lineCommentStart = "#";
this.blockComment = [
{start: "=begin", end: "=cut"},
{start: "=item", end: "=cut"}
];
this.getNextLineIndent = function(state, line, tab) {
var indent = this.$getIndent(line);

View file

@ -82,6 +82,10 @@ var PerlHighlightRules = function() {
{
token : "comment",
regex : "#.*$"
}, {
token : "comment.doc",
regex : "^=(?:begin|item)\\b",
next : "block_comment"
}, {
token : "string.regexp",
regex : "[/](?:(?:\\[(?:\\\\]|[^\\]])+\\])|(?:\\\\/|[^\\]/]))*[/]\\w*\\s*(?=[).,;]|$)"
@ -141,6 +145,16 @@ var PerlHighlightRules = function() {
token : "string",
regex : '.+'
}
],
"block_comment": [
{
token: "comment.doc",
regex: "^=cut\\b",
next: "start"
},
{
defaultToken: "comment.doc"
}
]
};
};

View file

@ -13,7 +13,7 @@ var Mode = function() {
this.$tokenizer = new Tokenizer(new PowershellHighlightRules().getRules());
this.$outdent = new MatchingBraceOutdent();
this.$behaviour = new CstyleBehaviour();
this.foldingRules = new CStyleFoldMode();
this.foldingRules = new CStyleFoldMode({start: "^\\s*(<#)", end: "^[#\\s]>\\s*$"});
};
oop.inherits(Mode, TextMode);

View file

@ -116,12 +116,7 @@ var Mode = function() {
};
this.$getIndent = function(line) {
var match = line.match(/^(\s+)/);
if (match) {
return match[1];
}
return "";
return line.match(/^\s*/)[0];
};
this.createWorker = function(session) {

View file

@ -50,22 +50,22 @@ oop.inherits(Mode, TextMode);
(function() {
this.getNextLineIndent = function(state, line, tab) {
var indent = this.$getIndent(line);
var match = line.match(/\s*(?:then|else|return|[{\(]|<\w+>)\s*$/);
if (match)
indent += tab;
var indent = this.$getIndent(line);
var match = line.match(/\s*(?:then|else|return|[{\(]|<\w+>)\s*$/);
if (match)
indent += tab;
return indent;
};
this.checkOutdent = function(state, line, input) {
if (! /^\s+$/.test(line))
if (! /^\s+$/.test(line))
return false;
return /^\s*[\}\)]/.test(input);
};
this.autoOutdent = function(state, doc, row) {
var line = doc.getLine(row);
var line = doc.getLine(row);
var match = line.match(/^(\s*[\}\)])/);
if (!match) return 0;
@ -79,15 +79,6 @@ oop.inherits(Mode, TextMode);
doc.replace(new Range(row, 0, row, column-1), indent);
};
this.$getIndent = function(line) {
var match = line.match(/^(\s+)/);
if (match) {
return match[1];
}
return "";
};
this.toggleCommentLines = function(state, doc, startRow, endRow) {
var i, line;
var outdent = true;

View file

@ -98,7 +98,7 @@ var Tokenizer = function(rules) {
matchcount = 1;
adjustedregex = this.removeCapturingGroups(rule.regex);
}
if (!rule.splitRegex)
if (!rule.splitRegex && typeof rule.token != "string")
rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
}
@ -167,7 +167,34 @@ var Tokenizer = function(rules) {
};
this.createSplitterRegexp = function(src, flag) {
src = src.replace(/\(\?=([^()]|\\.|\(([^()]|\\.)*?\))*\)(?=\)*$)/, "");
if (src.indexOf("(?=") != -1) {
var stack = 0;
var inChClass = false;
var lastCapture = {};
src.replace(/(\\.)|(\((?:\?[=!])?)|(\))|([])/g, function(
m, esc, parenOpen, parenClose, square, index
) {
if (inChClass) {
inChClass = square != "]";
} else if (square) {
inChClass = true;
} else if (parenClose) {
if (stack == lastCapture.stack)
lastCapture.end = index+1
stack--;
} else if (parenOpen) {
stack++;
if (parenOpen.length != 1) {
lastCapture.stack = stack
lastCapture.start = index;
}
}
return m;
});
if (lastCapture.end != null && /^\)*$/.test(src.substr(lastCapture.end)))
src = src.substring(0, lastCapture.start) + src.substr(lastCapture.end);
}
return new RegExp(src, (flag||"").replace("g", ""));
};