jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

update more highlighters

Browse source
This commit is contained in:
nightwing 2013-01-12 20:31:42 +04:00
commit c477184f2a
7 changed files with 258 additions and 213 deletions
Download patch file Download diff file Expand all files Collapse all files

84
lib/ace/mode/coffee_highlight_rules.js
View file

@ -38,10 +38,6 @@ define(function(require, exports, module) {
function CoffeeHighlightRules() { function CoffeeHighlightRules() {
var identifier = "[$A-Za-z_\\x7f-\\uffff][$\\w\\x7f-\\uffff]*"; var identifier = "[$A-Za-z_\\x7f-\\uffff][$\\w\\x7f-\\uffff]*";
var stringfill = {
token : "string",
regex : ".+"
};
var keywords = ( var keywords = (
"this|throw|then|try|typeof|super|switch|return|break|by|continue|" + "this|throw|then|try|typeof|super|switch|return|break|by|continue|" +
@ -91,31 +87,50 @@ define(function(require, exports, module) {
regex: /(\()((?:"[^")]*?"|'[^')]*?'|\/[^\/)]*?\/|[^()\"'\/])*?)(\))(\s*)([\-=]>)/ regex: /(\()((?:"[^")]*?"|'[^')]*?'|\/[^\/)]*?\/|[^()\"'\/])*?)(\))(\s*)([\-=]>)/
}; };
var stringEscape = /\\(?:x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)/;
this.$rules = { this.$rules = {
start : [ start : [
{ {
token : "constant.numeric", token : "constant.numeric",
regex : "(?:0x[\\da-fA-F]+|(?:\\d+(?:\\.\\d+)?|\\.\\d+)(?:[eE][+-]?\\d+)?)" regex : "(?:0x[\\da-fA-F]+|(?:\\d+(?:\\.\\d+)?|\\.\\d+)(?:[eE][+-]?\\d+)?)"
}, { }, {
token : "string", stateName: "qdoc",
regex : "'''", token : "string", regex : "'''", next : [
next : "qdoc" {token : "string", regex : "'''", next : "start"},
{token : "constant.language.escape", regex : stringEscape},
{defaultToken: "string"},
]
}, { }, {
stateName: "qqdoc",
token : "string", token : "string",
regex : '"""', regex : '"""',
next : "qqdoc" next : [
{token : "string", regex : '"""', next : "start"},
{token : "constant.language.escape", regex : stringEscape},
{defaultToken: "string"}
]
}, { }, {
token : "string", stateName: "qstring",
regex : "'", token : "string", regex : "'", next : [
next : "qstring" {token : "string", regex : "'", next : "start"},
{token : "constant.language.escape", regex : stringEscape},
{defaultToken: "string"},
]
}, { }, {
token : "string", stateName: "qqstring",
regex : '"', token : "string.start", regex : '"', next : [
next : "qqstring" {token : "string.end", regex : '"', next : "start"},
{token : "constant.language.escape", regex : stringEscape},
{defaultToken: "string"},
]
}, { }, {
token : "string", stateName: "js",
regex : "`", token : "string", regex : "`", next : [
next : "js" {token : "string", regex : "`", next : "start"},
{token : "constant.language.escape", regex : stringEscape},
{defaultToken: "string"},
]
}, { }, {
token : "string.regex", token : "string.regex",
regex : "///", regex : "///",
@ -173,35 +188,6 @@ define(function(require, exports, module) {
regex : "\\s+" regex : "\\s+"
}], }],
qdoc : [{
token : "string",
regex : ".*?'''",
next : "start"
}, stringfill],
qqdoc : [{
token : "string",
regex : '.*?"""',
next : "start"
}, stringfill],
qstring : [{
token : "string",
regex : "[^\\\\']*(?:\\\\.[^\\\\']*)*'",
next : "start"
}, stringfill],
qqstring : [{
token : "string",
regex : '[^\\\\"]*(?:\\\\.[^\\\\"]*)*"',
next : "start"
}, stringfill],
js : [{
token : "string",
regex : "[^\\\\`]*(?:\\\\.[^\\\\`]*)*`",
next : "start"
}, stringfill],
heregex : [{ heregex : [{
token : "string.regex", token : "string.regex",
@ -217,13 +203,13 @@ define(function(require, exports, module) {
comment : [{ comment : [{
token : "comment", token : "comment",
regex : '.*?###', regex : '###',
next : "start" next : "start"
}, { }, {
token : "comment", defaultToken : "comment",
regex : ".+"
}] }]
}; };
this.normalizeRules();
} }
exports.CoffeeHighlightRules = CoffeeHighlightRules; exports.CoffeeHighlightRules = CoffeeHighlightRules;

107
lib/ace/mode/markdown_highlight_rules.js
View file

@ -52,38 +52,9 @@ var MarkdownHighlightRules = function() {
// regexps are ordered -> the first match is used // regexps are ordered -> the first match is used
this.$rules = { this.$rules = {
"start" : [ { "basic" : [{ // code span `
token : "empty_line",
regex : '^$'
}, { // code span `
token : "support.function", token : "support.function",
regex : "(`+)(.*?[^`])(\\1)" regex : "(`+)(.*?[^`])(\\1)"
}, { // code block
token : "support.function",
regex : "^[ ]{4}.+"
}, { // h1
token: "markup.heading.1",
regex: "^=+(?=\\s*$)"
}, { // h2
token: "markup.heading.2",
regex: "^\\-+(?=\\s*$)"
}, { // header
token : function(value) {
return "markup.heading." + value.search(/[^#]/);
},
regex : "^#{1,6}(?:[^ #].*| +.*(?:[^ #].*|[^ ]+.* +#+ *))$"
}, github_embed("(?:javascript|js)", "js-"),
github_embed("xml", "xml-"),
github_embed("html", "html-"),
github_embed("css", "css-"),
{ // Github style block
token : "support.function",
regex : "^```\\s*[a-zA-Z]*(?:{.*?\\})?\\s*$",
next : "githubblock"
}, { // block quote
token : "string",
regex : "^>[ ].+$",
next : "blockquote"
}, { // reference }, { // reference
token : ["text", "constant", "text", "url", "string", "text"], token : ["text", "constant", "text", "url", "string", "text"],
regex : "^([ ]{0,3}\\[)([^\\]]+)(\\]:\\s*)([^ ]+)(\\s*(?:[\"][^\"]+[\"])?(\\s*))$" regex : "^([ ]{0,3}\\[)([^\\]]+)(\\]:\\s*)([^ ]+)(\\s*(?:[\"][^\"]+[\"])?(\\s*))$"
@ -98,19 +69,6 @@ var MarkdownHighlightRules = function() {
"(<?(?:(?:[^\\(]*?\\([^\\)]*?\\)\\S*?)|(?:.*?))>?)"+ "(<?(?:(?:[^\\(]*?\\([^\\)]*?\\)\\S*?)|(?:.*?))>?)"+
"((?:[ \t]*\"(?:.*?)\"[ \\t]*)?)"+ "((?:[ \t]*\"(?:.*?)\"[ \\t]*)?)"+
"(\\))" "(\\))"
}, { // HR *
token : "constant",
regex : "^[ ]{0,2}(?:[ ]?\\*[ ]?){3,}\\s*$"
}, { // HR -
token : "constant",
regex : "^[ ]{0,2}(?:[ ]?\\-[ ]?){3,}\\s*$"
}, { // HR _
token : "constant",
regex : "^[ ]{0,2}(?:[ ]?\\_[ ]?){3,}\\s*$"
}, { // list
token : "markup.list",
regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+",
next : "listblock"
}, { // strong ** __ }, { // strong ** __
token : "string", token : "string",
regex : "([*]{2}|[_]{2}(?=\\S))(.*?\\S[*_]*)(\\1)" regex : "([*]{2}|[_]{2}(?=\\S))(.*?\\S[*_]*)(\\1)"
@ -124,18 +82,72 @@ var MarkdownHighlightRules = function() {
"|"+ "|"+
"(?:mailto:)?[-.\\w]+\\@[-a-z0-9]+(?:\\.[-a-z0-9]+)*\\.[a-z]+"+ "(?:mailto:)?[-.\\w]+\\@[-a-z0-9]+(?:\\.[-a-z0-9]+)*\\.[a-z]+"+
")(>)" ")(>)"
}],
// code block
"allowBlock": [
{token : "support.function", regex : "^ {4}.+", next : "allowBlock"},
{token : "empty", regex : "", next : "start"}
],
"start" : [{
token : "empty_line",
regex : '^$',
next: "allowBlock"
}, { // h1
token: "markup.heading.1",
regex: "^=+(?=\\s*$)"
}, { // h2
token: "markup.heading.2",
regex: "^\\-+(?=\\s*$)"
}, { // header
token : function(value) {
return "markup.heading." + value.search(/[^#]/);
},
regex : "^#{1,6}(?:[^ #].*| +.*(?:[^ #].*|[^ ]+.* +#+ *))$"
},
github_embed("(?:javascript|js)", "js-"),
github_embed("xml", "xml-"),
github_embed("html", "html-"),
github_embed("css", "css-"),
{ // Github style block
token : "support.function",
regex : "^```\\s*[a-zA-Z]*(?:{.*?\\})?\\s*$",
next : "githubblock"
}, { // block quote
token : "string",
regex : "^>[ ].+$",
next : "blockquote"
}, { // HR * - _
token : "constant",
regex : "^ {0,2}(?:(?: ?\\* ?){3,}|(?: ?\\- ?){3,}|(?: ?\\_ ?){3,})\\s*$",
next: "allowBlock"
}, { // list
token : "markup.list",
regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+",
next : "listblock-start"
}, { }, {
token : "text", include : "basic"
regex : "[^\\*_%$`\\[#<>]+" }],
} ],
"listblock-start" : [{
token : "support.variable",
regex : /(?:\[[ x]\])?/,
next : "listblock"
}],
"listblock" : [ { // Lists only escape on completely blank lines. "listblock" : [ { // Lists only escape on completely blank lines.
token : "empty_line", token : "empty_line",
regex : "^$", regex : "^$",
next : "start" next : "start"
}, { }, {
include : "basic", noEscape: true
}, { // list
token : "markup.list", token : "markup.list",
regex : ".+" regex : "^\\s{0,3}(?:[*+-]|\\d+\\.)\\s+",
next : "listblock-start"
}, {
defaultToken : "markup.list"
} ], } ],
"blockquote" : [ { // BLockquotes only escape on blank lines. "blockquote" : [ { // BLockquotes only escape on blank lines.
@ -189,6 +201,7 @@ var MarkdownHighlightRules = function() {
this.$rules[i] = html[i]; this.$rules[i] = html[i];
} }
this.normalizeRules();
}; };
oop.inherits(MarkdownHighlightRules, TextHighlightRules); oop.inherits(MarkdownHighlightRules, TextHighlightRules);

27
lib/ace/mode/php_highlight_rules.js
View file

@ -1030,31 +1030,18 @@ var PhpLangHighlightRules = function() {
regex : '\\\\(?:[nrtvef\\\\"$]|[0-7]{1,3}|x[0-9A-Fa-f]{1,2})' regex : '\\\\(?:[nrtvef\\\\"$]|[0-7]{1,3}|x[0-9A-Fa-f]{1,2})'
}, { }, {
token : "constant.language.escape", token : "constant.language.escape",
regex : /\$[\w\d]+(?:\[[\w\d]+\])?/ regex : /\$[\w]+(?:\[[\w\]+]|=>\w+)?/
}, { }, {
token : "constant.language.escape", token : "constant.language.escape",
regex : /\$\{[^"\}]+\}?/ // this is wrong but ok for now regex : /\$\{[^"\}]+\}?/ // this is wrong but ok for now
}, { },
token : "string", {token : "string", regex : '"', next : "start"},
regex : '"', {defaultToken : "string"}
next : "start"
}, {
token : "string",
regex : '.+?'
}
], ],
"qstring" : [ "qstring" : [
{ {token : "constant.language.escape", regex : /\\['\\]/},
token : "constant.language.escape", {token : "string", regex : "'", next : "start"},
regex : "\\\\['\\\\]" {defaultToken : "string"}
}, {
token : "string",
regex : "'",
next : "start"
}, {
token : "string",
regex : ".+?"
}
] ]
}; };

28
lib/ace/mode/text_highlight_rules.js
View file

@ -99,24 +99,30 @@ var TextHighlightRules = function() {
this.normalizeRules = function() { this.normalizeRules = function() {
var id = 0; var id = 0;
for (var key in this.$rules) { for (var key in this.$rules) {
var rule = this.$rules[key]; var state = this.$rules[key];
for (var i = 0; i < rule.length; i++) { for (var i = 0; i < state.length; i++) {
var state = rule[i]; var rule = state[i];
if (state.next && Array.isArray(state.next)) { if (rule.next && Array.isArray(rule.next)) {
var stateName = state.stateName || ("state" + id++); var stateName = rule.stateName || ("state" + id++);
this.$rules[stateName] = state.next; this.$rules[stateName] = rule.next;
state.next = stateName; rule.next = stateName;
} }
if (state.rules) { if (rule.rules) {
for (var r in state.rules) { for (var r in rule.rules) {
if (this.$rules[r]) { if (this.$rules[r]) {
if (this.$rules[r].push) if (this.$rules[r].push)
this.$rules[r].push.apply(this.$rules[r], state.rules[r]); this.$rules[r].push.apply(this.$rules[r], rule.rules[r]);
} else { } else {
this.$rules[r] = state.rules[r]; this.$rules[r] = rule.rules[r];
} }
} }
} }
if (rule.include || typeof rule == "string") {
var args = [i, 1].concat(this.$rules[rule.include || rule]);
if (rule.noEscape)
args = args.filter(function(x) {return !x.next;});
state.splice.apply(state, args);
}
} }
} }
}; };

2
lib/ace/mode/xml_util.js
View file

@ -50,7 +50,7 @@ function multiLineString(quote, state) {
token : "constant.language.escape", token : "constant.language.escape",
regex : "(?:&#[0-9]+;)|(?:&#x[0-9a-fA-F]+;)|(?:&[a-zA-Z0-9_:\\.-]+;)" regex : "(?:&#[0-9]+;)|(?:&#x[0-9a-fA-F]+;)|(?:&[a-zA-Z0-9_:\\.-]+;)"
}, },
{token : "string", regex : '\\w+|.|\\s+'} {defaultToken : "string"}
]; ];
} }

197
lib/ace/tokenizer_dev.js
View file

@ -37,51 +37,111 @@ define(function(require, exports, module) {
**/ **/
var Tokenizer = function(rules, flag) { var Tokenizer = function(rules, flag) {
flag = flag ? "g" + flag : "g"; flag = flag ? "g" + flag : "g";
this.rules = rules; this.states = rules;
this.regExps = {}; this.regExps = {};
this.matchMappings = {}; this.matchMappings = {};
for ( var key in this.rules) { for (var key in this.states) {
var rule = this.rules[key]; var state = this.states[key];
var state = rule;
var ruleRegExps = []; var ruleRegExps = [];
var matchTotal = 0; var matchTotal = 0;
var mapping = this.matchMappings[key] = {}; var mapping = this.matchMappings[key] = {defaultToken: "default.text"};
for ( var i = 0; i < state.length; i++) { for (var i = 0; i < state.length; i++) {
var rule = state[i];
if (state[i].regex instanceof RegExp) if (rule.defaultToken) {
state[i].regex = state[i].regex.toString().slice(1, -1); mapping.defaultToken = rule.defaultToken;
continue;
}
if (rule.regex instanceof RegExp)
rule.regex = rule.regex.toString().slice(1, -1);
// Count number of matching groups. 2 extra groups from the full match // Count number of matching groups. 2 extra groups from the full match
// And the catch-all on the end (used to force a match); // And the catch-all on the end (used to force a match);
var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2; var adjustedregex = rule.regex;
var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2;
if (Array.isArray(rule.token)) {
if (rule.token.length == 1) {
rule.token = rule.token[0];
} else {
rule.tokenArray = rule.token;
rule.token = this.$arrayTokens;
}
}
// Replace any backreferences and offset appropriately. if (matchcount > 1) {
var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) { if (/\\\d/.test(rule.regex)) {
return "\\" + (parseInt(digit, 10) + matchTotal + 1); // Replace any backreferences and offset appropriately.
}); adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function (match, digit) {
return "\\" + (parseInt(digit, 10) + matchTotal + 1);
});
} else {
matchcount = 1;
adjustedregex = this.removeCapturingGroups(rule.regex);
}
if (!rule.splitRegex)
rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
}
if (matchcount > 1 && state[i].token.length !== matchcount-1) mapping[matchTotal] = i;
throw new Error("For " + state[i].regex + " the matching groups and length of the token array don't match (rule #" + i + " of state " + key + ")");
mapping[matchTotal] = {
rule: i,
len: matchcount
};
matchTotal += matchcount; matchTotal += matchcount;
ruleRegExps.push(adjustedregex); ruleRegExps.push(adjustedregex);
} }
this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag); this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag);
} }
}; };
(function() { (function() {
this.$arrayTokens = function(str) {
if (!str)
return [];
var values = str.split(this.splitRegex)
var tokens = [];
var types = this.tokenArray;
if (types.length != values.length - 2) {
if (window.console)
console.error(types.length , values.length - 2, str, this.splitRegex);
return [{type: "error.invalid", value: str}];
}
for (var i = 0; i < types.length; i++) {
if (values[i + 1]) {
tokens[tokens.length] = {
type: types[i],
value: values[i + 1]
};
}
}
return tokens;
};
this.removeCapturingGroups = function(src) {
var r = src.replace(
/\[(?:\\.|[^\]])*?\]|\\.|\(\?[:=!]|(\()/g,
function(x, y) {return y ? "(?:" : x;}
);
return r;
};
this.createSplitterRegexp = function(src, flag) {
src = src.replace(/\(\?=([^()]|\\.)*?\)$/, "");
return new RegExp(src, flag);
};
/**
* Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state.
* @returns {Object}
**/
this.getLineTokens = function(line, startState) { this.getLineTokens = function(line, startState) {
if (startState && typeof startState != "string") {
var stack = startState.slice(0);
startState = stack[0];
} else
var stack = [];
var currentState = startState || "start"; var currentState = startState || "start";
var state = this.rules[currentState]; var state = this.states[currentState];
var mapping = this.matchMappings[currentState]; var mapping = this.matchMappings[currentState];
var re = this.regExps[currentState]; var re = this.regExps[currentState];
re.lastIndex = 0; re.lastIndex = 0;
@ -110,92 +170,85 @@ var Tokenizer = function(rules, flag) {
var maxRecur = 10000; var maxRecur = 10000;
while (match = re.exec(line)) { while (match = re.exec(line)) {
var type = "default.text"; var type = mapping.defaultToken;
var rule = null; var rule = null;
var value = [match[0]]; var value = match[0];
var index = re.lastIndex;
if (index - value.length > lastIndex) {
var skipped = line.substring(lastIndex, index - value.length);
if (token.type == type) {
token.value += skipped;
} else {
if (token.type)
tokens.push(token);
token = {type: type, value: skipped};
}
}
for (var i = 0; i < match.length-2; i++) { for (var i = 0; i < match.length-2; i++) {
if (match[i + 1] === undefined) if (match[i + 1] === undefined)
continue; continue;
if (!maxRecur--) { if (!maxRecur--) {
throw "infinite" + mapping[i].rule + currentState throw "infinite" + state[mapping[i]] + currentState
} }
rule = state[mapping[i].rule]; rule = state[mapping[i]];
if (mapping[i].len > 1)
value = match.slice(i+2, i+1+mapping[i].len);
// compute token type // compute token type
if (typeof rule.token == "function") type = typeof rule.token == "function"
type = rule.token.apply(this, value); ? rule.token(value, currentState, stack)
else : rule.token;
type = rule.token;
if (rule.next) { if (rule.next) {
currentState = rule.next; currentState = rule.next;
state = this.rules[currentState]; state = this.states[currentState];
mapping = this.matchMappings[currentState]; if (!state) {
lastIndex = re.lastIndex; window.console && console.error && console.error(currentState, "doesn't exist");
currentState = "start";
re = this.regExps[currentState]; state = this.states[currentState];
if (re === undefined) {
throw new Error("You indicated a state of " + rule.next + " to go to, but it doesn't exist!");
} }
mapping = this.matchMappings[currentState];
re.lastIndex = lastIndex; lastIndex = index;
re = this.regExps[currentState];
re.lastIndex = index;
onStateChange(); onStateChange();
} }
break; break;
} }
if (value[0]) { if (value) {
if (typeof type == "string") { if (typeof type == "string") {
value = [value.join("")]; if ((!rule || rule.merge !== false) && token.type === type) {
type = [type]; token.value += value;
}
for (var i = 0; i < value.length; i++) {
if (!value[i])
continue;
var mergeable = (!rule || rule.merge || type[i] === "text") && token.type === type[i];
if (false && mergeable) {
token.value += value[i];
} else { } else {
if (token.type) { if (token.type)
token.stateTransitions = stateTransitions;
tokens.push(token); tokens.push(token);
initState() token = {type: type, value: value};
}
token = {
type: type[i],
value: value[i],
state: currentState,
mergeable: mergeable
};
} }
} else {
if (token.type)
tokens.push(token);
token = {type: null, value: ""};
for (var i = 0; i < type.length; i++)
tokens.push(type[i]);
} }
} }
if (lastIndex == line.length) if (lastIndex == line.length)
break; break;
lastIndex = re.lastIndex; lastIndex = index;
} }
if (token.type) { if (token.type)
token.stateTransitions = stateTransitions;
tokens.push(token); tokens.push(token);
}
return { return {
tokens : tokens, tokens : tokens,
state : currentState state : stack.length ? stack : currentState
}; };
}; };

2
tool/mode_creator.js
View file

@ -118,7 +118,7 @@ function run() {
var path = "ace/mode/new"; var path = "ace/mode/new";
var deps = getDeps(src, path); var deps = getDeps(src, path);
src = src.replace("define(", 'define("' + path +'", ["require","exports","module",' + deps +'],'); src = src.replace("define(", 'define("' + path +'", ["require","exports","module",' + deps +'],');
src += ';require(["ace/mode/new"], continueRun, function(e){console.log(e);require.undef("ace/mode/new")})'; src += ';require(["ace/mode/new"], continueRun, function(e){console.log(e);window.require.undef("ace/mode/new")})';
try { try {
eval(src); eval(src);
} catch(e) { } catch(e) {