simplify converter
This commit is contained in:
nightwing
parent
6001ba815c
commit
ff1282556b
2 changed files with 196 additions and 317 deletions
|
|
@ -26,35 +26,19 @@
|
||||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
*
|
*
|
||||||
*
|
|
||||||
* Contributor(s):
|
|
||||||
*
|
|
||||||
*
|
|
||||||
*
|
|
||||||
* ***** END LICENSE BLOCK ***** */
|
* ***** END LICENSE BLOCK ***** */
|
||||||
|
|
||||||
/*
|
/* THIS FILE WAS AUTOGENERATED FROM %name% (UUID: %uuid%) */
|
||||||
THIS FILE WAS AUTOGENERATED BY %name% (UUID: %uuid%) */
|
/****************************************************************
|
||||||
|
* IT MIGHT NOT BE PERFECT, PARTICULARLY: *
|
||||||
/*******
|
* IN DECIDING STATES TO TRANSITION TO, *
|
||||||
|
* IGNORING WHITESPACE, *
|
||||||
THIS FILE MIGHT NOT BE PERFECT, PARTICULARLY:
|
* IGNORING GROUPS WITH ?:, *
|
||||||
|
* EXTENDING EXISTING MODES, *
|
||||||
IN DECIDING STATES TO TRANSITION TO,
|
* GATHERING KEYWORDS, OR *
|
||||||
|
* DECIDING WHEN TO USE PUSH. *
|
||||||
IGNORING WHITESPACE,
|
* ...But it's a good start from an existing *.tmlanguage file. *
|
||||||
|
****************************************************************/
|
||||||
IGNORING GROUPS WITH ?:,
|
|
||||||
|
|
||||||
EXTENDING EXISTING MODES,
|
|
||||||
|
|
||||||
GATHERING KEYWORDS, OR
|
|
||||||
|
|
||||||
RULE PREFERENCE ORDER.
|
|
||||||
|
|
||||||
...But it's a good start from an existing *.tmlanguage file.
|
|
||||||
|
|
||||||
*******/
|
|
||||||
|
|
||||||
define(function(require, exports, module) {
|
define(function(require, exports, module) {
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
@ -63,14 +47,12 @@ var oop = require("../lib/oop");
|
||||||
var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
|
var TextHighlightRules = require("./text_highlight_rules").TextHighlightRules;
|
||||||
|
|
||||||
var %language%HighlightRules = function() {
|
var %language%HighlightRules = function() {
|
||||||
|
|
||||||
// regexp must not have capturing parentheses. Use (?:) instead.
|
// regexp must not have capturing parentheses. Use (?:) instead.
|
||||||
// regexps are ordered -> the first match is used
|
// regexps are ordered -> the first match is used
|
||||||
|
|
||||||
this.$rules =
|
this.$rules = %languageTokens%
|
||||||
%languageTokens%
|
|
||||||
|
|
||||||
%repositoryRules%
|
this.normalizeRules();
|
||||||
};
|
};
|
||||||
|
|
||||||
oop.inherits(%language%HighlightRules, TextHighlightRules);
|
oop.inherits(%language%HighlightRules, TextHighlightRules);
|
||||||
|
|
|
||||||
|
|
@ -1,89 +1,110 @@
|
||||||
var fs = require("fs");
|
var fs = require("fs");
|
||||||
var util = require("util");
|
var util = require("util");
|
||||||
|
var lib = require("./lib");
|
||||||
|
var parseLanguage = lib.parsePlist;
|
||||||
|
|
||||||
// for tracking token states
|
|
||||||
var startState = { start: [] }, statesObj = { };
|
|
||||||
|
|
||||||
|
|
||||||
var args = process.argv.splice(2);
|
|
||||||
var tmLanguageFile = args[0];
|
|
||||||
var devMode = args[1];
|
|
||||||
|
|
||||||
var parseString = require("plist").parseString;
|
|
||||||
function parseLanguage(languageXml, callback) {
|
|
||||||
parseString(languageXml, function(_, language) {
|
|
||||||
callback(language[0])
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function logDebug(string, obj) {
|
function logDebug(string, obj) {
|
||||||
console.log(string, obj);
|
console.log(string, obj);
|
||||||
}
|
}
|
||||||
|
|
||||||
String.prototype.splice = function( idx, rem, s ) {
|
|
||||||
return (this.slice(0,idx) + s + this.slice(idx + Math.abs(rem)));
|
|
||||||
};
|
|
||||||
|
|
||||||
String.prototype.replaceAt = function (index, char) {
|
// tmLanguage processor
|
||||||
return this.substr(0, index) + char + this.substr(index + 1);
|
|
||||||
|
// for tracking token states
|
||||||
|
var states = {start: []};
|
||||||
|
var stateName = "start";
|
||||||
|
|
||||||
|
function processRules(rules){
|
||||||
|
if (rules.patterns)
|
||||||
|
states.start = processPatterns(rules.patterns);
|
||||||
|
if (rules.repository)
|
||||||
|
processRepository(rules.repository);
|
||||||
|
return states;
|
||||||
|
}
|
||||||
|
function processRepository(r) {
|
||||||
|
for (var key in r) {
|
||||||
|
var p = r[key];
|
||||||
|
if (p.begin)
|
||||||
|
var stateObj = [processPattern(r[key])];
|
||||||
|
else if (p.patterns && !p.repository)
|
||||||
|
var stateObj = processPatterns(p.patterns);
|
||||||
|
else
|
||||||
|
var stateObj = [processPattern(r[key])];
|
||||||
|
|
||||||
|
if (stateObj)
|
||||||
|
states["#" + key] = stateObj;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function processPatterns(pl) {
|
||||||
|
return pl.map(processPattern);
|
||||||
|
}
|
||||||
|
function processPattern(p) {
|
||||||
|
|
||||||
|
if (p.end == "(?!\\G)" && p.patterns && p.patterns.length == 1) {
|
||||||
|
var rule = processPattern(p.patterns[0]);
|
||||||
|
}
|
||||||
|
else if (p.begin && p.end) {
|
||||||
|
var rule = simpleRule(p.begin, p.name, p.beginCaptures || p.captures)
|
||||||
|
|
||||||
|
var next = processPatterns(p.patterns || []);
|
||||||
|
var endRule = simpleRule(p.end, p.name, p.endCaptures || p.captures);
|
||||||
|
endRule.next = "pop";
|
||||||
|
next.push(endRule);
|
||||||
|
|
||||||
|
if (p.name || p.contentName)
|
||||||
|
next.push({defaultToken: p.name || p.contentName});
|
||||||
|
|
||||||
|
rule.push = next;
|
||||||
|
}
|
||||||
|
else if (p.match) {
|
||||||
|
var rule = simpleRule(p.match, p.name, p.captures)
|
||||||
|
}
|
||||||
|
else if (p.include) {
|
||||||
|
var rule = {include: p.include};
|
||||||
}
|
}
|
||||||
|
|
||||||
function keyCount(obj) {
|
if (p.comment)
|
||||||
return Object.keys(obj).length;
|
rule.comment = (rule.comment || "") + p.comment;
|
||||||
|
|
||||||
|
if (p.repository)
|
||||||
|
processRepository(p.repository);
|
||||||
|
return rule;
|
||||||
}
|
}
|
||||||
|
function simpleRule(regex, name, captures) {
|
||||||
|
name = name || "text";
|
||||||
|
var rule = {};
|
||||||
|
|
||||||
/**
|
regex = transformRegExp(regex, rule);
|
||||||
|
if (captures) {
|
||||||
Scrubbing is sometimes necessary, but there appears to be no
|
var tokenArray = [];
|
||||||
automated way to do it...
|
Object.keys(captures).forEach(function(x){
|
||||||
|
tokenArray[x] = captures[x] && captures[x].name;
|
||||||
|
});
|
||||||
function cleanSingleCapture(match) {
|
if (tokenArray.length == 1) {
|
||||||
// if there's a single "( )", screw that and make it "(?: )"
|
name = tokenArray[0];
|
||||||
return match.replace("(", "(?:");
|
} else {
|
||||||
}
|
for (var i = 0; i < tokenArray.length; i++)
|
||||||
|
if (!tokenArray[i])
|
||||||
function cleanMultiCapture(match) {
|
tokenArray[i] = name;
|
||||||
// regexp will be a quoted string, so turn "\" into "\\"
|
name = tokenArray;
|
||||||
var spaceFinderRegExp = new RegExp("\\\\s.| .", "g");
|
rule.todo = "fix grouping";
|
||||||
var m;
|
|
||||||
/*
|
|
||||||
essentially turns things like
|
|
||||||
|
|
||||||
\\s*(mixin) ([\\w\\-]+)\\s*(\\()
|
|
||||||
|
|
||||||
into
|
|
||||||
|
|
||||||
(\\s*mixin)( [\\w\\-]+)(\\s*\\()
|
|
||||||
|
|
||||||
so that mode parser stops complaining
|
|
||||||
|
|
||||||
while ((m = spaceFinderRegExp.exec(match)) != null) {
|
|
||||||
var idx = m.index;
|
|
||||||
var nextParenIdx = match.indexOf("(", idx);
|
|
||||||
|
|
||||||
if (nextParenIdx > idx) {
|
|
||||||
match = match.splice(idx, 0, "(").replaceAt(nextParenIdx + 1, '');
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//console.log("match", match);
|
try {new RegExp(regex);} catch(e) {
|
||||||
return match;
|
rule.TODO = "FIXME: regexp doesn't have js equivalent";
|
||||||
}
|
}
|
||||||
*/
|
rule.token = name;
|
||||||
|
rule.regex = regex;
|
||||||
// stupid yet necessary function, to transform JSON id comments into real comments
|
return rule;
|
||||||
function restoreComments(objStr) {
|
|
||||||
return objStr.replace(/"\s+(\/\/.+)",/g, "\$1").replace(/ \/\/ ERROR/g, '", // ERROR');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function checkForLookBehind(str) {
|
|
||||||
var lookbehindRegExp = new RegExp("\\?<[=|!]", "g");
|
// regex transformation
|
||||||
return lookbehindRegExp.test(str) ? str + " // ERROR: This contains a lookbehind, which JS does not support :(" : str;
|
|
||||||
}
|
|
||||||
|
|
||||||
function removeXFlag(str) {
|
function removeXFlag(str) {
|
||||||
if (str.slice(0,4) == "(?x)") {
|
if (str && str.slice(0,4) == "(?x)") {
|
||||||
str = str.replace(/\\.|\[([^\]\\]|\\.)*?\]|\s+|(?:#[^\n]*)/g, function(s) {
|
str = str.replace(/\\.|\[([^\]\\]|\\.)*?\]|\s+|(?:#[^\n]*)/g, function(s) {
|
||||||
if (s[0] == "[")
|
if (s[0] == "[")
|
||||||
return s;
|
return s;
|
||||||
|
|
@ -95,149 +116,31 @@ function removeXFlag(str) {
|
||||||
return str;
|
return str;
|
||||||
}
|
}
|
||||||
|
|
||||||
function transformRegExp(str) {
|
function transformRegExp(str, rule) {
|
||||||
str = removeXFlag(str);
|
str = removeXFlag(str);
|
||||||
str = checkForLookBehind(str);
|
str = str.replace(/\\n(?!\?).?/g, '$'); // replace newlines by $ except if its postfixed by ?
|
||||||
|
if (/\(\?[i]\:/g.test(str)) {
|
||||||
|
str = str.replace(/\(\?[ims\-]\:/g, "(?:"); // checkForInvariantRegex
|
||||||
|
rule && (rule.caseInsensitive = true);
|
||||||
|
}
|
||||||
return str;
|
return str;
|
||||||
}
|
}
|
||||||
|
|
||||||
function assembleStateObjs(strState, pattern) {
|
//
|
||||||
var patterns = pattern.patterns;
|
function extractPatterns(tmRules) {
|
||||||
var stateObj = {};
|
var patterns = processRules(tmRules);
|
||||||
var tokenElem = [];
|
return lib.restoreJSONComments(lib.formatJSON(patterns, " "));
|
||||||
|
|
||||||
if (patterns) {
|
|
||||||
for (var p in patterns) {
|
|
||||||
stateObj = {}; // this is apparently necessary
|
|
||||||
|
|
||||||
if (patterns[p].include) {
|
|
||||||
stateObj.include = patterns[p].include;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
stateObj.token = patterns[p].name;
|
|
||||||
stateObj.regex = transformRegExp(patterns[p].match);
|
|
||||||
}
|
|
||||||
statesObj[strState].push(stateObj);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
stateObj = {};
|
|
||||||
stateObj.token = "TODO";
|
|
||||||
stateObj.regex = transformRegExp(pattern.end);
|
|
||||||
stateObj.next = "start";
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
stateObj.token = "TODO";
|
|
||||||
stateObj.regex = transformRegExp(pattern.end);
|
|
||||||
stateObj.next = "start";
|
|
||||||
|
|
||||||
statesObj[strState].push(stateObj);
|
|
||||||
|
|
||||||
stateObj = {};
|
|
||||||
stateObj.token = "TODO";
|
|
||||||
stateObj.regex = ".+";
|
|
||||||
stateObj.next = strState;
|
|
||||||
}
|
|
||||||
|
|
||||||
return stateObj;
|
|
||||||
}
|
|
||||||
|
|
||||||
function extractPatterns(patterns) {
|
|
||||||
var state = 0;
|
|
||||||
patterns.forEach(function(pattern) {
|
|
||||||
state++;
|
|
||||||
var i = 1;
|
|
||||||
var tokenArray = [];
|
|
||||||
var tokenObj = { token: [] };
|
|
||||||
var stateObj = {};
|
|
||||||
|
|
||||||
if (pattern.comment) {
|
|
||||||
startState.start.push(" // " + pattern.comment.trim());
|
|
||||||
}
|
|
||||||
|
|
||||||
// it needs a state transition
|
|
||||||
if (pattern.begin && pattern.end) {
|
|
||||||
var strState = "state_" + state;
|
|
||||||
if ( pattern.beginCaptures === undefined && pattern.endCaptures === undefined) {
|
|
||||||
tokenObj.token.push(pattern.captures);
|
|
||||||
}
|
|
||||||
else if (pattern.beginCaptures) {
|
|
||||||
tokenObj.token.push(pattern.beginCaptures);
|
|
||||||
}
|
|
||||||
else if (pattern.endCaptures) {
|
|
||||||
tokenObj.token.push(pattern.endCaptures);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (tokenObj.token === undefined) {
|
|
||||||
if (pattern.name)
|
|
||||||
tokenObj.token.push(pattern.name);
|
|
||||||
else
|
|
||||||
logDebug("There's no token name for this state transition", pattern)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (tokenObj.token === undefined) {
|
|
||||||
tokenObj.token.push(pattern.name);
|
|
||||||
}
|
|
||||||
|
|
||||||
statesObj[strState] = [ ];
|
|
||||||
statesObj[strState].push(assembleStateObjs(strState, pattern));
|
|
||||||
|
|
||||||
tokenObj.regex = transformRegExp(pattern.begin);
|
|
||||||
tokenObj.next = strState;
|
|
||||||
}
|
|
||||||
else if( ( pattern.begin || pattern.end ) && !( pattern.begin && pattern.end ) ) {
|
|
||||||
logDebug("Somehow, there's pattern.begin or pattern.end--but not both?", pattern);
|
|
||||||
}
|
|
||||||
|
|
||||||
else if (pattern.captures) {
|
|
||||||
tokenObj.token.push([]);
|
|
||||||
tokenObj.token.push(pattern.captures);
|
|
||||||
tokenObj.regex = transformRegExp(pattern.match);
|
|
||||||
}
|
|
||||||
|
|
||||||
else if (pattern.match) {
|
|
||||||
tokenObj.token.push(pattern.name);
|
|
||||||
tokenObj.regex = transformRegExp(pattern.match);
|
|
||||||
}
|
|
||||||
|
|
||||||
else if (pattern.include) {
|
|
||||||
tokenObj.token.push(pattern.include);
|
|
||||||
tokenObj.regex = "";
|
|
||||||
}
|
|
||||||
|
|
||||||
else {
|
|
||||||
tokenObj.token.push("");
|
|
||||||
tokenObj.regex = "";
|
|
||||||
logDebug("I've gone through every choice, and have no clue what this is:", pattern);
|
|
||||||
}
|
|
||||||
|
|
||||||
// sometimes captures have names--not sure when or why
|
|
||||||
if (pattern.name) {
|
|
||||||
tokenObj.token.push(pattern.name);
|
|
||||||
}
|
|
||||||
|
|
||||||
startState.start.push(tokenObj);
|
|
||||||
});
|
|
||||||
|
|
||||||
var resultingObj = startState;
|
|
||||||
|
|
||||||
for (var state in statesObj) {
|
|
||||||
resultingObj[state] = statesObj[state];
|
|
||||||
}
|
|
||||||
|
|
||||||
return restoreComments(JSON.stringify(resultingObj, null, " "));
|
|
||||||
}
|
|
||||||
|
|
||||||
function fillTemplate(template, replacements) {
|
|
||||||
return template.replace(/%(.+?)%/g, function(str, m) {
|
|
||||||
return replacements[m] || "";
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
|
// cli stuff
|
||||||
var modeTemplate = fs.readFileSync(__dirname + "/mode.tmpl.js", "utf8");
|
var modeTemplate = fs.readFileSync(__dirname + "/mode.tmpl.js", "utf8");
|
||||||
var modeHighlightTemplate = fs.readFileSync(__dirname + "/mode_highlight_rules.tmpl.js", "utf8");
|
var modeHighlightTemplate = fs.readFileSync(__dirname + "/mode_highlight_rules.tmpl.js", "utf8");
|
||||||
|
|
||||||
function convertLanguage(name) {
|
function convertLanguageFile(name) {
|
||||||
var tmLanguage = fs.readFileSync(__dirname + "/" + name, "utf8");
|
var tmLanguage = fs.readFileSync(process.cwd() + "/" + name, "utf8");
|
||||||
parseLanguage(tmLanguage, function(language) {
|
parseLanguage(tmLanguage, function(language) {
|
||||||
var languageHighlightFilename = language.name.replace(/[-_]/g, "").toLowerCase();
|
var languageHighlightFilename = language.name.replace(/[-_]/g, "").toLowerCase();
|
||||||
var languageNameSanitized = language.name.replace(/-/g, "");
|
var languageNameSanitized = language.name.replace(/-/g, "");
|
||||||
|
|
@ -252,32 +155,23 @@ function convertLanguage(name) {
|
||||||
console.log(util.inspect(language.repository, false, 4));
|
console.log(util.inspect(language.repository, false, 4));
|
||||||
}
|
}
|
||||||
|
|
||||||
var languageMode = fillTemplate(modeTemplate, {
|
var languageMode = lib.fillTemplate(modeTemplate, {
|
||||||
language: languageNameSanitized,
|
language: languageNameSanitized,
|
||||||
languageHighlightFilename: languageHighlightFilename
|
languageHighlightFilename: languageHighlightFilename
|
||||||
});
|
});
|
||||||
|
|
||||||
var patterns = extractPatterns(language.patterns);
|
var patterns = extractPatterns(language);
|
||||||
var repository = {};
|
|
||||||
|
|
||||||
if (language.repository) {
|
var languageHighlightRules = lib.fillTemplate(modeHighlightTemplate, {
|
||||||
for (var r in language.repository) {
|
|
||||||
repository[r] = language.repository[r];
|
|
||||||
}
|
|
||||||
repository = restoreComments(JSON.stringify(repository, null, " "));
|
|
||||||
}
|
|
||||||
|
|
||||||
var languageHighlightRules = fillTemplate(modeHighlightTemplate, {
|
|
||||||
language: languageNameSanitized,
|
language: languageNameSanitized,
|
||||||
languageTokens: patterns,
|
languageTokens: patterns.trim(),
|
||||||
repositoryRules: "/*** START REPOSITORY RULES\n" + repository + "\nEND REPOSITORY RULES ***/",
|
|
||||||
uuid: language.uuid,
|
uuid: language.uuid,
|
||||||
name: name
|
name: name
|
||||||
});
|
});
|
||||||
|
|
||||||
if (devMode) {
|
if (devMode) {
|
||||||
console.log(languageMode)
|
console.log(languageMode);
|
||||||
console.log(languageHighlightRules)
|
console.log(languageHighlightRules);
|
||||||
console.log("Not writing, 'cause we're in dev mode, baby.");
|
console.log("Not writing, 'cause we're in dev mode, baby.");
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
|
|
@ -287,8 +181,11 @@ function convertLanguage(name) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var args = process.argv.splice(2);
|
||||||
|
var tmLanguageFile = args[0];
|
||||||
|
var devMode = args[1];
|
||||||
if (tmLanguageFile === undefined) {
|
if (tmLanguageFile === undefined) {
|
||||||
console.error("Please pass in a language file via the command line.");
|
console.error("Please pass in a language file via the command line.");
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
convertLanguage(tmLanguageFile);
|
convertLanguageFile(tmLanguageFile);
|
||||||
Loading…
Add table
Add a link
Reference in a new issue