use ace/tokenizer for parsing snippets

lib/ace/snippets.js

@@ -33,76 +33,78 @@ define(function(require, exports, module) {
 var Range = require("./range").Range
 var lang = require("./lib/lang")
 var HashHandler = require("./keyboard/hash_handler").HashHandler;
+var Tokenizer = require("./tokenizer").Tokenizer;
 
 var SnippetManager = function() {
     this.snippets = [];
 };
 
 (function() {
-    this.tokenizeTmSnippet = function(str) {
+    this.getTokenizer = function() {
-        var stringBuilder = [];
+        function TabstopToken(str, _, stack) {
-        var addText = function(str) {
+            str = str.substr(1);
-            str && stringBuilder.push(str);
+            if (/^\d+$/.test(str))
-        };
+                return [{tabstopId: parseInt(str, 10)}];
-        var addVar = function(text, placeholder) {
+            return [{text: str}]
-            if (/^\d+$/.test(text))
-                placeholder.tabstopId = parseInt(text, 10);
-            else
-                placeholder.text = text;
-            addText(placeholder);
-            return placeholder;
-        };
-        str = str.replace(/\r/g, "");
-        var stack = [], index = 0, m;
-        // m[1] = \\([\$}`\\]) escapes
-        // m[2] = }
-        // m[3] = \$(\w+)  variables or tabstops
-        // m[4] = \$\{([\dA-Z_]+)
-        // m[5] = format string
-        var re = /\\([\$}`\\])|(})|\n|\$(\d+|\w+)|\$\{([\dA-Z_]+)([\/|`])?(\:?)/g;
-        var formatters = {
-            "/": /((\/(?:\\.|[^\/])+){1,20})\/\w*/g, // replace or guard
-            "|": /|[^|}]+|/g, // choice list
-            "`": /`(?:\\`|[^`])+?`/g // interpolation
-        };
-        var readFormatString = function(quote, placeholder) {
-            var pos = re.lastIndex;
-            var formatRe = formatters[quote];
-            formatRe.lastIndex = pos - 1;
-            var m = formatRe.exec(str);
-            if (!m)
-                return;
-            if (quote == "`") {
-                placeholder.interpolation = m[0].replace(/\\`/g, "")
-            } else if (quote == "|") {
-                placeholder.choices = m[0].split(/,/);
-            } else if (quote == "/") {
-                placeholder[m[1] == m[2] ? "fmt" : "guard"] = m[0];
-            }
-            re.lastIndex = formatRe.lastIndex;
-        };
-        while (m = re.exec(str)) {
-            addText(str.substring(index, m.index)); // skipped text
-            index = m.index + m[0].length;
-            if (m[1]) { // escape
-                addText(m[1] == "}" && !stack.length ? m[0] : m[1]);
-            } else if (m[3]) { // variable
-                addVar(m[3], {});
-            } else if (m[4]) { // variable
-                var placeholder = addVar(m[4], {});
-                if (m[5])
-                    readFormatString(m[5], placeholder);
-                if (stack[0])
-                    stack[0].child = placeholder;
-                stack.unshift(placeholder);
-            } else if (m[2] && stack.length) {
-                addText(stack.shift());
-            } else {
-                addText(m[0]);
-            }
         }
-        addText(str.substring(index));
+        function escape(ch) {
-        return stringBuilder;
+            return "(?:[^\\\\" + ch + "]|\\\\.)";
+        }
+        SnippetManager.$tokenizer = new Tokenizer({
+            start: [
+                {regex: /\\[$}`\\]/, token: function(val, state, stack) {
+                    if (val[1] == "}" && !stack.length)
+                        return [val];
+                    return [val[1]];
+                }},
+                {regex: /}/, token: function(val, state, stack) {
+                    return [stack.length ? stack.shift() : val];
+                }},
+                {regex: /\$(?:\d+|\w+)/, token: TabstopToken},
+                {regex: /\$\{[\dA-Z_a-z]+/, token: function(str, state, stack) {
+                    var t = TabstopToken(str.substr(1), state, stack);
+                    stack.unshift(t[0]);
+                    return t;
+                }, next: "snippetVar"},
+                {regex: /\n/, token: "newline"}
+            ],
+            snippetVar: [
+                {regex: "\\|" + escape("\\|") + "*\\|", token: function(val, state, stack) {
+                    stack[0].choices = val.slice(1, -1).split(",");
+                }, next: "start"},
+                {regex: "/(" + escape("/") + "+)/(?:(" + escape("/") + "*)/)(\\w*):?",
+                 token: function(val, state, stack) {
+                    val = this.splitRegex.exec(val);
+                    var ts = stack[0];
+                    ts.guard = val[1];
+                    ts.fmt = val[2];
+                    ts.flag = val[3]
+                    return "";
+                }, next: "start"},
+                {regex: "`" + escape("`") + "*`", token: function(val, state, stack) {
+                    stack[0].code = val.splice(1, -1);
+                    return "";
+                }, next: "start"},
+                {regex: ":|", token: "", next: "start"}
+            ],
+            formatString: [
+				{regex: /\\[ulULEnt/]/, token: "escape"},
+				{include: "$"}
+            ],
+            formatStringVar: [
+
+            ]
+        });
+        SnippetManager.prototype.getTokenizer = function() {
+            return SnippetManager.$tokenizer;
+        }
+        return SnippetManager.$tokenizer;
+    };
+
+    this.tokenizeTmSnippet = function(str) {
+        return this.getTokenizer().getLineTokens(str).tokens.map(function(x) {
+            return x.value || x;
+        });
     };
 
     this.$getDefaultValue = function(editor, name) {
@@ -141,29 +143,9 @@ var SnippetManager = function() {
     };
 
     // returns string formatted according to http://manual.macromates.com/en/regular_expressions#replacement_string_syntax_format_strings
-    this.tmStrFormat = function(str, fmt) {
+    this.tmStrFormat = function(str, regex, fmt, flags) {
-        fmt = fmt.split("/");
+        var re = new RegExp(regex, flags.replace(/[^gi]/, ""));		
-        fmt.shift();
+        var fmtTokens = this.getTokenizer().getLineTokens(fmt, "formatString");
-        if (fmt.length < 3)
-            return str;
-        var flags = fmt.pop().replace(/[^gmi]/g, "");
-        var search = fmt.shift();
-        while (search[search.length - 1] == "\\")
-            search += "/" + fmt.shift();
-        fmt = fmt.join("/");
-        var re = new RegExp(search, flags);
-
-        var parseRe = /\\.|\\([ulULE])|\$(\d+)|\${(\d+)}|\(?(\d+):|\)/
-        var fmtParts = [];
-        while (m = parseRe.exec(fmt)) {
-            if (m[1]) {
-
-            } else if (m[2] || m[3]) {
-
-            } else if (m[4]) {
-
-            }
-        }
 
         return str.replace(re, function() {
             var matches = arguments;
@@ -180,22 +162,22 @@ var SnippetManager = function() {
             var ch = snippet[i]
             if (typeof ch == "string") {
                 result.push(ch);
-            } else if (typeof ch == "object" && !ch.processed) {
+            } else if (typeof ch != "object" || ch.processed) {
-                if (ch.text) {
+                continue;
-                    var value = this.getVariableValue(editor, ch.text);
+            } else if (ch.text) {
-                    if (value) {
+                var value = this.getVariableValue(editor, ch.text);
-                        var i1 = snippet.indexOf(ch, i + 1);
+                if (value) {
-                        if (i1 != -1)
+                    var i1 = snippet.indexOf(ch, i + 1);
-                            i = i1;
+                    if (i1 != -1)
-                        if (ch.fmt)
+                        i = i1;
-                            value = this.tmStrFormat(value, ch.fmt);
+                    if (ch.fmt)
-                        result.push(value);
+                        value = this.tmStrFormat(value, ch.fmt);
-                    } else {
+                    result.push(value);
-                        ch.processed = true;
+                } else {
-                    }
+                    ch.processed = true;
-                } else if (ch.tabstopId != null) {
-                    result.push(ch);
                 }
+            } else if (ch.tabstopId != null) {
+                result.push(ch);
             }
         }
         return result;
@@ -311,10 +293,9 @@ var SnippetManager = function() {
         if (!snippet)
             return false;
 
-        editor.session.doc.removeInLine(
+        editor.session.doc.removeInLine(cursor.row,
-            cursor.row, 
+            cursor.column - snippet.replaceBefore.length,
-            cursor.column - snippet.matchBefore[0].length,
+            cursor.column + snippet.replaceAfter.length
-            cursor.column - snippet.matchAfter[0].length
         );
         this.insertSnippet(editor, snippet.content, snippet.matchBefore, snippet.matchAfter);
         return true;
@@ -332,6 +313,8 @@ var SnippetManager = function() {
 
             s.matchBefore = s.startRe ? s.startRe.exec(before) : [""];
             s.matchAfter = s.endRe ? s.endRe.exec(after) : [""];
+            s.replaceBefore = s.triggerRe ? s.triggerRe.exec(before)[0] : "";
+            s.replaceAfter = s.endTriggerRe ? s.endTriggerRe.exec(after)[0] : "";
             return s;
         }
     };
@@ -362,6 +345,10 @@ var SnippetManager = function() {
                 s.endRe = "^" + s.endRe;
             if (s.endRe)
                 s.endRe = new RegExp(s.endRe);
+            if (s.trigger)
+                s.triggerRe = new RegExp(s.trigger + "$");
+            if (s.endTrigger)
+                s.endTriggerRe = new RegExp("^" + s.endTrigger);
         };
 
         if (snippets.content)
@@ -390,7 +377,7 @@ var SnippetManager = function() {
                     snippet.guard = val[1];
                     snippet.trigger = val[2];
                     snippet.endTrigger = val[3];
-                    snippet.endGuard = val[3];
+                    snippet.endGuard = val[4];
                 } else if (key == "snippet") {
                     val = val.split(/^(\S*)(?:\s(.*))?$/);
                     snippet.tabTrigger = val[1];
@@ -617,7 +604,7 @@ require("ace/lib/dom").importCssString("\
     -moz-box-sizing: border-box;\
     box-sizing: border-box;\
     background: rgba(194, 193, 208, 0.09);\
-    border: 1px dotted rgba(119, 116, 139, 0.5);\
+    border: 1px dotted rgba(211, 208, 235, 0.62);\
     position: absolute;\
 }");
 

lib/ace/snippets/javascript.snippets

@@ -1,7 +1,6 @@
 # Prototype
 snippet proto
-	${1:class_name}.prototype.${2:method_name} =
+	${1:class_name}.prototype.${2:method_name} = function(${3:first_argument}) {
-	function(${3:first_argument}) {
 		${4:// body...}
 	};
 # Function
@@ -10,16 +9,18 @@ snippet fun
 		${3:// body...}
 	}
 # Anonymous Function
-snippet f
+regex /\b(\()?/f//(\))?/
-	function(${1}) {
+name f
-		${3}
+	function($1) {
-	}${2:;}
+		${0:$TM_SELECTED_TEXT}
+	};
 # Immediate function
-regex //f\(/\)?/
+trigger \(?f\(
+endTrigger \)?
 snippet f(
 	(function(${1}) {
 		${3:/* code */}
-	}(${2}));
+	}(${1}));
 # if
 snippet if
 	if (${1:true}) {
@@ -92,7 +93,7 @@ snippet ret
 # for (property in object ) { ... }
 snippet fori
 	for (var ${1:prop} in ${2:Things}) {
-		${3:$2[$1]}
+		${0:$2[$1]}
 	}
 # hasOwnProperty
 snippet has
@@ -157,22 +158,47 @@ regex /^\s*/clas{0,2}/
 	}).call(${1:class}.prototype);
 # 
 snippet for-
-	for (var ${20:i} = ${1:Things}.length; ${20:i}--; ) {
+	for (var ${1:i} = ${2:Things}.length; ${1:i}--; ) {
-		${100:${1:Things}[${20:i}];
+		${0:${2:Things}[${1:i}];}
 	}
-	$0
-
-
 # for (...) {...}
 snippet for
-	for (var ${2:i} = 0; $2 < ${1:Things}.length; $2${3:++}) {
+	for (var ${1\n:i} = 0; $1 < ${2/\n/:Things}.length; $1${3/\n/:++}) {
-		${4:$1[$2]}
+		${0:$2[$1]}
 	}
 # for (...) {...} (Improved Native For-Loop)
 snippet forr
-	for (var ${2:i} = ${1:Things}.length - 1; $2 >= 0; $2${3:--}) {
+	for (var ${1:i} = ${2:Things}.length - 1; $1 >= 0; $1${3:--}) {
-		${4:$1[$2]}
+		${0:$2[$1]}
 	}
 snippet for-
 
 snippet for ?in
+
+
+regex /^\s*/for (\w+\b)?( \d+)( \d+)?/
+	
+regex /^\s*/for (\w+) in (\w+)/
+	${include:for $1=$M1,$2=$M2}
+regex /^\s*/for (\w*) (\w+)(.l?e?n?[ght]{0,3})/
+	${include:for $1=$M1,$2=$M2}
+regex /^\s*/for (\w+) in (\w+)/
+	${include:for $1=$M1,$2=$M2}
+
+
+#modules
+snippet def
+	define(function(require, exports, module) {
+	"use strict";
+	${includeRepeated:Req}
+	
+	$TM_SELECTED_TEXT
+	});
+snippet req
+guard ^\s*
+	var ${1/.*\///} = require("${1}");
+	$0
+snippet Req
+guard ^\s*
+	var ${1/.*\/(.)/\u$1/} = require("${1}").${1_0};
+	$0

lib/ace/snippets_test.js

@@ -39,14 +39,50 @@ var SnippetManager = require("./snippets").SnippetManager;
 var assert = require("./test/assertions");
 
 module.exports = {
-    "!test: textmate style format strings" : function() {
+    "test: textmate style format strings" : function() {
         var fmt = SnippetManager.tmStrFormat;
         assert.equal(fmt("abc", "/(.)(.)/$1/g"), "ac");
         assert.equal(fmt("abc", "/(.)(.)/$1(?2:Hello(1)2)/g"), "aHello(12)c2)");
-        assert.equal(fmt("abc", "/(.)(.)/\u$1+lL+\u$2e|/g"), "A+lL+be|C+lL+e|");
+        assert.equal(fmt("abc", "/(.)(.)/\\u$1+lL+\\u$2e|/g"), "A+lL+be|C+lL+e|");
-        assert.equal(fmt("aBCD", "/(.)(.)/\U$1+lL+\l$2e/g"), "A+LL+be");
+        assert.equal(fmt("aBCD", "/(.)(.)/\\U$1+lL+\\l$2e/g"), "A+LL+be");
     },
+    "test: parse snipmate file" : function() {
+        var expected = [{
+            name: "a",
+            guard: "(?:(=)|(:))?s*)",
+            trigger: "\\(?f",
+            endTrigger: "\\)",
+            endGuard: "\\)",
+            content: "{$0}\n"
+         }, {
+            tabTrigger: "f",
+            name: "function",
+            content: "function"
+        }];
+        var parsed = SnippetManager.parseSnippetFile(
+            "name a\nregex /(?:(=)|(:))?\s*)/\\(?f/\\)/\n\t{$0}" +
+            "\n\t\n\n#function\nsnippet f function\n\tfunction"
+        );
         
+        assert.equal(JSON.stringify(expected), JSON.stringify(parsed))
+    },
+	"test: parse snippet": function() {
+		var content = "-\\$$2a${1:x${$2:y$3}\\n\\}$TM_SELECTION}";
+		var tokens = SnippetManager.tokenizeTmSnippet(content);
+		assert.equal(tokens.length, 14);
+		assert.equal(tokens[4] == tokens[13]);
+		assert.equal(tokens[2].tabstopId == 2);
+
+		var content = "\\}${var/as\\/d/\\ul\\//g:s}"
+		var tokens = SnippetManager.tokenizeTmSnippet(content);
+		assert.equal(tokens.length, 4);
+		assert.equal(tokens[1], tokens[3]);
+		assert.equal(tokens[2], "s");
+		assert.equal(tokens[1].text, "var");
+		assert.equal(tokens[1].fmt, "\\ul\\/");
+		assert.equal(tokens[1].guard, "as\\/d");
+		assert.equal(tokens[1].flag, "g");
+	}
 };
 
 });