support multi line strings

BackgroundTokenizer.js

@@ -107,6 +107,9 @@ var keywords = {
 
 getLineTokens = function(line, startState)
 {
+  // regexp must not have capturing parentheses
+  // regexps are ordered -> the first match is used
+  
   var rules = {
     start :
     [
@@ -119,17 +122,27 @@ getLineTokens = function(line, startState)
         regex: "\\/\\*.*?\\*\\/"
       },      
       {
-        token: "comment", // multi line comment in several lines
+        token: "comment", // multi line comment start
         regex: "\\/\\*.*$",
         next: "comment"
       },
       {
         token: "string", // single line
-        regex: '["][^"]*["]'
+        regex: '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]'
+      },
+      {
+        token: "string", // multi line string start
+        regex: '["].*\\\\$',
+        next: "qqstring"
       },      
       {
         token: "string", // single line
-        regex: "['][^']*[']"
+        regex: "['](?:(?:\\\\.)|(?:[^'\\\\]))*?[']"
+      },
+      {
+        token: "string", // multi line string start
+        regex: "['].*\\\\$",
+        next: "qstring"
       },    
       {
         token: "number", // hex
@@ -173,6 +186,30 @@ getLineTokens = function(line, startState)
         token: "comment", // comment spanning whole line
         regex: ".+"
       }
+    ],
+    "qqstring":
+    [
+      {
+        token: "string",
+        regex: '(?:(?:\\\\.)|(?:[^"\\\\]))*?"',
+        next: "start"
+      },
+      {
+        token: "string",
+        regex: '.+'
+      }
+    ],
+    "qstring":
+    [
+      {
+        token: "string",
+        regex: "(?:(?:\\\\.)|(?:[^'\\\\]))*?'",
+        next: "start"
+      },
+      {
+        token: "string",
+        regex: '.+'
+      }
     ]    
   };
   
@@ -197,6 +234,8 @@ getLineTokens = function(line, startState)
   
   var match, tokens = [];
   
+  var lastIndex = 0;
+  
   while (match = re.exec(line))
   {
     var token = {
@@ -204,6 +243,11 @@ getLineTokens = function(line, startState)
       value: match[0]
     }
     
+    if (re.lastIndex == lastIndex) {
+      throw new Error("tokenizer error")
+    }
+    lastIndex = re.lastIndex;
+    
     //console.log(match);
     
     for (var i=0; i < state.length; i++)