Merge pull request #3760 from yglukhov/js-unicode-2

Fixed unicode strings in JS. Second try.

compiler/jsgen.nim

@@ -163,8 +163,31 @@ proc mangleName(s: PSym): Rope =
     add(result, rope(s.id))
     s.loc.r = result
 
-proc makeJSString(s: string): Rope =
+proc escapeJSString(s: string): string =
-  (if s.isNil: "null".rope else: strutils.escape(s).rope)
+   result = newStringOfCap(s.len + s.len shr 2)
+   result.add("\"")
+   for c in items(s):
+     case c
+     of '\l': result.add("\\n")
+     of '\r': result.add("\\r")
+     of '\t': result.add("\\t")
+     of '\b': result.add("\\b")
+     of '\a': result.add("\\a")
+     of '\e': result.add("\\e")
+     of '\v': result.add("\\v")
+     of '\\': result.add("\\\\")
+     of '\'': result.add("\\'")
+     of '\"': result.add("\\\"")
+     else: add(result, c)
+   result.add("\"")
+
+proc makeJSString(s: string, escapeNonAscii = true): Rope =
+  if s.isNil:
+    result = "null".rope
+  elif escapeNonAscii:
+    result = strutils.escape(s).rope
+  else:
+    result = escapeJSString(s).rope
 
 include jstypes
 
@@ -577,7 +600,7 @@ proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) =
           if stringSwitch:
             case e.kind
             of nkStrLit..nkTripleStrLit: addf(p.body, "case $1: ",
-                [makeJSString(e.strVal)])
+                [makeJSString(e.strVal, false)])
             else: internalError(e.info, "jsgen.genCaseStmt: 2")
           else:
             gen(p, e, cond)
@@ -1636,10 +1659,10 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
       r.kind = resExpr
   of nkStrLit..nkTripleStrLit:
     if skipTypes(n.typ, abstractVarRange).kind == tyString:
-      useMagic(p, "cstrToNimstr")
+      useMagic(p, "makeNimstrLit")
-      r.res = "cstrToNimstr($1)" % [makeJSString(n.strVal)]
+      r.res = "makeNimstrLit($1)" % [makeJSString(n.strVal)]
     else:
-      r.res = makeJSString(n.strVal)
+      r.res = makeJSString(n.strVal, false)
     r.kind = resExpr
   of nkFloatLit..nkFloat64Lit:
     let f = n.floatVal

lib/system/jssys.nim

@@ -165,15 +165,46 @@ proc SetConstr() {.varargs, asmNoStackFrame, compilerproc.} =
     return result;
   """
 
+proc makeNimstrLit(c: cstring): string {.asmNoStackFrame, compilerproc.} =
+  {.emit: """
+  var ln = `c`.length;
+  var result = new Array(ln + 1);
+  var i = 0;
+  for (; i < ln; ++i) {
+    result[i] = `c`.charCodeAt(i);
+  }
+  result[i] = 0; // terminating zero
+  return result;
+  """.}
+
 proc cstrToNimstr(c: cstring): string {.asmNoStackFrame, compilerproc.} =
-  asm """
+  {.emit: """
-    var result = [];
+  var ln = `c`.length;
-    for (var i = 0; i < `c`.length; ++i) {
+  var result = new Array(ln);
-      result[i] = `c`.charCodeAt(i);
+  var r = 0;
+  for (var i = 0; i < ln; ++i) {
+    var ch = `c`.charCodeAt(i);
+
+    if (ch < 128) {
+      result[r] = ch;
     }
-    result[result.length] = 0; // terminating zero
+    else if((ch > 127) && (ch < 2048)) {
-    return result;
+      result[r] = (ch >> 6) | 192;
-  """
+      ++r;
+      result[r] = (ch & 63) | 128;
+    }
+    else {
+      result[r] = (ch >> 12) | 224;
+      ++r;
+      result[r] = ((ch >> 6) & 63) | 128;
+      ++r;
+      result[r] = (ch & 63) | 128;
+    }
+    ++r;
+  }
+  result[r] = 0; // terminating zero
+  return result;
+  """.}
 
 proc toJSStr(s: string): cstring {.asmNoStackFrame, compilerproc.} =
   asm """

tests/js/tstringitems.nim

@@ -3,22 +3,76 @@ discard """
 Hello'''
 """
 
-# bug #2581
+block: # bug #2581
+  const someVars = [ "Hello" ]
+  var someVars2 = [ "Hello" ]
 
-const someVars = [ "Hello" ]
+  proc getSomeVar: string =
-var someVars2 = [ "Hello" ]
+      for i in someVars:
+          if i == "Hello":
+              result = i
+              break
 
-proc getSomeVar: string =
+  proc getSomeVar2: string =
-    for i in someVars:
+      for i in someVars2:
-        if i == "Hello":
+          if i == "Hello":
-            result = i
+              result = i
-            break
+              break
 
-proc getSomeVar2: string =
+  echo getSomeVar()
-    for i in someVars2:
+  echo getSomeVar2()
-        if i == "Hello":
-            result = i
-            break
 
-echo getSomeVar()
+block: # Test compile-time binary data generation, invalid unicode
-echo getSomeVar2()
+  proc signatureMaker(): string {. compiletime .} =
+    const signatureBytes = [137, 80, 78, 71, 13, 10, 26, 10]
+    result = ""
+    for c in signatureBytes: result.add chr(c)
+
+  const cSig = signatureMaker()
+
+  var rSig = newString(8)
+  rSig[0] = chr(137)
+  rSig[1] = chr(80)
+  rSig[2] = chr(78)
+  rSig[3] = chr(71)
+  rSig[4] = chr(13)
+  rSig[5] = chr(10)
+  rSig[6] = chr(26)
+  rSig[7] = chr(10)
+
+  doAssert(rSig == cSig)
+
+block: # Test unicode strings
+  const constStr = "Привет!"
+  var jsStr : cstring
+  {.emit: """`jsStr`[0] = "Привет!";""".}
+
+  doAssert($jsStr == constStr)
+  var runtimeStr = "При"
+  runtimeStr &= "вет!"
+
+  doAssert(runtimeStr == constStr)
+
+block: # Conversions from/to cstring
+  proc stringSaysHelloInRussian(s: cstring): bool =
+    {.emit: """`result` = (`s` === "Привет!");""".}
+
+  doAssert(stringSaysHelloInRussian("Привет!"))
+
+  const constStr = "Привет!"
+  doAssert(stringSaysHelloInRussian(constStr))
+
+  var rtStr = "Привет!"
+  doAssert(stringSaysHelloInRussian(rtStr))
+
+block: # String case of
+  const constStr = "Привет!"
+  var s = "Привет!"
+
+  case s
+  of constStr: discard
+  else: doAssert(false)
+
+  case s
+  of "Привет!": discard
+  else: doAssert(false)