Merge branch 'devel' into araq

compiler/ast.nim

@@ -1009,6 +1009,12 @@ proc safeLen*(n: PNode): int {.inline.} =
   if n.kind in {nkNone..nkNilLit} or isNil(n.sons): result = 0
   else: result = len(n.sons)
 
+proc safeArrLen*(n: PNode): int {.inline.} =
+  ## works for array-like objects (strings passed as openArray in VM).
+  if n.kind in {nkStrLit..nkTripleStrLit}:result = len(n.strVal)
+  elif n.kind in {nkNone..nkFloat128Lit}: result = 0
+  else: result = len(n)
+
 proc add*(father, son: PNode) =
   assert son != nil
   if isNil(father.sons): father.sons = @[]

compiler/vm.nim

@@ -508,7 +508,13 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         stackTrace(c, tos, pc, errIndexOutOfBounds)
       let idx = regs[rc].intVal.int
       let src = regs[rb].node
-      if src.kind notin {nkEmpty..nkNilLit} and idx <% src.len:
+      if src.kind in {nkStrLit..nkTripleStrLit}:
+        if idx <% src.strVal.len:
+          regs[ra].node = newNodeI(nkCharLit, c.debug[pc])
+          regs[ra].node.intVal = src.strVal[idx].ord
+        else:
+          stackTrace(c, tos, pc, errIndexOutOfBounds)
+      elif src.kind notin {nkEmpty..nkFloat128Lit} and idx <% src.len:
         regs[ra].node = src.sons[idx]
       else:
         stackTrace(c, tos, pc, errIndexOutOfBounds)
@@ -526,8 +532,14 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       # a[b] = c
       decodeBC(rkNode)
       let idx = regs[rb].intVal.int
-      if idx <% regs[ra].node.len:
+      let arr = regs[ra].node
-        putIntoNode(regs[ra].node.sons[idx], regs[rc])
+      if arr.kind in {nkStrLit..nkTripleStrLit}:
+        if idx <% arr.strVal.len:
+          arr.strVal[idx] = chr(regs[rc].intVal)
+        else:
+          stackTrace(c, tos, pc, errIndexOutOfBounds)
+      elif idx <% arr.len:
+        putIntoNode(arr.sons[idx], regs[rc])
       else:
         stackTrace(c, tos, pc, errIndexOutOfBounds)
     of opcLdObj:
@@ -641,8 +653,14 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcLenSeq:
       decodeBImm(rkInt)
       #assert regs[rb].kind == nkBracket
-      # also used by mNLen:
+      let high = (imm and 1) # discard flags
-      regs[ra].intVal = regs[rb].node.safeLen - imm
+      if (imm and nimNodeFlag) != 0:
+        # used by mNLen (NimNode.len)
+        regs[ra].intVal = regs[rb].node.safeLen - high
+      else:
+        # safeArrLen also return string node len
+        # used when string is passed as openArray in VM
+        regs[ra].intVal = regs[rb].node.safeArrLen - high
     of opcLenStr:
       decodeBImm(rkInt)
       assert regs[rb].kind == rkNode

compiler/vmdef.nim

@@ -234,6 +234,9 @@ const
   slotSomeTemp* = slotTempUnknown
   relativeJumps* = {opcTJmp, opcFJmp, opcJmp, opcJmpBack}
 
+# flag is used to signal opcSeqLen if node is NimNode.
+const nimNodeFlag* = 16
+
 template opcode*(x: TInstr): TOpcode = TOpcode(x.uint32 and 0xff'u32)
 template regA*(x: TInstr): TRegister = TRegister(x.uint32 shr 8'u32 and 0xff'u32)
 template regB*(x: TInstr): TRegister = TRegister(x.uint32 shr 16'u32 and 0xff'u32)

compiler/vmgen.nim

@@ -625,10 +625,10 @@ proc genUnaryABC(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
   c.gABC(n, opc, dest, tmp)
   c.freeTemp(tmp)
 
-proc genUnaryABI(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
+proc genUnaryABI(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; imm: BiggestInt=0) =
   let tmp = c.genx(n.sons[1])
   if dest < 0: dest = c.getTemp(n.typ)
-  c.gABI(n, opc, dest, tmp, 0)
+  c.gABI(n, opc, dest, tmp, imm)
   c.freeTemp(tmp)
 
 proc genBinaryABC(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
@@ -1021,7 +1021,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     c.freeTemp(tmp)
   of mSlurp: genUnaryABC(c, n, dest, opcSlurp)
   of mStaticExec: genBinaryABCD(c, n, dest, opcGorge)
-  of mNLen: genUnaryABI(c, n, dest, opcLenSeq)
+  of mNLen: genUnaryABI(c, n, dest, opcLenSeq, nimNodeFlag)
   of mGetImpl: genUnaryABC(c, n, dest, opcGetImpl)
   of mNChild: genBinaryABC(c, n, dest, opcNChild)
   of mNSetChild, mNDel:

lib/impure/db_postgres.nim

@@ -516,10 +516,13 @@ proc open*(connection, user, password, database: string): DbConn {.
   ##
   ## See http://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
   ## for more information.
-  ##
+  let
-  ## Note that the connection parameter is not used but exists to maintain
+    colonPos = connection.find(':')
-  ## the nim db api.
+    host = if colonPos < 0: connection
-  result = pqsetdbLogin(nil, nil, nil, nil, database, user, password)
+           else: substr(connection, 0, colonPos-1)
+    port = if colonPos < 0: ""
+           else: substr(connection, colonPos+1)
+  result = pqsetdbLogin(host, port, nil, nil, database, user, password)
   if pqStatus(result) != CONNECTION_OK: dbError(result) # result = nil
 
 proc setEncoding*(connection: DbConn, encoding: string): bool {.

lib/pure/asynchttpserver.nim

@@ -178,7 +178,12 @@ proc processClient(client: AsyncSocket, address: string,
         except ValueError:
           asyncCheck request.respondError(Http400)
           continue
-      of 1: parseUri(linePart, request.url)
+      of 1: 
+        try:
+          parseUri(linePart, request.url)
+        except ValueError:
+          asyncCheck request.respondError(Http400) 
+          continue
       of 2:
         try:
           request.protocol = parseProtocol(linePart)

lib/pure/collections/critbits.nim

@@ -110,6 +110,42 @@ proc rawInsert[T](c: var CritBitTree[T], key: string): Node[T] =
     wherep[] = inner
   inc c.count
 
+proc exclImpl[T](c: var CritBitTree[T], key: string) : int =
+  var p = c.root
+  var wherep = addr(c.root)
+  var whereq: ptr Node[T] = nil
+  if p == nil: return c.count
+  var dir = 0
+  var q: Node[T]
+  while not p.isLeaf:
+    whereq = wherep
+    q = p
+    let ch = if p.byte < key.len: key[p.byte] else: '\0'
+    dir = (1 + (ch.ord or p.otherBits.ord)) shr 8
+    wherep = addr(p.child[dir])
+    p = wherep[]
+  if p.key == key:
+    # else: not in tree at all
+    if whereq == nil:
+      c.root = nil
+    else:
+      whereq[] = q.child[1 - dir]
+    dec c.count
+
+  return c.count
+
+proc excl*[T](c: var CritBitTree[T], key: string) =
+  ## removes `key` (and its associated value) from the set `c`.
+  ## If the `key` does not exist, nothing happens.
+  discard exclImpl(c, key)
+
+proc missingOrExcl*[T](c: var CritBitTree[T], key: string): bool =
+  ## Returns true iff `c` does not contain the given `key`. If the key
+  ## does exist, c.excl(key) is performed. 
+  let oldCount = c.count 
+  var n = exclImpl(c, key)
+  result = c.count == oldCount
+
 proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: T): bool =
   ## returns true iff `c` contains the given `key`. If the key does not exist
   ## ``c[key] = val`` is performed.
@@ -171,30 +207,6 @@ proc mget*[T](c: var CritBitTree[T], key: string): var T {.inline, deprecated.}
   ## Use ```[]``` instead.
   get(c, key)
 
-proc excl*[T](c: var CritBitTree[T], key: string) =
-  ## removes `key` (and its associated value) from the set `c`.
-  ## If the `key` does not exist, nothing happens.
-  var p = c.root
-  var wherep = addr(c.root)
-  var whereq: ptr Node[T] = nil
-  if p == nil: return
-  var dir = 0
-  var q: Node[T]
-  while not p.isLeaf:
-    whereq = wherep
-    q = p
-    let ch = if p.byte < key.len: key[p.byte] else: '\0'
-    dir = (1 + (ch.ord or p.otherBits.ord)) shr 8
-    wherep = addr(p.child[dir])
-    p = wherep[]
-  if p.key == key:
-    # else: not in tree at all
-    if whereq == nil:
-      c.root = nil
-    else:
-      whereq[] = q.child[1 - dir]
-    dec c.count
-
 iterator leaves[T](n: Node[T]): Node[T] =
   if n != nil:
     # XXX actually we could compute the necessary stack size in advance:
@@ -326,10 +338,15 @@ when isMainModule:
   r.incl "def"
   r.incl "definition"
   r.incl "prefix"
+  r.incl "foo"
 
   doAssert r.contains"def"
 
   r.excl "def"
+  assert r.missingOrExcl("foo") == false
+  assert "foo" notin toSeq(r.items)
+
+  assert r.missingOrExcl("foo") == true
 
   assert toSeq(r.items) == @["abc", "definition", "prefix", "xyz"]
 

lib/pure/collections/intsets.nim

@@ -131,8 +131,7 @@ proc incl*(s: var IntSet, key: int) =
     # fall through:
   bitincl(s, key)
 
-proc excl*(s: var IntSet, key: int) =
+proc exclImpl(s: var IntSet, key: int) =
-  ## excludes `key` from the set `s`.
   if s.elems <= s.a.len:
     for i in 0..<s.elems:
       if s.a[i] == key:
@@ -146,6 +145,17 @@ proc excl*(s: var IntSet, key: int) =
       t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] and
           not `shl`(1, u and IntMask)
 
+proc excl*(s: var IntSet, key: int) =
+  ## excludes `key` from the set `s`.
+  exclImpl(s, key)
+
+proc missingOrExcl*(s: var IntSet, key: int) : bool =
+  ## returns true if `s` does not contain `key`, otherwise
+  ## `key` is removed from `s` and false is returned.
+  var count = s.elems
+  exclImpl(s, key)
+  result = count == s.elems 
+
 proc containsOrIncl*(s: var IntSet, key: int): bool =
   ## returns true if `s` contains `key`, otherwise `key` is included in `s`
   ## and false is returned.
@@ -270,6 +280,17 @@ when isMainModule:
   x.incl(7)
   x.incl(1056)
 
+  x.incl(1044)
+  x.excl(1044) 
+
+  assert x.containsOrIncl(888) == false
+  assert 888 in x
+  assert x.containsOrIncl(888) == true
+
+  assert x.missingOrExcl(888) == false
+  assert 888 notin x
+  assert x.missingOrExcl(888) == true
+
   var xs = toSeq(items(x))
   xs.sort(cmp[int])
   assert xs == @[1, 2, 7, 1056]

lib/pure/collections/sets.nim

@@ -278,21 +278,15 @@ template default[T](t: typedesc[T]): T =
   var v: T
   v
 
-proc excl*[A](s: var HashSet[A], key: A) =
+proc exclImpl[A](s: var HashSet[A], key: A) : bool {. inline .} =
-  ## Excludes `key` from the set `s`.
-  ##
-  ## This doesn't do anything if `key` is not found in `s`. Example:
-  ##
-  ## .. code-block::
-  ##   var s = toSet([2, 3, 6, 7])
-  ##   s.excl(2)
-  ##   s.excl(2)
-  ##   assert s.len == 3
   assert s.isValid, "The set needs to be initialized."
   var hc: Hash
   var i = rawGet(s, key, hc)
   var msk = high(s.data)
+  result = true
+
   if i >= 0:
+    result = false
     s.data[i].hcode = 0
     s.data[i].key = default(type(s.data[i].key))
     dec(s.counter)
@@ -308,6 +302,30 @@ proc excl*[A](s: var HashSet[A], key: A) =
         r = s.data[i].hcode and msk    # "home" location of key@i
       shallowCopy(s.data[j], s.data[i]) # data[j] will be marked EMPTY next loop
 
+proc missingOrExcl*[A](s: var HashSet[A], key: A): bool =
+  ## Excludes `key` in the set `s` and tells if `key` was removed from `s`.
+  ##
+  ## The difference with regards to the `excl() <#excl,TSet[A],A>`_ proc is
+  ## that this proc returns `true` if `key` was not present in `s`. Example:
+  ##
+  ## .. code-block::
+  ##  var s = toSet([2, 3, 6, 7])
+  ##  assert s.missingOrExcl(4) == true
+  ##  assert s.missingOrExcl(6) == false
+  exclImpl(s, key)
+
+proc excl*[A](s: var HashSet[A], key: A) =
+  ## Excludes `key` from the set `s`.
+  ##
+  ## This doesn't do anything if `key` is not found in `s`. Example:
+  ##
+  ## .. code-block::
+  ##   var s = toSet([2, 3, 6, 7])
+  ##   s.excl(2)
+  ##   s.excl(2)
+  ##   assert s.len == 3
+  discard exclImpl(s, key)
+
 proc excl*[A](s: var HashSet[A], other: HashSet[A]) =
   ## Excludes everything in `other` from `s`.
   ##
@@ -322,7 +340,7 @@ proc excl*[A](s: var HashSet[A], other: HashSet[A]) =
   ##   # --> {1, 3, 5}
   assert s.isValid, "The set `s` needs to be initialized."
   assert other.isValid, "The set `other` needs to be initialized."
-  for item in other: excl(s, item)
+  for item in other: discard exclImpl(s, item)
 
 proc containsOrIncl*[A](s: var HashSet[A], key: A): bool =
   ## Includes `key` in the set `s` and tells if `key` was added to `s`.

lib/pure/json.nim

@@ -996,24 +996,17 @@ proc nl(s: var string, ml: bool) =
 proc escapeJson*(s: string; result: var string) =
   ## Converts a string `s` to its JSON representation.
   ## Appends to ``result``.
-  const
-    HexChars = "0123456789ABCDEF"
   result.add("\"")
-  for x in runes(s):
+  for c in s:
-    var r = int(x)
-    if r >= 32 and r <= 126:
-      var c = chr(r)
     case c
+    of '\L': result.add("\\n")
+    of '\b': result.add("\\b")
+    of '\f': result.add("\\f")
+    of '\t': result.add("\\t")
+    of '\r': result.add("\\r")
     of '"': result.add("\\\"")
     of '\\': result.add("\\\\")
     else: result.add(c)
-    else:
-      # toHex inlined for more speed (saves stupid string allocations):
-      result.add("\\u0000")
-      let start = result.len - 4
-      for j in countdown(3, 0):
-        result[j+start] = HexChars[r and 0xF]
-        r = r shr 4
   result.add("\"")
 
 proc escapeJson*(s: string): string =
@@ -1925,7 +1918,7 @@ when isMainModule:
     var parsed2 = parseFile("tests/testdata/jsontest2.json")
     doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}")
 
-  doAssert escapeJson("\10FoobarÄ") == "\"\\u000AFoobar\\u00C4\""
+  doAssert escapeJson("\10Foo🎃barÄ") == "\"\\nFoo🎃barÄ\""
 
   # Test with extra data
   when not defined(js):

lib/pure/memfiles.nim

@@ -188,7 +188,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
     if low == INVALID_FILE_SIZE:
       fail(osLastError(), "error getting file size")
     else:
-      var fileSize = (int64(hi) shr 32) or low
+      var fileSize = (int64(hi) shl 32) or int64(uint32(low))
       if mappedSize != -1: result.size = min(fileSize, mappedSize).int
       else: result.size = fileSize.int
 

lib/pure/strutils.nim

@@ -776,7 +776,8 @@ proc countLines*(s: string): int {.noSideEffect,
   ##
   ## In this context, a line is any string seperated by a newline combination.
   ## A line can be an empty string.
-  var i = 1
+  result = 1
+  var i = 0
   while i < s.len:
     case s[i]
     of '\c':

lib/system.nim

@@ -3401,6 +3401,11 @@ template spliceImpl(s, a, L, b: untyped): untyped =
 when hasAlloc or defined(nimscript):
   proc `[]`*(s: string, x: Slice[int]): string {.inline.} =
     ## slice operation for strings.
+    ## returns the inclusive range [s[x.a], s[x.b]]:
+    ## 
+    ## .. code-block:: nim
+    ##    var s = "abcdef"
+    ##    assert s[1..3] == "bcd" 
     result = s.substr(x.a, x.b)
 
   proc `[]=`*(s: var string, x: Slice[int], b: string) =
@@ -3421,6 +3426,11 @@ when hasAlloc or defined(nimscript):
 
 proc `[]`*[Idx, T](a: array[Idx, T], x: Slice[int]): seq[T] =
   ## slice operation for arrays.
+  ## returns the inclusive range [a[x.a], a[x.b]]:
+  ## 
+  ## .. code-block:: nim
+  ##    var a = [1,2,3,4]
+  ##    assert a[0..2] == @[1,2,3]
   when low(a) < 0:
     {.error: "Slicing for arrays with negative indices is unsupported.".}
   var L = x.b - x.a + 1
@@ -3455,6 +3465,11 @@ proc `[]=`*[Idx, T](a: var array[Idx, T], x: Slice[Idx], b: openArray[T]) =
 
 proc `[]`*[T](s: seq[T], x: Slice[int]): seq[T] =
   ## slice operation for sequences.
+  ## returns the inclusive range [s[x.a], s[x.b]]:
+  ## 
+  ## .. code-block:: nim
+  ##    var s = @[1,2,3,4]
+  ##    assert s[0..2] == @[1,2,3]
   var a = x.a
   var L = x.b - a + 1
   newSeq(result, L)

nimsuggest/nimsuggest.nim

@@ -365,7 +365,7 @@ proc replEpc(x: ThreadParams) {.thread.} =
     of "epc-error":
       # an unhandled exception forces down the whole process anyway, so we
       # use 'quit' here instead of 'raise'
-      quit("recieved epc error: " & $messageBuffer)
+      quit("received epc error: " & $messageBuffer)
     else:
       let errMessage = case epcApi
                        of "return", "return-error":

tests/sets/tsets2.nim

@@ -37,16 +37,26 @@ block setTest2:
   t.incl("111")
   t.incl("123")
   t.excl("111")
-
   t.incl("012")
   t.incl("123") # test duplicates
 
   assert "123" in t
   assert "111" notin t # deleted
 
+  assert t.missingOrExcl("000") == true
+  assert "000" notin t
+  assert t.missingOrExcl("012") == false
+  assert "012" notin t
+
+  assert t.containsOrIncl("012") == false 
+  assert t.containsOrIncl("012") == true
+  assert "012" in t # added back 
+
   for key in items(data): t.incl(key)
   for key in items(data): assert key in t
 
+  for key in items(data): t.excl(key)
+  for key in items(data): assert key notin t
 
 block orderedSetTest1:
   var t = data.toOrderedSet

tests/stdlib/tmarshal.nim

@@ -1,5 +1,5 @@
 discard """
-  output: '''{"age": 12, "bio": "\u042F Cletus", "blob": [65, 66, 67, 128], "name": "Cletus"}
+  output: '''{"age": 12, "bio": "Я Cletus", "blob": [65, 66, 67, 128], "name": "Cletus"}
 true
 true
 alpha 100

tests/stdlib/tstrutil.nim

@@ -89,9 +89,21 @@ proc testRFind =
   assert "0123456789ABCDEFGAH".rfind({'A'..'C'}, 13) == 12
   assert "0123456789ABCDEFGAH".rfind({'G'..'H'}, 13) == -1
 
+proc testCountLines =
+  proc assertCountLines(s: string) = assert s.countLines == s.splitLines.len
+  assertCountLines("")
+  assertCountLines("\n")
+  assertCountLines("\n\n")
+  assertCountLines("abc")
+  assertCountLines("abc\n123")
+  assertCountLines("abc\n123\n")
+  assertCountLines("\nabc\n123")
+  assertCountLines("\nabc\n123\n")
+
 testDelete()
 testFind()
 testRFind()
+testCountLines()
 
 assert(insertSep($1000_000) == "1_000_000")
 assert(insertSep($232) == "232")

tests/vm/tstring_openarray.nim

@@ -0,0 +1,34 @@
+
+# tests various bug when passing string to openArray argument in VM.
+# bug #6086
+proc map*[T, S](data: openArray[T], op: proc (x: T): S {.closure.}):
+                                                            seq[S]{.inline.} =
+# map inlined from sequtils
+  newSeq(result, data.len)
+  for i in 0..data.len-1: result[i] = op(data[i])
+
+
+proc set_all[T](s: var openArray[T]; val: T) =
+  for i in 0..<s.len:
+    s[i] = val
+
+proc test() =
+    var a0 = "hello_world"
+    var a1 = [1,2,3,4,5,6,7,8,9]
+    var a2 = @[1,2,3,4,5,6,7,8,9]
+    a0.set_all('i')
+    a1.set_all(4)
+    a2.set_all(4)
+    doAssert a0 == "iiiiiiiiiii"
+    doAssert a1 == [4,4,4,4,4,4,4,4,4]
+    doAssert a2 == @[4,4,4,4,4,4,4,4,4]
+
+const constval0 = "hello".map(proc(x: char): char = x)
+const constval1 = [1,2,3,4].map(proc(x: int): int = x)
+
+doAssert("hello".map(proc(x: char): char = x) == constval0)
+doAssert([1,2,3,4].map(proc(x: int): int = x) == constval1)
+
+test()
+static:
+    test()