Merge branch 'devel' of https://github.com/nim-lang/Nim into devel

appveyor.yml

@@ -1,36 +0,0 @@
-clone_depth: 5
-
-artifacts:
-  - path: bin\nim.exe
-
-platform:
-  - x64
-
-before_build:
-  - git log -1
-  - C:\msys64\usr\bin\bash -lc "pacman --noconfirm -S zlib-devel"
-  - appveyor DownloadFile http://nim-lang.org/download/dlls.zip
-  - 7z e dlls.zip -odlls
-  - del dlls\libcurl.dll
-  - appveyor DownloadFile http://flatassembler.net/fasmw17139.zip
-  - 7z e fasmw17139.zip -obin fasm.exe
-
-build_script:
-  - SET PATH=C:\msys64\mingw64\bin;dlls;bin;%PATH%
-  - gcc -v
-  - git clone -q --depth 1 https://github.com/nim-lang/csources
-  - cd csources
-  - build64.bat
-  - cd ..
-  - nim c koch
-  - koch boot
-  - koch boot -d:release
-
-before_test:
-  - nim e install_nimble.nims
-  - nimble update
-  - nimble install zip
-
-test_script:
-  - nim c --taintMode:on tests/testament/tester
-  - tests\testament\tester --pedantic all

compiler/ast.nim

@@ -500,8 +500,7 @@ type
     skResult,             # special 'result' variable
     skProc,               # a proc
     skMethod,             # a method
-    skIterator,           # an inline iterator
+    skIterator,           # an iterator
-    skClosureIterator,    # a resumable closure iterator
     skConverter,          # a type converter
     skMacro,              # a macro
     skTemplate,           # a template; currently also misused for user-defined
@@ -518,7 +517,7 @@ type
   TSymKinds* = set[TSymKind]
 
 const
-  routineKinds* = {skProc, skMethod, skIterator, skClosureIterator,
+  routineKinds* = {skProc, skMethod, skIterator,
                    skConverter, skMacro, skTemplate}
   tfIncompleteStruct* = tfVarargs
   tfUncheckedArray* = tfVarargs
@@ -905,7 +904,7 @@ type
 # the poor naming choices in the standard library.
 
 const
-  OverloadableSyms* = {skProc, skMethod, skIterator, skClosureIterator,
+  OverloadableSyms* = {skProc, skMethod, skIterator,
     skConverter, skModule, skTemplate, skMacro}
 
   GenericTypes*: TTypeKinds = {tyGenericInvocation, tyGenericBody,
@@ -929,11 +928,11 @@ const
   NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr, tySequence,
     tyProc, tyString, tyError}
   ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType,
-    skIterator, skClosureIterator,
+    skIterator,
     skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
                                       nfDotSetter, nfDotField,
-                                      nfIsRef, nfIsCursor}
+                                      nfIsRef, nfIsCursor, nfLL}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2
@@ -958,11 +957,9 @@ const
   nkStrKinds* = {nkStrLit..nkTripleStrLit}
 
   skLocalVars* = {skVar, skLet, skForVar, skParam, skResult}
-  skProcKinds* = {skProc, skTemplate, skMacro, skIterator, skClosureIterator,
+  skProcKinds* = {skProc, skTemplate, skMacro, skIterator,
                   skMethod, skConverter}
 
-  skIterators* = {skIterator, skClosureIterator}
-
 var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things
 
 proc isCallExpr*(n: PNode): bool =
@@ -1558,12 +1555,13 @@ proc isGenericRoutine*(s: PSym): bool =
   else: discard
 
 proc skipGenericOwner*(s: PSym): PSym =
-  internalAssert s.kind in skProcKinds
   ## Generic instantiations are owned by their originating generic
   ## symbol. This proc skips such owners and goes straight to the owner
   ## of the generic itself (the module or the enclosing proc).
-  result = if sfFromGeneric in s.flags: s.owner.owner
+  result = if s.kind in skProcKinds and sfFromGeneric in s.flags:
-           else: s.owner
+             s.owner.owner
+           else:
+             s.owner
 
 proc originatingModule*(s: PSym): PSym =
   result = s.owner

compiler/ccgcalls.nim

@@ -118,6 +118,14 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
         result = "$1->data, $1->$2" % [a.rdLoc, lenField(p)]
     of tyArray, tyArrayConstr:
       result = "$1, $2" % [rdLoc(a), rope(lengthOrd(a.t))]
+    of tyPtr, tyRef:
+      case lastSon(a.t).kind
+      of tyString, tySequence:
+        result = "(*$1)->data, (*$1)->$2" % [a.rdLoc, lenField(p)]
+      of tyArray, tyArrayConstr:
+        result = "$1, $2" % [rdLoc(a), rope(lengthOrd(lastSon(a.t)))]
+      else: 
+        internalError("openArrayLoc: " & typeToString(a.t))
     else: internalError("openArrayLoc: " & typeToString(a.t))
 
 proc genArgStringToCString(p: BProc, n: PNode): Rope {.inline.} =
@@ -515,7 +523,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
     line(p, cpsStmts, pl)
 
 proc genCall(p: BProc, e: PNode, d: var TLoc) =
-  if e.sons[0].typ.callConv == ccClosure:
+  if e.sons[0].typ.skipTypes({tyGenericInst}).callConv == ccClosure:
     genClosureCall(p, nil, e, d)
   elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags:
     genInfixCall(p, nil, e, d)
@@ -528,7 +536,7 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) =
     if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d)
 
 proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
-  if ri.sons[0].typ.callConv == ccClosure:
+  if ri.sons[0].typ.skipTypes({tyGenericInst}).callConv == ccClosure:
     genClosureCall(p, le, ri, d)
   elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags:
     genInfixCall(p, le, ri, d)

compiler/ccgexprs.nim

@@ -1502,7 +1502,7 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   else:
     case op
     of mIncl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3] |=(1U<<($2&7U));$n")
-    of mExcl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3]] &= ~(1U<<($2&7U));$n")
+    of mExcl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3] &= ~(1U<<($2&7U));$n")
     of mCard: unaryExprChar(p, e, d, "#cardSet($1, " & $size & ')')
     of mLtSet, mLeSet:
       getTemp(p, getSysType(tyInt), i) # our counter
@@ -1838,9 +1838,9 @@ proc genClosure(p: BProc, n: PNode, d: var TLoc) =
   assert n.kind == nkClosure
 
   if isConstClosure(n):
-    inc(p.labels)
+    inc(p.module.labels)
-    var tmp = "LOC" & rope(p.labels)
+    var tmp = "CNSTCLOSURE" & rope(p.module.labels)
-    addf(p.module.s[cfsData], "NIM_CONST $1 $2 = $3;$n",
+    addf(p.module.s[cfsData], "static NIM_CONST $1 $2 = $3;$n",
         [getTypeDesc(p.module, n.typ), tmp, genConstExpr(p, n)])
     putIntoDest(p, d, n.typ, tmp, OnStatic)
   else:
@@ -1965,7 +1965,9 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       else:
         genProc(p.module, sym)
       putLocIntoDest(p, d, sym.loc)
-    of skProc, skConverter, skIterators:
+    of skProc, skConverter, skIterator:
+      #if sym.kind == skIterator:
+      #  echo renderTree(sym.getBody, {renderIds})
       if sfCompileTime in sym.flags:
         localError(n.info, "request to generate code for .compileTime proc: " &
            sym.name.s)
@@ -1987,6 +1989,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       if sfGlobal in sym.flags: genVarPrototype(p.module, sym)
       if sym.loc.r == nil or sym.loc.t == nil:
         #echo "FAILED FOR PRCO ", p.prc.name.s
+        #echo renderTree(p.prc.ast, {renderIds})
         internalError n.info, "expr: var not init " & sym.name.s & "_" & $sym.id
       if sfThread in sym.flags:
         accessThreadLocalVar(p, sym)

compiler/ccgstmts.nim

@@ -21,8 +21,8 @@ proc registerGcRoot(p: BProc, v: PSym) =
     # we register a specialized marked proc here; this has the advantage
     # that it works out of the box for thread local storage then :-)
     let prc = genTraverseProcForGlobal(p.module, v)
-    linefmt(p.module.initProc, cpsStmts,
+    appcg(p.module, p.module.initProc.procSec(cpsStmts),
-      "#nimRegisterGlobalMarker($1);$n", prc)
+      "#nimRegisterGlobalMarker($1);$n", [prc])
 
 proc isAssignedImmediately(n: PNode): bool {.inline.} =
   if n.kind == nkEmpty: return false
@@ -955,7 +955,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope =
       res.add(t.sons[i].strVal)
     of nkSym:
       var sym = t.sons[i].sym
-      if sym.kind in {skProc, skIterator, skClosureIterator, skMethod}:
+      if sym.kind in {skProc, skIterator, skMethod}:
         var a: TLoc
         initLocExpr(p, t.sons[i], a)
         res.add($rdLoc(a))

compiler/cgen.nim

@@ -594,7 +594,7 @@ proc cgsym(m: BModule, name: string): Rope =
   var sym = magicsys.getCompilerProc(name)
   if sym != nil:
     case sym.kind
-    of skProc, skMethod, skConverter, skIterators: genProc(m, sym)
+    of skProc, skMethod, skConverter, skIterator: genProc(m, sym)
     of skVar, skResult, skLet: genVarPrototype(m, sym)
     of skType: discard getTypeDesc(m, sym.typ)
     else: internalError("cgsym: " & name & ": " & $sym.kind)

compiler/cgmeth.nim

@@ -19,10 +19,7 @@ proc genConv(n: PNode, d: PType, downcast: bool): PNode =
   if (source.kind == tyObject) and (dest.kind == tyObject):
     var diff = inheritanceDiff(dest, source)
     if diff == high(int):
-      # see bug #3550 which triggers it. XXX This is a hack but I don't know yet
+      # no subtype relation, nothing to do
-      # how the real fix looks like:
-      localError(n.info, "there is no subtype relation between " &
-                 typeToString(d) & " and " & typeToString(n.typ))
       result = n
     elif diff < 0:
       result = newNodeIT(nkObjUpConv, n.info, d)

compiler/docgen.nim

@@ -149,10 +149,10 @@ proc ropeFormatNamedVars(frmt: FormatStr, varnames: openArray[string],
 proc genComment(d: PDoc, n: PNode): string =
   result = ""
   var dummyHasToc: bool
-  if n.comment != nil and startsWith(n.comment, "##"):
+  if n.comment != nil:
     renderRstToOut(d[], parseRst(n.comment, toFilename(n.info),
                                toLinenumber(n.info), toColumn(n.info),
-                               dummyHasToc, d.options + {roSkipPounds}), result)
+                               dummyHasToc, d.options), result)
 
 proc genRecComment(d: PDoc, n: PNode): Rope =
   if n == nil: return nil
@@ -537,7 +537,7 @@ proc generateJson(d: PDoc, n: PNode, jArray: JsonNode = nil): JsonNode =
 proc genSection(d: PDoc, kind: TSymKind) =
   const sectionNames: array[skModule..skTemplate, string] = [
     "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Methods",
-    "Iterators", "Iterators", "Converters", "Macros", "Templates"
+    "Iterators", "Converters", "Macros", "Templates"
   ]
   if d.section[kind] == nil: return
   var title = sectionNames[kind].rope

compiler/evaltempl.nim

@@ -38,7 +38,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
     if s.owner.id == c.owner.id:
       if s.kind == skParam and sfGenSym notin s.flags:
         handleParam actual.sons[s.position]
-      elif s.kind == skGenericParam:
+      elif s.kind == skGenericParam or
+           s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam:
         handleParam actual.sons[s.owner.typ.len + s.position - 1]
       else:
         internalAssert sfGenSym in s.flags

compiler/jsgen.nim

@@ -931,7 +931,7 @@ proc isIndirect(v: PSym): bool =
   result = {sfAddrTaken, sfGlobal} * v.flags != {} and
     #(mapType(v.typ) != etyObject) and
     {sfImportc, sfVolatile, sfExportc} * v.flags == {} and
-    v.kind notin {skProc, skConverter, skMethod, skIterator, skClosureIterator,
+    v.kind notin {skProc, skConverter, skMethod, skIterator,
                   skConst, skTemp, skLet}
 
 proc genAddr(p: PProc, n: PNode, r: var TCompRes) =
@@ -1636,6 +1636,9 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   of nkSym:
     genSym(p, n, r)
   of nkCharLit..nkInt64Lit:
+    if n.typ.kind == tyBool:
+      r.res = if n.intVal == 0: rope"false" else: rope"true"
+    else:
       r.res = rope(n.intVal)
     r.kind = resExpr
   of nkNilLit:

compiler/lexer.nim

@@ -769,24 +769,88 @@ proc getOperator(L: var TLexer, tok: var TToken) =
   if buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
     tok.strongSpaceB = -1
 
+proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
+                          isDoc: bool) =
+  var pos = start
+  var buf = L.buf
+  var toStrip = 0
+  # detect the amount of indentation:
+  if isDoc:
+    toStrip = getColNumber(L, pos)
+    while buf[pos] == ' ': inc pos
+    if buf[pos] in {CR, LF}:
+      pos = handleCRLF(L, pos)
+      buf = L.buf
+      toStrip = 0
+      while buf[pos] == ' ':
+        inc pos
+        inc toStrip
+  var nesting = 0
+  while true:
+    case buf[pos]
+    of '#':
+      if isDoc:
+        if buf[pos+1] == '#' and buf[pos+2] == '[':
+          inc nesting
+        tok.literal.add '#'
+      elif buf[pos+1] == '[':
+        inc nesting
+      inc pos
+    of ']':
+      if isDoc:
+        if buf[pos+1] == '#' and buf[pos+2] == '#':
+          if nesting == 0:
+            inc(pos, 3)
+            break
+          dec nesting
+        tok.literal.add ']'
+      elif buf[pos+1] == '#':
+        if nesting == 0:
+          inc(pos, 2)
+          break
+        dec nesting
+      inc pos
+    of '\t':
+      lexMessagePos(L, errTabulatorsAreNotAllowed, pos)
+      inc(pos)
+      if isDoc: tok.literal.add '\t'
+    of CR, LF:
+      pos = handleCRLF(L, pos)
+      buf = L.buf
+      # strip leading whitespace:
+      if isDoc:
+        tok.literal.add "\n"
+        inc tok.iNumber
+        var c = toStrip
+        while buf[pos] == ' ' and c > 0:
+          inc pos
+          dec c
+    of nimlexbase.EndOfFile:
+      lexMessagePos(L, errGenerated, pos, "end of multiline comment expected")
+      break
+    else:
+      if isDoc: tok.literal.add buf[pos]
+      inc(pos)
+  L.bufpos = pos
+
 proc scanComment(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
   var buf = L.buf
-  when not defined(nimfix):
-    assert buf[pos+1] == '#'
-    if buf[pos+2] == '[':
-      if buf[pos+3] == ']':
-        #  ##[] is the (rather complex) "cursor token" for idetools
-        tok.tokType = tkComment
-        tok.literal = "[]"
-        inc(L.bufpos, 4)
-        return
-      else:
-        lexMessagePos(L, warnDeprecated, pos, "use '## [' instead; '##['")
-
   tok.tokType = tkComment
   # iNumber contains the number of '\n' in the token
   tok.iNumber = 0
+  when not defined(nimfix):
+    assert buf[pos+1] == '#'
+    if buf[pos+2] == '[':
+      skipMultiLineComment(L, tok, pos+3, true)
+      return
+    inc(pos, 2)
+
+  var toStrip = 0
+  while buf[pos] == ' ':
+    inc pos
+    inc toStrip
+
   when defined(nimfix):
     var col = getColNumber(L, pos)
   while true:
@@ -820,6 +884,12 @@ proc scanComment(L: var TLexer, tok: var TToken) =
     if doContinue():
       tok.literal.add "\n"
       when defined(nimfix): col = indent
+      else:
+        inc(pos, 2)
+        var c = toStrip
+        while buf[pos] == ' ' and c > 0:
+          inc pos
+          dec c
       inc tok.iNumber
     else:
       if buf[pos] > ' ':
@@ -843,9 +913,16 @@ proc skip(L: var TLexer, tok: var TToken) =
       pos = handleCRLF(L, pos)
       buf = L.buf
       var indent = 0
-      while buf[pos] == ' ':
+      while true:
+        if buf[pos] == ' ':
           inc(pos)
           inc(indent)
+        elif buf[pos] == '#' and buf[pos+1] == '[':
+          skipMultiLineComment(L, tok, pos+2, false)
+          pos = L.bufpos
+          buf = L.buf
+        else:
+          break
       tok.strongSpaceA = 0
       when defined(nimfix):
         template doBreak(): expr = buf[pos] > ' '
@@ -863,7 +940,10 @@ proc skip(L: var TLexer, tok: var TToken) =
         # do not skip documentation comment:
         if buf[pos+1] == '#': break
         if buf[pos+1] == '[':
-          lexMessagePos(L, warnDeprecated, pos, "use '# [' instead; '#['")
+          skipMultiLineComment(L, tok, pos+2, false)
+          pos = L.bufpos
+          buf = L.buf
+        else:
           while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos)
     else:
       break                   # EndOfFile also leaves the loop

compiler/lookups.nim

@@ -133,7 +133,7 @@ type
 
 proc getSymRepr*(s: PSym): string =
   case s.kind
-  of skProc, skMethod, skConverter, skIterators: result = getProcHeader(s)
+  of skProc, skMethod, skConverter, skIterator: result = getProcHeader(s)
   else: result = s.name.s
 
 proc ensureNoMissingOrUnusedSymbols(scope: PScope) =

compiler/lowerings.nim

@@ -165,9 +165,10 @@ proc indirectAccess*(a: PNode, b: string, info: TLineInfo): PNode =
   deref.typ = a.typ.skipTypes(abstractInst).sons[0]
   var t = deref.typ.skipTypes(abstractInst)
   var field: PSym
+  let bb = getIdent(b)
   while true:
     assert t.kind == tyObject
-    field = getSymFromList(t.n, getIdent(b))
+    field = getSymFromList(t.n, bb)
     if field != nil: break
     t = t.sons[0]
     if t == nil: break
@@ -585,7 +586,7 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType;
     objType.addField(field)
     result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0])
     fn = indirectAccess(castExpr, field, n.info)
-  elif fn.kind == nkSym and fn.sym.kind in {skClosureIterator, skIterator}:
+  elif fn.kind == nkSym and fn.sym.kind == skIterator:
     localError(n.info, "iterator in spawn environment is not allowed")
   elif fn.typ.callConv == ccClosure:
     localError(n.info, "closure in spawn environment is not allowed")

compiler/nimsets.nim

@@ -106,13 +106,17 @@ proc toTreeSet(s: TBitSet, settype: PType, info: TLineInfo): PNode =
         inc(b)
         if (b >= len(s) * ElemSize) or not bitSetIn(s, b): break
       dec(b)
+      let aa = newIntTypeNode(nkIntLit, a + first, elemType)
+      aa.info = info
       if a == b:
-        addSon(result, newIntTypeNode(nkIntLit, a + first, elemType))
+        addSon(result, aa)
       else:
         n = newNodeI(nkRange, info)
         n.typ = elemType
-        addSon(n, newIntTypeNode(nkIntLit, a + first, elemType))
+        addSon(n, aa)
-        addSon(n, newIntTypeNode(nkIntLit, b + first, elemType))
+        let bb = newIntTypeNode(nkIntLit, b + first, elemType)
+        bb.info = info
+        addSon(n, bb)
         addSon(result, n)
       e = b
     inc(e)

compiler/parser.nim

@@ -112,11 +112,6 @@ proc rawSkipComment(p: var TParser, node: PNode) =
   if p.tok.tokType == tkComment:
     if node != nil:
       if node.comment == nil: node.comment = ""
-      if p.tok.literal == "[]":
-        node.flags.incl nfIsCursor
-        #echo "parser: "
-        #debug node
-      else:
       add(node.comment, p.tok.literal)
     else:
       parMessage(p, errInternal, "skipComment")
@@ -250,7 +245,9 @@ proc isUnary(p: TParser): bool =
   if p.tok.tokType in {tkOpr, tkDotDot} and
      p.tok.strongSpaceB == 0 and
      p.tok.strongSpaceA > 0:
-    # XXX change this after 0.10.4 is out
+      result = true
+      # versions prior to 0.13.0 used to do this:
+      when false:
         if p.strongSpaces:
           result = true
         else:

compiler/renderer.nim

@@ -167,33 +167,24 @@ proc makeNimString(s: string): string =
 proc putComment(g: var TSrcGen, s: string) =
   if s.isNil: return
   var i = 0
-  var comIndent = 1
   var isCode = (len(s) >= 2) and (s[1] != ' ')
   var ind = g.lineLen
-  var com = ""
+  var com = "## "
   while true:
     case s[i]
     of '\0':
       break
     of '\x0D':
       put(g, tkComment, com)
-      com = ""
+      com = "## "
       inc(i)
       if s[i] == '\x0A': inc(i)
       optNL(g, ind)
     of '\x0A':
       put(g, tkComment, com)
-      com = ""
+      com = "## "
       inc(i)
       optNL(g, ind)
-    of '#':
-      add(com, s[i])
-      inc(i)
-      comIndent = 0
-      while s[i] == ' ':
-        add(com, s[i])
-        inc(i)
-        inc(comIndent)
     of ' ', '\x09':
       add(com, s[i])
       inc(i)
@@ -206,7 +197,7 @@ proc putComment(g: var TSrcGen, s: string) =
       if not isCode and (g.lineLen + (j - i) > MaxLineLen):
         put(g, tkComment, com)
         optNL(g, ind)
-        com = '#' & spaces(comIndent)
+        com = "## "
       while s[i] > ' ':
         add(com, s[i])
         inc(i)
@@ -283,7 +274,7 @@ proc shouldRenderComment(g: var TSrcGen, n: PNode): bool =
   result = false
   if n.comment != nil:
     result = (renderNoComments notin g.flags) or
-        (renderDocComments in g.flags) and startsWith(n.comment, "##")
+        (renderDocComments in g.flags)
 
 proc gcom(g: var TSrcGen, n: PNode) =
   assert(n != nil)
@@ -1330,6 +1321,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     initContext c
     putWithSpace g, tkSymbol, if n.kind == nkState: "state" else: "goto"
     gsons(g, n, c)
+  of nkBreakState:
+    put(g, tkTuple, "breakstate")
   of nkTypeClassTy:
     gTypeClassTy(g, n)
   else:

compiler/sem.nim

@@ -186,6 +186,8 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
     result.owner = getCurrOwner()
   else:
     result = newSym(kind, considerQuotedIdent(n), getCurrOwner(), n.info)
+  #if kind in {skForVar, skLet, skVar} and result.owner.kind == skModule:
+  #  incl(result.flags, sfGlobal)
 
 proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
                  allowed: TSymFlags): PSym
@@ -202,7 +204,7 @@ proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind) =
                            "' in this context: '" & typeToString(typ) & "'")
 
 proc paramsTypeCheck(c: PContext, typ: PType) {.inline.} =
-  typeAllowedCheck(typ.n.info, typ, skConst)
+  typeAllowedCheck(typ.n.info, typ, skProc)
 
 proc expectMacroOrTemplateCall(c: PContext, n: PNode): PSym
 proc semDirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode

compiler/semcall.nim

@@ -75,7 +75,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
           errors.add(err)
     if z.state == csMatch:
       # little hack so that iterators are preferred over everything else:
-      if sym.kind in skIterators: inc(z.exactMatches, 200)
+      if sym.kind == skIterator: inc(z.exactMatches, 200)
       case best.state
       of csEmpty, csNoMatch: best = z
       of csMatch:
@@ -395,7 +395,7 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
     for i in countup(0, len(a)-1):
       var candidate = a.sons[i].sym
       if candidate.kind in {skProc, skMethod, skConverter,
-                            skIterator, skClosureIterator}:
+                            skIterator}:
         # it suffices that the candidate has the proper number of generic
         # type parameters:
         if safeLen(candidate.ast.sons[genericParamsPos]) == n.len-1:

compiler/semdata.nim

@@ -315,7 +315,7 @@ proc makeRangeType*(c: PContext; first, last: BiggestInt;
   addSonSkipIntLit(result, intType) # basetype of range
 
 proc markIndirect*(c: PContext, s: PSym) {.inline.} =
-  if s.kind in {skProc, skConverter, skMethod, skIterator, skClosureIterator}:
+  if s.kind in {skProc, skConverter, skMethod, skIterator}:
     incl(s.flags, sfAddrTaken)
     # XXX add to 'c' for global analysis
 

compiler/semexprs.nim

@@ -753,11 +753,11 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
                                      flags: TExprFlags): PNode =
   if flags*{efInTypeof, efWantIterator} != {}:
     # consider: 'for x in pReturningArray()' --> we don't want the restriction
-    # to 'skIterators' anymore; skIterators are preferred in sigmatch already
+    # to 'skIterator' anymore; skIterator is preferred in sigmatch already
     # for typeof support.
     # for ``type(countup(1,3))``, see ``tests/ttoseq``.
     result = semOverloadedCall(c, n, nOrig,
-      {skProc, skMethod, skConverter, skMacro, skTemplate}+skIterators)
+      {skProc, skMethod, skConverter, skMacro, skTemplate, skIterator})
   else:
     result = semOverloadedCall(c, n, nOrig,
       {skProc, skMethod, skConverter, skMacro, skTemplate})
@@ -770,7 +770,7 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
     case callee.kind
     of skMacro, skTemplate: discard
     else:
-      if callee.kind in skIterators and callee.id == c.p.owner.id:
+      if callee.kind == skIterator and callee.id == c.p.owner.id:
         localError(n.info, errRecursiveDependencyX, callee.name.s)
         # error correction, prevents endless for loop elimination in transf.
         # See bug #2051:
@@ -1201,7 +1201,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
     let s = if n.sons[0].kind == nkSym: n.sons[0].sym
             elif n[0].kind in nkSymChoices: n.sons[0][0].sym
             else: nil
-    if s != nil and s.kind in {skProc, skMethod, skConverter}+skIterators:
+    if s != nil and s.kind in {skProc, skMethod, skConverter, skIterator}:
       # type parameters: partial generic specialization
       n.sons[0] = semSymGenericInstantiation(c, n.sons[0], s)
       result = explicitGenericInstantiation(c, n, s)
@@ -1349,8 +1349,8 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
 proc semReturn(c: PContext, n: PNode): PNode =
   result = n
   checkSonsLen(n, 1)
-  if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro,
+  if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro} or (
-                        skClosureIterator}:
+     c.p.owner.kind == skIterator and c.p.owner.typ.callConv == ccClosure):
     if n.sons[0].kind != nkEmpty:
       # transform ``return expr`` to ``result = expr; return``
       if c.p.resultSym != nil:
@@ -1426,7 +1426,7 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) =
 proc semYield(c: PContext, n: PNode): PNode =
   result = n
   checkSonsLen(n, 1)
-  if c.p.owner == nil or c.p.owner.kind notin skIterators:
+  if c.p.owner == nil or c.p.owner.kind != skIterator:
     localError(n.info, errYieldNotAllowedHere)
   elif c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline:
     localError(n.info, errYieldNotAllowedInTryStmt)
@@ -1817,6 +1817,7 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode =
       whenNimvm = lookUp(c, exprNode).magic == mNimvm
     elif exprNode.kind == nkSym:
       whenNimvm = exprNode.sym.magic == mNimvm
+    if whenNimvm: n.flags.incl nfLL
 
   for i in countup(0, sonsLen(n) - 1):
     var it = n.sons[i]
@@ -2124,7 +2125,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     var s = lookUp(c, n)
     if c.inTypeClass == 0: semCaptureSym(s, c.p.owner)
     result = semSym(c, n, s, flags)
-    if s.kind in {skProc, skMethod, skConverter}+skIterators:
+    if s.kind in {skProc, skMethod, skConverter, skIterator}:
       #performProcvarCheck(c, n, s)
       result = symChoice(c, n, s, scClosed)
       if result.kind == nkSym:
@@ -2212,7 +2213,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
           localError(n.info, errUseQualifier, s.name.s)
         elif s.magic == mNone: result = semDirectOp(c, n, flags)
         else: result = semMagic(c, n, s, flags)
-      of skProc, skMethod, skConverter, skIterators:
+      of skProc, skMethod, skConverter, skIterator:
         if s.magic == mNone: result = semDirectOp(c, n, flags)
         else: result = semMagic(c, n, s, flags)
       else:

compiler/semgnrc.nim

@@ -58,7 +58,7 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
   of skUnknown:
     # Introduced in this pass! Leave it as an identifier.
     result = n
-  of skProc, skMethod, skIterators, skConverter, skModule:
+  of skProc, skMethod, skIterator, skConverter, skModule:
     result = symChoice(c, n, s, scOpen)
   of skTemplate:
     if macroToExpand(s):
@@ -226,7 +226,7 @@ proc semGenericStmt(c: PContext, n: PNode,
       of skUnknown, skParam:
         # Leave it as an identifier.
         discard
-      of skProc, skMethod, skIterators, skConverter, skModule:
+      of skProc, skMethod, skIterator, skConverter, skModule:
         result.sons[0] = symChoice(c, fn, s, scOption)
         # do not check of 's.magic==mRoof' here because it might be some
         # other '^' but after overload resolution the proper one:

compiler/semmagic.nim

@@ -207,7 +207,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
         result = n.sons[1]
       else:
         result = newNodeIT(nkCall, n.info, getSysType(tyInt))
-        result.add newSymNode(createMagic("-", mSubI), n.info)
+        result.add newSymNode(getSysMagic("-", mSubI), n.info)
         result.add lenExprB
         result.add n.sons[1]
   of mPlugin:

compiler/semstmts.nim

@@ -84,7 +84,7 @@ proc performProcvarCheck(c: PContext, n: PNode, s: PSym) =
 proc semProcvarCheck(c: PContext, n: PNode) =
   let n = n.skipConv
   if n.kind == nkSym and n.sym.kind in {skProc, skMethod, skConverter,
-                                        skIterator, skClosureIterator}:
+                                        skIterator}:
     performProcvarCheck(c, n, n.sym)
 
 proc semProc(c: PContext, n: PNode): PNode
@@ -326,6 +326,8 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym =
     incl(result.flags, sfGlobal)
   else:
     result = semIdentWithPragma(c, kind, n, {})
+    if result.owner.kind == skModule:
+      incl(result.flags, sfGlobal)
   suggestSym(n.info, result)
   styleCheckDef(result)
 
@@ -598,7 +600,7 @@ proc semFor(c: PContext, n: PNode): PNode =
     # first class iterator:
     result = semForVars(c, n)
   elif not isCallExpr or call.sons[0].kind != nkSym or
-      call.sons[0].sym.kind notin skIterators:
+      call.sons[0].sym.kind != skIterator:
     if length == 3:
       n.sons[length-2] = implicitIterator(c, "items", n.sons[length-2])
     elif length == 4:
@@ -958,15 +960,17 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode =
   var n = n
 
   let original = n.sons[namePos].sym
-  let s = copySym(original, false)
+  let s = original #copySym(original, false)
-  incl(s.flags, sfFromGeneric)
+  #incl(s.flags, sfFromGeneric)
+  #s.owner = original
 
   n = replaceTypesInBody(c, pt, n, original)
   result = n
   s.ast = result
   n.sons[namePos].sym = s
   n.sons[genericParamsPos] = emptyNode
-  let params = n.typ.n
+  # for LL we need to avoid wrong aliasing
+  let params = copyTree n.typ.n
   n.sons[paramsPos] = params
   s.typ = n.typ
   for i in 1..<params.len:
@@ -974,6 +978,7 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode =
                               tyFromExpr, tyFieldAccessor}+tyTypeClasses:
       localError(params[i].info, "cannot infer type of parameter: " &
                  params[i].sym.name.s)
+    #params[i].sym.owner = s
   openScope(c)
   pushOwner(s)
   addParams(c, params, skProc)
@@ -1006,7 +1011,8 @@ proc activate(c: PContext, n: PNode) =
       discard
 
 proc maybeAddResult(c: PContext, s: PSym, n: PNode) =
-  if s.typ.sons[0] != nil and s.kind != skIterator:
+  if s.typ.sons[0] != nil and not
+      (s.kind == skIterator and s.typ.callConv != ccClosure):
     addResult(c, s.typ.sons[0], n.info, s.kind)
     addResultNode(c, n)
 
@@ -1143,13 +1149,15 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   if tfTriggersCompileTime in s.typ.flags: incl(s.flags, sfCompileTime)
   if n.sons[patternPos].kind != nkEmpty:
     n.sons[patternPos] = semPattern(c, n.sons[patternPos])
-  if s.kind in skIterators:
+  if s.kind == skIterator:
     s.typ.flags.incl(tfIterator)
 
   var proto = searchForProc(c, oldScope, s)
   if proto == nil:
-    if s.kind == skClosureIterator: s.typ.callConv = ccClosure
+    if s.kind == skIterator and s.typ.callConv == ccClosure:
-    else: s.typ.callConv = lastOptionEntry(c).defaultCC
+      discard
+    else:
+      s.typ.callConv = lastOptionEntry(c).defaultCC
     # add it here, so that recursive procs are possible:
     if sfGenSym in s.flags: discard
     elif kind in OverloadableSyms:
@@ -1209,7 +1217,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
         n.sons[bodyPos] = transformBody(c.module, semBody, s)
       popProcCon(c)
     else:
-      if s.typ.sons[0] != nil and kind notin skIterators:
+      if s.typ.sons[0] != nil and kind != skIterator:
         addDecl(c, newSym(skUnknown, getIdent"result", nil, n.info))
       openScope(c)
       n.sons[bodyPos] = semGenericStmt(c, n.sons[bodyPos])
@@ -1230,9 +1238,9 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   if n.sons[patternPos].kind != nkEmpty:
     c.patterns.add(s)
   if isAnon: result.typ = s.typ
-  if isTopLevel(c) and s.kind != skClosureIterator and
+  if isTopLevel(c) and s.kind != skIterator and
       s.typ.callConv == ccClosure:
-    message(s.info, warnDeprecated, "top level '.closure' calling convention")
+    localError(s.info, "'.closure' calling convention for top level routines is invalid")
 
 proc determineType(c: PContext, s: PSym) =
   if s.typ != nil: return
@@ -1240,15 +1248,12 @@ proc determineType(c: PContext, s: PSym) =
   discard semProcAux(c, s.ast, s.kind, {}, stepDetermineType)
 
 proc semIterator(c: PContext, n: PNode): PNode =
-  let kind = if hasPragma(n[pragmasPos], wClosure) or
-                n[namePos].kind == nkEmpty: skClosureIterator
-             else: skIterator
   # gensym'ed iterator?
   if n[namePos].kind == nkSym:
     # gensym'ed iterators might need to become closure iterators:
     n[namePos].sym.owner = getCurrOwner()
-    n[namePos].sym.kind = kind
+    n[namePos].sym.kind = skIterator
-  result = semProcAux(c, n, kind, iteratorPragmas)
+  result = semProcAux(c, n, skIterator, iteratorPragmas)
   var s = result.sons[namePos].sym
   var t = s.typ
   if t.sons[0] == nil and s.typ.callConv != ccClosure:

compiler/semtempl.nim

@@ -228,10 +228,7 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym): PNode =
   of skParam:
     result = n
   of skType:
-    if (s.typ != nil) and (s.typ.kind != tyGenericParam):
     result = newSymNodeTypeDesc(s, n.info)
-    else:
-      result = n
   else:
     result = newSymNode(s, n.info)
 
@@ -456,9 +453,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
   of nkMethodDef:
     result = semRoutineInTemplBody(c, n, skMethod)
   of nkIteratorDef:
-    let kind = if hasPragma(n[pragmasPos], wClosure): skClosureIterator
+    result = semRoutineInTemplBody(c, n, skIterator)
-               else: skIterator
-    result = semRoutineInTemplBody(c, n, kind)
   of nkTemplateDef:
     result = semRoutineInTemplBody(c, n, skTemplate)
   of nkMacroDef:

compiler/semtypes.nim

@@ -960,10 +960,6 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
   var r: PType
   if n.sons[0].kind != nkEmpty:
     r = semTypeNode(c, n.sons[0], nil)
-  elif kind == skIterator:
-    # XXX This is special magic we should likely get rid of
-    r = newTypeS(tyExpr, c)
-    message(n.info, warnDeprecated, "implicit return type for 'iterator'")
 
   if r != nil:
     # turn explicit 'void' return type into 'nil' because the rest of the
@@ -1296,7 +1292,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
       child.flags.incl tfIterator
       result.addSonSkipIntLit(child)
     else:
-      result = semProcTypeWithScope(c, n, prev, skClosureIterator)
+      result = semProcTypeWithScope(c, n, prev, skIterator)
       result.flags.incl(tfIterator)
       if n.lastSon.kind == nkPragma and hasPragma(n.lastSon, wInline):
         result.callConv = ccInline

compiler/sigmatch.nim

@@ -167,12 +167,12 @@ proc sumGeneric(t: PType): int =
       t = t.lastSon
       if t.kind == tyEmpty: break
       inc result
-    of tyGenericInvocation, tyTuple:
+    of tyGenericInvocation, tyTuple, tyProc:
       result += ord(t.kind == tyGenericInvocation)
       for i in 0 .. <t.len: result += t.sons[i].sumGeneric
       break
     of tyGenericParam, tyExpr, tyStatic, tyStmt: break
-    of tyBool, tyChar, tyEnum, tyObject, tyProc, tyPointer,
+    of tyBool, tyChar, tyEnum, tyObject, tyPointer,
         tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128,
         tyUInt..tyUInt64:
       return isvar
@@ -1442,7 +1442,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
     z.calleeSym = m.calleeSym
     var best = -1
     for i in countup(0, sonsLen(arg) - 1):
-      if arg.sons[i].sym.kind in {skProc, skMethod, skConverter}+skIterators:
+      if arg.sons[i].sym.kind in {skProc, skMethod, skConverter, skIterator}:
         copyCandidate(z, m)
         z.callee = arg.sons[i].typ
         z.calleeSym = arg.sons[i].sym

compiler/transf.nim

@@ -45,7 +45,7 @@ type
     inlining: int            # > 0 if we are in inlining context (copy vars)
     nestedProcs: int         # > 0 if we are in a nested proc
     contSyms, breakSyms: seq[PSym]  # to transform 'continue' and 'break'
-    deferDetected: bool
+    deferDetected, tooEarly: bool
   PTransf = ref TTransfContext
 
 proc newTransNode(a: PNode): PTransNode {.inline.} =
@@ -93,10 +93,15 @@ proc getCurrOwner(c: PTransf): PSym =
   if c.transCon != nil: result = c.transCon.owner
   else: result = c.module
 
-proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PSym =
+proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode =
-  result = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info)
+  let r = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info)
-  result.typ = skipTypes(typ, {tyGenericInst})
+  r.typ = skipTypes(typ, {tyGenericInst})
-  incl(result.flags, sfFromGeneric)
+  incl(r.flags, sfFromGeneric)
+  let owner = getCurrOwner(c)
+  if owner.isIterator and not c.tooEarly:
+    result = freshVarForClosureIter(r, owner)
+  else:
+    result = newSymNode(r)
 
 proc transform(c: PTransf, n: PNode): PTransNode
 
@@ -111,13 +116,22 @@ proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode =
   result[1] = ri
 
 proc transformSymAux(c: PTransf, n: PNode): PNode =
-  #if n.sym.kind == skClosureIterator:
+  let s = n.sym
-  #  return liftIterSym(n)
+  if s.typ != nil and s.typ.callConv == ccClosure:
+    if s.kind == skIterator:
+      if c.tooEarly: return n
+      else: return liftIterSym(n, getCurrOwner(c))
+    elif s.kind in {skProc, skConverter, skMethod} and not c.tooEarly:
+      # top level .closure procs are still somewhat supported for 'Nake':
+      return makeClosure(s, nil, n.info)
+  #elif n.sym.kind in {skVar, skLet} and n.sym.typ.callConv == ccClosure:
+  #  echo n.info, " come heer for ", c.tooEarly
+  #  if not c.tooEarly:
   var b: PNode
   var tc = c.transCon
-  if sfBorrow in n.sym.flags and n.sym.kind in routineKinds:
+  if sfBorrow in s.flags and s.kind in routineKinds:
     # simply exchange the symbol:
-    b = n.sym.getBody
+    b = s.getBody
     if b.kind != nkSym: internalError(n.info, "wrong AST for borrowed symbol")
     b = newSymNode(b.sym)
     b.info = n.info
@@ -132,6 +146,16 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
 proc transformSym(c: PTransf, n: PNode): PTransNode =
   result = PTransNode(transformSymAux(c, n))
 
+proc freshVar(c: PTransf; v: PSym): PNode =
+  let owner = getCurrOwner(c)
+  if owner.isIterator and not c.tooEarly:
+    result = freshVarForClosureIter(v, owner)
+  else:
+    var newVar = copySym(v)
+    incl(newVar.flags, sfFromGeneric)
+    newVar.owner = owner
+    result = newSymNode(newVar)
+
 proc transformVarSection(c: PTransf, v: PNode): PTransNode =
   result = newTransNode(v)
   for i in countup(0, sonsLen(v)-1):
@@ -141,35 +165,30 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode =
     elif it.kind == nkIdentDefs:
       if it.sons[0].kind == nkSym:
         internalAssert(it.len == 3)
-        var newVar = copySym(it.sons[0].sym)
+        let x = freshVar(c, it.sons[0].sym)
-        incl(newVar.flags, sfFromGeneric)
+        idNodeTablePut(c.transCon.mapping, it.sons[0].sym, x)
-        # fixes a strange bug for rodgen:
-        #include(it.sons[0].sym.flags, sfFromGeneric);
-        newVar.owner = getCurrOwner(c)
-        idNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar))
         var defs = newTransNode(nkIdentDefs, it.info, 3)
         if importantComments():
           # keep documentation information:
           PNode(defs).comment = it.comment
-        defs[0] = newSymNode(newVar).PTransNode
+        defs[0] = x.PTransNode
         defs[1] = it.sons[1].PTransNode
         defs[2] = transform(c, it.sons[2])
-        newVar.ast = defs[2].PNode
+        if x.kind == nkSym: x.sym.ast = defs[2].PNode
         result[i] = defs
       else:
-        # has been transformed into 'param.x' for closure iterators, so keep it:
+        # has been transformed into 'param.x' for closure iterators, so just
-        result[i] = PTransNode(it)
+        # transform it:
+        result[i] = transform(c, it)
     else:
       if it.kind != nkVarTuple:
         internalError(it.info, "transformVarSection: not nkVarTuple")
       var L = sonsLen(it)
       var defs = newTransNode(it.kind, it.info, L)
       for j in countup(0, L-3):
-        var newVar = copySym(it.sons[j].sym)
+        let x = freshVar(c, it.sons[j].sym)
-        incl(newVar.flags, sfFromGeneric)
+        idNodeTablePut(c.transCon.mapping, it.sons[j].sym, x)
-        newVar.owner = getCurrOwner(c)
+        defs[j] = x.PTransNode
-        idNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar))
-        defs[j] = newSymNode(newVar).PTransNode
       assert(it.sons[L-2].kind == nkEmpty)
       defs[L-2] = ast.emptyNode.PTransNode
       defs[L-1] = transform(c, it.sons[L-1])
@@ -294,6 +313,14 @@ proc introduceNewLocalVars(c: PTransf, n: PNode): PTransNode =
     result = PTransNode(n)
   of nkVarSection, nkLetSection:
     result = transformVarSection(c, n)
+  of nkClosure:
+    # it can happen that for-loop-inlining produced a fresh
+    # set of variables, including some computed environment
+    # (bug #2604). We need to patch this environment here too:
+    let a = n[1]
+    if a.kind == nkSym:
+      n.sons[1] = transformSymAux(c, a)
+    return PTransNode(n)
   else:
     result = newTransNode(n)
     for i in countup(0, sonsLen(n)-1):
@@ -348,6 +375,20 @@ proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PTransNode =
       # addr ( deref ( x )) --> x
       result = PTransNode(n.sons[0].sons[0])
 
+proc generateThunk(prc: PNode, dest: PType): PNode =
+  ## Converts 'prc' into '(thunk, nil)' so that it's compatible with
+  ## a closure.
+
+  # we cannot generate a proper thunk here for GC-safety reasons
+  # (see internal documentation):
+  if gCmd == cmdCompileToJS: return prc
+  result = newNodeIT(nkClosure, prc.info, dest)
+  var conv = newNodeIT(nkHiddenStdConv, prc.info, dest)
+  conv.add(emptyNode)
+  conv.add(prc)
+  result.add(conv)
+  result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil)))
+
 proc transformConv(c: PTransf, n: PNode): PTransNode =
   # numeric types need range checks:
   var dest = skipTypes(n.typ, abstractVarRange)
@@ -428,6 +469,10 @@ proc transformConv(c: PTransf, n: PNode): PTransNode =
   of tyGenericParam, tyOrdinal:
     result = transform(c, n.sons[1])
     # happens sometimes for generated assignments, etc.
+  of tyProc:
+    result = transformSons(c, n)
+    if dest.callConv == ccClosure and source.callConv == ccDefault:
+      result = generateThunk(result[1].PNode, dest).PTransNode
   else:
     result = transformSons(c, n)
 
@@ -479,9 +524,13 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
     return result
   c.breakSyms.add(labl)
   if call.kind notin nkCallKinds or call.sons[0].kind != nkSym or
-      call.sons[0].sym.kind != skIterator:
+      call.sons[0].typ.callConv == ccClosure:
     n.sons[length-1] = transformLoopBody(c, n.sons[length-1]).PNode
-    result[1] = lambdalifting.liftForLoop(n).PTransNode
+    if not c.tooEarly:
+      n.sons[length-2] = transform(c, n.sons[length-2]).PNode
+      result[1] = lambdalifting.liftForLoop(n, getCurrOwner(c)).PTransNode
+    else:
+      result[1] = newNode(nkEmpty).PTransNode
     discard c.breakSyms.pop
     return result
 
@@ -517,9 +566,9 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
     of paFastAsgn:
       # generate a temporary and produce an assignment statement:
       var temp = newTemp(c, formal.typ, formal.info)
-      addVar(v, newSymNode(temp))
+      addVar(v, temp)
-      add(stmtList, newAsgnStmt(c, newSymNode(temp), arg.PTransNode))
+      add(stmtList, newAsgnStmt(c, temp, arg.PTransNode))
-      idNodeTablePut(newC.mapping, formal, newSymNode(temp))
+      idNodeTablePut(newC.mapping, formal, temp)
     of paVarAsgn:
       assert(skipTypes(formal.typ, abstractInst).kind == tyVar)
       idNodeTablePut(newC.mapping, formal, arg)
@@ -700,17 +749,12 @@ proc transform(c: PTransf, n: PNode): PTransNode =
     result = PTransNode(n)
   of nkBracketExpr: result = transformArrayAccess(c, n)
   of procDefs:
-    when false:
-      if n.sons[genericParamsPos].kind == nkEmpty:
     var s = n.sons[namePos].sym
-        n.sons[bodyPos] = PNode(transform(c, s.getBody))
+    if n.typ != nil and s.typ.callConv == ccClosure:
-        if s.ast.sons[bodyPos] != n.sons[bodyPos]:
+      result = transformSym(c, n.sons[namePos])
-          # somehow this can happen ... :-/
+      # use the same node as before if still a symbol:
-          s.ast.sons[bodyPos] = n.sons[bodyPos]
+      if result.PNode.kind == nkSym: result = PTransNode(n)
-        #n.sons[bodyPos] = liftLambdas(s, n)
+    else:
-        #if n.kind == nkMethodDef: methodDef(s, false)
-    #if n.kind == nkIteratorDef and n.typ != nil:
-    #  return liftIterSym(n.sons[namePos]).PTransNode
       result = PTransNode(n)
   of nkMacroDef:
     # XXX no proper closure support yet:
@@ -748,7 +792,7 @@ proc transform(c: PTransf, n: PNode): PTransNode =
         result = newTransNode(nkCommentStmt, n.info, 0)
       tryStmt.addSon(deferPart)
       # disable the original 'defer' statement:
-      n.kind = nkCommentStmt
+      n.kind = nkEmpty
   of nkContinueStmt:
     result = PTransNode(newNodeI(nkBreakStmt, n.info))
     var labl = c.contSyms[c.contSyms.high]
@@ -794,7 +838,14 @@ proc transform(c: PTransf, n: PNode): PTransNode =
     # XXX comment handling really sucks:
     if importantComments():
       PNode(result).comment = n.comment
-  of nkClosure: return PTransNode(n)
+  of nkClosure:
+    # it can happen that for-loop-inlining produced a fresh
+    # set of variables, including some computed environment
+    # (bug #2604). We need to patch this environment here too:
+    let a = n[1]
+    if a.kind == nkSym:
+      n.sons[1] = transformSymAux(c, a)
+    return PTransNode(n)
   else:
     result = transformSons(c, n)
   when false:
@@ -866,11 +917,11 @@ proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
     result = n
   else:
     var c = openTransf(module, "")
-    result = processTransf(c, n, prc)
+    result = liftLambdas(prc, n, c.tooEarly)
+    #result = n
+    result = processTransf(c, result, prc)
     liftDefer(c, result)
-    result = liftLambdas(prc, result)
+    #result = liftLambdas(prc, result)
-    #if prc.kind == skClosureIterator:
-    #  result = lambdalifting.liftIterator(prc, result)
     incl(result.flags, nfTransf)
     when useEffectSystem: trackProc(prc, result)
     #if prc.name.s == "testbody":
@@ -883,9 +934,11 @@ proc transformStmt*(module: PSym, n: PNode): PNode =
     var c = openTransf(module, "")
     result = processTransf(c, n, module)
     liftDefer(c, result)
-    result = liftLambdasForTopLevel(module, result)
+    #result = liftLambdasForTopLevel(module, result)
     incl(result.flags, nfTransf)
     when useEffectSystem: trackTopLevelStmt(module, result)
+    #if n.info ?? "temp.nim":
+    #  echo renderTree(result, {renderIds})
 
 proc transformExpr*(module: PSym, n: PNode): PNode =
   if nfTransf in n.flags:

compiler/types.nim

@@ -1061,7 +1061,8 @@ proc typeAllowedNode(marker: var IntSet, n: PNode, kind: TSymKind,
       else:
         for i in countup(0, sonsLen(n) - 1):
           let it = n.sons[i]
-          if it.kind == nkRecCase and kind == skConst: return n.typ
+          if it.kind == nkRecCase and kind in {skProc, skConst}:
+            return n.typ
           result = typeAllowedNode(marker, it, kind, flags)
           if result != nil: break
 
@@ -1076,7 +1077,7 @@ proc matchType*(a: PType, pattern: openArray[tuple[k:TTypeKind, i:int]],
 
 proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
                     flags: TTypeAllowedFlags = {}): PType =
-  assert(kind in {skVar, skLet, skConst, skParam, skResult})
+  assert(kind in {skVar, skLet, skConst, skProc, skParam, skResult})
   # if we have already checked the type, return true, because we stop the
   # evaluation if something is wrong:
   result = nil
@@ -1085,7 +1086,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
   var t = skipTypes(typ, abstractInst-{tyTypeDesc})
   case t.kind
   of tyVar:
-    if kind == skConst: return t
+    if kind in {skProc, skConst}: return t
     var t2 = skipTypes(t.sons[0], abstractInst-{tyTypeDesc})
     case t2.kind
     of tyVar:
@@ -1097,6 +1098,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
       if kind notin {skParam, skResult}: result = t
       else: result = typeAllowedAux(marker, t2, kind, flags)
   of tyProc:
+    if kind == skConst and t.callConv == ccClosure: return t
     for i in countup(1, sonsLen(t) - 1):
       result = typeAllowedAux(marker, t.sons[i], skParam, flags)
       if result != nil: break
@@ -1144,7 +1146,8 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
       result = typeAllowedAux(marker, t.sons[i], kind, flags)
       if result != nil: break
   of tyObject, tyTuple:
-    if kind == skConst and t.kind == tyObject and t.sons[0] != nil: return t
+    if kind in {skProc, skConst} and
+        t.kind == tyObject and t.sons[0] != nil: return t
     let flags = flags+{taField}
     for i in countup(0, sonsLen(t) - 1):
       result = typeAllowedAux(marker, t.sons[i], kind, flags)

compiler/vm.nim

@@ -359,7 +359,14 @@ proc opConv*(dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): bool =
       of tyFloat..tyFloat64:
         dest.intVal = int(src.floatVal)
       else:
-        dest.intVal = src.intVal and ((1 shl (desttyp.size*8))-1)
+        let srcDist = (sizeof(src.intVal) - srctyp.size) * 8
+        let destDist = (sizeof(dest.intVal) - desttyp.size) * 8
+        when system.cpuEndian == bigEndian:
+          dest.intVal = (src.intVal shr srcDist) shl srcDist
+          dest.intVal = (dest.intVal shr destDist) shl destDist
+        else:
+          dest.intVal = (src.intVal shl srcDist) shr srcDist
+          dest.intVal = (dest.intVal shl destDist) shr destDist
     of tyFloat..tyFloat64:
       if dest.kind != rkFloat:
         myreset(dest); dest.kind = rkFloat
@@ -611,7 +618,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       addSon(regs[ra].node, r.copyTree)
     of opcExcl:
       decodeB(rkNode)
-      var b = newNodeIT(nkCurly, regs[rb].node.info, regs[rb].node.typ)
+      var b = newNodeIT(nkCurly, regs[ra].node.info, regs[ra].node.typ)
       addSon(b, regs[rb].regToNode)
       var r = diffSets(regs[ra].node, b)
       discardSons(regs[ra].node)

compiler/vmdeps.nim

@@ -70,7 +70,7 @@ proc atomicTypeX(name: string; t: PType; info: TLineInfo): PNode =
 proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode
 
 proc mapTypeToBracket(name: string; t: PType; info: TLineInfo): PNode =
-  result = newNodeIT(nkBracketExpr, info, t)
+  result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
   result.add atomicTypeX(name, t, info)
   for i in 0 .. < t.len:
     if t.sons[i] == nil:
@@ -92,19 +92,19 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode =
   of tyStmt: result = atomicType("stmt")
   of tyEmpty: result = atomicType"void"
   of tyArrayConstr, tyArray:
-    result = newNodeIT(nkBracketExpr, info, t)
+    result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
     result.add atomicType("array")
     result.add mapTypeToAst(t.sons[0], info)
     result.add mapTypeToAst(t.sons[1], info)
   of tyTypeDesc:
     if t.base != nil:
-      result = newNodeIT(nkBracketExpr, info, t)
+      result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
       result.add atomicType("typeDesc")
       result.add mapTypeToAst(t.base, info)
     else:
       result = atomicType"typeDesc"
   of tyGenericInvocation:
-    result = newNodeIT(nkBracketExpr, info, t)
+    result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
     for i in 0 .. < t.len:
       result.add mapTypeToAst(t.sons[i], info)
   of tyGenericInst, tyGenericBody, tyOrdinal, tyUserTypeClassInst:
@@ -117,7 +117,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode =
   of tyGenericParam, tyForward: result = atomicType(t.sym.name.s)
   of tyObject:
     if allowRecursion:
-      result = newNodeIT(nkObjectTy, info, t)
+      result = newNodeIT(nkObjectTy, if t.n.isNil: info else: t.n.info, t)
       if t.sons[0] == nil:
         result.add ast.emptyNode
       else:
@@ -126,7 +126,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode =
     else:
       result = atomicType(t.sym.name.s)
   of tyEnum:
-    result = newNodeIT(nkEnumTy, info, t)
+    result = newNodeIT(nkEnumTy, if t.n.isNil: info else: t.n.info, t)
     result.add copyTree(t.n)
   of tyTuple: result = mapTypeToBracket("tuple", t, info)
   of tySet: result = mapTypeToBracket("set", t, info)
@@ -137,7 +137,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode =
   of tyProc: result = mapTypeToBracket("proc", t, info)
   of tyOpenArray: result = mapTypeToBracket("openArray", t, info)
   of tyRange:
-    result = newNodeIT(nkBracketExpr, info, t)
+    result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
     result.add atomicType("range")
     result.add t.n.sons[0].copyTree
     result.add t.n.sons[1].copyTree
@@ -174,7 +174,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode =
   of tyNot: result = mapTypeToBracket("not", t, info)
   of tyAnything: result = atomicType"anything"
   of tyStatic, tyFromExpr, tyFieldAccessor:
-    result = newNodeIT(nkBracketExpr, info, t)
+    result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t)
     result.add atomicType("static")
     if t.n != nil:
       result.add t.n.copyTree

compiler/vmgen.nim

@@ -1209,7 +1209,7 @@ proc checkCanEval(c: PCtx; n: PNode) =
       not s.isOwnedBy(c.prc.sym) and s.owner != c.module and c.mode != emRepl:
     cannotEval(n)
   elif s.kind in {skProc, skConverter, skMethod,
-                  skIterator, skClosureIterator} and sfForward in s.flags:
+                  skIterator} and sfForward in s.flags:
     cannotEval(n)
 
 proc isTemp(c: PCtx; dest: TDest): bool =
@@ -1638,7 +1638,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     case s.kind
     of skVar, skForVar, skTemp, skLet, skParam, skResult:
       genRdVar(c, n, dest, flags)
-    of skProc, skConverter, skMacro, skTemplate, skMethod, skIterators:
+    of skProc, skConverter, skMacro, skTemplate, skMethod, skIterator:
       # 'skTemplate' is only allowed for 'getAst' support:
       if procIsCallback(c, s): discard
       elif sfImportc in s.flags: c.importcSym(n.info, s)

config/nimdoc.cfg

@@ -102,8 +102,8 @@ doc.file = """<?xml version="1.0" encoding="utf-8" ?>
 <link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
 
 <!-- Google fonts -->
-<link href='http://fonts.googleapis.com/css?family=Raleway:400,600,900' rel='stylesheet' type='text/css'>
+<link href='http://fonts.googleapis.com/css?family=Raleway:400,600,900' rel='stylesheet' type='text/css'/>
-<link href='http://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'>
+<link href='http://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 
 <!-- CSS -->
 <title>$title</title>
@@ -1246,7 +1246,7 @@ dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator
     <div class="row">
       <div class="twelve-columns footer">
         <span class="nim-sprite"></span>
-        <br>
+        <br/>
         <small>Made with Nim. Generated: $date $time UTC</small>
       </div>
     </div>

doc/manual/lexing.txt

@@ -69,6 +69,34 @@ Documentation comments are tokens; they are only allowed at certain places in
 the input file as they belong to the syntax tree!
 
 
+Multiline comments
+------------------
+
+Starting with version 0.13.0 of the language Nim supports multiline comments.
+They look like:
+
+.. code-block:: nim
+  #[Comment here.
+  Multiple lines
+  are not a problem.]#
+
+Multiline comments support nesting:
+
+.. code-block:: nim
+  #[  #[ Multiline comment in already
+     commented out code. ]#
+  proc p[T](x: T) = discard
+  ]#
+
+Multiline documentation comments look like and support nesting too:
+
+.. code-block:: nim
+  proc foo =
+    ##[Long documentation comment
+    here.
+    ]##
+
+
 Identifiers & Keywords
 ----------------------
 

doc/manual/procs.txt

@@ -236,8 +236,6 @@ executable code.
 Do notation
 -----------
 
-**Note:** The future of the ``do`` notation is uncertain.
-
 As a special more convenient notation, proc expressions involved in procedure
 calls can use the ``do`` keyword:
 
@@ -251,10 +249,12 @@ calls can use the ``do`` keyword:
 ``do`` is written after the parentheses enclosing the regular proc params.
 The proc expression represented by the do block is appended to them.
 
-More than one ``do`` block can appear in a single call:
+``do`` with parentheses is an anonymous ``proc``; however a ``do`` without
+parentheses is just a block of code. The ``do`` notation can be used to
+pass multiple blocks to a macro:
 
 .. code-block:: nim
-  proc performWithUndo(task: proc(), undo: proc()) = ...
+  macro performWithUndo(task, undo: untyped) = ...
 
   performWithUndo do:
     # multiple-line block of code

doc/manual/syntax.txt

@@ -64,6 +64,14 @@ Precedence level    Operators                                      First charact
 ================  ===============================================  ==================  ===============
 
 
+Whether an operator is used a prefix operator is also affected by preceeding whitespace (this parsing change was introduced with version 0.13.0):
+
+.. code-block:: nim
+  echo $foo
+  # is parsed as
+  echo($foo)
+
+
 Strong spaces
 -------------
 

doc/tut2.txt

@@ -990,3 +990,17 @@ generated by `treeRepr <macros.html#treeRepr>`_. If at the end of the this
 example you add ``echo treeRepr(result)`` you should get the same output as
 using the ``dumpTree`` macro, but of course you can call that at any point of
 the macro where you might be having troubles.
+
+
+Compilation to JavaScript
+=========================
+
+Nim code can be compiled to JavaScript. However in order to write
+JavaScript-compatible code you should remember the following:
+- ``addr`` and ``ptr`` have slightly different semantic meaning in JavaScript.
+  It is recommended to avoid those if you're not sure how they are translated
+  to JavaScript.
+- ``cast[T](x)`` in JavaScript is translated to ``(x)``.
+- ``cstring`` in JavaScript means JavaScript string. It is a good practice to
+  use ``cstring`` only when it is semantically appropriate. E.g. don't use
+  ``cstring`` as a binary data buffer.

examples/statcsv.nim

@@ -3,7 +3,7 @@
 # the standard deviation of its columns.
 # The CSV file can have a header which is then used for the output.
 
-import os, streams, parsecsv, strutils, math
+import os, streams, parsecsv, strutils, math, stats
 
 if paramCount() < 1:
   quit("Usage: statcsv filename[.csv]")

install_nimble.nims

@@ -1,4 +1,6 @@
 
+import ospaths
+
 mode = ScriptMode.Verbose
 
 var id = 0
@@ -10,4 +12,8 @@ exec "git clone https://github.com/nim-lang/nimble.git nimble" & $id
 withDir "nimble" & $id & "/src":
   exec "nim c nimble"
 
+mkDir "bin/nimblepkg"
+for file in listFiles("nimble" & $id & "/src/nimblepkg/"):
+  cpFile file, "bin/nimblepkg/" & file.extractFilename
+
 mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe

lib/core/macros.nim

@@ -97,7 +97,7 @@ type
     nskUnknown, nskConditional, nskDynLib, nskParam,
     nskGenericParam, nskTemp, nskModule, nskType, nskVar, nskLet,
     nskConst, nskResult,
-    nskProc, nskMethod, nskIterator, nskClosureIterator,
+    nskProc, nskMethod, nskIterator,
     nskConverter, nskMacro, nskTemplate, nskField,
     nskEnumField, nskForVar, nskLabel,
     nskStub

lib/core/typeinfo.nim

@@ -61,7 +61,10 @@ type
                           ## wrapped value and **must not** live longer than
                           ## its wrapped value.
     value: pointer
+    when defined(js):
       rawType: PNimType
+    else:
+      rawTypePtr: pointer
 
   ppointer = ptr pointer
   pbyteArray = ptr array[0.. 0xffff, int8]
@@ -71,6 +74,14 @@ type
     when defined(gogc):
       elemSize: int
   PGenSeq = ptr TGenericSeq
+
+when not defined(js):
+  template rawType(x: Any): PNimType =
+    cast[PNimType](x.rawTypePtr)
+
+  template `rawType=`(x: var Any, p: PNimType) =
+    x.rawTypePtr = cast[pointer](p)
+
 {.deprecated: [TAny: Any, TAnyKind: AnyKind].}
 
 when defined(gogc):
@@ -108,7 +119,7 @@ proc selectBranch(aa: pointer, n: ptr TNimNode): ptr TNimNode =
   else:
     result = n.sons[n.len]
 
-proc newAny(value: pointer, rawType: PNimType): Any =
+proc newAny(value: pointer, rawType: PNimType): Any {.inline.} =
   result.value = value
   result.rawType = rawType
 
@@ -126,8 +137,7 @@ proc toAny*[T](x: var T): Any {.inline.} =
   ## constructs a ``Any`` object from `x`. This captures `x`'s address, so
   ## `x` can be modified with its ``Any`` wrapper! The client needs to ensure
   ## that the wrapper **does not** live longer than `x`!
-  result.value = addr(x)
+  newAny(addr(x), cast[PNimType](getTypeInfo(x)))
-  result.rawType = cast[PNimType](getTypeInfo(x))
 
 proc kind*(x: Any): AnyKind {.inline.} =
   ## get the type kind
@@ -345,7 +355,7 @@ proc `[]`*(x: Any, fieldName: string): Any =
     result.value = x.value +!! n.offset
     result.rawType = n.typ
   elif x.rawType.kind == tyObject and x.rawType.base != nil:
-    return `[]`(Any(value: x.value, rawType: x.rawType.base), fieldName)
+    return `[]`(newAny(x.value, x.rawType.base), fieldName)
   else:
     raise newException(ValueError, "invalid field name: " & fieldName)
 

lib/impure/db_mysql.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nim's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2015 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -43,45 +43,26 @@
 
 import strutils, mysql
 
+import db_common
+export db_common
+
 type
   DbConn* = PMySQL     ## encapsulates a database connection
   Row* = seq[string]   ## a row of a dataset. NULL database values will be
-                       ## transformed always to the empty string.
+                       ## converted to nil.
-  InstantRow* = tuple[row: cstringArray, len: int]  ## a handle that can be
+  InstantRow* = object ## a handle that can be used to get a row's
-                                                    ## used to get a row's
                        ## column text on demand
-  EDb* = object of IOError ## exception that is raised if a database error occurs
+    row: cstringArray
+    len: int
+{.deprecated: [TRow: Row, TDbConn: DbConn].}
 
-  SqlQuery* = distinct string ## an SQL query string
+proc dbError*(db: DbConn) {.noreturn.} =
-
+  ## raises a DbError exception.
-  FDb* = object of IOEffect ## effect that denotes a database operation
+  var e: ref DbError
-  FReadDb* = object of FDb   ## effect that denotes a read operation
-  FWriteDb* = object of FDb  ## effect that denotes a write operation
-{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].}
-
-proc sql*(query: string): SqlQuery {.noSideEffect, inline.} =
-  ## constructs a SqlQuery from the string `query`. This is supposed to be
-  ## used as a raw-string-literal modifier:
-  ## ``sql"update user set counter = counter + 1"``
-  ##
-  ## If assertions are turned off, it does nothing. If assertions are turned
-  ## on, later versions will check the string for valid syntax.
-  result = SqlQuery(query)
-
-proc dbError(db: DbConn) {.noreturn.} =
-  ## raises an EDb exception.
-  var e: ref EDb
   new(e)
   e.msg = $mysql.error(db)
   raise e
 
-proc dbError*(msg: string) {.noreturn.} =
-  ## raises an EDb exception with message `msg`.
-  var e: ref EDb
-  new(e)
-  e.msg = msg
-  raise e
-
 when false:
   proc dbQueryOpt*(db: DbConn, query: string, args: varargs[string, `$`]) =
     var stmt = mysql_stmt_init(db)
@@ -114,7 +95,7 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string =
       add(result, c)
 
 proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {.
-  tags: [FReadDB, FWriteDb].} =
+  tags: [ReadDbEffect, WriteDbEffect].} =
   ## tries to execute the query and returns true if successful, false otherwise.
   var q = dbFormat(query, args)
   return mysql.realQuery(db, q, q.len) == 0'i32
@@ -124,7 +105,7 @@ proc rawExec(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) =
   if mysql.realQuery(db, q, q.len) != 0'i32: dbError(db)
 
 proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
-  tags: [FReadDB, FWriteDb].} =
+  tags: [ReadDbEffect, WriteDbEffect].} =
   ## executes the query and raises EDB if not successful.
   var q = dbFormat(query, args)
   if mysql.realQuery(db, q, q.len) != 0'i32: dbError(db)
@@ -139,7 +120,7 @@ proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) =
   mysql.freeResult(sqlres)
 
 iterator fastRows*(db: DbConn, query: SqlQuery,
-                   args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+                   args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## executes the query and iterates over the result dataset.
   ##
   ## This is very fast, but potentially dangerous.  Use this iterator only
@@ -167,9 +148,9 @@ iterator fastRows*(db: DbConn, query: SqlQuery,
 
 iterator instantRows*(db: DbConn, query: SqlQuery,
                       args: varargs[string, `$`]): InstantRow
-                      {.tags: [FReadDb].} =
+                      {.tags: [ReadDbEffect].} =
-  ## same as fastRows but returns a handle that can be used to get column text
+  ## Same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within the interator body.
+  ## on demand using []. Returned handle is valid only within the iterator body.
   rawExec(db, query, args)
   var sqlres = mysql.useResult(db)
   if sqlres != nil:
@@ -178,20 +159,102 @@ iterator instantRows*(db: DbConn, query: SqlQuery,
     while true:
       row = mysql.fetchRow(sqlres)
       if row == nil: break
-      yield (row: row, len: L)
+      yield InstantRow(row: row, len: L)
     properFreeResult(sqlres, row)
 
+proc setTypeName(t: var DbType; f: PFIELD) =
+  shallowCopy(t.name, $f.name)
+  t.maxReprLen = Natural(f.max_length)
+  if (NOT_NULL_FLAG and f.flags) != 0: t.notNull = true
+  case f.ftype
+  of TYPE_DECIMAL:
+    t.kind = dbDecimal
+  of TYPE_TINY:
+    t.kind = dbInt
+    t.size = 1
+  of TYPE_SHORT:
+    t.kind = dbInt
+    t.size = 2
+  of TYPE_LONG:
+    t.kind = dbInt
+    t.size = 4
+  of TYPE_FLOAT:
+    t.kind = dbFloat
+    t.size = 4
+  of TYPE_DOUBLE:
+    t.kind = dbFloat
+    t.size = 8
+  of TYPE_NULL:
+    t.kind = dbNull
+  of TYPE_TIMESTAMP:
+    t.kind = dbTimestamp
+  of TYPE_LONGLONG:
+    t.kind = dbInt
+    t.size = 8
+  of TYPE_INT24:
+    t.kind = dbInt
+    t.size = 3
+  of TYPE_DATE:
+    t.kind = dbDate
+  of TYPE_TIME:
+    t.kind = dbTime
+  of TYPE_DATETIME:
+    t.kind = dbDatetime
+  of TYPE_YEAR:
+    t.kind = dbDate
+  of TYPE_NEWDATE:
+    t.kind = dbDate
+  of TYPE_VARCHAR, TYPE_VAR_STRING, TYPE_STRING:
+    t.kind = dbVarchar
+  of TYPE_BIT:
+    t.kind = dbBit
+  of TYPE_NEWDECIMAL:
+    t.kind = dbDecimal
+  of TYPE_ENUM: t.kind = dbEnum
+  of TYPE_SET: t.kind = dbSet
+  of TYPE_TINY_BLOB, TYPE_MEDIUM_BLOB, TYPE_LONG_BLOB,
+     TYPE_BLOB: t.kind = dbBlob
+  of TYPE_GEOMETRY:
+    t.kind = dbGeometry
+
+proc setColumnInfo(columns: var DbColumns; res: PRES; L: int) =
+  setLen(columns, L)
+  for i in 0..<L:
+    let fp = mysql.fetch_field_direct(res, cint(i))
+    setTypeName(columns[i].typ, fp)
+    columns[i].name = $fp.name
+    columns[i].tableName = $fp.table
+    columns[i].primaryKey = (fp.flags and PRI_KEY_FLAG) != 0
+    #columns[i].foreignKey = there is no such thing in mysql
+
+iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery;
+                      args: varargs[string, `$`]): InstantRow =
+  ## Same as fastRows but returns a handle that can be used to get column text
+  ## on demand using []. Returned handle is valid only within the iterator body.
+  rawExec(db, query, args)
+  var sqlres = mysql.useResult(db)
+  if sqlres != nil:
+    let L = int(mysql.numFields(sqlres))
+    setColumnInfo(columns, sqlres, L)
+    var row: cstringArray
+    while true:
+      row = mysql.fetchRow(sqlres)
+      if row == nil: break
+      yield InstantRow(row: row, len: L)
+    properFreeResult(sqlres, row)
+
+
 proc `[]`*(row: InstantRow, col: int): string {.inline.} =
-  ## returns text for given column of the row
+  ## Returns text for given column of the row.
   $row.row[col]
 
 proc len*(row: InstantRow): int {.inline.} =
-  ## returns number of columns in the row
+  ## Returns number of columns in the row.
   row.len
 
 proc getRow*(db: DbConn, query: SqlQuery,
-             args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+             args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
-  ## retrieves a single row. If the query doesn't return any rows, this proc
+  ## Retrieves a single row. If the query doesn't return any rows, this proc
   ## will return a Row with empty strings for each column.
   rawExec(db, query, args)
   var sqlres = mysql.useResult(db)
@@ -209,7 +272,7 @@ proc getRow*(db: DbConn, query: SqlQuery,
     properFreeResult(sqlres, row)
 
 proc getAllRows*(db: DbConn, query: SqlQuery,
-                 args: varargs[string, `$`]): seq[Row] {.tags: [FReadDB].} =
+                 args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} =
   ## executes the query and returns the whole result dataset.
   result = @[]
   rawExec(db, query, args)
@@ -232,19 +295,19 @@ proc getAllRows*(db: DbConn, query: SqlQuery,
     mysql.freeResult(sqlres)
 
 iterator rows*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+               args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## same as `fastRows`, but slower and safe.
   for r in items(getAllRows(db, query, args)): yield r
 
 proc getValue*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): string {.tags: [FReadDB].} =
+               args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} =
   ## executes the query and returns the first column of the first row of the
   ## result dataset. Returns "" if the dataset contains no rows or the database
   ## value is NULL.
   result = getRow(db, query, args)[0]
 
 proc tryInsertId*(db: DbConn, query: SqlQuery,
-                  args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} =
+                  args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} =
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row or -1 in case of an error.
   var q = dbFormat(query, args)
@@ -254,7 +317,7 @@ proc tryInsertId*(db: DbConn, query: SqlQuery,
     result = mysql.insertId(db)
 
 proc insertId*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} =
+               args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} =
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row.
   result = tryInsertID(db, query, args)
@@ -262,18 +325,18 @@ proc insertId*(db: DbConn, query: SqlQuery,
 
 proc execAffectedRows*(db: DbConn, query: SqlQuery,
                        args: varargs[string, `$`]): int64 {.
-                       tags: [FReadDB, FWriteDb].} =
+                       tags: [ReadDbEffect, WriteDbEffect].} =
   ## runs the query (typically "UPDATE") and returns the
   ## number of affected rows
   rawExec(db, query, args)
   result = mysql.affectedRows(db)
 
-proc close*(db: DbConn) {.tags: [FDb].} =
+proc close*(db: DbConn) {.tags: [DbEffect].} =
   ## closes the database connection.
   if db != nil: mysql.close(db)
 
 proc open*(connection, user, password, database: string): DbConn {.
-  tags: [FDb].} =
+  tags: [DbEffect].} =
   ## opens a database connection. Raises `EDb` if the connection could not
   ## be established.
   result = mysql.init(nil)
@@ -291,7 +354,7 @@ proc open*(connection, user, password, database: string): DbConn {.
     dbError(errmsg)
 
 proc setEncoding*(connection: DbConn, encoding: string): bool {.
-  tags: [FDb].} =
+  tags: [DbEffect].} =
   ## sets the encoding of a database connection, returns true for
   ## success, false for failure.
   result = mysql.set_character_set(connection, encoding) == 0

lib/impure/db_odbc.nim

@@ -0,0 +1,463 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2015 Nim Contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## A higher level `ODBC` database wrapper.
+##
+## This is the same interface that is implemented for other databases.
+##
+## This has NOT yet been (extensively) tested agains ODBC drivers for
+## Teradata, Oracle, Sybase, MSSqlvSvr, et. al.  databases
+##
+## Currently all queries are ANSI calls, not Unicode.
+##
+## Example:
+##
+## .. code-block:: Nim
+##
+##  import db_odbc, math
+##
+##  let theDb = open("localhost", "nim", "nim", "test")
+##
+##  theDb.exec(sql"Drop table if exists myTestTbl")
+##  theDb.exec(sql("create table myTestTbl (" &
+##      " Id    INT(11)     NOT NULL AUTO_INCREMENT PRIMARY KEY, " &
+##      " Name  VARCHAR(50) NOT NULL, " &
+##      " i     INT(11), " &
+##      " f     DECIMAL(18,10))"))
+##
+##  theDb.exec(sql"START TRANSACTION")
+##  for i in 1..1000:
+##    theDb.exec(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)",
+##          "Item#" & $i, i, sqrt(i.float))
+##  theDb.exec(sql"COMMIT")
+##
+##  for x in theDb.fastRows(sql"select * from myTestTbl"):
+##    echo x
+##
+##  let id = theDb.tryInsertId(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)",
+##          "Item#1001", 1001, sqrt(1001.0))
+##  echo "Inserted item: ", theDb.getValue(sql"SELECT name FROM myTestTbl WHERE id=?", id)
+##
+##  theDb.close()
+
+
+import strutils, odbcsql
+
+import db_common
+export db_common
+
+type
+  OdbcConnTyp = tuple[hDb: SqlHDBC, env: SqlHEnv, stmt: SqlHStmt]
+  DbConn* = OdbcConnTyp    ## encapsulates a database connection
+  Row* = seq[string]   ## a row of a dataset. NULL database values will be
+                       ## converted to nil.
+  InstantRow* = tuple[row: seq[string], len: int]  ## a handle that can be
+                                                    ## used to get a row's
+                                                    ## column text on demand
+
+{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].}
+
+var
+  buf: array[0..4096, char]
+
+proc properFreeResult(hType: int, sqlres: var SqlHandle) {.
+          tags: [WriteDbEffect], raises: [].} =
+  try:
+    discard SQLFreeHandle(hType.TSqlSmallInt, sqlres)
+    sqlres = nil
+  except: discard
+
+proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {.
+          tags: [ReadDbEffect], raises: [].} =
+  ## Returns ODBC error information
+  var
+    sqlState: array[0..512, char]
+    nativeErr: array[0..512, char]
+    errMsg: array[0..512, char]
+    retSz: TSqlSmallInt = 0
+    res: TSqlSmallInt = 0
+  try:
+    sqlState[0] = '\0'
+    nativeErr[0] = '\0'
+    errMsg[0] = '\0'
+    res = SQLErr(db.env, db.hDb, db.stmt,
+              cast[PSQLCHAR](sqlState.addr),
+              cast[PSQLCHAR](nativeErr.addr),
+              cast[PSQLCHAR](errMsg.addr),
+              511.TSqlSmallInt, retSz.addr.PSQLSMALLINT)
+  except:
+    discard
+  return (res.int, $sqlState, $nativeErr, $errMsg)
+
+proc dbError*(db: var DbConn) {.
+          tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} =
+  ## Raises an `[DbError]` exception with ODBC error information
+  var
+    e: ref DbError
+    ss, ne, msg: string = ""
+    isAnError = false
+    res: int = 0
+    prevSs = ""
+  while true:
+    prevSs = ss
+    (res, ss, ne, msg) = db.getErrInfo()
+    if prevSs == ss:
+      break
+    # sqlState of 00000 is not an error
+    elif ss == "00000":
+      break
+    elif ss == "01000":
+      echo "\nWarning: ", ss, " ", msg
+      continue
+    else:
+      isAnError = true
+      echo "\nError: ", ss, " ", msg
+  if isAnError:
+    new(e)
+    e.msg = "ODBC Error"
+    if db.stmt != nil:
+      properFreeResult(SQL_HANDLE_STMT, db.stmt)
+    properFreeResult(SQL_HANDLE_DBC, db.hDb)
+    properFreeResult(SQL_HANDLE_ENV, db.env)
+    raise e
+
+proc SqlCheck(db: var DbConn, resVal: TSqlSmallInt) {.raises: [DbError]} =
+  ## Wrapper that checks if ``resVal`` is not SQL_SUCCESS and if so, raises [EDb]
+  if resVal != SQL_SUCCESS: dbError(db)
+
+proc SqlGetDBMS(db: var DbConn): string {.
+        tags: [ReadDbEffect, WriteDbEffect], raises: [] .} =
+  ## Returns the ODBC SQL_DBMS_NAME string
+  const
+    SQL_DBMS_NAME = 17.SqlUSmallInt
+  var
+    sz: TSqlSmallInt = 0
+  buf[0] = '\0'
+  try:
+    db.SqlCheck(SQLGetInfo(db.hDb, SQL_DBMS_NAME, cast[SqlPointer](buf.addr),
+                        4095.TSqlSmallInt, sz.addr))
+  except: discard
+  return $buf.cstring
+
+proc dbQuote*(s: string): string {.noSideEffect.} =
+  ## DB quotes the string.
+  result = "'"
+  for c in items(s):
+    if c == '\'': add(result, "''")
+    else: add(result, c)
+  add(result, '\'')
+
+proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string {.
+                  noSideEffect.} =
+  ## Replace any ``?`` placeholders with `args`,
+  ## and quotes the arguments
+  result = ""
+  var a = 0
+  for c in items(string(formatstr)):
+    if c == '?':
+      if args[a] == nil:
+        add(result, "NULL")
+      else:
+        add(result, dbQuote(args[a]))
+      inc(a)
+    else:
+      add(result, c)
+
+proc prepareFetch(db: var DbConn, query: SqlQuery,
+                args: varargs[string, `$`]) {.
+                tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  # Prepare a statement, execute it and fetch the data to the driver
+  # ready for retrieval of the data
+  # Used internally by iterators and retrieval procs
+  # requires calling
+  #      properFreeResult(SQL_HANDLE_STMT, db.stmt)
+  # when finished
+  db.SqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt))
+  var q = dbFormat(query, args)
+  db.SqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt))
+  db.SqlCheck(SQLExecute(db.stmt))
+  db.SqlCheck(SQLFetch(db.stmt))
+
+proc prepareFetchDirect(db: var DbConn, query: SqlQuery,
+                args: varargs[string, `$`]) {.
+                tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  # Prepare a statement, execute it and fetch the data to the driver
+  # ready for retrieval of the data
+  # Used internally by iterators and retrieval procs
+  # requires calling
+  #      properFreeResult(SQL_HANDLE_STMT, db.stmt)
+  # when finished
+  db.SqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt))
+  var q = dbFormat(query, args)
+  db.SqlCheck(SQLExecDirect(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt))
+  db.SqlCheck(SQLFetch(db.stmt))
+
+proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {.
+  tags: [ReadDbEffect, WriteDbEffect], raises: [].} =
+  ## Tries to execute the query and returns true if successful, false otherwise.
+  var
+    res:TSqlSmallInt = -1
+  try:
+    db.prepareFetchDirect(query, args)
+    var
+      rCnt = -1
+    res = SQLRowCount(db.stmt, rCnt)
+    if res != SQL_SUCCESS: dbError(db)
+    properFreeResult(SQL_HANDLE_STMT, db.stmt)
+  except: discard
+  return res == SQL_SUCCESS
+
+proc rawExec(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
+            tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  db.prepareFetchDirect(query, args)
+
+proc exec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
+            tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Executes the query and raises EDB if not successful.
+  db.prepareFetchDirect(query, args)
+  properFreeResult(SQL_HANDLE_STMT, db.stmt)
+
+proc newRow(L: int): Row {.noSideEFfect.} =
+  newSeq(result, L)
+  for i in 0..L-1: result[i] = ""
+
+iterator fastRows*(db: var DbConn, query: SqlQuery,
+                   args: varargs[string, `$`]): Row {.
+                tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Executes the query and iterates over the result dataset.
+  ##
+  ## This is very fast, but potentially dangerous.  Use this iterator only
+  ## if you require **ALL** the rows.
+  ##
+  ## Breaking the fastRows() iterator during a loop may cause a driver error
+  ## for subsequenct queries
+  ##
+  ## Rows are retrieved from the server at each iteration.
+  var
+    rowRes: Row
+    sz: TSqlSmallInt = 0
+    cCnt: TSqlSmallInt = 0.TSqlSmallInt
+    rCnt = -1
+
+  db.prepareFetch(query, args)
+  db.SqlCheck(SQLNumResultCols(db.stmt, cCnt))
+  db.SqlCheck(SQLRowCount(db.stmt, rCnt))
+  rowRes = newRow(cCnt)
+  for rNr in 1..rCnt:
+    for colId in 1..cCnt:
+      buf[0] = '\0'
+      db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
+                               cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr))
+      rowRes[colId-1] = $buf.cstring
+    db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1))
+    yield rowRes
+  properFreeResult(SQL_HANDLE_STMT, db.stmt)
+
+iterator instantRows*(db: var DbConn, query: SqlQuery,
+                      args: varargs[string, `$`]): InstantRow
+                {.tags: [ReadDbEffect, WriteDbEffect].} =
+  ## Same as fastRows but returns a handle that can be used to get column text
+  ## on demand using []. Returned handle is valid only within the interator body.
+  var
+    rowRes: Row
+    sz: TSqlSmallInt = 0
+    cCnt: TSqlSmallInt = 0.TSqlSmallInt
+    rCnt = -1
+  db.prepareFetch(query, args)
+  db.SqlCheck(SQLNumResultCols(db.stmt, cCnt))
+  db.SqlCheck(SQLRowCount(db.stmt, rCnt))
+  rowRes = newRow(cCnt)
+  for rNr in 1..rCnt:
+    for colId in 1..cCnt:
+      buf[0] = '\0'
+      db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
+                               cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr))
+      rowRes[colId-1] = $buf.cstring
+    db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1))
+    yield (row: rowRes, len: cCnt.int)
+  properFreeResult(SQL_HANDLE_STMT, db.stmt)
+
+proc `[]`*(row: InstantRow, col: int): string {.inline.} =
+  ## Returns text for given column of the row
+  row.row[col]
+
+proc len*(row: InstantRow): int {.inline.} =
+  ## Returns number of columns in the row
+  row.len
+
+proc getRow*(db: var DbConn, query: SqlQuery,
+             args: varargs[string, `$`]): Row {.
+          tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Retrieves a single row. If the query doesn't return any rows, this proc
+  ## will return a Row with empty strings for each column.
+  var
+    sz: TSqlSmallInt = 0.TSqlSmallInt
+    cCnt: TSqlSmallInt = 0.TSqlSmallInt
+    rCnt = -1
+  result = @[]
+  db.prepareFetch(query, args)
+  db.SqlCheck(SQLNumResultCols(db.stmt, cCnt))
+
+  db.SqlCheck(SQLRowCount(db.stmt, rCnt))
+  for colId in 1..cCnt:
+    db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
+                             cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr))
+    result.add($buf.cstring)
+  db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1))
+  properFreeResult(SQL_HANDLE_STMT, db.stmt)
+
+proc getAllRows*(db: var DbConn, query: SqlQuery,
+                 args: varargs[string, `$`]): seq[Row] {.
+           tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Executes the query and returns the whole result dataset.
+  var
+    rowRes: Row
+    sz: TSqlSmallInt = 0
+    cCnt: TSqlSmallInt = 0.TSqlSmallInt
+    rCnt = -1
+  db.prepareFetch(query, args)
+  db.SqlCheck(SQLNumResultCols(db.stmt, cCnt))
+  db.SqlCheck(SQLRowCount(db.stmt, rCnt))
+  result = @[]
+  for rNr in 1..rCnt:
+    rowRes = @[]
+    buf[0] = '\0'
+    for colId in 1..cCnt:
+      db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
+                               cast[SqlPointer](buf.addr), 4095.TSqlSmallInt, sz.addr))
+      rowRes.add($buf.cstring)
+    db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1))
+    result.add(rowRes)
+  properFreeResult(SQL_HANDLE_STMT, db.stmt)
+
+iterator rows*(db: var DbConn, query: SqlQuery,
+               args: varargs[string, `$`]): Row {.
+         tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Same as `fastRows`, but slower and safe.
+  ##
+  ## This retrieves ALL rows into memory before
+  ## iterating through the rows.
+  ## Large dataset queries will impact on memory usage.
+  for r in items(getAllRows(db, query, args)): yield r
+
+proc getValue*(db: var DbConn, query: SqlQuery,
+               args: varargs[string, `$`]): string {.
+           tags: [ReadDbEffect, WriteDbEffect], raises: [].} =
+  ## Executes the query and returns the first column of the first row of the
+  ## result dataset. Returns "" if the dataset contains no rows or the database
+  ## value is NULL.
+  result = ""
+  try:
+    result = getRow(db, query, args)[0]
+  except: discard
+
+proc tryInsertId*(db: var DbConn, query: SqlQuery,
+                  args: varargs[string, `$`]): int64 {.
+            tags: [ReadDbEffect, WriteDbEffect], raises: [].} =
+  ## Executes the query (typically "INSERT") and returns the
+  ## generated ID for the row or -1 in case of an error.
+  if not tryExec(db, query, args):
+    result = -1'i64
+  else:
+    echo "DBMS: ",SqlGetDBMS(db).toLower()
+    result = -1'i64
+    try:
+      case SqlGetDBMS(db).toLower():
+      of "postgresql":
+        result = getValue(db, sql"SELECT LASTVAL();", []).parseInt
+      of "mysql":
+        result = getValue(db, sql"SELECT LAST_INSERT_ID();", []).parseInt
+      of "sqlite":
+        result = getValue(db, sql"SELECT LAST_INSERT_ROWID();", []).parseInt
+      of "microsoft sql server":
+        result = getValue(db, sql"SELECT SCOPE_IDENTITY();", []).parseInt
+      of "oracle":
+        result = getValue(db, sql"SELECT id.currval FROM DUAL;", []).parseInt
+      else: result = -1'i64
+    except: discard
+
+proc insertId*(db: var DbConn, query: SqlQuery,
+               args: varargs[string, `$`]): int64 {.
+         tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Executes the query (typically "INSERT") and returns the
+  ## generated ID for the row.
+  result = tryInsertID(db, query, args)
+  if result < 0: dbError(db)
+
+proc execAffectedRows*(db: var DbConn, query: SqlQuery,
+                       args: varargs[string, `$`]): int64 {.
+             tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Runs the query (typically "UPDATE") and returns the
+  ## number of affected rows
+  result = -1
+  var res = SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt.SqlHandle)
+  if res != SQL_SUCCESS: dbError(db)
+  var q = dbFormat(query, args)
+  res = SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)
+  if res != SQL_SUCCESS: dbError(db)
+  rawExec(db, query, args)
+  var rCnt = -1
+  result = SQLRowCount(db.hDb, rCnt)
+  if res != SQL_SUCCESS: dbError(db)
+  properFreeResult(SQL_HANDLE_STMT, db.stmt)
+  result = rCnt
+
+proc close*(db: var DbConn) {.
+      tags: [WriteDbEffect], raises: [].} =
+  ## Closes the database connection.
+  if db.hDb != nil:
+    try:
+      var res = SQLDisconnect(db.hDb)
+      if db.stmt != nil:
+        res = SQLFreeHandle(SQL_HANDLE_STMT, db.stmt)
+      res = SQLFreeHandle(SQL_HANDLE_DBC, db.hDb)
+      res = SQLFreeHandle(SQL_HANDLE_ENV, db.env)
+      db = (hDb: nil, env: nil, stmt: nil)
+    except:
+      discard
+
+proc open*(connection, user, password, database: string): DbConn {.
+  tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Opens a database connection.
+  ##
+  ## Raises `EDb` if the connection could not be established.
+  ##
+  ## Currently the database parameter is ignored,
+  ## but included to match ``open()`` in the other db_xxxxx library modules.
+  var
+    val: TSqlInteger = SQL_OV_ODBC3
+    resLen = 0
+  result = (hDb: nil, env: nil, stmt: nil)
+  # allocate environment handle
+  var res = SQLAllocHandle(SQL_HANDLE_ENV, result.env, result.env)
+  if res != SQL_SUCCESS: dbError("Error: unable to initialise ODBC environment.")
+  res = SQLSetEnvAttr(result.env,
+                      SQL_ATTR_ODBC_VERSION.TSqlInteger,
+                      val, resLen.TSqlInteger)
+  if res != SQL_SUCCESS: dbError("Error: unable to set ODBC driver version.")
+  # allocate hDb handle
+  res = SQLAllocHandle(SQL_HANDLE_DBC, result.env, result.hDb)
+  if res != SQL_SUCCESS: dbError("Error: unable to allocate connection handle.")
+
+  # Connect: connection = dsn str,
+  res = SQLConnect(result.hDb,
+                  connection.PSQLCHAR , connection.len.TSqlSmallInt,
+                  user.PSQLCHAR, user.len.TSqlSmallInt,
+                  password.PSQLCHAR, password.len.TSqlSmallInt)
+  if res != SQL_SUCCESS:
+    result.dbError()
+
+proc setEncoding*(connection: DbConn, encoding: string): bool {.
+  tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} =
+  ## Currently not implemented for ODBC.
+  ##
+  ## Sets the encoding of a database connection, returns true for
+  ## success, false for failure.
+  #result = set_character_set(connection, encoding) == 0
+  dbError("setEncoding() is currently not implemented by the db_odbc module")

lib/impure/db_postgres.nim

@@ -62,47 +62,28 @@
 ##             "Dominik")
 import strutils, postgres
 
+import db_common
+export db_common
+
 type
   DbConn* = PPGconn   ## encapsulates a database connection
   Row* = seq[string]  ## a row of a dataset. NULL database values will be
-                       ## transformed always to the empty string.
+                      ## converted to nil.
   InstantRow* = tuple[res: PPGresult, line: int32]  ## a handle that can be
                                                     ## used to get a row's
                                                     ## column text on demand
-  EDb* = object of IOError ## exception that is raised if a database error occurs
-
-  SqlQuery* = distinct string ## an SQL query string
   SqlPrepared* = distinct string ## a identifier for the prepared queries
 
-  FDb* = object of IOEffect ## effect that denotes a database operation
+{.deprecated: [TRow: Row, TDbConn: DbConn,
-  FReadDb* = object of FDb   ## effect that denotes a read operation
-  FWriteDb* = object of FDb  ## effect that denotes a write operation
-{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn,
               TSqlPrepared: SqlPrepared].}
 
-proc sql*(query: string): SqlQuery {.noSideEffect, inline.} =
-  ## constructs a SqlQuery from the string `query`. This is supposed to be
-  ## used as a raw-string-literal modifier:
-  ## ``sql"update user set counter = counter + 1"``
-  ##
-  ## If assertions are turned off, it does nothing. If assertions are turned
-  ## on, later versions will check the string for valid syntax.
-  result = SqlQuery(query)
-
 proc dbError*(db: DbConn) {.noreturn.} =
-  ## raises an EDb exception.
+  ## raises a DbError exception.
-  var e: ref EDb
+  var e: ref DbError
   new(e)
   e.msg = $pqErrorMessage(db)
   raise e
 
-proc dbError*(msg: string) {.noreturn.} =
-  ## raises an EDb exception with message `msg`.
-  var e: ref EDb
-  new(e)
-  e.msg = msg
-  raise e
-
 proc dbQuote*(s: string): string =
   ## DB quotes the string.
   result = "'"
@@ -127,7 +108,7 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string =
       add(result, c)
 
 proc tryExec*(db: DbConn, query: SqlQuery,
-              args: varargs[string, `$`]): bool {.tags: [FReadDB, FWriteDb].} =
+              args: varargs[string, `$`]): bool {.tags: [ReadDbEffect, WriteDbEffect].} =
   ## tries to execute the query and returns true if successful, false otherwise.
   var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil,
                         nil, nil, 0)
@@ -135,7 +116,8 @@ proc tryExec*(db: DbConn, query: SqlQuery,
   pqclear(res)
 
 proc tryExec*(db: DbConn, stmtName: SqlPrepared,
-              args: varargs[string, `$`]): bool {.tags: [FReadDB, FWriteDb].} =
+              args: varargs[string, `$`]): bool {.tags: [
+              ReadDbEffect, WriteDbEffect].} =
   ## tries to execute the query and returns true if successful, false otherwise.
   var arr = allocCStringArray(args)
   var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr,
@@ -145,7 +127,7 @@ proc tryExec*(db: DbConn, stmtName: SqlPrepared,
   pqclear(res)
 
 proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
-  tags: [FReadDB, FWriteDb].} =
+  tags: [ReadDbEffect, WriteDbEffect].} =
   ## executes the query and raises EDB if not successful.
   var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil,
                         nil, nil, 0)
@@ -153,7 +135,7 @@ proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
   pqclear(res)
 
 proc exec*(db: DbConn, stmtName: SqlPrepared,
-          args: varargs[string]) {.tags: [FReadDB, FWriteDb].} =
+          args: varargs[string]) {.tags: [ReadDbEffect, WriteDbEffect].} =
   var arr = allocCStringArray(args)
   var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr,
                            nil, nil, 0)
@@ -196,7 +178,7 @@ proc setRow(res: PPGresult, r: var Row, line, cols: int32) =
       add(r[col], x)
 
 iterator fastRows*(db: DbConn, query: SqlQuery,
-                   args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+                   args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## executes the query and iterates over the result dataset. This is very
   ## fast, but potenially dangerous: If the for-loop-body executes another
   ## query, the results can be undefined. For Postgres it is safe though.
@@ -209,7 +191,7 @@ iterator fastRows*(db: DbConn, query: SqlQuery,
   pqclear(res)
 
 iterator fastRows*(db: DbConn, stmtName: SqlPrepared,
-                   args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+                   args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## executes the prepared query and iterates over the result dataset.
   var res = setupQuery(db, stmtName, args)
   var L = pqNfields(res)
@@ -221,9 +203,9 @@ iterator fastRows*(db: DbConn, stmtName: SqlPrepared,
 
 iterator instantRows*(db: DbConn, query: SqlQuery,
                       args: varargs[string, `$`]): InstantRow
-                      {.tags: [FReadDb].} =
+                      {.tags: [ReadDbEffect].} =
   ## same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within interator body.
+  ## on demand using []. Returned handle is valid only within iterator body.
   var res = setupQuery(db, query, args)
   for i in 0..pqNtuples(res)-1:
     yield (res: res, line: i)
@@ -231,9 +213,9 @@ iterator instantRows*(db: DbConn, query: SqlQuery,
 
 iterator instantRows*(db: DbConn, stmtName: SqlPrepared,
                       args: varargs[string, `$`]): InstantRow
-                      {.tags: [FReadDb].} =
+                      {.tags: [ReadDbEffect].} =
   ## same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within interator body.
+  ## on demand using []. Returned handle is valid only within iterator body.
   var res = setupQuery(db, stmtName, args)
   for i in 0..pqNtuples(res)-1:
     yield (res: res, line: i)
@@ -248,7 +230,7 @@ proc len*(row: InstantRow): int32 {.inline.} =
   pqNfields(row.res)
 
 proc getRow*(db: DbConn, query: SqlQuery,
-             args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+             args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## retrieves a single row. If the query doesn't return any rows, this proc
   ## will return a Row with empty strings for each column.
   var res = setupQuery(db, query, args)
@@ -258,7 +240,7 @@ proc getRow*(db: DbConn, query: SqlQuery,
   pqclear(res)
 
 proc getRow*(db: DbConn, stmtName: SqlPrepared,
-             args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+             args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   var res = setupQuery(db, stmtName, args)
   var L = pqNfields(res)
   result = newRow(L)
@@ -266,31 +248,34 @@ proc getRow*(db: DbConn, stmtName: SqlPrepared,
   pqClear(res)
 
 proc getAllRows*(db: DbConn, query: SqlQuery,
-                 args: varargs[string, `$`]): seq[Row] {.tags: [FReadDB].} =
+                 args: varargs[string, `$`]): seq[Row] {.
+                 tags: [ReadDbEffect].} =
   ## executes the query and returns the whole result dataset.
   result = @[]
   for r in fastRows(db, query, args):
     result.add(r)
 
 proc getAllRows*(db: DbConn, stmtName: SqlPrepared,
-                 args: varargs[string, `$`]): seq[Row] {.tags: [FReadDB].} =
+                 args: varargs[string, `$`]): seq[Row] {.tags:
+                 [ReadDbEffect].} =
   ## executes the prepared query and returns the whole result dataset.
   result = @[]
   for r in fastRows(db, stmtName, args):
     result.add(r)
 
 iterator rows*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+               args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## same as `fastRows`, but slower and safe.
   for r in items(getAllRows(db, query, args)): yield r
 
 iterator rows*(db: DbConn, stmtName: SqlPrepared,
-               args: varargs[string, `$`]): Row {.tags: [FReadDB].} =
+               args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## same as `fastRows`, but slower and safe.
   for r in items(getAllRows(db, stmtName, args)): yield r
 
 proc getValue*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): string {.tags: [FReadDB].} =
+               args: varargs[string, `$`]): string {.
+               tags: [ReadDbEffect].} =
   ## executes the query and returns the first column of the first row of the
   ## result dataset. Returns "" if the dataset contains no rows or the database
   ## value is NULL.
@@ -298,7 +283,8 @@ proc getValue*(db: DbConn, query: SqlQuery,
   result = if isNil(x): "" else: $x
 
 proc tryInsertID*(db: DbConn, query: SqlQuery,
-                  args: varargs[string, `$`]): int64  {.tags: [FWriteDb].}=
+                  args: varargs[string, `$`]): int64 {.
+                  tags: [WriteDbEffect].}=
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row or -1 in case of an error. For Postgre this adds
   ## ``RETURNING id`` to the query, so it only works if your primary key is
@@ -311,7 +297,8 @@ proc tryInsertID*(db: DbConn, query: SqlQuery,
     result = -1
 
 proc insertID*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} =
+               args: varargs[string, `$`]): int64 {.
+               tags: [WriteDbEffect].} =
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row. For Postgre this adds
   ## ``RETURNING id`` to the query, so it only works if your primary key is
@@ -321,7 +308,7 @@ proc insertID*(db: DbConn, query: SqlQuery,
 
 proc execAffectedRows*(db: DbConn, query: SqlQuery,
                        args: varargs[string, `$`]): int64 {.tags: [
-                       FReadDB, FWriteDb].} =
+                       ReadDbEffect, WriteDbEffect].} =
   ## executes the query (typically "UPDATE") and returns the
   ## number of affected rows.
   var q = dbFormat(query, args)
@@ -332,7 +319,7 @@ proc execAffectedRows*(db: DbConn, query: SqlQuery,
 
 proc execAffectedRows*(db: DbConn, stmtName: SqlPrepared,
                        args: varargs[string, `$`]): int64 {.tags: [
-                       FReadDB, FWriteDb].} =
+                       ReadDbEffect, WriteDbEffect].} =
   ## executes the query (typically "UPDATE") and returns the
   ## number of affected rows.
   var arr = allocCStringArray(args)
@@ -343,12 +330,12 @@ proc execAffectedRows*(db: DbConn, stmtName: SqlPrepared,
   result = parseBiggestInt($pqcmdTuples(res))
   pqclear(res)
 
-proc close*(db: DbConn) {.tags: [FDb].} =
+proc close*(db: DbConn) {.tags: [DbEffect].} =
   ## closes the database connection.
   if db != nil: pqfinish(db)
 
 proc open*(connection, user, password, database: string): DbConn {.
-  tags: [FDb].} =
+  tags: [DbEffect].} =
   ## opens a database connection. Raises `EDb` if the connection could not
   ## be established.
   ##
@@ -370,7 +357,7 @@ proc open*(connection, user, password, database: string): DbConn {.
   if pqStatus(result) != CONNECTION_OK: dbError(result) # result = nil
 
 proc setEncoding*(connection: DbConn, encoding: string): bool {.
-  tags: [FDb].} =
+  tags: [DbEffect].} =
   ## sets the encoding of a database connection, returns true for
   ## success, false for failure.
   return pqsetClientEncoding(connection, encoding) == 0

lib/impure/db_sqlite.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nim's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2015 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -40,47 +40,30 @@
 ##
 ##  theDb.close()
 
+{.deadCodeElim:on.}
+
 import strutils, sqlite3
 
+import db_common
+export db_common
+
 type
   DbConn* = PSqlite3  ## encapsulates a database connection
   Row* = seq[string]  ## a row of a dataset. NULL database values will be
-                       ## transformed always to the empty string.
+                       ## converted to nil.
   InstantRow* = Pstmt  ## a handle that can be used to get a row's column
                        ## text on demand
-  EDb* = object of IOError ## exception that is raised if a database error occurs
+{.deprecated: [TRow: Row, TDbConn: DbConn].}
 
-  SqlQuery* = distinct string ## an SQL query string
+proc dbError*(db: DbConn) {.noreturn.} =
-
+  ## raises a DbError exception.
-  FDb* = object of IOEffect ## effect that denotes a database operation
+  var e: ref DbError
-  FReadDb* = object of FDb   ## effect that denotes a read operation
-  FWriteDb* = object of FDb  ## effect that denotes a write operation
-{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].}
-
-proc sql*(query: string): SqlQuery {.noSideEffect, inline.} =
-  ## constructs a SqlQuery from the string `query`. This is supposed to be
-  ## used as a raw-string-literal modifier:
-  ## ``sql"update user set counter = counter + 1"``
-  ##
-  ## If assertions are turned off, it does nothing. If assertions are turned
-  ## on, later versions will check the string for valid syntax.
-  result = SqlQuery(query)
-
-proc dbError(db: DbConn) {.noreturn.} =
-  ## raises an EDb exception.
-  var e: ref EDb
   new(e)
   e.msg = $sqlite3.errmsg(db)
   raise e
 
-proc dbError*(msg: string) {.noreturn.} =
+proc dbQuote*(s: string): string =
-  ## raises an EDb exception with message `msg`.
+  ## DB quotes the string.
-  var e: ref EDb
-  new(e)
-  e.msg = msg
-  raise e
-
-proc dbQuote(s: string): string =
   if s.isNil: return "NULL"
   result = "'"
   for c in items(s):
@@ -99,7 +82,8 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string =
       add(result, c)
 
 proc tryExec*(db: DbConn, query: SqlQuery,
-              args: varargs[string, `$`]): bool {.tags: [FReadDb, FWriteDb].} =
+              args: varargs[string, `$`]): bool {.
+              tags: [ReadDbEffect, WriteDbEffect].} =
   ## tries to execute the query and returns true if successful, false otherwise.
   var q = dbFormat(query, args)
   var stmt: sqlite3.Pstmt
@@ -108,8 +92,8 @@ proc tryExec*(db: DbConn, query: SqlQuery,
       result = finalize(stmt) == SQLITE_OK
 
 proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`])  {.
-  tags: [FReadDb, FWriteDb].} =
+  tags: [ReadDbEffect, WriteDbEffect].} =
-  ## executes the query and raises EDB if not successful.
+  ## executes the query and raises DbError if not successful.
   if not tryExec(db, query, args): dbError(db)
 
 proc newRow(L: int): Row =
@@ -129,14 +113,14 @@ proc setRow(stmt: Pstmt, r: var Row, cols: cint) =
     if not isNil(x): add(r[col], x)
 
 iterator fastRows*(db: DbConn, query: SqlQuery,
-                   args: varargs[string, `$`]): Row  {.tags: [FReadDb].} =
+                   args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## Executes the query and iterates over the result dataset.
   ##
   ## This is very fast, but potentially dangerous.  Use this iterator only
   ## if you require **ALL** the rows.
   ##
   ## Breaking the fastRows() iterator during a loop will cause the next
-  ## database query to raise an [EDb] exception ``unable to close due to ...``.
+  ## database query to raise a DbError exception ``unable to close due to ...``.
   var stmt = setupQuery(db, query, args)
   var L = (column_count(stmt))
   var result = newRow(L)
@@ -147,14 +131,47 @@ iterator fastRows*(db: DbConn, query: SqlQuery,
 
 iterator instantRows*(db: DbConn, query: SqlQuery,
                       args: varargs[string, `$`]): InstantRow
-                      {.tags: [FReadDb].} =
+                      {.tags: [ReadDbEffect].} =
   ## same as fastRows but returns a handle that can be used to get column text
-  ## on demand using []. Returned handle is valid only within the interator body.
+  ## on demand using []. Returned handle is valid only within the iterator body.
   var stmt = setupQuery(db, query, args)
   while step(stmt) == SQLITE_ROW:
     yield stmt
   if finalize(stmt) != SQLITE_OK: dbError(db)
 
+proc toTypeKind(t: var DbType; x: int32) =
+  case x
+  of SQLITE_INTEGER:
+    t.kind = dbInt
+    t.size = 8
+  of SQLITE_FLOAT:
+    t.kind = dbFloat
+    t.size = 8
+  of SQLITE_BLOB: t.kind = dbBlob
+  of SQLITE_NULL: t.kind = dbNull
+  of SQLITE_TEXT: t.kind = dbVarchar
+  else: t.kind = dbUnknown
+
+proc setColumns(columns: var DbColumns; x: PStmt) =
+  let L = column_count(x)
+  setLen(columns, L)
+  for i in 0'i32 ..< L:
+    columns[i].name = $column_name(x, i)
+    columns[i].typ.name = $column_decltype(x, i)
+    toTypeKind(columns[i].typ, column_type(x, i))
+    columns[i].tableName = $column_table_name(x, i)
+
+iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery,
+                      args: varargs[string, `$`]): InstantRow
+                      {.tags: [ReadDbEffect].} =
+  ## same as fastRows but returns a handle that can be used to get column text
+  ## on demand using []. Returned handle is valid only within the iterator body.
+  var stmt = setupQuery(db, query, args)
+  setColumns(columns, stmt)
+  while step(stmt) == SQLITE_ROW:
+    yield stmt
+  if finalize(stmt) != SQLITE_OK: dbError(db)
+
 proc `[]`*(row: InstantRow, col: int32): string {.inline.} =
   ## returns text for given column of the row
   $column_text(row, col)
@@ -164,7 +181,7 @@ proc len*(row: InstantRow): int32 {.inline.} =
   column_count(row)
 
 proc getRow*(db: DbConn, query: SqlQuery,
-             args: varargs[string, `$`]): Row {.tags: [FReadDb].} =
+             args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## retrieves a single row. If the query doesn't return any rows, this proc
   ## will return a Row with empty strings for each column.
   var stmt = setupQuery(db, query, args)
@@ -175,19 +192,19 @@ proc getRow*(db: DbConn, query: SqlQuery,
   if finalize(stmt) != SQLITE_OK: dbError(db)
 
 proc getAllRows*(db: DbConn, query: SqlQuery,
-                 args: varargs[string, `$`]): seq[Row] {.tags: [FReadDb].} =
+                 args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} =
   ## executes the query and returns the whole result dataset.
   result = @[]
   for r in fastRows(db, query, args):
     result.add(r)
 
 iterator rows*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): Row {.tags: [FReadDb].} =
+               args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} =
   ## same as `FastRows`, but slower and safe.
   for r in fastRows(db, query, args): yield r
 
 proc getValue*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): string {.tags: [FReadDb].} =
+               args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} =
   ## executes the query and returns the first column of the first row of the
   ## result dataset. Returns "" if the dataset contains no rows or the database
   ## value is NULL.
@@ -205,7 +222,7 @@ proc getValue*(db: DbConn, query: SqlQuery,
 
 proc tryInsertID*(db: DbConn, query: SqlQuery,
                   args: varargs[string, `$`]): int64
-                  {.tags: [FWriteDb], raises: [].} =
+                  {.tags: [WriteDbEffect], raises: [].} =
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row or -1 in case of an error.
   var q = dbFormat(query, args)
@@ -218,7 +235,7 @@ proc tryInsertID*(db: DbConn, query: SqlQuery,
       result = -1
 
 proc insertID*(db: DbConn, query: SqlQuery,
-               args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} =
+               args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} =
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row. For Postgre this adds
   ## ``RETURNING id`` to the query, so it only works if your primary key is
@@ -228,18 +245,18 @@ proc insertID*(db: DbConn, query: SqlQuery,
 
 proc execAffectedRows*(db: DbConn, query: SqlQuery,
                        args: varargs[string, `$`]): int64 {.
-                       tags: [FReadDb, FWriteDb].} =
+                       tags: [ReadDbEffect, WriteDbEffect].} =
   ## executes the query (typically "UPDATE") and returns the
   ## number of affected rows.
   exec(db, query, args)
   result = changes(db)
 
-proc close*(db: DbConn) {.tags: [FDb].} =
+proc close*(db: DbConn) {.tags: [DbEffect].} =
   ## closes the database connection.
   if sqlite3.close(db) != SQLITE_OK: dbError(db)
 
 proc open*(connection, user, password, database: string): DbConn {.
-  tags: [FDb].} =
+  tags: [DbEffect].} =
   ## opens a database connection. Raises `EDb` if the connection could not
   ## be established. Only the ``connection`` parameter is used for ``sqlite``.
   var db: DbConn
@@ -249,7 +266,7 @@ proc open*(connection, user, password, database: string): DbConn {.
     dbError(db)
 
 proc setEncoding*(connection: DbConn, encoding: string): bool {.
-  tags: [FDb].} =
+  tags: [DbEffect].} =
   ## sets the encoding of a database connection, returns true for
   ## success, false for failure.
   ##

lib/nimbase.h

@@ -418,10 +418,6 @@ typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(
 #  define NIM_EXTERNC
 #endif
 
-/* we have to tinker with TNimType as it's both part of system.nim and
-   typeinfo.nim but system.nim doesn't export it cleanly... */
-typedef struct TNimType TNimType;
-
 /* ---------------- platform specific includes ----------------------- */
 
 /* VxWorks related includes */

lib/packages/docutils/highlite.nim

@@ -173,7 +173,41 @@ proc nimNextToken(g: var GeneralTokenizer) =
       while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos)
     of '#':
       g.kind = gtComment
-      while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos)
+      inc(pos)
+      var isDoc = false
+      if g.buf[pos] == '#':
+        inc(pos)
+        isDoc = true
+      if g.buf[pos] == '[':
+        g.kind = gtLongComment
+        var nesting = 0
+        while true:
+          case g.buf[pos]
+          of '\0': break
+          of '#':
+            if isDoc:
+              if g.buf[pos+1] == '#' and g.buf[pos+2] == '[':
+                inc nesting
+            elif g.buf[pos+1] == '[':
+              inc nesting
+            inc pos
+          of ']':
+            if isDoc:
+              if g.buf[pos+1] == '#' and g.buf[pos+2] == '#':
+                if nesting == 0:
+                  inc(pos, 3)
+                  break
+                dec nesting
+            elif g.buf[pos+1] == '#':
+              if nesting == 0:
+                inc(pos, 2)
+                break
+              dec nesting
+            inc pos
+          else:
+            inc pos
+      else:
+        while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos)
     of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF':
       var id = ""
       while g.buf[pos] in SymChars + {'_'}:

lib/packages/docutils/rstgen.nim

@@ -534,7 +534,7 @@ proc generateDocumentationJumps(docs: IndexedDocs): string =
   for title in titles:
     chunks.add("<a href=\"" & title.link & "\">" & title.keyword & "</a>")
 
-  result.add(chunks.join(", ") & ".<br>")
+  result.add(chunks.join(", ") & ".<br/>")
 
 proc generateModuleJumps(modules: seq[string]): string =
   ## Returns a plain list of hyperlinks to the list of modules.
@@ -544,7 +544,7 @@ proc generateModuleJumps(modules: seq[string]): string =
   for name in modules:
     chunks.add("<a href=\"" & name & ".html\">" & name & "</a>")
 
-  result.add(chunks.join(", ") & ".<br>")
+  result.add(chunks.join(", ") & ".<br/>")
 
 proc readIndexDir(dir: string):
     tuple[modules: seq[string], symbols: seq[IndexEntry], docs: IndexedDocs] =

lib/posix/posix.nim

@@ -2165,6 +2165,10 @@ proc pwrite*(a1: cint, a2: pointer, a3: int, a4: Off): int {.
   importc, header: "<unistd.h>".}
 proc read*(a1: cint, a2: pointer, a3: int): int {.importc, header: "<unistd.h>".}
 proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "<unistd.h>".}
+proc ioctl*(f: FileHandle, device: uint): int {.importc: "ioctl",
+      header: "<sys/ioctl.h>", varargs, tags: [WriteIOEffect].}
+  ## A system call for device-specific input/output operations and other
+  ## operations which cannot be expressed by regular system calls
 
 proc rmdir*(a1: cstring): cint {.importc, header: "<unistd.h>".}
 proc setegid*(a1: Gid): cint {.importc, header: "<unistd.h>".}

lib/pure/asyncftpclient.nim

@@ -288,7 +288,7 @@ proc defaultOnProgressChanged*(total, progress: BiggestInt,
   result.complete()
 
 proc retrFile*(ftp: AsyncFtpClient, file, dest: string,
-               onProgressChanged = defaultOnProgressChanged) {.async.} =
+               onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} =
   ## Downloads ``file`` and saves it to ``dest``.
   ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function
   ## when the download is finished. The event's ``filename`` field will be equal
@@ -339,7 +339,7 @@ proc doUpload(ftp: AsyncFtpClient, file: File,
     await countdownFut or sendFut
 
 proc store*(ftp: AsyncFtpClient, file, dest: string,
-            onProgressChanged = defaultOnProgressChanged) {.async.} =
+            onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} =
   ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this
   ## function asynchronously is recommended to view the progress of
   ## the download.

lib/pure/db_common.nim

@@ -0,0 +1,103 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Common datatypes and definitions for all ``db_*.nim`` (
+## `db_mysql <db_mysql.html>`_, `db_postgres <db_postgres.html>`_,
+## and `db_sqlite <db_sqlite.html>`_) modules.
+
+type
+  DbError* = object of IOError ## exception that is raised if a database error occurs
+
+  SqlQuery* = distinct string ## an SQL query string
+
+
+  DbEffect* = object of IOEffect ## effect that denotes a database operation
+  ReadDbEffect* = object of DbEffect   ## effect that denotes a read operation
+  WriteDbEffect* = object of DbEffect  ## effect that denotes a write operation
+
+  DbTypeKind* = enum  ## a superset of datatypes that might be supported.
+    dbUnknown,        ## unknown datatype
+    dbSerial,         ## datatype used for primary auto-increment keys
+    dbNull,           ## datatype used for the NULL value
+    dbBit,            ## bit datatype
+    dbBool,           ## boolean datatype
+    dbBlob,           ## blob datatype
+    dbFixedChar,      ## string of fixed length
+    dbVarchar,        ## string datatype
+    dbJson,           ## JSON datatype
+    dbXml,            ## XML datatype
+    dbInt,            ## some integer type
+    dbUInt,           ## some unsigned integer type
+    dbDecimal,        ## decimal numbers (fixed-point number)
+    dbFloat,          ## some floating point type
+    dbDate,           ## a year-month-day description
+    dbTime,           ## HH:MM:SS information
+    dbDatetime,       ## year-month-day and HH:MM:SS information,
+                      ## plus optional time or timezone information
+    dbTimestamp,      ## Timestamp values are stored as the number of seconds
+                      ## since the epoch ('1970-01-01 00:00:00' UTC).
+    dbTimeInterval,   ## an interval [a,b] of times
+    dbEnum,           ## some enum
+    dbSet,            ## set of enum values
+    dbArray,          ## an array of values
+    dbComposite,      ## composite type (record, struct, etc)
+    dbUrl,            ## a URL
+    dbUuid,           ## a UUID
+    dbInet,           ## an IP address
+    dbMacAddress,     ## a MAC address
+    dbGeometry,       ## some geometric type
+    dbPoint,          ## Point on a plane   (x,y)
+    dbLine,           ## Infinite line ((x1,y1),(x2,y2))
+    dbLseg,           ## Finite line segment   ((x1,y1),(x2,y2))
+    dbBox,            ## Rectangular box   ((x1,y1),(x2,y2))
+    dbPath,           ## Closed or open path (similar to polygon) ((x1,y1),...)
+    dbPolygon,        ## Polygon (similar to closed path)   ((x1,y1),...)
+    dbCircle,         ## Circle   <(x,y),r> (center point and radius)
+    dbUser1,          ## user definable datatype 1 (for unknown extensions)
+    dbUser2,          ## user definable datatype 2 (for unknown extensions)
+    dbUser3,          ## user definable datatype 3 (for unknown extensions)
+    dbUser4,          ## user definable datatype 4 (for unknown extensions)
+    dbUser5           ## user definable datatype 5 (for unknown extensions)
+
+  DbType* = object    ## describes a database type
+    kind*: DbTypeKind ## the kind of the described type
+    notNull*: bool    ## does the type contain NULL?
+    name*: string     ## the name of the type
+    size*: Natural    ## the size of the datatype; 0 if of variable size
+    maxReprLen*: Natural ## maximal length required for the representation
+    precision*, scale*: Natural ## precision and scale of the number
+    min*, max*: BiggestInt ## the minimum and maximum of allowed values
+    validValues*: seq[string] ## valid values of an enum or a set
+
+  DbColumn* = object   ## information about a database column
+    name*: string      ## name of the column
+    tableName*: string ## name of the table the column belongs to (optional)
+    typ*: DbType       ## type of the column
+    primaryKey*: bool  ## is this a primary key?
+    foreignKey*: bool  ## is this a foreign key?
+  DbColumns* = seq[DbColumn]
+
+{.deprecated: [EDb: DbError, TSqlQuery: SqlQuery, FDb: DbEffect,
+              FReadDb: ReadDbEffect, FWriteDb: WriteDbEffect].}
+
+template sql*(query: string): SqlQuery =
+  ## constructs a SqlQuery from the string `query`. This is supposed to be
+  ## used as a raw-string-literal modifier:
+  ## ``sql"update user set counter = counter + 1"``
+  ##
+  ## If assertions are turned off, it does nothing. If assertions are turned
+  ## on, later versions will check the string for valid syntax.
+  SqlQuery(query)
+
+proc dbError*(msg: string) {.noreturn, noinline.} =
+  ## raises an DbError exception with message `msg`.
+  var e: ref DbError
+  new(e)
+  e.msg = msg
+  raise e

lib/pure/httpclient.nim

@@ -110,7 +110,7 @@ type
   EInvalidProtocol: ProtocolError, EHttpRequestErr: HttpRequestError
 ].}
 
-const defUserAgent* = "Nim httpclient/0.1"
+const defUserAgent* = "Nim httpclient/" & NimVersion
 
 proc httpError(msg: string) =
   var e: ref ProtocolError
@@ -389,6 +389,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "",
   ## | An optional timeout can be specified in milliseconds, if reading from the
   ## server takes longer than specified an ETimeout exception will be raised.
   var r = if proxy == nil: parseUri(url) else: proxy.url
+  var hostUrl = if proxy == nil: r else: parseUri(url)
   var headers = substr(httpMethod, len("http"))
   # TODO: Use generateHeaders further down once it supports proxies.
   if proxy == nil:
@@ -402,10 +403,10 @@ proc request*(url: string, httpMethod: string, extraHeaders = "",
 
   headers.add(" HTTP/1.1\c\L")
 
-  if r.port == "":
+  if hostUrl.port == "":
-    add(headers, "Host: " & r.hostname & "\c\L")
+    add(headers, "Host: " & hostUrl.hostname & "\c\L")
   else:
-    add(headers, "Host: " & r.hostname & ":" & r.port & "\c\L")
+    add(headers, "Host: " & hostUrl.hostname & ":" & hostUrl.port & "\c\L")
 
   if userAgent != "":
     add(headers, "User-Agent: " & userAgent & "\c\L")
@@ -414,7 +415,6 @@ proc request*(url: string, httpMethod: string, extraHeaders = "",
     add(headers, "Proxy-Authorization: basic " & auth & "\c\L")
   add(headers, extraHeaders)
   add(headers, "\c\L")
-
   var s = newSocket()
   if s == nil: raiseOSError(osLastError())
   var port = net.Port(80)

lib/pure/lexbase.nim

@@ -28,7 +28,10 @@ type
   BaseLexer* = object of RootObj ## the base lexer. Inherit your lexer from
                                  ## this object.
     bufpos*: int              ## the current position within the buffer
-    buf*: cstring             ## the buffer itself
+    when defined(js):         ## the buffer itself
+      buf*: string
+    else:
+      buf*: cstring
     bufLen*: int              ## length of buffer in characters
     input: Stream            ## the input stream
     lineNumber*: int          ## the current line number
@@ -43,6 +46,7 @@ const
 
 proc close*(L: var BaseLexer) =
   ## closes the base lexer. This closes `L`'s associated stream too.
+  when not defined(js):
     dealloc(L.buf)
   close(L.input)
 
@@ -58,8 +62,11 @@ proc fillBuffer(L: var BaseLexer) =
   toCopy = L.bufLen - L.sentinel - 1
   assert(toCopy >= 0)
   if toCopy > 0:
-    moveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize)
+    when defined(js):
+      for i in 0 ..< toCopy: L.buf[i] = L.buf[L.sentinel + 1 + i]
+    else:
       # "moveMem" handles overlapping regions
+      moveMem(L.buf, addr L.buf[L.sentinel + 1], toCopy * chrSize)
   charsRead = readData(L.input, addr(L.buf[toCopy]),
                        (L.sentinel + 1) * chrSize) div chrSize
   s = toCopy + charsRead
@@ -81,6 +88,9 @@ proc fillBuffer(L: var BaseLexer) =
         # double the buffer's size and try again:
         oldBufLen = L.bufLen
         L.bufLen = L.bufLen * 2
+        when defined(js):
+          L.buf.setLen(L.bufLen)
+        else:
           L.buf = cast[cstring](realloc(L.buf, L.bufLen * chrSize))
         assert(L.bufLen - oldBufLen == oldBufLen)
         charsRead = readData(L.input, addr(L.buf[oldBufLen]),
@@ -139,6 +149,9 @@ proc open*(L: var BaseLexer, input: Stream, bufLen: int = 8192;
   L.bufpos = 0
   L.bufLen = bufLen
   L.refillChars = refillChars
+  when defined(js):
+    L.buf = newString(bufLen)
+  else:
     L.buf = cast[cstring](alloc(bufLen * chrSize))
   L.sentinel = bufLen - 1
   L.lineStart = 0

lib/pure/math.nim

@@ -118,26 +118,6 @@ proc sum*[T](x: openArray[T]): T {.noSideEffect.} =
   ## If `x` is empty, 0 is returned.
   for i in items(x): result = result + i
 
-template toFloat(f: float): float = f
-
-proc mean*[T](x: openArray[T]): float {.noSideEffect.} =
-  ## Computes the mean of the elements in `x`, which are first converted to floats.
-  ## If `x` is empty, NaN is returned.
-  ## ``toFloat(x: T): float`` must be defined.
-  for i in items(x): result = result + toFloat(i)
-  result = result / toFloat(len(x))
-
-proc variance*[T](x: openArray[T]): float {.noSideEffect.} =
-  ## Computes the variance of the elements in `x`.
-  ## If `x` is empty, NaN is returned.
-  ## ``toFloat(x: T): float`` must be defined.
-  result = 0.0
-  var m = mean(x)
-  for i in items(x):
-    var diff = toFloat(i) - m
-    result = result + diff*diff
-  result = result / toFloat(len(x))
-
 proc random*(max: int): int {.benign.}
   ## Returns a random number in the range 0..max-1. The sequence of
   ## random number is always the same, unless `randomize` is called
@@ -376,48 +356,6 @@ proc random*[T](a: openArray[T]): T =
   ## returns a random element from the openarray `a`.
   result = a[random(a.low..a.len)]
 
-type
-  RunningStat* = object                 ## an accumulator for statistical data
-    n*: int                             ## number of pushed data
-    sum*, min*, max*, mean*: float      ## self-explaining
-    oldM, oldS, newS: float
-
-{.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].}
-
-proc push*(s: var RunningStat, x: float) =
-  ## pushes a value `x` for processing
-  inc(s.n)
-  # See Knuth TAOCP vol 2, 3rd edition, page 232
-  if s.n == 1:
-    s.min = x
-    s.max = x
-    s.oldM = x
-    s.mean = x
-    s.oldS = 0.0
-  else:
-    if s.min > x: s.min = x
-    if s.max < x: s.max = x
-    s.mean = s.oldM + (x - s.oldM)/toFloat(s.n)
-    s.newS = s.oldS + (x - s.oldM)*(x - s.mean)
-
-    # set up for next iteration:
-    s.oldM = s.mean
-    s.oldS = s.newS
-  s.sum = s.sum + x
-
-proc push*(s: var RunningStat, x: int) =
-  ## pushes a value `x` for processing. `x` is simply converted to ``float``
-  ## and the other push operation is called.
-  push(s, toFloat(x))
-
-proc variance*(s: RunningStat): float =
-  ## computes the current variance of `s`
-  if s.n > 1: result = s.newS / (toFloat(s.n - 1))
-
-proc standardDeviation*(s: RunningStat): float =
-  ## computes the current standard deviation of `s`
-  result = sqrt(variance(s))
-
 {.pop.}
 {.pop.}
 

lib/pure/nativesockets.nim

@@ -206,7 +206,8 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET,
   # OpenBSD doesn't support AI_V4MAPPED and doesn't define the macro AI_V4MAPPED.
   # FreeBSD doesn't support AI_V4MAPPED but defines the macro.
   # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=198092
-  when not defined(freebsd) or defined(openbsd):
+  when not defined(freebsd) and not defined(openbsd) and not defined(netbsd):
+    if domain == AF_INET6:
       hints.ai_flags = AI_V4MAPPED
   var gaiResult = getaddrinfo(address, $port, addr(hints), result)
   if gaiResult != 0'i32:

lib/pure/nimprof.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nim's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2015 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -117,24 +117,38 @@ when defined(memProfiler):
   var
     gTicker {.threadvar.}: int
 
-  proc hook(st: StackTrace, size: int) {.nimcall.} =
+  proc requestedHook(): bool {.nimcall.} =
     if gTicker == 0:
-      gTicker = -1
+      gTicker = SamplingInterval
+      result = true
+    dec gTicker
+
+  proc hook(st: StackTrace, size: int) {.nimcall.} =
     when defined(ignoreAllocationSize):
       hookAux(st, 1)
     else:
       hookAux(st, size)
-      gTicker = SamplingInterval
-    dec gTicker
 
 else:
   var
     t0 {.threadvar.}: Ticks
+    gTicker: int # we use an additional counter to
+                 # avoid calling 'getTicks' too frequently
+
+  proc requestedHook(): bool {.nimcall.} =
+    if interval == 0: result = true
+    elif gTicker == 0:
+      gTicker = 500
+      if getTicks() - t0 > interval:
+        result = true
+    else:
+      dec gTicker
 
   proc hook(st: StackTrace) {.nimcall.} =
+    #echo "profiling! ", interval
     if interval == 0:
       hookAux(st, 1)
-    elif int64(t0) == 0 or getTicks() - t0 > interval:
+    else:
       hookAux(st, 1)
       t0 = getTicks()
 
@@ -145,9 +159,10 @@ proc cmpEntries(a, b: ptr ProfileEntry): int =
   result = b.getTotal - a.getTotal
 
 proc `//`(a, b: int): string =
-  result = format("$1/$2 = $3%", a, b, formatFloat(a / b * 100.0, ffDefault, 2))
+  result = format("$1/$2 = $3%", a, b, formatFloat(a / b * 100.0, ffDecimal, 2))
 
 proc writeProfile() {.noconv.} =
+  system.profilingRequestedHook = nil
   when declared(system.StackTrace):
     system.profilerHook = nil
   const filename = "profile_results.txt"
@@ -193,14 +208,15 @@ var
 proc disableProfiling*() =
   when declared(system.StackTrace):
     atomicDec disabled
-    system.profilerHook = nil
+    system.profilingRequestedHook = nil
 
 proc enableProfiling*() =
   when declared(system.StackTrace):
     if atomicInc(disabled) >= 0:
-      system.profilerHook = hook
+      system.profilingRequestedHook = requestedHook
 
 when declared(system.StackTrace):
+  system.profilingRequestedHook = requestedHook
   system.profilerHook = hook
   addQuitProc(writeProfile)
 

lib/pure/osproc.nim

@@ -886,7 +886,7 @@ elif not defined(useNimRtl):
     discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error))
     exitnow(1)
 
-  when defined(macosx) or defined(freebsd):
+  when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(android):
     var environ {.importc.}: cstringArray
 
   proc startProcessAfterFork(data: ptr StartProcessData) =
@@ -916,7 +916,7 @@ elif not defined(useNimRtl):
     discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC)
 
     if data.optionPoUsePath:
-      when defined(macosx) or defined(freebsd):
+      when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(android):
         # MacOSX doesn't have execvpe, so we need workaround.
         # On MacOSX we can arrive here only from fork, so this is safe:
         environ = data.sysEnv

lib/pure/parseutils.nim

@@ -25,7 +25,7 @@ const
 proc toLower(c: char): char {.inline.} =
   result = if c in {'A'..'Z'}: chr(ord(c)-ord('A')+ord('a')) else: c
 
-proc parseHex*(s: string, number: var int, start = 0): int {.
+proc parseHex*(s: string, number: var int, start = 0; maxLen = 0): int {.
   rtl, extern: "npuParseHex", noSideEffect.}  =
   ## Parses a hexadecimal number and stores its value in ``number``.
   ##
@@ -45,11 +45,14 @@ proc parseHex*(s: string, number: var int, start = 0): int {.
   ##   discard parseHex("0x38", value)
   ##   assert value == -200
   ##
+  ## If 'maxLen==0' the length of the hexadecimal number has no
+  ## upper bound. Not more than ```maxLen`` characters are parsed.
   var i = start
   var foundDigit = false
   if s[i] == '0' and (s[i+1] == 'x' or s[i+1] == 'X'): inc(i, 2)
   elif s[i] == '#': inc(i)
-  while true:
+  let last = if maxLen == 0: s.len else: i+maxLen
+  while i < last:
     case s[i]
     of '_': discard
     of '0'..'9':

lib/pure/stats.nim

@@ -0,0 +1,348 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+## Statistical analysis framework for performing
+## basic statistical analysis of data.
+## The data is analysed in a single pass, when a data value
+## is pushed to the ``RunningStat`` or ``RunningRegress`` objects
+##
+## ``RunningStat`` calculates for a single data set
+## - n (data count)
+## - min  (smallest value)
+## - max  (largest value)
+## - sum
+## - mean
+## - variance
+## - varianceS (sample var)
+## - standardDeviation
+## - standardDeviationS  (sample stddev)
+## - skewness (the third statistical moment)
+## - kurtosis (the fourth statistical moment)
+##
+## ``RunningRegress`` calculates for two sets of data
+## - n
+## - slope
+## - intercept
+## - correlation
+##
+## Procs have been provided to calculate statistics on arrays and sequences.
+##
+## However, if more than a single statistical calculation is required, it is more
+## efficient to push the data once to the RunningStat object, and
+## call the numerous statistical procs for the RunningStat object.
+##
+## .. code-block:: Nim
+##
+##  var rs: RunningStat
+##  rs.push(MySeqOfData)
+##  rs.mean()
+##  rs.variance()
+##  rs.skewness()
+##  rs.kurtosis()
+
+from math import FloatClass, sqrt, pow, round
+
+{.push debugger:off .} # the user does not want to trace a part
+                       # of the standard library!
+{.push checks:off, line_dir:off, stack_trace:off.}
+
+type
+  RunningStat* = object             ## an accumulator for statistical data
+    n*: int                         ## number of pushed data
+    min*, max*, sum*: float         ## self-explaining
+    mom1, mom2, mom3, mom4: float   ## statistical moments, mom1 is mean
+
+
+  RunningRegress* = object  ## an accumulator for regression calculations
+    n*: int                 ## number of pushed data
+    x_stats*: RunningStat   ## stats for first set of data
+    y_stats*: RunningStat   ## stats for second set of data
+    s_xy: float             ## accumulated data for combined xy
+
+{.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].}
+
+# ----------- RunningStat --------------------------
+proc clear*(s: var RunningStat) =
+  ## reset `s`
+  s.n = 0
+  s.min = toBiggestFloat(int.high)
+  s.max = 0.0
+  s.sum = 0.0
+  s.mom1 = 0.0
+  s.mom2 = 0.0
+  s.mom3 = 0.0
+  s.mom4 = 0.0
+
+proc push*(s: var RunningStat, x: float) =
+  ## pushes a value `x` for processing
+  if s.n == 0: s.min = x
+  inc(s.n)
+  # See Knuth TAOCP vol 2, 3rd edition, page 232
+  if s.min > x: s.min = x
+  if s.max < x: s.max = x
+  s.sum += x
+  let n = toFloat(s.n)
+  let delta = x - s.mom1
+  let delta_n = delta / toFloat(s.n)
+  let delta_n2 = delta_n * delta_n
+  let term1 = delta * delta_n * toFloat(s.n - 1)
+  s.mom4 += term1 * delta_n2 * (n*n - 3*n + 3) +
+              6*delta_n2*s.mom2 - 4*delta_n*s.mom3
+  s.mom3 += term1 * delta_n * (n - 2) - 3*delta_n*s.mom2
+  s.mom2 += term1
+  s.mom1 += delta_n
+
+proc push*(s: var RunningStat, x: int) =
+  ## pushes a value `x` for processing.
+  ##
+  ## `x` is simply converted to ``float``
+  ## and the other push operation is called.
+  s.push(toFloat(x))
+
+proc push*(s: var RunningStat, x: openarray[float|int]) =
+  ## pushes all values of `x` for processing.
+  ##
+  ## Int values of `x` are simply converted to ``float`` and
+  ## the other push operation is called.
+  for val in x:
+    s.push(val)
+
+proc mean*(s: RunningStat): float =
+  ## computes the current mean of `s`
+  result = s.mom1
+
+proc variance*(s: RunningStat): float =
+  ## computes the current population variance of `s`
+  result = s.mom2 / toFloat(s.n)
+
+proc varianceS*(s: RunningStat): float =
+  ## computes the current sample variance of `s`
+  if s.n > 1: result = s.mom2 / toFloat(s.n - 1)
+
+proc standardDeviation*(s: RunningStat): float =
+  ## computes the current population standard deviation of `s`
+  result = sqrt(variance(s))
+
+proc standardDeviationS*(s: RunningStat): float =
+  ## computes the current sample standard deviation of `s`
+  result = sqrt(varianceS(s))
+
+proc skewness*(s: RunningStat): float =
+  ## computes the current population skewness of `s`
+  result = sqrt(toFloat(s.n)) * s.mom3 / pow(s.mom2, 1.5)
+
+proc skewnessS*(s: RunningStat): float =
+  ## computes the current sample skewness of `s`
+  let s2 = skewness(s)
+  result = sqrt(toFloat(s.n*(s.n-1)))*s2 / toFloat(s.n-2)
+
+proc kurtosis*(s: RunningStat): float =
+  ## computes the current population kurtosis of `s`
+  result = toFloat(s.n) * s.mom4 / (s.mom2 * s.mom2) - 3.0
+
+proc kurtosisS*(s: RunningStat): float =
+  ## computes the current sample kurtosis of `s`
+  result = toFloat(s.n-1) / toFloat((s.n-2)*(s.n-3)) *
+              (toFloat(s.n+1)*kurtosis(s) + 6)
+
+proc `+`*(a, b: RunningStat): RunningStat =
+  ## combine two RunningStats.
+  ##
+  ## Useful if performing parallel analysis of data series
+  ## and need to re-combine parallel result sets
+  result.clear()
+  result.n = a.n + b.n
+
+  let delta = b.mom1 - a.mom1
+  let delta2 = delta*delta
+  let delta3 = delta*delta2
+  let delta4 = delta2*delta2
+  let n = toFloat(result.n)
+
+  result.mom1 = (a.n.float*a.mom1 + b.n.float*b.mom1) / n
+  result.mom2 = a.mom2 + b.mom2 + delta2 * a.n.float * b.n.float / n
+  result.mom3 = a.mom3 + b.mom3 +
+                delta3 * a.n.float * b.n.float * (a.n.float - b.n.float)/(n*n);
+  result.mom3 += 3.0*delta * (a.n.float*b.mom2 - b.n.float*a.mom2) / n
+  result.mom4 = a.mom4 + b.mom4 +
+            delta4*a.n.float*b.n.float * toFloat(a.n*a.n - a.n*b.n + b.n*b.n) /
+                (n*n*n)
+  result.mom4 += 6.0*delta2 * (a.n.float*a.n.float*b.mom2 + b.n.float*b.n.float*a.mom2) /
+                (n*n) +
+                4.0*delta*(a.n.float*b.mom3 - b.n.float*a.mom3) / n
+  result.max = max(a.max, b.max)
+  result.min = max(a.min, b.min)
+
+proc `+=`*(a: var RunningStat, b: RunningStat) {.inline.} =
+  ## add a second RunningStats `b` to `a`
+  a = a + b
+# ---------------------- standalone array/seq stats ---------------------
+proc mean*[T](x: openArray[T]): float =
+  ## computes the mean of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.mean()
+
+proc variance*[T](x: openArray[T]): float =
+  ## computes the population variance of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.variance()
+
+proc varianceS*[T](x: openArray[T]): float =
+  ## computes the sample variance of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.varianceS()
+
+proc standardDeviation*[T](x: openArray[T]): float =
+  ## computes the population standardDeviation of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.standardDeviation()
+
+proc standardDeviationS*[T](x: openArray[T]): float =
+  ## computes the sanple standardDeviation of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.standardDeviationS()
+
+proc skewness*[T](x: openArray[T]): float =
+  ## computes the population skewness of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.skewness()
+
+proc skewnessS*[T](x: openArray[T]): float =
+  ## computes the sample skewness of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.skewnessS()
+
+proc kurtosis*[T](x: openArray[T]): float =
+  ## computes the population kurtosis of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.kurtosis()
+
+proc kurtosisS*[T](x: openArray[T]): float =
+  ## computes the sample kurtosis of `x`
+  var rs: RunningStat
+  rs.push(x)
+  result = rs.kurtosisS()
+
+# ---------------------- Running Regression -----------------------------
+
+proc clear*(r: var RunningRegress) =
+  ## reset `r`
+  r.x_stats.clear()
+  r.y_stats.clear()
+  r.s_xy = 0.0
+  r.n = 0
+
+proc push*(r: var RunningRegress, x, y: float) =
+  ## pushes two values `x` and `y` for processing
+  r.s_xy += (r.x_stats.mean() - x)*(r.y_stats.mean() - y)*
+                toFloat(r.n) / toFloat(r.n + 1)
+  r.x_stats.push(x)
+  r.y_stats.push(y)
+  inc(r.n)
+
+proc push*(r: var RunningRegress, x, y: int) {.inline.} =
+  ## pushes two values `x` and `y` for processing.
+  ##
+  ## `x` and `y` are converted to ``float``
+  ## and the other push operation is called.
+  r.push(toFloat(x), toFloat(y))
+
+proc push*(r: var RunningRegress, x, y: openarray[float|int]) =
+  ## pushes two sets of values `x` and `y` for processing.
+  assert(x.len == y.len)
+  for i in 0..<x.len:
+    r.push(x[i], y[i])
+
+proc slope*(r: RunningRegress): float =
+  ## computes the current slope of `r`
+  let s_xx = r.x_stats.varianceS()*toFloat(r.n - 1)
+  result = r.s_xy / s_xx
+
+proc intercept*(r: RunningRegress): float =
+  ## computes the current intercept of `r`
+  result = r.y_stats.mean() - r.slope()*r.x_stats.mean()
+
+proc correlation*(r: RunningRegress): float =
+  ## computes the current correlation of the two data
+  ## sets pushed into `r`
+  let t = r.x_stats.standardDeviation() * r.y_stats.standardDeviation()
+  result = r.s_xy / ( toFloat(r.n) * t )
+
+proc `+`*(a, b: RunningRegress):  RunningRegress =
+  ## combine two `RunningRegress` objects.
+  ##
+  ## Useful if performing parallel analysis of data series
+  ## and need to re-combine parallel result sets
+  result.clear()
+  result.x_stats = a.x_stats + b.x_stats
+  result.y_stats = a.y_stats + b.y_stats
+  result.n = a.n + b.n
+
+  let delta_x = b.x_stats.mean() - a.x_stats.mean()
+  let delta_y = b.y_stats.mean() - a.y_stats.mean()
+  result.s_xy = a.s_xy + b.s_xy +
+      toFloat(a.n*b.n)*delta_x*delta_y/toFloat(result.n)
+
+proc `+=`*(a: var RunningRegress, b: RunningRegress) =
+  ## add RunningRegress `b` to `a`
+  a = a + b
+
+{.pop.}
+{.pop.}
+
+when isMainModule:
+  proc clean(x: float): float =
+    result = round(1.0e8*x).float * 1.0e-8
+
+  var rs: RunningStat
+  rs.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0])
+  doAssert(rs.n == 8)
+  doAssert(clean(rs.mean) == 2.0)
+  doAssert(clean(rs.variance()) == 1.5)
+  doAssert(clean(rs.varianceS()) == 1.71428571)
+  doAssert(clean(rs.skewness()) == 0.81649658)
+  doAssert(clean(rs.skewnessS()) == 1.01835015)
+  doAssert(clean(rs.kurtosis()) == -1.0)
+  doAssert(clean(rs.kurtosisS()) == -0.7000000000000001)
+
+  var rs1, rs2: RunningStat
+  rs1.push(@[1.0, 2.0, 1.0, 4.0])
+  rs2.push(@[1.0, 4.0, 1.0, 2.0])
+  let rs3 = rs1 + rs2
+  doAssert(clean(rs3.mom2) == clean(rs.mom2))
+  doAssert(clean(rs3.mom3) == clean(rs.mom3))
+  doAssert(clean(rs3.mom4) == clean(rs.mom4))
+  rs1 += rs2
+  doAssert(clean(rs1.mom2) == clean(rs.mom2))
+  doAssert(clean(rs1.mom3) == clean(rs.mom3))
+  doAssert(clean(rs1.mom4) == clean(rs.mom4))
+  rs1.clear()
+  rs1.push(@[1.0, 2.2, 1.4, 4.9])
+  doAssert(rs1.sum == 9.5)
+  doAssert(rs1.mean() == 2.375)
+
+  var rr: RunningRegress
+  rr.push(@[0.0,1.0,2.8,3.0,4.0], @[0.0,1.0,2.3,3.0,4.0])
+  doAssert(rr.slope() == 0.9695585996955861)
+  doAssert(rr.intercept() == -0.03424657534246611)
+  doAssert(rr.correlation() == 0.9905100362239381)
+  var rr1, rr2: RunningRegress
+  rr1.push(@[0.0,1.0], @[0.0,1.0])
+  rr2.push(@[2.8,3.0,4.0], @[2.3,3.0,4.0])
+  let rr3 = rr1 + rr2
+  doAssert(rr3.correlation() == rr.correlation())
+  doAssert(clean(rr3.slope()) == clean(rr.slope()))
+  doAssert(clean(rr3.intercept()) == clean(rr.intercept()))

lib/pure/strutils.nim

@@ -1210,22 +1210,21 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect,
   ## If `s` does not begin with ``prefix`` and end with ``suffix`` a
   ## ValueError exception will be raised.
   result = newStringOfCap(s.len)
-  var i = 0
+  var i = prefix.len
   if not s.startsWith(prefix):
     raise newException(ValueError,
                        "String does not start with a prefix of: " & prefix)
-  inc(i)
   while true:
     if i == s.len-suffix.len: break
     case s[i]
     of '\\':
       case s[i+1]:
       of 'x':
-        inc i
+        inc i, 2
         var c: int
-        i += parseutils.parseHex(s, c, i)
+        i += parseutils.parseHex(s, c, i, maxLen=2)
         result.add(chr(c))
-        inc(i, 2)
+        dec i, 2
       of '\\':
         result.add('\\')
       of '\'':
@@ -1721,3 +1720,4 @@ when isMainModule:
   doAssert isUpper("ABC")
   doAssert(not isUpper("AAcc"))
   doAssert(not isUpper("A#$"))
+  doAssert(unescape(r"\x013", "", "") == "\x013")

lib/pure/times.nim

@@ -29,7 +29,7 @@
 ##  echo "epochTime() float value: ", epochTime()
 ##  echo "getTime()   float value: ", toSeconds(getTime())
 ##  echo "cpuTime()   float value: ", cpuTime()
-##  echo "An hour from now      : ", getLocalTime(getTime()) + initInterval(0,0,0,1)
+##  echo "An hour from now      : ", getLocalTime(getTime()) + 1.hours
 ##  echo "An hour from (UTC) now: ", getGmTime(getTime()) + initInterval(0,0,0,1)
 
 {.push debugger:off.} # the user does not want to trace a part
@@ -171,11 +171,6 @@ type
 {.deprecated: [TMonth: Month, TWeekDay: WeekDay, TTime: Time,
     TTimeInterval: TimeInterval, TTimeInfo: TimeInfo].}
 
-proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds
-
-proc `miliseconds=`*(t:var TimeInterval, milliseconds: int) {.deprecated.} =
-  t.milliseconds = milliseconds
-
 proc getTime*(): Time {.tags: [TimeEffect], benign.}
   ## gets the current calendar time as a UNIX epoch value (number of seconds
   ## elapsed since 1970) with integer precission. Use epochTime for higher
@@ -245,13 +240,59 @@ proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.}
 proc initInterval*(milliseconds, seconds, minutes, hours, days, months,
                    years: int = 0): TimeInterval =
   ## creates a new ``TimeInterval``.
-  result.milliseconds = milliseconds
+  ##
-  result.seconds = seconds
+  ## You can also use the convenience procedures called ``milliseconds``,
-  result.minutes = minutes
+  ## ``seconds``, ``minutes``, ``hours``, ``days``, ``months``, and ``years``.
-  result.hours = hours
+  ##
-  result.days = days
+  ## Example:
-  result.months = months
+  ##
-  result.years = years
+  ## .. code-block:: nim
+  ##
+  ##     let day = initInterval(hours=24)
+  ##     let tomorrow = getTime() + day
+  ##     echo(tomorrow)
+  var carryO = 0
+  result.milliseconds = `mod`(milliseconds, 1000)
+  carryO = `div`(milliseconds, 1000)
+  result.seconds = `mod`(carryO + seconds, 60)
+  carryO = `div`(seconds, 60)
+  result.minutes = `mod`(carryO + minutes, 60)
+  carryO = `div`(minutes, 60)
+  result.hours = `mod`(carryO + hours, 24)
+  carryO = `div`(hours, 24)
+  result.days = carryO + days
+  carryO = 0
+  result.months = `mod`(months, 12)
+  carryO = `div`(months, 12)
+  result.years = carryO + years
+
+proc `+`*(ti1, ti2: TimeInterval): TimeInterval =
+  ## Adds two ``TimeInterval`` objects together.
+  var carryO = 0
+  result.milliseconds = `mod`(ti1.milliseconds + ti2.milliseconds, 1000)
+  carryO = `div`(ti1.milliseconds + ti2.milliseconds, 1000)
+  result.seconds = `mod`(carryO + ti1.seconds + ti2.seconds, 60)
+  carryO = `div`(ti1.seconds + ti2.seconds, 60)
+  result.minutes = `mod`(carryO + ti1.minutes + ti2.minutes, 60)
+  carryO = `div`(ti1.minutes + ti2.minutes, 60)
+  result.hours = `mod`(carryO + ti1.hours + ti2.hours, 24)
+  carryO = `div`(ti1.hours + ti2.hours, 24)
+  result.days = carryO + ti1.days + ti2.days
+  carryO = 0
+  result.months = `mod`(ti1.months + ti2.months, 12)
+  carryO = `div`(ti1.months + ti2.months, 12)
+  result.years = carryO + ti1.years + ti2.years
+
+proc `-`*(ti1, ti2: TimeInterval): TimeInterval =
+  ## Subtracts TimeInterval ``ti1`` from ``ti2``.
+  result = ti1
+  result.milliseconds -= ti2.milliseconds
+  result.seconds -= ti2.seconds
+  result.minutes -= ti2.minutes
+  result.hours -= ti2.hours
+  result.days -= ti2.days
+  result.months -= ti2.months
+  result.years -= ti2.years
 
 proc isLeapYear*(year: int): bool =
   ## returns true if ``year`` is a leap year
@@ -288,6 +329,15 @@ proc toSeconds(a: TimeInfo, interval: TimeInterval): float =
 
   newinterv.months += interval.years * 12
   var curMonth = anew.month
+  if newinterv.months < 0:   # subtracting
+    for mth in countDown(-1 * newinterv.months, 1):
+      result -= float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60)
+      if curMonth == mJan:
+        curMonth = mDec
+        anew.year.dec()
+      else:
+        curMonth.dec()
+  else:  # adding
     for mth in 1 .. newinterv.months:
       result += float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60)
       if curMonth == mDec:
@@ -302,28 +352,39 @@ proc toSeconds(a: TimeInfo, interval: TimeInterval): float =
   result += newinterv.milliseconds / 1000
 
 proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo =
-  ## adds ``interval`` time.
+  ## adds ``interval`` time from TimeInfo ``a``.
   ##
   ## **Note:** This has been only briefly tested and it may not be
   ## very accurate.
   let t = toSeconds(timeInfoToTime(a))
   let secs = toSeconds(a, interval)
-  #if a.tzname == "UTC":
-  #  result = getGMTime(fromSeconds(t + secs))
-  #else:
   result = getLocalTime(fromSeconds(t + secs))
 
 proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo =
-  ## subtracts ``interval`` time.
+  ## subtracts ``interval`` time from TimeInfo ``a``.
   ##
   ## **Note:** This has been only briefly tested, it is inaccurate especially
   ## when you subtract so much that you reach the Julian calendar.
   let t = toSeconds(timeInfoToTime(a))
-  let secs = toSeconds(a, interval)
+  var intval: TimeInterval
-  #if a.tzname == "UTC":
+  intval.milliseconds = - interval.milliseconds
-  #  result = getGMTime(fromSeconds(t - secs))
+  intval.seconds = - interval.seconds
-  #else:
+  intval.minutes = - interval.minutes
-  result = getLocalTime(fromSeconds(t - secs))
+  intval.hours = - interval.hours
+  intval.days = - interval.days
+  intval.months = - interval.months
+  intval.years = - interval.years
+  let secs = toSeconds(a, intval)
+  result = getLocalTime(fromSeconds(t + secs))
+
+proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds
+
+proc `miliseconds=`*(t: var TimeInterval, milliseconds: int) {.deprecated.} =
+  ## An alias for a misspelled field in ``TimeInterval``.
+  ##
+  ## **Warning:** This should not be used! It will be removed in the next
+  ## version.
+  t.milliseconds = milliseconds
 
 when not defined(JS):
   proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].}
@@ -603,6 +664,69 @@ proc `$`*(m: Month): string =
       "November", "December"]
   return lookup[m]
 
+proc milliseconds*(ms: int): TimeInterval {.inline.} =
+  ## TimeInterval of `ms` milliseconds
+  ##
+  ## Note: not all time functions have millisecond resolution
+  initInterval(`mod`(ms,1000), `div`(ms,1000))
+
+proc seconds*(s: int): TimeInterval {.inline.} =
+  ## TimeInterval of `s` seconds
+  ##
+  ## ``echo getTime() + 5.second``
+  initInterval(0,`mod`(s,60), `div`(s,60))
+
+proc minutes*(m: int): TimeInterval {.inline.} =
+  ## TimeInterval of `m` minutes
+  ##
+  ## ``echo getTime() + 5.minutes``
+  initInterval(0,0,`mod`(m,60), `div`(m,60))
+
+proc hours*(h: int): TimeInterval {.inline.} =
+  ## TimeInterval of `h` hours
+  ##
+  ## ``echo getTime() + 2.hours``
+  initInterval(0,0,0,`mod`(h,24),`div`(h,24))
+
+proc days*(d: int): TimeInterval {.inline.} =
+  ## TimeInterval of `d` days
+  ##
+  ## ``echo getTime() + 2.days``
+  initInterval(0,0,0,0,d)
+
+proc months*(m: int): TimeInterval {.inline.} =
+  ## TimeInterval of `m` months
+  ##
+  ## ``echo getTime() + 2.months``
+  initInterval(0,0,0,0,0,`mod`(m,12),`div`(m,12))
+
+proc years*(y: int): TimeInterval {.inline.} =
+  ## TimeInterval of `y` years
+  ##
+  ## ``echo getTime() + 2.years``
+  initInterval(0,0,0,0,0,0,y)
+
+proc `+=`*(t: var Time, ti: TimeInterval) =
+  ## modifies `t` by adding the interval `ti`
+  t = timeInfoToTime(getLocalTime(t) + ti)
+
+proc `+`*(t: Time, ti: TimeInterval): Time =
+  ## adds the interval `ti` to Time `t`
+  ## by converting to localTime, adding the interval, and converting back
+  ##
+  ## ``echo getTime() + 1.day``
+  result = timeInfoToTime(getLocalTime(t) + ti)
+
+proc `-=`*(t: var Time, ti: TimeInterval) =
+  ## modifies `t` by subtracting the interval `ti`
+  t = timeInfoToTime(getLocalTime(t) - ti)
+
+proc `-`*(t: Time, ti: TimeInterval): Time =
+  ## adds the interval `ti` to Time `t`
+  ##
+  ## ``echo getTime() - 1.day``
+  result = timeInfoToTime(getLocalTime(t) - ti)
+
 proc formatToken(info: TimeInfo, token: string, buf: var string) =
   ## Helper of the format proc to parse individual tokens.
   ##

lib/system.nim

@@ -840,7 +840,7 @@ proc `div` *(x, y: int32): int32 {.magic: "DivI", noSideEffect.}
   ##   1 div 2 == 0
   ##   2 div 2 == 1
   ##   3 div 2 == 1
-  ##   7 div 5 == 2
+  ##   7 div 5 == 1
 
 when defined(nimnomagic64):
   proc `div` *(x, y: int64): int64 {.magic: "DivI", noSideEffect.}

lib/system/dyncalls.nim

@@ -68,6 +68,7 @@ when defined(posix):
 
   proc nimLoadLibrary(path: string): LibHandle =
     result = dlopen(path, RTLD_NOW)
+    when defined(nimDebugDlOpen):
       let error = dlerror()
       if error != nil:
         c_fprintf(c_stdout, "%s\n", error)

lib/system/jssys.nim

@@ -533,6 +533,10 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef =
     nimCopyAux(result, src, ti.node)
   of tySequence, tyArrayConstr, tyOpenArray, tyArray:
     asm """
+      if (`src` === null) {
+        `result` = null;
+      }
+      else {
         if (`dest` === null || `dest` === undefined) {
           `dest` = new Array(`src`.length);
         }
@@ -543,6 +547,7 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef =
         for (var i = 0; i < `src`.length; ++i) {
           `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base);
         }
+      }
     """
   of tyString:
     asm """

lib/system/nimscript.nim

@@ -242,7 +242,7 @@ template task*(name: untyped; description: string; body: untyped): untyped =
   ## .. code-block:: nim
   ##  task build, "default build is via the C backend":
   ##    setCommand "c"
-  proc `name Task`() = body
+  proc `name Task`*() = body
 
   let cmd = getCommand()
   if cmd.len == 0 or cmd ==? "help":

lib/system/profiler.nim

@@ -50,10 +50,15 @@ proc captureStackTrace(f: PFrame, st: var StackTrace) =
     inc(i)
     b = b.prev
 
+var
+  profilingRequestedHook*: proc (): bool {.nimcall, benign.}
+    ## set this variable to provide a procedure that implements a profiler in
+    ## user space. See the `nimprof` module for a reference implementation.
+
 when defined(memProfiler):
   type
     MemProfilerHook* = proc (st: StackTrace, requestedSize: int) {.nimcall, benign.}
-  {.deprecated: [TMemProfilerHook: MemProfilerHook].}
+
   var
     profilerHook*: MemProfilerHook
       ## set this variable to provide a procedure that implements a profiler in
@@ -65,17 +70,13 @@ when defined(memProfiler):
     hook(st, requestedSize)
 
   proc nimProfile(requestedSize: int) =
-    if not isNil(profilerHook):
+    if not isNil(profilingRequestedHook) and profilingRequestedHook():
       callProfilerHook(profilerHook, requestedSize)
 else:
-  const
-    SamplingInterval = 50_000
-      # set this to change the default sampling interval
   var
     profilerHook*: ProfilerHook
       ## set this variable to provide a procedure that implements a profiler in
       ## user space. See the `nimprof` module for a reference implementation.
-    gTicker {.threadvar.}: int
 
   proc callProfilerHook(hook: ProfilerHook) {.noinline.} =
     # 'noinline' so that 'nimProfile' does not perform the stack allocation
@@ -86,16 +87,7 @@ else:
 
   proc nimProfile() =
     ## This is invoked by the compiler in every loop and on every proc entry!
-    if gTicker == 0:
+    if not isNil(profilingRequestedHook) and profilingRequestedHook():
-      gTicker = -1
+      callProfilerHook(profilerHook)
-      if not isNil(profilerHook):
-        # disable recursive calls: XXX should use try..finally,
-        # but that's too expensive!
-        let oldHook = profilerHook
-        profilerHook = nil
-        callProfilerHook(oldHook)
-        profilerHook = oldHook
-      gTicker = SamplingInterval
-    dec gTicker
 
 {.pop.}

lib/system/repr.nim

@@ -259,8 +259,10 @@ when not defined(useNimRtl):
     of tyInt16: add result, $int(cast[ptr int16](p)[])
     of tyInt32: add result, $int(cast[ptr int32](p)[])
     of tyInt64: add result, $(cast[ptr int64](p)[])
-    of tyUInt8: add result, $ze(cast[ptr int8](p)[])
+    of tyUInt8: add result, $(cast[ptr uint8](p)[])
-    of tyUInt16: add result, $ze(cast[ptr int16](p)[])
+    of tyUInt16: add result, $(cast[ptr uint16](p)[])
+    of tyUInt32: add result, $(cast[ptr uint32](p)[])
+    of tyUInt64: add result, $(cast[ptr uint64](p)[])
 
     of tyFloat: add result, $(cast[ptr float](p)[])
     of tyFloat32: add result, $(cast[ptr float32](p)[])

lib/wrappers/linenoise/clinenoise.c

@@ -139,7 +139,7 @@ struct linenoiseState {
     int ofd;            /* Terminal stdout file descriptor. */
     char *buf;          /* Edited line buffer. */
     size_t buflen;      /* Edited line buffer size. */
-    const char *prompt; /* Prompt to display. */
+    char *prompt; /* Prompt to display. */
     size_t plen;        /* Prompt length. */
     size_t pos;         /* Current cursor position. */
     size_t oldpos;      /* Previous refresh cursor position. */
@@ -172,7 +172,7 @@ enum KEY_ACTION{
 };
 
 static void linenoiseAtExit(void);
-int linenoiseHistoryAdd(const char *line);
+int linenoiseHistoryAdd(char *line);
 static void refreshLine(struct linenoiseState *l);
 
 /* Debugging macro. */
@@ -413,14 +413,14 @@ void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) {
  * in order to add completion options given the input string when the
  * user typed <tab>. See the example.c source code for a very easy to
  * understand example. */
-void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) {
+void linenoiseAddCompletion(linenoiseCompletions *lc, char *str) {
     size_t len = strlen(str);
     char *copy, **cvec;
 
-    copy = malloc(len+1);
+    copy = (char*)malloc(len+1);
     if (copy == NULL) return;
     memcpy(copy,str,len+1);
-    cvec = realloc(lc->cvec,sizeof(char*)*(lc->len+1));
+    cvec = (char**)realloc(lc->cvec,sizeof(char*)*(lc->len+1));
     if (cvec == NULL) {
         free(copy);
         return;
@@ -445,12 +445,12 @@ static void abInit(struct abuf *ab) {
     ab->len = 0;
 }
 
-static void abAppend(struct abuf *ab, const char *s, int len) {
+static void abAppend(struct abuf *ab, char *s, int len) {
-    char *new = realloc(ab->b,ab->len+len);
+    char *neww = (char*)realloc(ab->b,ab->len+len);
 
-    if (new == NULL) return;
+    if (neww == NULL) return;
-    memcpy(new+ab->len,s,len);
+    memcpy(neww+ab->len,s,len);
-    ab->b = new;
+    ab->b = neww;
     ab->len += len;
 }
 
@@ -723,7 +723,7 @@ void linenoiseEditDeletePrevWord(struct linenoiseState *l) {
  * when ctrl+d is typed.
  *
  * The function returns the length of the current buffer. */
-static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt)
+static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, char *prompt)
 {
     struct linenoiseState l;
 
@@ -929,7 +929,7 @@ void linenoisePrintKeyCodes(void) {
 
 /* This function calls the line editing function linenoiseEdit() using
  * the STDIN file descriptor set in raw mode. */
-static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) {
+static int linenoiseRaw(char *buf, size_t buflen, char *prompt) {
     int count;
 
     if (buflen == 0) {
@@ -959,7 +959,7 @@ static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) {
  * for a blacklist of stupid terminals, and later either calls the line
  * editing function or uses dummy fgets() so that you will be able to type
  * something even in the most desperate of the conditions. */
-char *linenoise(const char *prompt) {
+char *linenoise(char *prompt) {
     char buf[LINENOISE_MAX_LINE];
     int count;
 
@@ -1009,14 +1009,14 @@ static void linenoiseAtExit(void) {
  * histories, but will work well for a few hundred of entries.
  *
  * Using a circular buffer is smarter, but a bit more complex to handle. */
-int linenoiseHistoryAdd(const char *line) {
+int linenoiseHistoryAdd(char *line) {
     char *linecopy;
 
     if (history_max_len == 0) return 0;
 
     /* Initialization on first call. */
     if (history == NULL) {
-        history = malloc(sizeof(char*)*history_max_len);
+        history = (char**)malloc(sizeof(char*)*history_max_len);
         if (history == NULL) return 0;
         memset(history,0,(sizeof(char*)*history_max_len));
     }
@@ -1043,14 +1043,14 @@ int linenoiseHistoryAdd(const char *line) {
  * just the latest 'len' elements if the new history length value is smaller
  * than the amount of items already inside the history. */
 int linenoiseHistorySetMaxLen(int len) {
-    char **new;
+    char **neww;
 
     if (len < 1) return 0;
     if (history) {
         int tocopy = history_len;
 
-        new = malloc(sizeof(char*)*len);
+        neww = (char**)malloc(sizeof(char*)*len);
-        if (new == NULL) return 0;
+        if (neww == NULL) return 0;
 
         /* If we can't copy everything, free the elements we'll not use. */
         if (len < tocopy) {
@@ -1059,10 +1059,10 @@ int linenoiseHistorySetMaxLen(int len) {
             for (j = 0; j < tocopy-len; j++) free(history[j]);
             tocopy = len;
         }
-        memset(new,0,sizeof(char*)*len);
+        memset(neww,0,sizeof(char*)*len);
-        memcpy(new,history+(history_len-tocopy), sizeof(char*)*tocopy);
+        memcpy(neww,history+(history_len-tocopy), sizeof(char*)*tocopy);
         free(history);
-        history = new;
+        history = neww;
     }
     history_max_len = len;
     if (history_len > history_max_len)
@@ -1072,7 +1072,7 @@ int linenoiseHistorySetMaxLen(int len) {
 
 /* Save the history in the specified file. On success 0 is returned
  * otherwise -1 is returned. */
-int linenoiseHistorySave(const char *filename) {
+int linenoiseHistorySave(char *filename) {
     FILE *fp = fopen(filename,"w");
     int j;
 
@@ -1088,7 +1088,7 @@ int linenoiseHistorySave(const char *filename) {
  *
  * If the file exists and the operation succeeded 0 is returned, otherwise
  * on error -1 is returned. */
-int linenoiseHistoryLoad(const char *filename) {
+int linenoiseHistoryLoad(char *filename) {
     FILE *fp = fopen(filename,"r");
     char buf[LINENOISE_MAX_LINE];
 

lib/wrappers/linenoise/clinenoise.h

@@ -39,30 +39,22 @@
 #ifndef __LINENOISE_H
 #define __LINENOISE_H
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
 typedef struct linenoiseCompletions {
   size_t len;
   char **cvec;
 } linenoiseCompletions;
 
-typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *);
+typedef void(linenoiseCompletionCallback)(char *, linenoiseCompletions *);
 void linenoiseSetCompletionCallback(linenoiseCompletionCallback *);
-void linenoiseAddCompletion(linenoiseCompletions *, const char *);
+void linenoiseAddCompletion(linenoiseCompletions *, char *);
 
-char *linenoise(const char *prompt);
+char *linenoise(char *prompt);
-int linenoiseHistoryAdd(const char *line);
+int linenoiseHistoryAdd(char *line);
 int linenoiseHistorySetMaxLen(int len);
-int linenoiseHistorySave(const char *filename);
+int linenoiseHistorySave(char *filename);
-int linenoiseHistoryLoad(const char *filename);
+int linenoiseHistoryLoad(char *filename);
 void linenoiseClearScreen(void);
 void linenoiseSetMultiLine(int ml);
 void linenoisePrintKeyCodes(void);
 
-#ifdef __cplusplus
-}
-#endif
-
 #endif /* __LINENOISE_H */

lib/wrappers/mysql.nim

@@ -418,6 +418,7 @@ type
     decimals*: cuint          # Number of decimals in field
     charsetnr*: cuint         # Character set
     ftype*: Enum_field_types  # Type of field. See mysql_com.h for types
+    extension*: pointer
 
   FIELD* = St_mysql_field
   PFIELD* = ptr FIELD

lib/wrappers/odbcsql.nim

@@ -641,11 +641,42 @@ const
   ODBC_CONFIG_SYS_DSN* = 5
   ODBC_REMOVE_SYS_DSN* = 6
 
+  SQL_ACTIVE_CONNECTIONS* = 0   # SQLGetInfo
+  SQL_DATA_SOURCE_NAME* = 2
+  SQL_DATA_SOURCE_READ_ONLY* = 25
+  SQL_DATABASE_NAME* = 2
+  SQL_DBMS_NAME* = 17
+  SQL_DBMS_VERSION* = 18
+  SQL_DRIVER_HDBC* = 3
+  SQL_DRIVER_HENV* = 4
+  SQL_DRIVER_HSTMT* = 5
+  SQL_DRIVER_NAME* = 6
+  SQL_DRIVER_VER* = 7
+  SQL_FETCH_DIRECTION* = 8
+  SQL_ODBC_VER* = 10
+  SQL_DRIVER_ODBC_VER* = 77
+  SQL_SERVER_NAME* = 13
+  SQL_ACTIVE_ENVIRONMENTS* = 116
+  SQL_ACTIVE_STATEMENTS* = 1
+  SQL_SQL_CONFORMANCE* = 118
+  SQL_DATETIME_LITERALS* = 119
+  SQL_ASYNC_MODE* = 10021
+  SQL_BATCH_ROW_COUNT* = 120
+  SQL_BATCH_SUPPORT* = 121
+  SQL_CATALOG_LOCATION* = 114
+  #SQL_CATALOG_NAME* = 10003
+  SQL_CATALOG_NAME_SEPARATOR* = 41
+  SQL_CATALOG_TERM* = 42
+  SQL_CATALOG_USAGE* = 92
+  #SQL_COLLATION_SEQ* = 10004
+  SQL_COLUMN_ALIAS* = 87
+  #SQL_USER_NAME* = 47
+
 proc SQLAllocHandle*(HandleType: TSqlSmallInt, InputHandle: SqlHandle,
                      OutputHandlePtr: var SqlHandle): TSqlSmallInt{.
     dynlib: odbclib, importc.}
 proc SQLSetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger,
-                    Value: SqlPointer, StringLength: TSqlInteger): TSqlSmallInt{.
+                    Value: TSqlInteger, StringLength: TSqlInteger): TSqlSmallInt{.
     dynlib: odbclib, importc.}
 proc SQLGetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger,
                     Value: SqlPointer, BufferLength: TSqlInteger,
@@ -807,5 +838,10 @@ proc SQLStatistics*(hstmt: SqlHStmt, CatalogName: PSQLCHAR,
                     NameLength3: TSqlSmallInt, Unique: SqlUSmallInt,
                     Reserved: SqlUSmallInt): TSqlSmallInt {.
                     dynlib: odbclib, importc.}
+proc SQLErr*(henv: SqlHEnv, hdbc: SqlHDBC, hstmt: SqlHStmt,
+              szSqlState, pfNativeError, szErrorMsg: PSQLCHAR,
+              cbErrorMsgMax: TSqlSmallInt,
+              pcbErrorMsg: PSQLINTEGER): TSqlSmallInt {.
+                    dynlib: odbclib, importc: "SQLError".}
 
 {.pop.}

lib/wrappers/sqlite3.nim

@@ -239,6 +239,8 @@ proc column_count*(pStmt: Pstmt): int32{.cdecl, dynlib: Lib,
     importc: "sqlite3_column_count".}
 proc column_name*(para1: Pstmt, para2: int32): cstring{.cdecl, dynlib: Lib,
     importc: "sqlite3_column_name".}
+proc column_table_name*(para1: Pstmt; para2: int32): cstring{.cdecl, dynlib: Lib,
+    importc: "sqlite3_column_table_name".}
 proc column_name16*(para1: Pstmt, para2: int32): pointer{.cdecl, dynlib: Lib,
     importc: "sqlite3_column_name16".}
 proc column_decltype*(para1: Pstmt, i: int32): cstring{.cdecl, dynlib: Lib,

readme.md

@@ -1,6 +1,14 @@
 # Nim Compiler
 
-[![Join the chat at https://gitter.im/nim-lang/Nim](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/nim-lang/Nim?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Join the Chat at irc.freenode.net#nim](https://img.shields.io/badge/IRC-join_chat_in_%23nim-blue.svg)](https://webchat.freenode.net/?channels=nim)
+[![Get help](https://img.shields.io/badge/Forum-get%20help-4eb899.svg)](http://forum.nim-lang.org)
+[![Stackoverflow](https://img.shields.io/badge/stackoverflow-use_%23nim_tag-yellow.svg)](http://stackoverflow.com/questions/tagged/nim?sort=newest&pageSize=15)
+[![Follow @nim_lang!](https://img.shields.io/twitter/follow/nim_lang.svg?style=social)](https://twitter.com/nim_lang)
+
+[![Travis](https://img.shields.io/travis/nim-lang/Nim.svg)](https://travis-ci.org/nim-lang/Nim)
+
+[![Contribute to Nim via Gratipay!](https://img.shields.io/gratipay/team/nim.svg)](https://gratipay.com/nim/)
+[![Bountysource](https://img.shields.io/bountysource/team/nim/activity.svg)](https://www.bountysource.com/teams/nim)
 
 
 This repo contains the Nim compiler, Nim's stdlib, tools and

tests/async/tlambda.nim

@@ -0,0 +1,55 @@
+
+# bug 2007
+
+import asyncdispatch, asyncnet, logging, json, uri, strutils, future
+
+type
+  Builder = ref object
+    client: Client
+    build: Build
+
+  ProgressCB* = proc (message: string): Future[void] {.closure, gcsafe.}
+
+  Build* = ref object
+    onProgress*: ProgressCB
+
+  Client = ref ClientObj
+  ClientObj = object
+    onMessage: proc (client: Client, msg: JsonNode): Future[void]
+
+proc newClient*(name: string,
+                onMessage: (Client, JsonNode) -> Future[void]): Client =
+  new result
+  result.onMessage = onMessage
+
+proc newBuild*(onProgress: ProgressCB): Build =
+  new result
+  result.onProgress = onProgress
+
+proc start(build: Build, repo, hash: string) {.async.} =
+  let path = repo.parseUri().path.toLower()
+
+proc onProgress(builder: Builder, message: string) {.async.} =
+  debug($message)
+
+proc onMessage(builder: Builder, message: JsonNode) {.async.} =
+  debug("onMessage")
+
+proc newBuilder(): Builder =
+  var cres: Builder
+  new cres
+
+  cres.client = newClient("builder", (client, msg) => (onMessage(cres, msg)))
+  cres.build = newBuild(
+      proc (msg: string): Future[void] {.closure, gcsafe.} = onProgress(cres, msg))
+  return cres
+
+proc main() =
+  # Set up logging.
+  var console = newConsoleLogger(fmtStr = verboseFmtStr)
+  addHandler(console)
+
+  var builder = newBuilder()
+
+
+main()

tests/async/tnimcall_to_closure.nim

@@ -0,0 +1,17 @@
+
+import asyncdispatch
+
+proc defaultOnProgressChanged() = discard
+
+proc ask(x: proc()) = x()
+
+proc retrFile*(onProgressChanged: proc() {.nimcall.}): Future[void] =
+  var retFuture = newFuture[void]("retrFile")
+  iterator retrFileIter(): FutureBase {.closure.} =
+    ask(onProgressChanged)
+    complete(retFuture)
+
+  var nameIterVar = retrFileIter
+  return retFuture
+
+discard retrFile(defaultOnProgressChanged)

tests/ccgbugs/tgeneric_closure.nim

@@ -0,0 +1,28 @@
+
+
+# bug 2659
+
+type
+  GenProcType[T,U] = proc(x:T, y:var U)
+  IntProcType = proc(x:int, y:var int)
+
+proc mult(x:int, y:var int) =
+  y = 2 * x
+
+when isMainModule:
+
+  var input = 1
+  var output = 0
+
+  var someIntProc:IntProcType = mult
+  var someGenProc:GenProcType[int,int] = mult
+
+  mult(input, output)
+  echo output
+
+  someIntProc(input, output)
+  echo output
+
+  # Uncommenting causes an error in the C compiler.
+  someGenProc(input, output)
+  echo output

tests/closure/tclosure0.nim

@@ -0,0 +1,87 @@
+discard """
+  output: '''foo88
+23 24foo 88
+18
+18
+99
+99
+99
+99 99
+99 99
+12 99 99
+12 99 99'''
+"""
+
+when true:
+  # test simple closure within dummy 'main':
+  proc dummy =
+    proc main2(param: int) =
+      var fooB = 23
+      proc outer(outerParam: string) =
+        var outerVar = 88
+        echo outerParam, outerVar
+        proc inner() =
+          block Test:
+            echo fooB, " ", param, outerParam, " ", outerVar
+        inner()
+      outer("foo")
+    main2(24)
+
+  dummy()
+
+when true:
+  proc outer2(x:int) : proc(y:int):int =   # curry-ed application
+      return proc(y:int):int = x*y
+
+  var fn = outer2(6)  # the closure
+  echo fn(3)   # it works
+
+  var rawP = fn.rawProc()
+  var rawE = fn.rawEnv()
+
+  # A type to cast the function pointer into a nimcall
+  type
+    TimesClosure = proc(a: int, x: pointer): int {.nimcall.}
+
+  # Call the function with its closure
+  echo cast[TimesClosure](rawP)(3, rawE)
+
+when true:
+  proc outer =
+    var x, y: int = 99
+    proc innerA = echo x
+    proc innerB =
+      echo y
+      innerA()
+
+    innerA()
+    innerB()
+
+  outer()
+
+when true:
+  proc indirectDep =
+    var x, y: int = 99
+    proc innerA = echo x, " ", y
+    proc innerB =
+      innerA()
+
+    innerA()
+    innerB()
+
+  indirectDep()
+
+when true:
+  proc needlessIndirection =
+    var x, y: int = 99
+    proc indirection =
+      var z = 12
+      proc innerA = echo z, " ", x, " ", y
+      proc innerB =
+        innerA()
+
+      innerA()
+      innerB()
+    indirection()
+
+  needlessIndirection()

tests/closure/tclosure2.nim

@@ -87,7 +87,7 @@ when true:
     proc py() {.closure.} =
       echo "py"
 
-    const
+    let
       mapping = {
         "abc": px,
         "xyz": py

tests/closure/tclosure3.nim

@@ -8,8 +8,9 @@ proc main =
   for iterations in 0..50_000:
     var s: seq[proc(): string {.closure.}] = @[]
     for i in 0 .. n-1:
+      (proc () =
         let ii = i
-      s.add(proc(): string = return $(ii*ii))
+        s.add(proc(): string = return $(ii*ii)))()
     for i in 0 .. n-1:
       let val = s[i]()
       if val != $(i*i): echo "bug  ", val

tests/closure/tclosurebug2.nim

@@ -19,11 +19,11 @@ proc mustRehash(length, counter: int): bool {.inline.} =
   assert(length > counter)
   result = (length * 2 < counter * 3) or (length - counter < 4)
 
-proc nextTry(h, maxHash: THash): THash {.inline.} =
+proc nextTry(h, maxHash: Hash): Hash {.inline.} =
   result = ((5 * h) + 1) and maxHash
 
 template rawGetImpl() {.dirty.} =
-  var h: THash = hash(key) and high(t.data) # start with real hash value
+  var h: Hash = hash(key) and high(t.data) # start with real hash value
   while t.data[h].slot != seEmpty:
     if t.data[h].key == key and t.data[h].slot == seFilled:
       return h
@@ -31,7 +31,7 @@ template rawGetImpl() {.dirty.} =
   result = -1
 
 template rawInsertImpl() {.dirty.} =
-  var h: THash = hash(key) and high(data)
+  var h: Hash = hash(key) and high(data)
   while data[h].slot == seFilled:
     h = nextTry(h, high(data))
   data[h].key = key

tests/closure/tclosureinference3304.nim

@@ -0,0 +1,15 @@
+discard """
+  output: '''@[1, 2, 5]'''
+"""
+
+import future, sequtils
+
+type
+  List[T] = ref object
+    val: T
+  
+proc foo[T](l: List[T]): seq[int] =
+  @[1,2,3,5].filter(x => x != l.val)
+
+when isMainModule:
+  echo(foo(List[int](val: 3)))

tests/closure/tcodegenerr1923.nim

@@ -0,0 +1,9 @@
+type
+  Foo[M] = proc() : M
+
+proc bar[M](f : Foo[M]) =
+  discard f()
+
+proc baz() : int = 42
+
+bar(baz)

tests/closure/texplicit_dummy_closure.nim

@@ -0,0 +1,22 @@
+
+# This is a regression of the new lambda lifting; detected by Aporia
+import asyncio, sockets
+import os
+
+type
+  Window = object
+    oneInstSock*: PAsyncSocket
+    IODispatcher*: PDispatcher
+
+var
+  win: Window
+
+proc initSocket() =
+  win.oneInstSock = asyncSocket()
+  #win.oneInstSock.handleAccept =
+  proc test(s: PAsyncSocket) =
+    var client: PAsyncSocket
+    proc dummy(c: PAsyncSocket) {.closure.} =
+      discard
+    client.handleRead = dummy
+  test(win.oneInstSock)

tests/closure/tfutclosure2138.nim

@@ -0,0 +1,10 @@
+import future, sequtils
+
+proc any[T](list: varargs[T], pred: (T) -> bool): bool =
+    for item in list:
+        if pred(item):
+            result = true
+            break
+
+proc contains(s: string, words: varargs[string]): bool =
+  any(words, (word) => s.contains(word))

tests/closure/tinvalidclosure.nim

@@ -1,9 +1,9 @@
 discard """
   line: 12
-  errormsg: "type mismatch: got (proc (x: int){.closure, gcsafe, locks: 0.})"
+  errormsg: "type mismatch: got (proc (x: int){.gcsafe, locks: 0.})"
 """
 
-proc ugh[T](x: T) {.closure.} =
+proc ugh[T](x: T) {.nimcall.} =
   echo "ugha"
 
 

tests/closure/tissue1502def.nim

@@ -0,0 +1,6 @@
+import sequtils
+let xs: seq[tuple[key: string, val: seq[string]]] = @[("foo", @["bar"])]
+
+let maps = xs.map(
+  proc(x: auto): tuple[typ: string, maps: seq[string]] =
+    (x.key, x.val.map(proc(x: string): string = x)))

tests/closure/tissue1642.nim

@@ -1,7 +1,3 @@
-discard """
-  file: "tissue1642.nim"
-  disabled: true
-"""
 block:
     var i = 0
     proc p() = inc(i)

tests/closure/tissue1846.nim

@@ -0,0 +1,16 @@
+type
+    TBinOp*[T] = proc (x,y: T): bool
+
+    THeap*[T] = object
+        cmp*:   TBinOp[T]
+
+proc less*[T](x,y: T): bool =
+    x < y
+
+proc initHeap*[T](cmp: TBinOp[T]): THeap[T] =
+    result.cmp = cmp
+
+when isMainModule:
+    var h = initHeap[int](less[int])
+
+    echo h.cmp(2,3)

tests/closure/tissue1911.nim

@@ -0,0 +1,7 @@
+proc foo(x: int) : auto =
+
+  proc helper() : int = x
+  proc bar() : int = helper()
+  proc baz() : int = helper()
+
+  return (bar, baz)

tests/closure/tissue600.nim

@@ -0,0 +1,4 @@
+for i in 1..1:
+  var reported = false
+  proc report() =
+    reported = true

tests/closure/tjester.nim

@@ -7,7 +7,7 @@ type
     data: T
     callback: proc () {.closure.}
 
-proc cbOuter(response: string) {.closure, discardable.} =
+proc cbOuter(response: string) {.discardable.} =
   iterator cbIter(): Future[int] {.closure.} =
     for i in 0..7:
       proc foo(): int =

tests/closure/tmacrobust1512.nim

@@ -0,0 +1,137 @@
+import macros, strutils
+
+# https://github.com/nim-lang/Nim/issues/1512
+
+proc macrobust0 (raw_input: string) =
+  var output = ""
+  proc p1 (a:string) =
+    output.add (a)
+
+  proc p2 (a:string) = p1 (a)
+  proc p3 (a:string) = p2 (a)
+  proc p4 (a:string) = p3 (a)
+  proc p5 (a:string) = p4 (a)
+  proc p6 (a:string) = p5 (a)
+  proc p7 (a:string) = p6 (a)
+  proc p8 (a:string) = p7 (a)
+  proc p9 (a:string) = p8 (a)
+  proc p10 (a:string) = p9 (a)
+  proc p11 (a:string) = p10 (a)
+  proc p12 (a:string) = p11 (a)
+  proc p13 (a:string) = p12 (a)
+  proc p14 (a:string) = p13 (a)
+  proc p15 (a:string) = p14 (a)
+  proc p16 (a:string) = p15 (a)
+  proc p17 (a:string) = p16 (a)
+  proc p18 (a:string) = p17 (a)
+  proc p19 (a:string) = p18 (a)
+  proc p20 (a:string) = p19 (a)
+
+  let input = $raw_input
+
+  for a in input.split ():
+    p20 (a)
+    p19 (a)
+
+
+    p18 (a)
+    p17 (a)
+    p16 (a)
+    p15 (a)
+    p14 (a)
+    p13 (a)
+    p12 (a)
+    p11 (a)
+    p10 (a)
+    p9 (a)
+    p8 (a)
+    p7 (a)
+    p6 (a)
+    p5 (a)
+    p4 (a)
+    p3 (a)
+    p2 (a)
+    p1 (a)
+
+
+  echo output
+
+macro macrobust (raw_input: expr) : stmt =
+
+  var output = ""
+  proc p1 (a:string) =
+    output.add (a)
+
+  proc p2 (a:string) = p1 (a)
+  proc p3 (a:string) = p2 (a)
+  proc p4 (a:string) = p3 (a)
+  proc p5 (a:string) = p4 (a)
+  proc p6 (a:string) = p5 (a)
+  proc p7 (a:string) = p6 (a)
+  proc p8 (a:string) = p7 (a)
+  proc p9 (a:string) = p8 (a)
+  proc p10 (a:string) = p9 (a)
+  proc p11 (a:string) = p10 (a)
+  proc p12 (a:string) = p11 (a)
+  proc p13 (a:string) = p12 (a)
+  proc p14 (a:string) = p13 (a)
+  proc p15 (a:string) = p14 (a)
+  proc p16 (a:string) = p15 (a)
+  proc p17 (a:string) = p16 (a)
+  proc p18 (a:string) = p17 (a)
+  proc p19 (a:string) = p18 (a)
+  proc p20 (a:string) = p19 (a)
+
+  let input = $raw_input
+
+  for a in input.split ():
+    p20 (a)
+    p19 (a)
+
+    p18 (a)
+    p17 (a)
+    p16 (a)
+    p15 (a)
+    p14 (a)
+    p13 (a)
+    p12 (a)
+    p11 (a)
+    p10 (a)
+    p9 (a)
+    p8 (a)
+    p7 (a)
+    p6 (a)
+    p5 (a)
+    p4 (a)
+    p3 (a)
+    p2 (a)
+
+  echo output
+  discard result
+
+macrobust """
+  fdsasadfsdfa sadfsdafsdaf
+  dsfsdafdsfadsfa fsdaasdfasdf
+  fsdafsadfsad asdfasdfasdf
+  fdsasdfasdfa sadfsadfsadf
+  sadfasdfsdaf sadfsdafsdaf dsfasdaf
+  sadfsdafsadf fdsasdafsadf fdsasadfsdaf
+  sdfasadfsdafdfsa sadfsadfsdaf
+  sdafsdaffsda sdfasadfsadf
+  fsdasdafsdfa sdfasdfafsda
+  sdfasdafsadf sdfasdafsdaf sdfasdafsdaf
+"""
+
+
+macrobust0 """
+  fdsasadfsdfa sadfsdafsdaf
+  dsfsdafdsfadsfa fsdaasdfasdf
+  fsdafsadfsad asdfasdfasdf
+  fdsasdfasdfa sadfsadfsadf
+  sadfasdfsdaf sadfsdafsdaf dsfasdaf
+  sadfsdafsadf fdsasdafsadf fdsasadfsdaf
+  sdfasadfsdafdfsa sadfsadfsdaf
+  sdafsdaffsda sdfasadfsadf
+  fsdasdafsdfa sdfasdfafsda
+  sdfasdafsadf sdfasdafsdaf sdfasdafsdaf
+"""

tests/closure/tnestedclosure.nim

@@ -21,13 +21,13 @@ proc main(param: int) =
 # test simple closure within dummy 'main':
 proc dummy =
   proc main2(param: int) =
-    var foo = 23
+    var fooB = 23
     proc outer(outerParam: string) =
       var outerVar = 88
       echo outerParam, outerVar
       proc inner() =
         block Test:
-          echo foo, " ", param, outerParam, " ", outerVar
+          echo fooB, " ", param, outerParam, " ", outerVar
       inner()
     outer("foo")
   main2(24)

tests/closure/tnoclosure.nim

@@ -0,0 +1,25 @@
+discard """
+  output: '''@[1]
+@[1, 1]
+@[1, 2, 1]
+@[1, 3, 3, 1]
+@[1, 4, 6, 4, 1]
+@[1, 5, 10, 10, 5, 1]
+@[1, 6, 15, 20, 15, 6, 1]
+@[1, 7, 21, 35, 35, 21, 7, 1]
+@[1, 8, 28, 56, 70, 56, 28, 8, 1]
+@[1, 9, 36, 84, 126, 126, 84, 36, 9, 1]'''
+"""
+
+import sequtils
+
+proc pascal(n: int) =
+  var row = @[1]
+  for r in 1..n:
+    echo row
+    row = zip(row & @[0], @[0] & row).mapIt(it[0] + it[1])
+
+pascal(10)
+
+# bug #3499 last snippet fixed
+# bug 705  last snippet fixed

tests/destructor/tdestructor3.nim

@@ -19,10 +19,11 @@ proc `=`(lhs: var T, rhs: T) =
 proc `=destroy`(v: var T) =
     echo "destroy"
 
-block:
+proc usedToBeBlock =
     var v1 : T
     var v2 : T = v1
 
+usedToBeBlock()
 
 # bug #1632
 

tests/generics/tgenerictmpl.nim

@@ -1,12 +1,21 @@
+discard """
+  output: '''0
+123'''
+"""
 
-template tmp[T](x: var seq[T]) =
+# bug #3498
-  #var yz: T  # XXX doesn't work yet
+
+template defaultOf[T](t: T): expr = (var d: T; d)
+
+echo defaultOf(1) #<- excpected 0
+
+# assignment using template
+
+template tassign[T](x: var seq[T]) =
   x = @[1, 2, 3]
 
-macro tmp2[T](x: var seq[T]): stmt =
-  nil
-
 var y: seq[int]
-tmp(y)
+tassign(y) #<- x is expected = @[1, 2, 3]
-tmp(y)
+tassign(y)
-echo y.repr
+
+echo y[0], y[1], y[2]

tests/generics/tspecialized_procvar.nim

@@ -0,0 +1,17 @@
+discard """
+  output: '''concrete 88'''
+"""
+
+# Another regression triggered by changed closure computations:
+
+proc foo[T](x: proc(): T) =
+  echo "generic ", x()
+
+proc foo(x: proc(): int) =
+  echo "concrete ", x()
+
+# note the following 'proc' is not .closure!
+foo(proc (): auto {.nimcall.} = 88)
+
+# bug #3499 last snippet fixed
+# bug 705  last snippet fixed

tests/iter/tclosureiters.nim

@@ -0,0 +1,73 @@
+discard """
+  output: '''0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+5 5
+7 7
+9 9
+0
+0
+0
+0
+1
+2'''
+"""
+
+when true:
+  proc main() =
+    let
+      lo=0
+      hi=10
+
+    iterator itA(): int =
+      for x in lo..hi:
+        yield x
+
+    for x in itA():
+      echo x
+
+    var y: int
+
+    iterator itB(): int =
+      while y <= hi:
+        yield y
+        inc y
+
+    y = 5
+    for x in itB():
+      echo x, " ", y
+      inc y
+
+  main()
+
+
+iterator infinite(): int {.closure.} =
+  var i = 0
+  while true:
+    yield i
+    inc i
+
+iterator take[T](it: iterator (): T, numToTake: int): T {.closure.} =
+  var i = 0
+  for x in it():
+    if i >= numToTake:
+      break
+    yield x
+    inc i
+
+# gives wrong reasult (3 times 0)
+for x in infinite.take(3):
+  echo x
+
+# does what we want
+let inf = infinite
+for x in inf.take(3):
+  echo x

tests/iter/timplicit_auto.nim

@@ -9,7 +9,7 @@ proc univ(x, y: int): State = Tree
 
 var w, h = 30
 
-iterator fields(a = (0,0), b = (h-1,w-1)) =
+iterator fields(a = (0,0), b = (h-1,w-1)): auto =
   for y in max(a[0], 0) .. min(b[0], h-1):
     for x in max(a[1], 0) .. min(b[1], w-1):
       yield (y,x)