Merge branch 'devel' into alloc-overloads

compiler/ast.nim

@@ -62,7 +62,7 @@ type
     nkTripleStrLit,       # a triple string literal """
     nkNilLit,             # the nil literal
                           # end of atoms
-    nkMetaNode,           # difficult to explain; represents itself
+    nkMetaNode_Obsolete,  # difficult to explain; represents itself
                           # (used for macros)
     nkDotCall,            # used to temporarily flag a nkCall node; 
                           # this is used
@@ -409,7 +409,9 @@ type
                 # efficiency
     nfTransf,   # node has been transformed
     nfSem       # node has been checked for semantics
-    nfDelegate  # the call can use a delegator
+    nfDotField  # the call can use a dot operator
+    nfDotSetter # the call can use a setter dot operarator
+    nfExplicitCall # x.y() was used instead of x.y
     nfExprCall  # this is an attempt to call a regular expression
     nfIsRef     # this node is a 'ref' node; used for the VM
 
@@ -479,12 +481,15 @@ type
     skStub,               # symbol is a stub and not yet loaded from the ROD
                           # file (it is loaded on demand, which may
                           # mean: never)
+    skPackage             # symbol is a package (used for canonicalization)
   TSymKinds* = set[TSymKind]
 
 const
   routineKinds* = {skProc, skMethod, skIterator, skConverter,
     skMacro, skTemplate}
   tfIncompleteStruct* = tfVarargs
+  tfUncheckedArray* = tfVarargs
+  tfUnion* = tfNoSideEffect
   skError* = skUnknown
   
   # type flags that are essential for type equality:
@@ -843,7 +848,8 @@ const
   ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType, skIterator, 
     skMacro, skTemplate, skConverter, skEnumField, skLet, skStub}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
-                                      nfAllConst, nfDelegate, nfIsRef}
+                                      nfDotSetter, nfDotField,
+                                      nfAllConst,nfIsRef}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2
@@ -1044,6 +1050,10 @@ proc newStrNode(kind: TNodeKind, strVal: string): PNode =
   result = newNode(kind)
   result.strVal = strVal
 
+proc withInfo*(n: PNode, info: TLineInfo): PNode =
+  n.info = info
+  return n
+
 proc newIdentNode(ident: PIdent, info: TLineInfo): PNode = 
   result = newNode(nkIdent)
   result.ident = ident
@@ -1105,10 +1115,6 @@ proc newNodeIT(kind: TNodeKind, info: TLineInfo, typ: PType): PNode =
   result.info = info
   result.typ = typ
 
-proc newMetaNodeIT*(tree: PNode, info: TLineInfo, typ: PType): PNode =
-  result = newNodeIT(nkMetaNode, info, typ)
-  result.add(tree)
-
 var emptyParams = newNode(nkFormalParams)
 emptyParams.addSon(emptyNode)
 

compiler/astalgo.nim

@@ -337,6 +337,9 @@ proc treeToYamlAux(n: PNode, marker: var TIntSet, indent: int,
         appf(result, ",$N$1\"floatVal\": $2", 
             [istr, toRope(n.floatVal.toStrMaxPrecision)])
       of nkStrLit..nkTripleStrLit: 
+        if n.strVal.isNil:
+          appf(result, ",$N$1\"strVal\": null", [istr])
+        else:
           appf(result, ",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
       of nkSym: 
         appf(result, ",$N$1\"sym\": $2", 
@@ -407,6 +410,9 @@ proc debugTree(n: PNode, indent: int, maxRecDepth: int): PRope =
         appf(result, ",$N$1\"floatVal\": $2", 
             [istr, toRope(n.floatVal.toStrMaxPrecision)])
       of nkStrLit..nkTripleStrLit: 
+        if n.strVal.isNil:
+          appf(result, ",$N$1\"strVal\": null", [istr])
+        else:
           appf(result, ",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
       of nkSym: 
         appf(result, ",$N$1\"sym\": $2_$3", 

compiler/c2nim/cparse.nim

@@ -540,7 +540,7 @@ proc typeAtom(p: var TParser): PNode =
       if p.tok.s == "unsigned":
         isUnsigned = true
       elif p.tok.s == "signed" or p.tok.s == "int":
-        nil
+        discard
       else:
         add(x, p.tok.s)
       getTok(p, nil)
@@ -681,11 +681,6 @@ proc parseField(p: var TParser, kind: TNodeKind): PNode =
     else: result = mangledIdent(p.tok.s, p)
     getTok(p, result)
 
-proc takeOnlyFirstField(p: TParser, isUnion: bool): bool =
-  # if we generate an interface to a header file, *all* fields can be 
-  # generated:
-  result = isUnion and p.options.header.len == 0
-
 proc parseStructBody(p: var TParser, isUnion: bool,
                      kind: TNodeKind = nkRecList): PNode =
   result = newNodeP(kind, p)
@@ -698,7 +693,6 @@ proc parseStructBody(p: var TParser, isUnion: bool,
       var i = parseField(p, kind)
       t = parseTypeSuffix(p, t)
       addSon(def, i, t, ast.emptyNode)
-      if not takeOnlyFirstField(p, isUnion) or sonsLen(result) < 1: 
       addSon(result, def)
       if p.tok.xkind != pxComma: break
       getTok(p, def)
@@ -710,11 +704,12 @@ proc structPragmas(p: TParser, name: PNode, origName: string): PNode =
   result = newNodeP(nkPragmaExpr, p)
   addSon(result, exportSym(p, name, origName))
   var pragmas = newNodeP(nkPragma, p)
-  addSon(pragmas, newIdentNodeP("pure", p), newIdentNodeP("final", p))
+  #addSon(pragmas, newIdentNodeP("pure", p), newIdentNodeP("final", p))
   if p.options.header.len > 0:
     addSon(pragmas, newIdentStrLitPair("importc", origName, p),
                     newIdentStrLitPair("header", p.options.header, p))
-  addSon(result, pragmas)
+  if pragmas.len > 0: addSon(result, pragmas)
+  else: addSon(result, ast.emptyNode)
 
 proc enumPragmas(p: TParser, name: PNode): PNode =
   result = newNodeP(nkPragmaExpr, p)
@@ -728,7 +723,11 @@ proc enumPragmas(p: TParser, name: PNode): PNode =
 
 proc parseStruct(p: var TParser, isUnion: bool): PNode =
   result = newNodeP(nkObjectTy, p)
-  addSon(result, ast.emptyNode, ast.emptyNode) # no pragmas, no inheritance 
+  var pragmas = ast.emptyNode
+  if isUnion:
+    pragmas = newNodeP(nkPragma, p)
+    addSon(pragmas, newIdentNodeP("union", p))
+  addSon(result, pragmas, ast.emptyNode) # no inheritance 
   if p.tok.xkind == pxCurlyLe:
     addSon(result, parseStructBody(p, isUnion))
   else: 
@@ -746,7 +745,7 @@ proc directDeclarator(p: var TParser, a: PNode, ident: ptr PNode): PNode =
       result = declarator(p, a, ident)
       eat(p, pxParRi, result)
   else:
-    nil
+    discard
   return parseTypeSuffix(p, a)
 
 proc declarator(p: var TParser, a: PNode, ident: ptr PNode): PNode =
@@ -1165,7 +1164,7 @@ proc enumSpecifier(p: var TParser): PNode =
 
 proc setBaseFlags(n: PNode, base: TNumericalBase) = 
   case base
-  of base10: nil
+  of base10: discard
   of base2: incl(n.flags, nfBase2)
   of base8: incl(n.flags, nfBase8)
   of base16: incl(n.flags, nfBase16)
@@ -1686,7 +1685,7 @@ proc switchStatement(p: var TParser): PNode =
       break
     of "case", "default":
       break
-    else: nil
+    else: discard
     addSon(result, statement(p))
   if sonsLen(result) == 0:
     # translate empty statement list to Nimrod's ``nil`` statement

compiler/c2nim/cpp.nim

@@ -103,7 +103,7 @@ proc parseDefBody(p: var TParser, m: var TMacro, params: seq[string]) =
       m.body.add(tok)
     of pxDirConc: 
       # just ignore this token: this implements token merging correctly
-      nil
+      discard
     else:
       m.body.add(p.tok)
     # we do not want macro expansion here:
@@ -166,7 +166,7 @@ proc parseStmtList(p: var TParser): PNode =
     of pxDirectiveParLe, pxDirective: 
       case p.tok.s
       of "else", "endif", "elif": break
-    else: nil
+    else: discard
     addSon(result, statement(p))
   
 proc eatEndif(p: var TParser) =

compiler/canonicalizer.nim

@@ -0,0 +1,416 @@
+#
+#
+#           The Nimrod Compiler
+#        (c) Copyright 2014 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements the canonalization for the various caching mechanisms.
+
+import strutils, db_sqlite, md5
+
+var db: TDbConn
+
+# We *hash* the relevant information into 128 bit hashes. This should be good
+# enough to prevent any collisions.
+
+type
+  TUid = distinct MD5Digest
+
+# For name mangling we encode these hashes via a variant of base64 (called
+# 'base64a') and prepend the *primary* identifier to ease the debugging pain.
+# So a signature like:
+#
+#   proc gABI(c: PCtx; n: PNode; opc: TOpcode; a, b: TRegister; imm: BiggestInt)
+#
+# is mangled into:
+#   gABI_MTdmOWY5MTQ1MDcyNGQ3ZA
+#
+# This is a good compromise between correctness and brevity. ;-)
+
+const
+  cb64 = [
+    "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N",
+    "O", "P", "Q", "R", "S", "T" "U", "V", "W", "X", "Y", "Z", 
+    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", 
+    "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", 
+    "0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
+    "_A", "_B"]
+
+proc toBase64a(s: cstring, len: int): string =
+  ## encodes `s` into base64 representation. After `lineLen` characters, a 
+  ## `newline` is added.
+  result = newStringOfCap(((len + 2) div 3) * 4)
+  var i = 0
+  while i < s.len - 2:
+    let a = ord(s[i])
+    let b = ord(s[i+1])
+    let c = ord(s[i+2])
+    result.add cb64[a shr 2]
+    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
+    result.add cb64[((b and 0x0F) shl 2) or ((c and 0xC0) shr 6)]
+    result.add cb64[c and 0x3F]
+    inc(i, 3)
+  if i < s.len-1:
+    let a = ord(s[i])
+    let b = ord(s[i+1])
+    result.add cb64[a shr 2]
+    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
+    result.add cb64[((b and 0x0F) shl 2)]
+  elif i < s.len:
+    let a = ord(s[i])
+    result.add cb64[a shr 2]
+    result.add cb64[(a and 3) shl 4]
+
+proc toBase64a(u: TUid): string = toBase64a(cast[cstring](u), sizeof(u))
+
+proc `&=`(c: var MD5Context, s: string) = md5Update(c, s, s.len)
+
+proc hashSym(c: var MD5Context, s: PSym) =
+  if sfAnon in s.flags or s.kind == skGenericParam:
+    c &= ":anon"
+  else:
+    var it = s.owner
+    while it != nil: 
+      hashSym(c, it)
+      c &= "."
+      it = s.owner
+    c &= s.name.s
+
+proc hashTree(c: var MD5Context, n: PNode) =
+  if n == nil:
+    c &= "noTreeKind"
+    return
+  var k = n.kind
+  md5Update(c, cast[cstring](addr(k)), 1)
+  # we really must not hash line information. 'n.typ' is debatable but
+  # shouldn't be necessary for now and avoids potential infinite recursions.
+  case n.kind
+  of nkEmpty, nkNilLit, nkType: discard
+  of nkIdent:
+    c &= n.ident.s
+  of nkSym:
+    hashSym(c, n.sym)
+  of nkCharLit..nkUInt64Lit:
+    var v = n.intVal
+    md5Update(c, cast[cstring](addr(v)), sizeof(v))
+  of nkFloatLit..nkFloat64Lit:
+    var v = n.floatVal
+    md5Update(c, cast[cstring](addr(v)), sizeof(v))
+  of nkStrLit..nkTripleStrLit:
+    c &= n.strVal
+  else:
+    for i in 0.. <n.len: hashTree(c, n.sons[i])
+
+proc hashType(c: var MD5Context, t: PType) =
+  # modelled after 'typeToString'
+  if t == nil: 
+    c &= "noTypeKind"
+    return
+
+  var k = t.kind
+  md5Update(c, cast[cstring](addr(k)), 1)
+  
+  if t.sym != nil and sfAnon notin t.sym.flags:
+    # t.n for literals, but not for e.g. objects!
+    if t.kind in {tyFloat, tyInt}: c.hashNode(t.n)
+    c.hashSym(t.sym)
+    
+  case t.kind
+  of tyGenericBody, tyGenericInst, tyGenericInvokation:
+    for i in countup(0, sonsLen(t) -1 -ord(t.kind != tyGenericInvokation)):
+      c.hashType t.sons[i]
+  of tyUserTypeClass:
+    internalAssert t.sym != nil and t.sym.owner != nil
+    c &= t.sym.owner.name.s
+  of tyUserTypeClassInst:
+    let body = t.base
+    c.hashSym body.sym
+    for i in countup(1, sonsLen(t) - 2):
+      c.hashType t.sons[i]
+  of tyFromExpr, tyFieldAccessor:
+    c.hashTree(t.n)
+  of tyArrayConstr:
+    c.hashTree(t.sons[0].n)
+    c.hashType(t.sons[1])
+  of tyTuple: 
+    if t.n != nil:
+      assert(sonsLen(t.n) == sonsLen(t))
+      for i in countup(0, sonsLen(t.n) - 1): 
+        assert(t.n.sons[i].kind == nkSym)
+        c &= t.n.sons[i].sym.name.s
+        c &= ":"
+        c.hashType(t.sons[i])
+        c &= ","
+    else:
+      for i in countup(0, sonsLen(t) - 1): c.hashType t.sons[i]
+  of tyRange:
+    c.hashTree(t.n)
+    c.hashType(t.sons[0])
+  of tyProc:
+    c &= (if tfIterator in t.flags: "iterator " else: "proc ")
+    for i in 0.. <t.len: c.hashType(t.sons[i])
+    md5Update(c, cast[cstring](addr(t.callConv)), 1)
+
+    if tfNoSideEffect in t.flags: c &= ".noSideEffect"
+    if tfThread in t.flags: c &= ".thread"
+  else:
+    for i in 0.. <t.len: c.hashType(t.sons[i])
+  if tfShared in t.flags: c &= "shared"
+  if tfNotNil in t.flags: c &= "not nil"
+
+proc canonConst(n: PNode): TUid =
+  var c: MD5Context
+  md5Init(c)
+  c.hashTree(n)
+  c.hashType(n.typ)
+  md5Final(c, MD5Digest(result))
+
+proc canonSym(s: PSym): TUid
+  var c: MD5Context
+  md5Init(c)
+  c.hashSym(s)
+  md5Final(c, MD5Digest(result))
+
+proc pushType(w: PRodWriter, t: PType) =
+  # check so that the stack does not grow too large:
+  if iiTableGet(w.index.tab, t.id) == InvalidKey:
+    w.tstack.add(t)
+
+proc pushSym(w: PRodWriter, s: PSym) =
+  # check so that the stack does not grow too large:
+  if iiTableGet(w.index.tab, s.id) == InvalidKey:
+    w.sstack.add(s)
+
+proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode, 
+                result: var string) = 
+  if n == nil: 
+    # nil nodes have to be stored too:
+    result.add("()")
+    return
+  result.add('(')
+  encodeVInt(ord(n.kind), result) 
+  # we do not write comments for now
+  # Line information takes easily 20% or more of the filesize! Therefore we
+  # omit line information if it is the same as the father's line information:
+  if fInfo.fileIndex != n.info.fileIndex: 
+    result.add('?')
+    encodeVInt(n.info.col, result)
+    result.add(',')
+    encodeVInt(n.info.line, result)
+    result.add(',')
+    encodeVInt(fileIdx(w, toFilename(n.info)), result)
+  elif fInfo.line != n.info.line:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+    result.add(',')
+    encodeVInt(n.info.line, result)
+  elif fInfo.col != n.info.col:
+    result.add('?')
+    encodeVInt(n.info.col, result)
+  var f = n.flags * PersistentNodeFlags
+  if f != {}: 
+    result.add('$')
+    encodeVInt(cast[int32](f), result)
+  if n.typ != nil:
+    result.add('^')
+    encodeVInt(n.typ.id, result)
+    pushType(w, n.typ)
+  case n.kind
+  of nkCharLit..nkInt64Lit: 
+    if n.intVal != 0:
+      result.add('!')
+      encodeVBiggestInt(n.intVal, result)
+  of nkFloatLit..nkFloat64Lit: 
+    if n.floatVal != 0.0: 
+      result.add('!')
+      encodeStr($n.floatVal, result)
+  of nkStrLit..nkTripleStrLit:
+    if n.strVal != "": 
+      result.add('!')
+      encodeStr(n.strVal, result)
+  of nkIdent:
+    result.add('!')
+    encodeStr(n.ident.s, result)
+  of nkSym:
+    result.add('!')
+    encodeVInt(n.sym.id, result)
+    pushSym(w, n.sym)
+  else:
+    for i in countup(0, sonsLen(n) - 1): 
+      encodeNode(w, n.info, n.sons[i], result)
+  add(result, ')')
+
+proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) = 
+  var oldLen = result.len
+  result.add('<')
+  if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
+  if loc.s != low(loc.s): 
+    add(result, '*')
+    encodeVInt(ord(loc.s), result)
+  if loc.flags != {}: 
+    add(result, '$')
+    encodeVInt(cast[int32](loc.flags), result)
+  if loc.t != nil:
+    add(result, '^')
+    encodeVInt(cast[int32](loc.t.id), result)
+    pushType(w, loc.t)
+  if loc.r != nil: 
+    add(result, '!')
+    encodeStr(ropeToStr(loc.r), result)
+  if loc.a != 0: 
+    add(result, '?')
+    encodeVInt(loc.a, result)
+  if oldLen + 1 == result.len:
+    # no data was necessary, so remove the '<' again:
+    setLen(result, oldLen)
+  else:
+    add(result, '>')
+  
+proc encodeType(w: PRodWriter, t: PType, result: var string) = 
+  if t == nil: 
+    # nil nodes have to be stored too:
+    result.add("[]")
+    return
+  # we need no surrounding [] here because the type is in a line of its own
+  if t.kind == tyForward: internalError("encodeType: tyForward")
+  # for the new rodfile viewer we use a preceeding [ so that the data section
+  # can easily be disambiguated:
+  add(result, '[')
+  encodeVInt(ord(t.kind), result)
+  add(result, '+')
+  encodeVInt(t.id, result)
+  if t.n != nil: 
+    encodeNode(w, unknownLineInfo(), t.n, result)
+  if t.flags != {}: 
+    add(result, '$')
+    encodeVInt(cast[int32](t.flags), result)
+  if t.callConv != low(t.callConv): 
+    add(result, '?')
+    encodeVInt(ord(t.callConv), result)
+  if t.owner != nil: 
+    add(result, '*')
+    encodeVInt(t.owner.id, result)
+    pushSym(w, t.owner)
+  if t.sym != nil: 
+    add(result, '&')
+    encodeVInt(t.sym.id, result)
+    pushSym(w, t.sym)
+  if t.size != - 1: 
+    add(result, '/')
+    encodeVBiggestInt(t.size, result)
+  if t.align != 2: 
+    add(result, '=')
+    encodeVInt(t.align, result)
+  encodeLoc(w, t.loc, result)
+  for i in countup(0, sonsLen(t) - 1): 
+    if t.sons[i] == nil: 
+      add(result, "^()")
+    else: 
+      add(result, '^') 
+      encodeVInt(t.sons[i].id, result)
+      pushType(w, t.sons[i])
+
+proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) = 
+  add(result, '|')
+  encodeVInt(ord(lib.kind), result)
+  add(result, '|')
+  encodeStr(ropeToStr(lib.name), result)
+  add(result, '|')
+  encodeNode(w, info, lib.path, result)
+
+proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
+  if s == nil:
+    # nil nodes have to be stored too:
+    result.add("{}")
+    return
+  # we need no surrounding {} here because the symbol is in a line of its own
+  encodeVInt(ord(s.kind), result)
+  result.add('+')
+  encodeVInt(s.id, result)
+  result.add('&')
+  encodeStr(s.name.s, result)
+  if s.typ != nil:
+    result.add('^')
+    encodeVInt(s.typ.id, result)
+    pushType(w, s.typ)
+  result.add('?')
+  if s.info.col != -1'i16: encodeVInt(s.info.col, result)
+  result.add(',')
+  if s.info.line != -1'i16: encodeVInt(s.info.line, result)
+  result.add(',')
+  encodeVInt(fileIdx(w, toFilename(s.info)), result)
+  if s.owner != nil:
+    result.add('*')
+    encodeVInt(s.owner.id, result)
+    pushSym(w, s.owner)
+  if s.flags != {}:
+    result.add('$')
+    encodeVInt(cast[int32](s.flags), result)
+  if s.magic != mNone:
+    result.add('@')
+    encodeVInt(ord(s.magic), result)
+  if s.options != w.options: 
+    result.add('!')
+    encodeVInt(cast[int32](s.options), result)
+  if s.position != 0: 
+    result.add('%')
+    encodeVInt(s.position, result)
+  if s.offset != - 1:
+    result.add('`')
+    encodeVInt(s.offset, result)
+  encodeLoc(w, s.loc, result)
+  if s.annex != nil: encodeLib(w, s.annex, s.info, result)
+  if s.constraint != nil:
+    add(result, '#')
+    encodeNode(w, unknownLineInfo(), s.constraint, result)
+  # lazy loading will soon reload the ast lazily, so the ast needs to be
+  # the last entry of a symbol:
+  if s.ast != nil:
+    # we used to attempt to save space here by only storing a dummy AST if
+    # it is not necessary, but Nimrod's heavy compile-time evaluation features
+    # make that unfeasible nowadays:
+    encodeNode(w, s.info, s.ast, result)
+
+
+proc createDb() =
+  db.exec(sql"""
+    create table if not exists Module(
+      id integer primary key,
+      name varchar(256) not null,
+      fullpath varchar(256) not null,
+      interfHash varchar(256) not null,
+      fullHash varchar(256) not null,
+      
+      created timestamp not null default (DATETIME('now')),
+    );""")
+
+  db.exec(sql"""
+    create table if not exists Symbol(
+      id integer primary key,
+      module integer not null,
+      name varchar(max) not null,
+      data varchar(max) not null,
+      created timestamp not null default (DATETIME('now')),
+
+      foreign key (module) references module(id)
+    );""")
+    
+  db.exec(sql"""
+    create table if not exists Type(
+      id integer primary key,
+      module integer not null,
+      name varchar(max) not null,
+      data varchar(max) not null,
+      created timestamp not null default (DATETIME('now')),
+
+      foreign key (module) references module(id)
+    );""")
+
+
+  #db.exec(sql"""
+  #  --create unique index if not exists TsstNameIx on TestResult(name);
+  #  """, [])
+

compiler/ccgexprs.nim

@@ -593,7 +593,7 @@ proc unaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
 
 proc genDeref(p: BProc, e: PNode, d: var TLoc) =
   var a: TLoc
-  if mapType(e.sons[0].typ) == ctArray:
+  if mapType(e.sons[0].typ) in {ctArray, ctPtrToArray}:
     # XXX the amount of hacks for C's arrays is incredible, maybe we should
     # simply wrap them in a struct? --> Losing auto vectorization then?
     expr(p, e.sons[0], d)
@@ -736,7 +736,7 @@ proc genArrayElem(p: BProc, e: PNode, d: var TLoc) =
   var ty = skipTypes(skipTypes(a.t, abstractVarRange), abstractPtrs)
   var first = intLiteral(firstOrd(ty))
   # emit range check:
-  if (optBoundsCheck in p.options):
+  if optBoundsCheck in p.options and tfUncheckedArray notin ty.flags:
     if not isConstExpr(e.sons[1]):
       # semantic pass has already checked for const index expressions
       if firstOrd(ty) == 0:
@@ -1049,7 +1049,7 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) =
     app(tmp2.r, field.loc.r)
     tmp2.k = locTemp
     tmp2.t = field.loc.t
-    tmp2.s = OnHeap
+    tmp2.s = if isRef: OnHeap else: OnStack
     tmp2.heapRoot = tmp.r
     expr(p, it.sons[1], tmp2)
   if d.k == locNone:
@@ -1376,7 +1376,7 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       getTemp(p, getSysType(tyInt), i) # our counter
       initLocExpr(p, e.sons[1], a)
       initLocExpr(p, e.sons[2], b)
-      if d.k == locNone: getTemp(p, a.t, d)
+      if d.k == locNone: getTemp(p, getSysType(tyBool), d)
       lineF(p, cpsStmts, lookupOpr[op],
            [rdLoc(i), toRope(size), rdLoc(d), rdLoc(a), rdLoc(b)])
     of mEqSet:
@@ -1915,7 +1915,6 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       internalError(n.info, "expr: proc not init " & sym.name.s)
     putLocIntoDest(p, d, sym.loc)
   of nkClosure: genClosure(p, n, d)
-  of nkMetaNode: expr(p, n.sons[0], d)
 
   of nkEmpty: discard
   of nkWhileStmt: genWhileStmt(p, n)

compiler/ccgmerge.nim

@@ -145,33 +145,6 @@ proc atEndMark(buf: cstring, pos: int): bool =
   while s < NimMergeEndMark.len and buf[pos+s] == NimMergeEndMark[s]: inc s
   result = s == NimMergeEndMark.len
 
-when false:
-  proc readVerbatimSection(L: var TBaseLexer): PRope = 
-    var pos = L.bufpos
-    var buf = L.buf
-    result = newMutableRope(30_000)
-    while true:
-      case buf[pos]
-      of CR:
-        pos = nimlexbase.HandleCR(L, pos)
-        buf = L.buf
-        result.data.add(tnl)
-      of LF:
-        pos = nimlexbase.HandleLF(L, pos)
-        buf = L.buf
-        result.data.add(tnl)
-      of '\0':
-        InternalError("ccgmerge: expected: " & NimMergeEndMark)
-        break
-      else: 
-        if atEndMark(buf, pos):
-          inc pos, NimMergeEndMark.len
-          break
-        result.data.add(buf[pos])
-        inc pos
-    L.bufpos = pos
-    freezeMutableRope(result)
-
 proc readVerbatimSection(L: var TBaseLexer): PRope = 
   var pos = L.bufpos
   var buf = L.buf

compiler/ccgstmts.nim

@@ -65,6 +65,7 @@ proc startBlock(p: BProc, start: TFormatStr = "{$n",
   setLen(p.blocks, result + 1)
   p.blocks[result].id = p.labels
   p.blocks[result].nestedTryStmts = p.nestedTryStmts.len.int16
+  p.blocks[result].nestedExceptStmts = p.inExceptBlock.int16
 
 proc assignLabel(b: var TBlock): PRope {.inline.} =
   b.label = con("LA", b.id.toRope)
@@ -260,37 +261,57 @@ proc genIf(p: BProc, n: PNode, d: var TLoc) =
     else: internalError(n.info, "genIf()")
   if sonsLen(n) > 1: fixLabel(p, lend)
 
-proc blockLeaveActions(p: BProc, howMany: int) = 
+
-  var L = p.nestedTryStmts.len
+proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) = 
-  # danger of endless recursion! we workaround this here by a temp stack
+  # Called by return and break stmts.
+  # Deals with issues faced when jumping out of try/except/finally stmts,
+
   var stack: seq[PNode]
-  newSeq(stack, howMany)
+  newSeq(stack, 0)
-  for i in countup(1, howMany): 
-    stack[i-1] = p.nestedTryStmts[L-i]
-  setLen(p.nestedTryStmts, L-howMany)
   
   var alreadyPoppedCnt = p.inExceptBlock
-  for tryStmt in items(stack):
+  for i in countup(1, howManyTrys):
+
     if gCmd != cmdCompileToCpp:
+      # Pop safe points generated by try
       if alreadyPoppedCnt > 0:
         dec alreadyPoppedCnt
       else:
         linefmt(p, cpsStmts, "#popSafePoint();$n")
+
+    # Pop this try-stmt of the list of nested trys
+    # so we don't infinite recurse on it in the next step.
+    var tryStmt = p.nestedTryStmts.pop
+    stack.add(tryStmt)
+
+    # Find finally-stmt for this try-stmt
+    # and generate a copy of its sons
     var finallyStmt = lastSon(tryStmt)
     if finallyStmt.kind == nkFinally: 
       genStmts(p, finallyStmt.sons[0])
+
   # push old elements again:
-  for i in countdown(howMany-1, 0): 
+  for i in countdown(howManyTrys-1, 0): 
     p.nestedTryStmts.add(stack[i])
+
   if gCmd != cmdCompileToCpp:
-    for i in countdown(p.inExceptBlock-1, 0):
+    # Pop exceptions that was handled by the
+    # except-blocks we are in
+    for i in countdown(howManyExcepts-1, 0):
       linefmt(p, cpsStmts, "#popCurrentException();$n")
 
 proc genReturnStmt(p: BProc, t: PNode) =
   p.beforeRetNeeded = true
   genLineDir(p, t)
   if (t.sons[0].kind != nkEmpty): genStmts(p, t.sons[0])
-  blockLeaveActions(p, min(1, p.nestedTryStmts.len))
+  blockLeaveActions(p, 
+    howManyTrys    = p.nestedTryStmts.len,
+    howManyExcepts = p.inExceptBlock)
+  if (p.finallySafePoints.len > 0):
+    # If we're in a finally block, and we came here by exception
+    # consume it before we return.
+    var safePoint = p.finallySafePoints[p.finallySafePoints.len-1]
+    linefmt(p, cpsStmts, "if ($1.status != 0) #popCurrentException();$n", safePoint)    
   lineFF(p, cpsStmts, "goto BeforeRet;$n", "br label %BeforeRet$n", [])
 
 proc genComputedGoto(p: BProc; n: PNode) =
@@ -450,7 +471,9 @@ proc genBreakStmt(p: BProc, t: PNode) =
     if idx < 0 or not p.blocks[idx].isLoop:
       internalError(t.info, "no loop to break")
   let label = assignLabel(p.blocks[idx])
-  blockLeaveActions(p, p.nestedTryStmts.len - p.blocks[idx].nestedTryStmts)
+  blockLeaveActions(p, 
+    p.nestedTryStmts.len - p.blocks[idx].nestedTryStmts,
+    p.inExceptBlock - p.blocks[idx].nestedExceptStmts)
   genLineDir(p, t)
   lineF(p, cpsStmts, "goto $1;$n", [label])
 
@@ -827,7 +850,9 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
   discard pop(p.nestedTryStmts)
   endBlock(p) # end of else block
   if i < length and t.sons[i].kind == nkFinally:
+    p.finallySafePoints.add(safePoint)
     exprBlock(p, t.sons[i].sons[0], d)
+    discard pop(p.finallySafePoints)
   linefmt(p, cpsStmts, "if ($1.status != 0) #reraiseException();$n", safePoint)
 
 proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): PRope =

compiler/ccgtypes.nim

@@ -185,7 +185,7 @@ proc mapType(typ: PType): TCTypeKind =
   of tyPtr, tyVar, tyRef:
     var base = skipTypes(typ.sons[0], typedescInst)
     case base.kind
-    of tyOpenArray, tyArrayConstr, tyArray, tyVarargs: result = ctArray
+    of tyOpenArray, tyArrayConstr, tyArray, tyVarargs: result = ctPtrToArray
     else: result = ctPtr
   of tyPointer: result = ctPtr
   of tySequence: result = ctNimSeq
@@ -377,9 +377,12 @@ proc getTypePre(m: BModule, typ: PType): PRope =
     result = getSimpleTypeDesc(m, typ)
     if result == nil: result = cacheGetType(m.typeCache, typ)
 
+proc structOrUnion(t: PType): PRope =
+  (if tfUnion in t.flags: toRope("union") else: toRope("struct"))
+
 proc getForwardStructFormat(): string = 
-  if gCmd == cmdCompileToCpp: result = "struct $1;$n"
+  if gCmd == cmdCompileToCpp: result = "$1 $2;$n"
-  else: result = "typedef struct $1 $1;$n"
+  else: result = "typedef $1 $2 $2;$n"
   
 proc getTypeForward(m: BModule, typ: PType): PRope = 
   result = cacheGetType(m.forwTypeCache, typ)
@@ -390,7 +393,8 @@ proc getTypeForward(m: BModule, typ: PType): PRope =
   of tySequence, tyTuple, tyObject: 
     result = getTypeName(typ)
     if not isImportedType(typ): 
-      appf(m.s[cfsForwardTypes], getForwardStructFormat(), [result])
+      appf(m.s[cfsForwardTypes], getForwardStructFormat(),
+          [structOrUnion(typ), result])
     idTablePut(m.forwTypeCache, typ, result)
   else: internalError("getTypeForward(" & $typ.kind & ')')
   
@@ -445,7 +449,12 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
     if accessExpr != nil: ae = ropef("$1.$2", [accessExpr, sname])
     else: ae = sname
     fillLoc(field.loc, locField, field.typ, ae, OnUnknown)
-    appf(result, "$1 $2;$n", [getTypeDescAux(m, field.loc.t, check), sname])
+    let fieldType = field.loc.t
+    if fieldType.kind == tyArray and tfUncheckedArray in fieldType.flags:
+      appf(result, "$1 $2[SEQ_DECL_SIZE];$n",
+          [getTypeDescAux(m, fieldType.elemType, check), sname])
+    else:
+      appf(result, "$1 $2;$n", [getTypeDescAux(m, fieldType, check), sname])
   else: internalError(n.info, "genRecordFieldsAux()")
   
 proc getRecordFields(m: BModule, typ: PType, check: var TIntSet): PRope = 
@@ -455,23 +464,24 @@ proc getRecordDesc(m: BModule, typ: PType, name: PRope,
                    check: var TIntSet): PRope = 
   # declare the record:
   var hasField = false
+  let aStruct = structOrUnion(typ)
   if typ.kind == tyObject: 
     if typ.sons[0] == nil: 
       if (typ.sym != nil and sfPure in typ.sym.flags) or tfFinal in typ.flags: 
-        result = ropecg(m, "struct $1 {$n", [name])
+        result = ropecg(m, "$1 $2 {$n", [aStruct, name])
       else: 
-        result = ropecg(m, "struct $1 {$n#TNimType* m_type;$n", [name])
+        result = ropecg(m, "$1 $2 {$n#TNimType* m_type;$n", [aStruct, name])
         hasField = true
     elif gCmd == cmdCompileToCpp: 
-      result = ropecg(m, "struct $1 : public $2 {$n", 
+      result = ropecg(m, "$1 $2 : public $3 {$n", 
-                      [name, getTypeDescAux(m, typ.sons[0], check)])
+                      [aStruct, name, getTypeDescAux(m, typ.sons[0], check)])
       hasField = true
     else: 
-      result = ropecg(m, "struct $1 {$n  $2 Sup;$n", 
+      result = ropecg(m, "$1 $2 {$n  $3 Sup;$n", 
-                      [name, getTypeDescAux(m, typ.sons[0], check)])
+                      [aStruct, name, getTypeDescAux(m, typ.sons[0], check)])
       hasField = true
   else: 
-    result = ropef("struct $1 {$n", [name])
+    result = ropef("$1 $2 {$n", [aStruct, name])
   var desc = getRecordFields(m, typ, check)
   if (desc == nil) and not hasField: 
     appf(result, "char dummy;$n", [])
@@ -481,7 +491,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: PRope,
 
 proc getTupleDesc(m: BModule, typ: PType, name: PRope, 
                   check: var TIntSet): PRope =
-  result = ropef("struct $1 {$n", [name])
+  result = ropef("$1 $2 {$n", [structOrUnion(typ), name])
   var desc: PRope = nil
   for i in countup(0, sonsLen(typ) - 1): 
     appf(desc, "$1 Field$2;$n", 
@@ -557,7 +567,8 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var TIntSet): PRope =
     if result == nil: 
       result = getTypeName(t)
       if not isImportedType(t): 
-        appf(m.s[cfsForwardTypes], getForwardStructFormat(), [result])
+        appf(m.s[cfsForwardTypes], getForwardStructFormat(),
+            [structOrUnion(t), result])
       idTablePut(m.forwTypeCache, t, result)
     assert(cacheGetType(m.typeCache, t) == nil)
     idTablePut(m.typeCache, t, con(result, "*"))
@@ -588,7 +599,8 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var TIntSet): PRope =
     if result == nil: 
       result = getTypeName(t)
       if not isImportedType(t): 
-        appf(m.s[cfsForwardTypes], getForwardStructFormat(), [result])
+        appf(m.s[cfsForwardTypes], getForwardStructFormat(),
+           [structOrUnion(t), result])
       idTablePut(m.forwTypeCache, t, result)
     idTablePut(m.typeCache, t, result) # always call for sideeffects:
     if t.kind != tyTuple: recdesc = getRecordDesc(m, t, result, check)

compiler/ccgutils.nim

@@ -130,7 +130,6 @@ proc getUniqueType*(key: PType): PType =
       idTablePut(gTypeTable[k], key, key)
       result = key
   of tyProc:
-    # tyVar is not 100% correct, but would speeds things up a little:
     if key.callConv != ccClosure:
       result = key
     else:

compiler/cgen.nim

@@ -761,6 +761,8 @@ proc genProcAux(m: BModule, prc: PSym) =
   var returnStmt: PRope = nil
   assert(prc.ast != nil)
   if sfPure notin prc.flags and prc.typ.sons[0] != nil:
+    if resultPos >= prc.ast.len:
+      internalError(prc.info, "proc has no result symbol")
     var res = prc.ast.sons[resultPos].sym # get result symbol
     if not isInvalidReturnType(prc.typ.sons[0]):
       if sfNoInit in prc.flags: incl(res.flags, sfNoInit)
@@ -962,8 +964,8 @@ proc genMainProc(m: BModule) =
     NimMainBody =
       "N_CDECL(void, NimMain)(void) {$N" &
         "\tPreMain();$N" &
-        "$1$N" &
+        "$1" &
-      "}$N"
+      "}$N$N"
 
     PosixNimMain =
       "int cmdCount;$N" &
@@ -977,20 +979,20 @@ proc genMainProc(m: BModule) =
         "\tcmdCount = argc;$N" &
         "\tgEnv = env;$N" &
         MainProcsWithResult &
-      "}$N"
+      "}$N$N"
   
     StandaloneCMain =
       "int main(void) {$N" &
         MainProcs &
         "\treturn 0;$N" &
-      "}$N"
+      "}$N$N"
     
     WinNimMain = NimMainBody
     
     WinCMain = "N_STDCALL(int, WinMain)(HINSTANCE hCurInstance, $N" &
       "                        HINSTANCE hPrevInstance, $N" &
       "                        LPSTR lpCmdLine, int nCmdShow) {$N" &
-      MainProcsWithResult & "}$N"
+      MainProcsWithResult & "}$N$N"
   
     WinNimDllMain = "N_LIB_EXPORT " & NimMainBody
 
@@ -998,14 +1000,14 @@ proc genMainProc(m: BModule) =
       "BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fwdreason, $N" &
       "                    LPVOID lpvReserved) {$N" &
       "\tif(fwdreason == DLL_PROCESS_ATTACH) {$N" & MainProcs & "}$N" &
-      "\treturn 1;$N}$N"
+      "\treturn 1;$N}$N$N"
 
     PosixNimDllMain = WinNimDllMain
     
     PosixCDllMain =
       "void NIM_POSIX_INIT NimMainInit(void) {$N" &
         MainProcs &
-      "}$N"
+      "}$N$N"
 
   var nimMain, otherMain: TFormatStr
   if platform.targetOS == osWindows and
@@ -1034,7 +1036,7 @@ proc genMainProc(m: BModule) =
                               platform.targetOS == osStandalone: "".toRope
                             else: ropecg(m, "\t#initStackBottom();$N")
   inc(m.labels)
-  appcg(m, m.s[cfsProcs], "void PreMain() {$N" & PreMainBody & "}$N", [
+  appcg(m, m.s[cfsProcs], "void PreMain() {$N" & PreMainBody & "}$N$N", [
     mainDatInit, initStackBottomCall, gBreakpoints, otherModsInit])
 
   appcg(m, m.s[cfsProcs], nimMain, [mainModInit, toRope(m.labels)])
@@ -1042,8 +1044,10 @@ proc genMainProc(m: BModule) =
     appcg(m, m.s[cfsProcs], otherMain, [])
 
 proc getSomeInitName(m: PSym, suffix: string): PRope =
+  assert m.kind == skModule
+  assert m.owner.kind == skPackage
   if {sfSystemModule, sfMainModule} * m.flags == {}:
-    result = m.info.toFullPath.getPackageName.mangle.toRope
+    result = m.owner.name.s.mangle.toRope
   result.app m.name.s
   result.app suffix
   

compiler/cgendata.nim

@@ -41,7 +41,8 @@ type
     ctInt, ctInt8, ctInt16, ctInt32, ctInt64,
     ctFloat, ctFloat32, ctFloat64, ctFloat128,
     ctUInt, ctUInt8, ctUInt16, ctUInt32, ctUInt64,
-    ctArray, ctStruct, ctPtr, ctNimStr, ctNimSeq, ctProc, ctCString
+    ctArray, ctPtrToArray, ctStruct, ctPtr, ctNimStr, ctNimSeq, ctProc,
+    ctCString
   TCFileSections* = array[TCFileSection, PRope] # represents a generated C file
   TCProcSection* = enum       # the sections a generated C proc consists of
     cpsLocals,                # section of local variables for C proc
@@ -57,6 +58,7 @@ type
     sections*: TCProcSections # the code beloging
     isLoop*: bool             # whether block is a loop
     nestedTryStmts*: int16    # how many try statements is it nested into
+    nestedExceptStmts*: int16 # how many except statements is it nested into
     frameLen*: int16
   
   TCProc{.final.} = object    # represents C proc that is currently generated
@@ -68,6 +70,8 @@ type
     inExceptBlock*: int       # are we currently inside an except block?
                               # leaving such scopes by raise or by return must
                               # execute any applicable finally blocks
+    finallySafePoints*: seq[PRope]  # For correctly cleaning up exceptions when
+                                    # using return in finally statements
     labels*: Natural          # for generating unique labels in the C proc
     blocks*: seq[TBlock]      # nested blocks
     breakIdx*: int            # the block that will be exited
@@ -140,6 +144,7 @@ proc newProc*(prc: PSym, module: BModule): BProc =
   else: result.options = gOptions
   newSeq(result.blocks, 1)
   result.nestedTryStmts = @[]
+  result.finallySafePoints = @[]
 
 iterator cgenModules*: var BModule =
   for i in 0..high(gModules):

compiler/condsyms.nim

@@ -47,6 +47,7 @@ proc initDefines*() =
   defineSymbol("nimeffects")
   defineSymbol("nimbabel")
   defineSymbol("nimcomputedgoto")
+  defineSymbol("nimunion")
   
   # add platform specific symbols:
   case targetCPU

compiler/docgen.nim

@@ -42,17 +42,23 @@ proc compilerMsgHandler(filename: string, line, col: int,
   of mwUnsupportedLanguage: k = warnLanguageXNotSupported
   globalError(newLineInfo(filename, line, col), k, arg)
 
+proc docgenFindFile(s: string): string {.procvar.} =
+  result = options.findFile(s)
+  if result.len == 0:
+    result = getCurrentDir() / s
+    if not existsFile(result): result = ""
+
 proc parseRst(text, filename: string,
               line, column: int, hasToc: var bool,
               rstOptions: TRstParseOptions): PRstNode =
   result = rstParse(text, filename, line, column, hasToc, rstOptions,
-                    options.findFile, compilerMsgHandler)
+                    docgenFindFile, compilerMsgHandler)
 
 proc newDocumentor*(filename: string, config: PStringTable): PDoc =
   new(result)
   initRstGenerator(result[], (if gCmd != cmdRst2tex: outHtml else: outLatex),
                    options.gConfigVars, filename, {roSupportRawDirective},
-                   options.findFile, compilerMsgHandler)
+                   docgenFindFile, compilerMsgHandler)
   result.id = 100
 
 proc dispA(dest: var PRope, xml, tex: string, args: openArray[PRope]) =

compiler/guards.nim

@@ -9,7 +9,7 @@
 
 ## This module implements the 'implies' relation for guards.
 
-import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer
+import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents
 
 const
   someEq = {mEqI, mEqI64, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
@@ -69,9 +69,23 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
 
 proc interestingCaseExpr*(m: PNode): bool = isLetLocation(m, true)
 
-proc swapArgs(fact: PNode, newOp: string, m: TMagic): PNode =
+proc getMagicOp(name: string, m: TMagic): PSym =
+  result = newSym(skProc, getIdent(name), nil, unknownLineInfo())
+  result.magic = m
+
+let
+  opLe = getMagicOp("<=", mLeI)
+  opLt = getMagicOp("<", mLtI)
+  opAnd = getMagicOp("and", mAnd)
+  opOr = getMagicOp("or", mOr)
+  opNot = getMagicOp("not", mNot)
+  opIsNil = getMagicOp("isnil", mIsNil)
+  opContains = getMagicOp("contains", mInSet)
+  opEq = getMagicOp("==", mEqI)
+
+proc swapArgs(fact: PNode, newOp: PSym): PNode =
   result = newNodeI(nkCall, fact.info, 3)
-  result.sons[0] = newSymNode(getSysMagic(newOp, m))
+  result.sons[0] = newSymNode(newOp)
   result.sons[1] = fact.sons[2]
   result.sons[2] = fact.sons[1]
 
@@ -82,9 +96,9 @@ proc neg(n: PNode): PNode =
     result = n.sons[1]
   of someLt:
     # not (a < b)  ==  a >= b  ==  b <= a
-    result = swapArgs(n, "<=", mLeI)
+    result = swapArgs(n, opLe)
   of someLe:
-    result = swapArgs(n, "<", mLtI)
+    result = swapArgs(n, opLt)
   of mInSet:
     if n.sons[1].kind != nkCurly: return nil
     let t = n.sons[2].typ.skipTypes(abstractInst)
@@ -110,7 +124,7 @@ proc neg(n: PNode): PNode =
       b = n.sons[2].neg
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(getSysMagic("and", mAnd))
+      result.sons[0] = newSymNode(opAnd)
       result.sons[1] = a
       result.sons[2] = b
     elif a != nil:
@@ -120,12 +134,12 @@ proc neg(n: PNode): PNode =
   else:
     # leave  not (a == 4)  as it is
     result = newNodeI(nkCall, n.info, 2)
-    result.sons[0] = newSymNode(getSysMagic("not", mNot))
+    result.sons[0] = newSymNode(opNot)
     result.sons[1] = n
 
 proc buildIsNil(arg: PNode): PNode =
   result = newNodeI(nkCall, arg.info, 2)
-  result.sons[0] = newSymNode(getSysMagic("isNil", mIsNil))
+  result.sons[0] = newSymNode(opIsNil)
   result.sons[1] = arg
 
 proc usefulFact(n: PNode): PNode =
@@ -154,7 +168,7 @@ proc usefulFact(n: PNode): PNode =
       b = usefulFact(n.sons[2])
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(getSysMagic("and", mAnd))
+      result.sons[0] = newSymNode(opAnd)
       result.sons[1] = a
       result.sons[2] = b
     elif a != nil:
@@ -177,7 +191,7 @@ proc usefulFact(n: PNode): PNode =
       b = usefulFact(n.sons[2]).neg
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(getSysMagic("and", mAnd))
+      result.sons[0] = newSymNode(opAnd)
       result.sons[1] = a
       result.sons[2] = b
       result = result.neg
@@ -520,7 +534,7 @@ proc buildOf(it, loc: PNode): PNode =
   s.typ = settype(loc)
   for i in 0..it.len-2: s.sons[i] = it.sons[i]
   result = newNodeI(nkCall, it.info, 3)
-  result.sons[0] = newSymNode(getSysMagic("contains", mInSet))
+  result.sons[0] = newSymNode(opContains)
   result.sons[1] = s
   result.sons[2] = loc
 
@@ -532,20 +546,20 @@ proc buildElse(n: PNode): PNode =
     for j in 0..branch.len-2:
       s.add(branch.sons[j])
   result = newNodeI(nkCall, n.info, 3)
-  result.sons[0] = newSymNode(getSysMagic("contains", mInSet))
+  result.sons[0] = newSymNode(opContains)
   result.sons[1] = s
   result.sons[2] = n.sons[0]
 
 proc addDiscriminantFact*(m: var TModel, n: PNode) =
   var fact = newNodeI(nkCall, n.info, 3)
-  fact.sons[0] = newSymNode(getSysMagic("==", mEqI))
+  fact.sons[0] = newSymNode(opEq)
   fact.sons[1] = n.sons[0]
   fact.sons[2] = n.sons[1]
   m.add fact
 
 proc addAsgnFact*(m: var TModel, key, value: PNode) =
   var fact = newNodeI(nkCall, key.info, 3)
-  fact.sons[0] = newSymNode(getSysMagic("==", mEqI))
+  fact.sons[0] = newSymNode(opEq)
   fact.sons[1] = key
   fact.sons[2] = value
   m.add fact

compiler/jsgen.nim

@@ -275,11 +275,11 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxI64
     ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinF64
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxF64
-    ["AddU", "AddU", "AddU($1, $2)", "AddU($1, $2)"], # AddU
+    ["addU", "addU", "addU($1, $2)", "addU($1, $2)"], # addU
-    ["SubU", "SubU", "SubU($1, $2)", "SubU($1, $2)"], # SubU
+    ["subU", "subU", "subU($1, $2)", "subU($1, $2)"], # subU
-    ["MulU", "MulU", "MulU($1, $2)", "MulU($1, $2)"], # MulU
+    ["mulU", "mulU", "mulU($1, $2)", "mulU($1, $2)"], # mulU
-    ["DivU", "DivU", "DivU($1, $2)", "DivU($1, $2)"], # DivU
+    ["divU", "divU", "divU($1, $2)", "divU($1, $2)"], # divU
-    ["ModU", "ModU", "ModU($1, $2)", "ModU($1, $2)"], # ModU
+    ["modU", "modU", "modU($1, $2)", "modU($1, $2)"], # modU
     ["", "", "($1 == $2)", "($1 == $2)"], # EqI
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeI
     ["", "", "($1 < $2)", "($1 < $2)"], # LtI
@@ -289,10 +289,10 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 == $2)", "($1 == $2)"], # EqF64
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeF64
     ["", "", "($1 < $2)", "($1 < $2)"], # LtF64
-    ["LeU", "LeU", "LeU($1, $2)", "LeU($1, $2)"], # LeU
+    ["leU", "leU", "leU($1, $2)", "leU($1, $2)"], # leU
-    ["LtU", "LtU", "LtU($1, $2)", "LtU($1, $2)"], # LtU
+    ["ltU", "ltU", "ltU($1, $2)", "ltU($1, $2)"], # ltU
-    ["LeU64", "LeU64", "LeU64($1, $2)", "LeU64($1, $2)"], # LeU64
+    ["leU64", "leU64", "leU64($1, $2)", "leU64($1, $2)"], # leU64
-    ["LtU64", "LtU64", "LtU64($1, $2)", "LtU64($1, $2)"], # LtU64
+    ["ltU64", "ltU64", "ltU64($1, $2)", "ltU64($1, $2)"], # ltU64
     ["", "", "($1 == $2)", "($1 == $2)"], # EqEnum
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeEnum
     ["", "", "($1 < $2)", "($1 < $2)"], # LtEnum
@@ -309,10 +309,10 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 == $2)", "($1 == $2)"], # EqCString
     ["", "", "($1 != $2)", "($1 != $2)"], # Xor
     ["", "", "($1 == $2)", "($1 == $2)"], # EqProc
-    ["NegInt", "", "NegInt($1)", "-($1)"], # UnaryMinusI
+    ["negInt", "", "negInt($1)", "-($1)"], # UnaryMinusI
-    ["NegInt64", "", "NegInt64($1)", "-($1)"], # UnaryMinusI64
+    ["negInt64", "", "negInt64($1)", "-($1)"], # UnaryMinusI64
-    ["AbsInt", "", "AbsInt($1)", "Math.abs($1)"], # AbsI
+    ["absInt", "", "absInt($1)", "Math.abs($1)"], # AbsI
-    ["AbsInt64", "", "AbsInt64($1)", "Math.abs($1)"], # AbsI64
+    ["absInt64", "", "absInt64($1)", "Math.abs($1)"], # AbsI64
     ["", "", "!($1)", "!($1)"], # Not
     ["", "", "+($1)", "+($1)"], # UnaryPlusI
     ["", "", "~($1)", "~($1)"], # BitnotI
@@ -327,9 +327,9 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["Ze16ToI64", "Ze16ToI64", "Ze16ToI64($1)", "Ze16ToI64($1)"], # mZe16ToI64
     ["Ze32ToI64", "Ze32ToI64", "Ze32ToI64($1)", "Ze32ToI64($1)"], # mZe32ToI64
     ["ZeIToI64", "ZeIToI64", "ZeIToI64($1)", "ZeIToI64($1)"], # mZeIToI64
-    ["ToU8", "ToU8", "ToU8($1)", "ToU8($1)"], # ToU8
+    ["toU8", "toU8", "toU8($1)", "toU8($1)"], # toU8
-    ["ToU16", "ToU16", "ToU16($1)", "ToU16($1)"], # ToU16
+    ["toU16", "toU16", "toU16($1)", "toU16($1)"], # toU16
-    ["ToU32", "ToU32", "ToU32($1)", "ToU32($1)"], # ToU32
+    ["toU32", "toU32", "toU32($1)", "toU32($1)"], # toU32
     ["", "", "$1", "$1"],     # ToFloat
     ["", "", "$1", "$1"],     # ToBiggestFloat
     ["", "", "Math.floor($1)", "Math.floor($1)"], # ToInt
@@ -375,11 +375,11 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxI64
     ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinF64
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxF64
-    ["AddU", "AddU", "AddU($1, $2)", "AddU($1, $2)"], # AddU
+    ["addU", "addU", "addU($1, $2)", "addU($1, $2)"], # addU
-    ["SubU", "SubU", "SubU($1, $2)", "SubU($1, $2)"], # SubU
+    ["subU", "subU", "subU($1, $2)", "subU($1, $2)"], # subU
-    ["MulU", "MulU", "MulU($1, $2)", "MulU($1, $2)"], # MulU
+    ["mulU", "mulU", "mulU($1, $2)", "mulU($1, $2)"], # mulU
-    ["DivU", "DivU", "DivU($1, $2)", "DivU($1, $2)"], # DivU
+    ["divU", "divU", "divU($1, $2)", "divU($1, $2)"], # divU
-    ["ModU", "ModU", "ModU($1, $2)", "ModU($1, $2)"], # ModU
+    ["modU", "modU", "modU($1, $2)", "modU($1, $2)"], # modU
     ["", "", "($1 == $2)", "($1 == $2)"], # EqI
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeI
     ["", "", "($1 < $2)", "($1 < $2)"], # LtI
@@ -389,10 +389,10 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 == $2)", "($1 == $2)"], # EqF64
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeF64
     ["", "", "($1 < $2)", "($1 < $2)"], # LtF64
-    ["LeU", "LeU", "LeU($1, $2)", "LeU($1, $2)"], # LeU
+    ["leU", "leU", "leU($1, $2)", "leU($1, $2)"], # leU
-    ["LtU", "LtU", "LtU($1, $2)", "LtU($1, $2)"], # LtU
+    ["ltU", "ltU", "ltU($1, $2)", "ltU($1, $2)"], # ltU
-    ["LeU64", "LeU64", "LeU64($1, $2)", "LeU64($1, $2)"], # LeU64
+    ["leU64", "leU64", "leU64($1, $2)", "leU64($1, $2)"], # leU64
-    ["LtU64", "LtU64", "LtU64($1, $2)", "LtU64($1, $2)"], # LtU64
+    ["ltU64", "ltU64", "ltU64($1, $2)", "ltU64($1, $2)"], # ltU64
     ["", "", "($1 == $2)", "($1 == $2)"], # EqEnum
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeEnum
     ["", "", "($1 < $2)", "($1 < $2)"], # LtEnum
@@ -409,10 +409,10 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 == $2)", "($1 == $2)"], # EqCString
     ["", "", "($1 != $2)", "($1 != $2)"], # Xor
     ["", "", "($1 == $2)", "($1 == $2)"], # EqProc
-    ["NegInt", "", "NegInt($1)", "-($1)"], # UnaryMinusI
+    ["negInt", "", "negInt($1)", "-($1)"], # UnaryMinusI
-    ["NegInt64", "", "NegInt64($1)", "-($1)"], # UnaryMinusI64
+    ["negInt64", "", "negInt64($1)", "-($1)"], # UnaryMinusI64
-    ["AbsInt", "", "AbsInt($1)", "Math.abs($1)"], # AbsI
+    ["absInt", "", "absInt($1)", "Math.abs($1)"], # AbsI
-    ["AbsInt64", "", "AbsInt64($1)", "Math.abs($1)"], # AbsI64
+    ["absInt64", "", "absInt64($1)", "Math.abs($1)"], # AbsI64
     ["", "", "not ($1)", "not ($1)"], # Not
     ["", "", "+($1)", "+($1)"], # UnaryPlusI
     ["", "", "~($1)", "~($1)"], # BitnotI
@@ -427,9 +427,9 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["Ze16ToI64", "Ze16ToI64", "Ze16ToI64($1)", "Ze16ToI64($1)"], # mZe16ToI64
     ["Ze32ToI64", "Ze32ToI64", "Ze32ToI64($1)", "Ze32ToI64($1)"], # mZe32ToI64
     ["ZeIToI64", "ZeIToI64", "ZeIToI64($1)", "ZeIToI64($1)"], # mZeIToI64
-    ["ToU8", "ToU8", "ToU8($1)", "ToU8($1)"], # ToU8
+    ["toU8", "toU8", "toU8($1)", "toU8($1)"], # toU8
-    ["ToU16", "ToU16", "ToU16($1)", "ToU16($1)"], # ToU16
+    ["toU16", "toU16", "toU16($1)", "toU16($1)"], # toU16
-    ["ToU32", "ToU32", "ToU32($1)", "ToU32($1)"], # ToU32
+    ["toU32", "toU32", "toU32($1)", "toU32($1)"], # toU32
     ["", "", "$1", "$1"],     # ToFloat
     ["", "", "$1", "$1"],     # ToBiggestFloat
     ["", "", "Math.floor($1)", "Math.floor($1)"], # ToInt
@@ -812,8 +812,8 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
     if needsNoCopy(y) or noCopyNeeded:
       appf(p.body, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
-      useMagic(p, "NimCopy")
+      useMagic(p, "nimCopy")
-      appf(p.body, "$1 = NimCopy($2, $3);$n",
+      appf(p.body, "$1 = nimCopy($2, $3);$n",
            [a.res, b.res, genTypeInfo(p, y.typ)])
   of etyBaseIndex: 
     if a.typ != etyBaseIndex or b.typ != etyBaseIndex: 
@@ -1113,8 +1113,8 @@ proc createVar(p: PProc, typ: PType, indirect: bool): PRope =
     var length = int(lengthOrd(t))
     var e = elemType(t)
     if length > 32: 
-      useMagic(p, "ArrayConstr")
+      useMagic(p, "arrayConstr")
-      result = ropef("ArrayConstr($1, $2, $3)", [toRope(length), 
+      result = ropef("arrayConstr($1, $2, $3)", [toRope(length), 
           createVar(p, e, false), genTypeInfo(p, e)])
     else: 
       result = toRope("[")
@@ -1171,8 +1171,8 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
       if needsNoCopy(n): 
         s = a.res
       else: 
-        useMagic(p, "NimCopy")
+        useMagic(p, "nimCopy")
-        s = ropef("NimCopy($1, $2)", [a.res, genTypeInfo(p, n.typ)])
+        s = ropef("nimCopy($1, $2)", [a.res, genTypeInfo(p, n.typ)])
     of etyBaseIndex: 
       if (a.typ != etyBaseIndex): internalError(n.info, "genVarInit")
       if {sfAddrTaken, sfGlobal} * v.flags != {}: 
@@ -1600,7 +1600,6 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
     if lfNoDecl in s.loc.flags or s.magic != mNone: discard
     elif not p.g.generatedSyms.containsOrIncl(s.id):
       app(p.locals, genProc(p, s))
-  of nkMetaNode: gen(p, n.sons[0], r)
   of nkType: r.res = genTypeInfo(p, n.typ)
   of nkStmtList, nkStmtListExpr:
     # this shows the distinction is nice for backends and should be kept

compiler/lambdalifting.nim

@@ -117,6 +117,7 @@ type
   TEnv {.final.} = object of TObject
     attachedNode: PNode
     createdVar: PSym        # if != nil it is a used environment
+    createdVarComesFromIter: bool
     capturedVars: seq[PSym] # captured variables in this environment
     deps: seq[TDep]         # dependencies
     up: PEnv
@@ -571,7 +572,14 @@ proc rawClosureCreation(o: POuterContext, scope: PEnv; env: PSym): PNode =
       # maybe later: (sfByCopy in local.flags)
       # add ``env.param = param``
       result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info))
+    # it can happen that we already captured 'local' in some other environment
+    # then we capture by copy for now. This is not entirely correct but better
+    # than nothing:
+    let existing = idNodeTableGet(o.localsToAccess, local)
+    if existing.isNil:
       idNodeTablePut(o.localsToAccess, local, fieldAccess)
+    else:
+      result.add(newAsgnStmt(fieldAccess, existing, env.info))
   # add support for 'up' references:
   for e, field in items(scope.deps):
     # add ``env.up = env2``
@@ -584,6 +592,8 @@ proc generateClosureCreation(o: POuterContext, scope: PEnv): PNode =
 
 proc generateIterClosureCreation(o: POuterContext; env: PEnv;
                                  scope: PNode): PSym =
+  if env.createdVarComesFromIter or env.createdVar.isNil:
+    # we have to create a new closure:
     result = newClosureCreationVar(o, env)
     let cc = rawClosureCreation(o, env, result)
     var insertPoint = scope.sons[0]
@@ -592,6 +602,9 @@ proc generateIterClosureCreation(o: POuterContext; env: PEnv;
       assert cc.kind == nkStmtList and insertPoint.kind == nkStmtList
       for x in cc: insertPoint.add(x)
     if env.createdVar == nil: env.createdVar = result
+  else:
+    result = env.createdVar
+  env.createdVarComesFromIter = true
 
 proc interestingIterVar(s: PSym): bool {.inline.} =
   result = s.kind in {skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags
@@ -637,6 +650,22 @@ proc outerProcSons(o: POuterContext, n: PNode) =
     let x = transformOuterProc(o, n.sons[i])
     if x != nil: n.sons[i] = x
 
+proc liftIterSym*(n: PNode): PNode =
+  # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env)) 
+  let iter = n.sym
+  assert iter.kind == skIterator
+
+  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
+  
+  var env = copySym(getHiddenParam(iter))
+  env.kind = skLet
+  var v = newNodeI(nkVarSection, n.info)
+  addVar(v, newSymNode(env))
+  result.add(v)
+  # add 'new' statement:
+  result.add(newCall(getSysSym"internalNew", env))
+  result.add makeClosure(iter, env, n.info)
+
 proc transformOuterProc(o: POuterContext, n: PNode): PNode =
   if n == nil: return nil
   case n.kind
@@ -649,18 +678,23 @@ proc transformOuterProc(o: POuterContext, n: PNode): PNode =
       return indirectAccess(newSymNode(o.closureParam), local, n.info)
 
     var closure = PEnv(idTableGet(o.lambdasToEnv, local))
-    if closure != nil:
+
-      # we need to replace the lambda with '(lambda, env)':
     if local.kind == skIterator and local.typ.callConv == ccClosure:
       # consider: [i1, i2, i1]  Since we merged the iterator's closure
       # with the captured owning variables, we need to generate the
       # closure generation code again:
-        #if local == o.fn: message(n.info, errRecursiveDependencyX, local.name.s)
+      if local == o.fn: message(n.info, errRecursiveDependencyX, local.name.s)
       # XXX why doesn't this work?
+      if closure.isNil:
+        return liftIterSym(n)
+      else:
         let createdVar = generateIterClosureCreation(o, closure,
                                                      closure.attachedNode)
         return makeClosure(local, createdVar, n.info)
 
+    if closure != nil:
+      # we need to replace the lambda with '(lambda, env)':
+      
       let a = closure.createdVar
       if a != nil:
         return makeClosure(local, a, n.info)
@@ -773,22 +807,6 @@ proc liftLambdasForTopLevel*(module: PSym, body: PNode): PNode =
 
 # ------------------- iterator transformation --------------------------------
 
-proc liftIterSym*(n: PNode): PNode =
-  # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env)) 
-  let iter = n.sym
-  assert iter.kind == skIterator
-
-  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
-  
-  var env = copySym(getHiddenParam(iter))
-  env.kind = skLet
-  var v = newNodeI(nkVarSection, n.info)
-  addVar(v, newSymNode(env))
-  result.add(v)
-  # add 'new' statement:
-  result.add(newCall(getSysSym"internalNew", env))
-  result.add makeClosure(iter, env, n.info)
-
 proc liftForLoop*(body: PNode): PNode =
   # problem ahead: the iterator could be invoked indirectly, but then
   # we don't know what environment to create here: 

compiler/lookups.nim

@@ -32,6 +32,7 @@ proc considerAcc*(n: PNode): PIdent =
         of nkSym: id.add(x.sym.name.s)
         else: globalError(n.info, errIdentifierExpected, renderTree(n))
       result = getIdent(id)
+  of nkOpenSymChoice, nkClosedSymChoice: result = n.sons[0].sym.name
   else:
     globalError(n.info, errIdentifierExpected, renderTree(n))
  
@@ -91,7 +92,7 @@ proc errorSym*(c: PContext, n: PNode): PSym =
   result.typ = errorType(c)
   incl(result.flags, sfDiscardable)
   # pretend it's imported from some unknown module to prevent cascading errors:
-  if gCmd != cmdInteractive:
+  if gCmd != cmdInteractive and c.inCompilesContext == 0:
     c.importTable.addSym(result)
 
 type 

compiler/modules.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2013 Andreas Rumpf
+#        (c) Copyright 2014 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -20,7 +20,7 @@ type
   TModuleInMemory* = object
     compiledAt*: float
     crc*: TCrc32
-    deps*: seq[int32] ## XXX: slurped files are not currently tracked
+    deps*: seq[int32] ## XXX: slurped files are currently not tracked
     needsRecompile*: TNeedRecompile
     crcStatus*: TCrcStatus
 
@@ -83,7 +83,7 @@ proc resetAllModules* =
   for i in 0..gCompiledModules.high:
     if gCompiledModules[i] != nil:
       resetModule(i.int32)
-
+  resetPackageCache()
   # for m in cgenModules(): echo "CGEN MODULE FOUND"
 
 proc checkDepMem(fileIdx: int32): TNeedRecompile =
@@ -120,8 +120,9 @@ proc newModule(fileIdx: int32): PSym =
   if not isNimrodIdentifier(result.name.s):
     rawMessage(errInvalidModuleName, result.name.s)
   
-  result.owner = result       # a module belongs to itself
   result.info = newLineInfo(fileIdx, 1, 1)
+  result.owner = newSym(skPackage, getIdent(getPackageName(filename)), nil,
+                        result.info)
   result.position = fileIdx
   
   growCache gMemCacheData, fileIdx

compiler/msgs.nim

@@ -67,7 +67,7 @@ type
     errAmbiguousCallXYZ, errWrongNumberOfArguments, 
     errXCannotBePassedToProcVar, 
     errXCannotBeInParamDecl, errPragmaOnlyInHeaderOfProc, errImplOfXNotAllowed, 
-    errImplOfXexpected, errNoSymbolToBorrowFromFound, errDiscardValue, 
+    errImplOfXexpected, errNoSymbolToBorrowFromFound, errDiscardValueX, 
     errInvalidDiscard, errIllegalConvFromXtoY, errCannotBindXTwice, 
     errInvalidOrderInArrayConstructor,
     errInvalidOrderInEnumX, errEnumXHasHoles, errExceptExpected, errInvalidTry, 
@@ -89,10 +89,12 @@ type
     errInvalidIndexValueForTuple, errCommandExpectsFilename,
     errMainModuleMustBeSpecified,
     errXExpected,
+    errTIsNotAConcreteType,
     errInvalidSectionStart, errGridTableNotImplemented, errGeneralParseError, 
     errNewSectionExpected, errWhitespaceExpected, errXisNoValidIndexFile, 
     errCannotRenderX, errVarVarTypeNotAllowed, errInstantiateXExplicitely,
     errOnlyACallOpCanBeDelegator, errUsingNoSymbol,
+    errMacroBodyDependsOnGenericTypes,
     errDestructorNotGenericEnough,
     
     errXExpectsTwoArguments, 
@@ -103,6 +105,7 @@ type
     errXhasSideEffects, errIteratorExpected, errLetNeedsInit,
     errThreadvarCannotInit, errWrongSymbolX, errIllegalCaptureX,
     errXCannotBeClosure, errXMustBeCompileTime,
+    errCannotInferTypeOfTheLiteral,
     errUser,
     warnCannotOpenFile, 
     warnOctalEscape, warnXIsNeverRead, warnXmightNotBeenInit, 
@@ -263,7 +266,7 @@ const
     errImplOfXNotAllowed: "implementation of \'$1\' is not allowed", 
     errImplOfXexpected: "implementation of \'$1\' expected", 
     errNoSymbolToBorrowFromFound: "no symbol to borrow from found", 
-    errDiscardValue: "value returned by statement has to be discarded", 
+    errDiscardValueX: "value of type '$1' has to be discarded", 
     errInvalidDiscard: "statement returns no value that can be discarded", 
     errIllegalConvFromXtoY: "conversion from $1 to $2 is invalid",
     errCannotBindXTwice: "cannot bind parameter \'$1\' twice", 
@@ -312,6 +315,7 @@ const
     errCommandExpectsFilename: "command expects a filename argument",
     errMainModuleMustBeSpecified: "please, specify a main module in the project configuration file",
     errXExpected: "\'$1\' expected", 
+    errTIsNotAConcreteType: "\'$1\' is not a concrete type.",
     errInvalidSectionStart: "invalid section start",
     errGridTableNotImplemented: "grid table is not implemented", 
     errGeneralParseError: "general parse error",
@@ -323,6 +327,8 @@ const
     errInstantiateXExplicitely: "instantiate '$1' explicitely",
     errOnlyACallOpCanBeDelegator: "only a call operator can be a delegator",
     errUsingNoSymbol: "'$1' is not a variable, constant or a proc name",
+    errMacroBodyDependsOnGenericTypes: "the macro body cannot be compiled, " &
+                                       "because the parameter '$1' has a generic type",
     errDestructorNotGenericEnough: "Destructor signarue is too specific. " &
                                    "A destructor must be associated will all instantiations of a generic type",
     errXExpectsTwoArguments: "\'$1\' expects two arguments", 
@@ -346,6 +352,7 @@ const
     errIllegalCaptureX: "illegal capture '$1'",
     errXCannotBeClosure: "'$1' cannot have 'closure' calling convention",
     errXMustBeCompileTime: "'$1' can only be used in compile-time context",
+    errCannotInferTypeOfTheLiteral: "cannot infer the type of the $1",
     errUser: "$1", 
     warnCannotOpenFile: "cannot open \'$1\' [CannotOpenFile]",
     warnOctalEscape: "octal escape sequences do not exist; leading zero is ignored [OctalEscape]", 

compiler/nimeval.nim

@@ -17,11 +17,11 @@ proc execute*(program: string) =
   passes.gIncludeFile = includeModule
   passes.gImportModule = importModule
   initDefines()
-  LoadConfigs(DefaultConfig)
+  loadConfigs(DefaultConfig)
 
   initDefines()
-  DefineSymbol("nimrodvm")
+  defineSymbol("nimrodvm")
-  when hasFFI: DefineSymbol("nimffi")
+  when hasFFI: defineSymbol("nimffi")
   registerPass(verbosePass)
   registerPass(semPass)
   registerPass(vmPass)
@@ -30,4 +30,4 @@ proc execute*(program: string) =
   compileSystemModule()
   var m = makeStdinModule()
   incl(m.flags, sfMainModule)
-  processModule(m, LLStreamOpen(program), nil)
+  processModule(m, llStreamOpen(program), nil)

compiler/nimrod.nimrod.cfg

@@ -17,4 +17,6 @@ import:testability
   cincludes: "$lib/wrappers/libffi/common"
 @end
 
+define:useStdoutAsStdmsg
+
 cs:partial

compiler/options.nim

@@ -209,21 +209,42 @@ proc getGeneratedPath: string =
   result = if nimcacheDir.len > 0: nimcacheDir else: gProjectPath.shortenDir /
                                                          genSubDir
 
+template newPackageCache(): expr =
+  newStringTable(when FileSystemCaseSensitive:
+                   modeCaseInsensitive
+                 else:
+                   modeCaseSensitive)
+
+var packageCache = newPackageCache()
+
+proc resetPackageCache*() = packageCache = newPackageCache()
+
+iterator myParentDirs(p: string): string =
+  # XXX os's parentDirs is stupid (multiple yields) and triggers an old bug...
+  var current = p
+  while true:
+    current = current.parentDir
+    if current.len == 0: break
+    yield current
+
 proc getPackageName*(path: string): string =
-  var q = 1
+  var parents = 0
-  var b = 0
+  block packageSearch:
-  if path[len(path)-1] in {DirSep, AltSep}: q = 2
+    for d in myParentDirs(path):
-  for i in countdown(len(path)-q, 0):
+      if packageCache.hasKey(d):
-    if path[i] in {DirSep, AltSep}:
+        #echo "from cache ", d, " |", packageCache[d], "|", path.splitFile.name
-      if b == 0: b = i
+        return packageCache[d]
-      else:
+      inc parents
-        let x = path.substr(i+1, b-1)
+      for file in walkFiles(d / "*.babel"):
-        case x.normalize
+        result = file.splitFile.name
-        of "lib", "src", "source", "package", "pckg", "library", "private":
+        break packageSearch
-          b = i
+  # we also store if we didn't find anything:
-        else:
+  if result.isNil: result = ""
-          return x.replace('.', '_')
+  for d in myParentDirs(path):
-  result = ""
+    #echo "set cache ", d, " |", result, "|", parents
+    packageCache[d] = result
+    dec parents
+    if parents <= 0: break
 
 proc withPackageName*(path: string): string =
   let x = path.getPackageName

compiler/parser.nim

@@ -281,7 +281,7 @@ proc parseSymbol(p: var TParser): PNode =
         add(result, newIdentNodeP(getIdent"{}", p))
         getTok(p)
         eat(p, tkCurlyRi)
-      of tokKeywordLow..tokKeywordHigh, tkSymbol, tkOpr, tkDotDot:
+      of tokKeywordLow..tokKeywordHigh, tkSymbol, tkOpr, tkDot, tkDotDot:
         add(result, newIdentNodeP(p.tok.ident, p))
         getTok(p)
       of tkIntLit..tkCharLit:

compiler/pas2nim/paslex.nim

@@ -342,7 +342,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
       h = h +% ord(c)
       h = h +% h shl 10
       h = h xor (h shr 6)
-    of '_': nil
+    of '_': discard
     else: break
     inc(pos)
   h = h +% h shl 3

compiler/pas2nim/pasparse.nim

@@ -335,7 +335,7 @@ proc exprColonEqExprList(p: var TParser, kind, elemKind: TNodeKind,
 
 proc setBaseFlags(n: PNode, base: TNumericalBase) = 
   case base
-  of base10: nil
+  of base10: discard
   of base2: incl(n.flags, nfBase2)
   of base8: incl(n.flags, nfBase8)
   of base16: incl(n.flags, nfBase16)
@@ -466,7 +466,7 @@ proc lowestExprAux(p: var TParser, v: var PNode, limit: int): TTokKind =
         eat(p, pxCurlyDirRi)
         opNode.ident = getIdent("&")
       else: 
-        nil
+        discard
     of pxMinus: 
       if p.tok.xkind == pxPer: 
         getTok(p)
@@ -477,7 +477,7 @@ proc lowestExprAux(p: var TParser, v: var PNode, limit: int): TTokKind =
     of pxNeq: 
       opNode.ident = getIdent("!=")
     else: 
-      nil
+      discard
     skipCom(p, opNode)        # read sub-expression with higher priority
     nextop = lowestExprAux(p, v2, opPred)
     addSon(node, opNode)
@@ -505,7 +505,7 @@ proc fixExpr(n: PNode): PNode =
             (n.sons[2].kind in {nkCharLit, nkStrLit}): 
           n.sons[0].ident = getIdent("&") # fix operator
   else: 
-    nil
+    discard
   if not (n.kind in {nkEmpty..nkNilLit}): 
     for i in countup(0, sonsLen(n) - 1): result.sons[i] = fixExpr(n.sons[i])
   
@@ -603,7 +603,7 @@ proc parseStmtList(p: var TParser): PNode =
     of pxCurlyDirLe, pxStarDirLe: 
       if not isHandledDirective(p): break 
     else: 
-      nil
+      discard
     addSon(result, parseStmt(p))
   if sonsLen(result) == 1: result = result.sons[0]
   
@@ -732,7 +732,7 @@ proc parseRepeat(p: var TParser): PNode =
   addSon(b, c)
   addSon(a, b)
   if b.sons[0].kind == nkIdent and b.sons[0].ident.id == getIdent("false").id: 
-    nil
+    discard
   else: 
     addSon(s, a)
   addSon(result, s)
@@ -840,7 +840,7 @@ proc parseParam(p: var TParser): PNode =
     getTok(p)
     v = newNodeP(nkVarTy, p)
   else: 
-    nil
+    discard
   while true: 
     case p.tok.xkind
     of pxSymbol: a = createIdentNodeP(p.tok.ident, p)
@@ -1133,7 +1133,7 @@ proc parseRecordPart(p: var TParser): PNode =
 proc exSymbol(n: var PNode) = 
   case n.kind
   of nkPostfix: 
-    nil
+    discard
   of nkPragmaExpr: 
     exSymbol(n.sons[0])
   of nkIdent, nkAccQuoted: 
@@ -1154,7 +1154,7 @@ proc fixRecordDef(n: var PNode) =
     for i in countup(0, sonsLen(n) - 1): fixRecordDef(n.sons[i])
   of nkIdentDefs: 
     for i in countup(0, sonsLen(n) - 3): exSymbol(n.sons[i])
-  of nkNilLit, nkEmpty: nil
+  of nkNilLit, nkEmpty: discard
   else: internalError(n.info, "fixRecordDef(): " & $n.kind)
   
 proc addPragmaToIdent(ident: var PNode, pragma: PNode) = 
@@ -1191,7 +1191,7 @@ proc parseRecordBody(p: var TParser, result, definition: PNode) =
     if definition != nil: addPragmaToIdent(definition.sons[0], parseCommand(p))
     else: internalError(result.info, "anonymous record is not supported")
   else: 
-    nil
+    discard
   opt(p, pxSemicolon)
   skipCom(p, result)
 
@@ -1399,7 +1399,7 @@ proc fixVarSection(p: var TParser, counter: PNode) =
 
 proc exSymbols(n: PNode) = 
   case n.kind
-  of nkEmpty..nkNilLit: nil
+  of nkEmpty..nkNilLit: discard
   of nkProcDef..nkIteratorDef: exSymbol(n.sons[namePos])
   of nkWhenStmt, nkStmtList: 
     for i in countup(0, sonsLen(n) - 1): exSymbols(n.sons[i])
@@ -1410,7 +1410,7 @@ proc exSymbols(n: PNode) =
       exSymbol(n.sons[i].sons[0])
       if n.sons[i].sons[2].kind == nkObjectTy: 
         fixRecordDef(n.sons[i].sons[2])
-  else: nil
+  else: discard
 
 proc parseBegin(p: var TParser, result: PNode) = 
   getTok(p)

compiler/pragmas.nim

@@ -23,7 +23,7 @@ const
     wMagic, wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader, 
     wCompilerproc, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge, 
     wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC,
-    wNoStackFrame, wError, wDiscardable, wNoInit, wDestructor, wCodegenDecl,
+    wAsmNoStackFrame, wError, wDiscardable, wNoInit, wDestructor, wCodegenDecl,
     wGensym, wInject, wRaises, wTags, wOperator, wDelegator}
   converterPragmas* = procPragmas
   methodPragmas* = procPragmas
@@ -47,12 +47,12 @@ const
     wInjectStmt}
   lambdaPragmas* = {FirstCallConv..LastCallConv, wImportc, wExportc, wNodecl, 
     wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader, 
-    wDeprecated, wExtern, wThread, wImportCpp, wImportObjC, wNoStackFrame,
+    wDeprecated, wExtern, wThread, wImportCpp, wImportObjC, wAsmNoStackFrame,
     wRaises, wTags}
   typePragmas* = {wImportc, wExportc, wDeprecated, wMagic, wAcyclic, wNodecl, 
     wPure, wHeader, wCompilerproc, wFinal, wSize, wExtern, wShallow,
     wImportCpp, wImportObjC, wError, wIncompleteStruct, wByCopy, wByRef,
-    wInheritable, wGensym, wInject, wRequiresInit}
+    wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion}
   fieldPragmas* = {wImportc, wExportc, wDeprecated, wExtern, 
     wImportCpp, wImportObjC, wError}
   varPragmas* = {wImportc, wExportc, wVolatile, wRegister, wThreadVar, wNodecl, 
@@ -97,8 +97,25 @@ proc makeExternImport(s: PSym, extname: string) =
   incl(s.flags, sfImportc)
   excl(s.flags, sfForward)
 
-proc makeExternExport(s: PSym, extname: string) = 
+const invalidIdentChars = AllChars - IdentChars
+
+proc validateExternCName(s: PSym, info: TLineInfo) =
+  ## Validates that the symbol name in s.loc.r is a valid C identifier.
+  ##
+  ## Valid identifiers are those alphanumeric including the underscore not
+  ## starting with a number. If the check fails, a generic error will be
+  ## displayed to the user.
+  let target = ropeToStr(s.loc.r)
+  if target.len < 1 or (not (target[0] in IdentStartChars)) or
+      (not target.allCharsInSet(IdentChars)):
+    localError(info, errGenerated, "invalid exported symbol")
+
+proc makeExternExport(s: PSym, extname: string, info: TLineInfo) =
   setExternName(s, extname)
+  case gCmd
+  of cmdCompileToC, cmdCompileToCpp, cmdCompileToOC:
+    validateExternCName(s, info)
+  else: discard
   incl(s.flags, sfExportc)
 
 proc processImportCompilerProc(s: PSym, extname: string) =
@@ -515,7 +532,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
       if k in validPragmas: 
         case k
         of wExportc: 
-          makeExternExport(sym, getOptionalStr(c, it, "$1"))
+          makeExternExport(sym, getOptionalStr(c, it, "$1"), it.info)
           incl(sym.flags, sfUsed) # avoid wrong hints
         of wImportc: makeExternImport(sym, getOptionalStr(c, it, "$1"))
         of wImportCompilerProc:
@@ -548,9 +565,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         of wNodecl: 
           noVal(it)
           incl(sym.loc.flags, lfNoDecl)
-        of wPure, wNoStackFrame:
+        of wPure, wAsmNoStackFrame:
           noVal(it)
-          if sym != nil: incl(sym.flags, sfPure)
+          if sym != nil:
+            if k == wPure and sym.kind in routineKinds: invalidPragma(it)
+            else: incl(sym.flags, sfPure)
         of wVolatile: 
           noVal(it)
           incl(sym.flags, sfVolatile)
@@ -601,7 +620,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
           processDynLib(c, it, sym)
         of wCompilerproc: 
           noVal(it)           # compilerproc may not get a string!
-          makeExternExport(sym, "$1")
+          makeExternExport(sym, "$1", it.info)
           incl(sym.flags, sfCompilerProc)
           incl(sym.flags, sfUsed) # suppress all those stupid warnings
           registerCompilerProc(sym)
@@ -699,6 +718,14 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
           noVal(it)
           if sym.typ == nil: invalidPragma(it)
           else: incl(sym.typ.flags, tfIncompleteStruct)
+        of wUnchecked:
+          noVal(it)
+          if sym.typ == nil: invalidPragma(it)
+          else: incl(sym.typ.flags, tfUncheckedArray)
+        of wUnion:
+          noVal(it)
+          if sym.typ == nil: invalidPragma(it)
+          else: incl(sym.typ.flags, tfUnion)
         of wRequiresInit:
           noVal(it)
           if sym.typ == nil: invalidPragma(it)

compiler/renderer.nim

@@ -558,15 +558,18 @@ proc longMode(n: PNode, start: int = 0, theEnd: int = - 1): bool =
         result = true
         break 
 
-proc gstmts(g: var TSrcGen, n: PNode, c: TContext) = 
+proc gstmts(g: var TSrcGen, n: PNode, c: TContext, doIndent=true) =
   if n.kind == nkEmpty: return
   if n.kind in {nkStmtList, nkStmtListExpr, nkStmtListType}:
-    indentNL(g)
+    if doIndent: indentNL(g)
     for i in countup(0, sonsLen(n) - 1):
       optNL(g)
+      if n.sons[i].kind in {nkStmtList, nkStmtListExpr, nkStmtListType}:
+        gstmts(g, n.sons[i], c, doIndent=false)
+      else:
         gsub(g, n.sons[i])
       gcoms(g)
-    dedent(g)
+    if doIndent: dedent(g)
   else:
     if rfLongMode in c.flags: indentNL(g)
     gsub(g, n)
@@ -1268,7 +1271,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
       put(g, tkBracketLe, "[")
       gcomma(g, n)
       put(g, tkBracketRi, "]")
-  of nkMetaNode:
+  of nkMetaNode_Obsolete:
     put(g, tkParLe, "(META|")
     gsub(g, n.sons[0])
     put(g, tkParRi, ")")

compiler/rodread.nim

@@ -890,7 +890,7 @@ proc loadStub*(s: PSym) =
   
   # deactivate the GC here because we do a deep recursion and generate no
   # garbage when restoring parts of the object graph anyway.
-  # Since we die with internal errors if this fails, so no try-finally is
+  # Since we die with internal errors if this fails, no try-finally is
   # necessary.
   GC_disable()
   rawLoadStub(s)

compiler/sem.nim

@@ -120,7 +120,8 @@ proc commonType*(x, y: PType): PType =
     if a.kind == tyObject and b.kind == tyObject:
       result = commonSuperclass(a, b)
       # this will trigger an error later:
-      if result.isNil: return x
+      if result.isNil or result == a: return x
+      if result == b: return y
       if k != tyNone:
         let r = result
         result = newType(k, r.owner)
@@ -219,7 +220,7 @@ proc tryConstExpr(c: PContext, n: PNode): PNode =
       return nil
 
     result = fixupTypeAfterEval(c, result, e)
-  except:
+  except ERecoverableError:
     return nil
 
 proc semConstExpr(c: PContext, n: PNode): PNode =
@@ -332,6 +333,8 @@ proc myOpen(module: PSym): PPassContext =
   c.semOperand = semOperand
   c.semConstBoolExpr = semConstBoolExpr
   c.semOverloadedCall = semOverloadedCall
+  c.semInferredLambda = semInferredLambda
+  c.semGenerateInstance = generateInstance
   c.semTypeNode = semTypeNode
   pushProcCon(c, module)
   pushOwner(c.module)

compiler/semcall.nim

@@ -82,7 +82,7 @@ proc notFoundError*(c: PContext, n: PNode, errors: seq[string]) =
     # fail fast:
     globalError(n.info, errTypeMismatch, "")
   var result = msgKindToString(errTypeMismatch)
-  add(result, describeArgs(c, n, 1 + ord(nfDelegate in n.flags)))
+  add(result, describeArgs(c, n, 1 + ord(nfDotField in n.flags)))
   add(result, ')')
   
   var candidates = ""
@@ -138,15 +138,33 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
 
   let overloadsState = result.state
   if overloadsState != csMatch:
-    if nfDelegate in n.flags:
+    if nfDotField in n.flags:
-      internalAssert f.kind == nkIdent
+      internalAssert f.kind == nkIdent and n.sonsLen >= 2
-      let calleeName = newStrNode(nkStrLit, f.ident.s)
+      let calleeName = newStrNode(nkStrLit, f.ident.s).withInfo(n.info)
-      calleeName.info = n.info
 
-      let callOp = newIdentNode(idDelegator, n.info)
+      # leave the op head symbol empty,
-      n.sons[0..0] = [callOp, calleeName]
+      # we are going to try multiple variants
-      orig.sons[0..0] = [callOp, calleeName]
+      n.sons[0..1] = [nil, n[1], calleeName]
+      orig.sons[0..1] = [nil, orig[1], calleeName]
       
+      template tryOp(x) =
+        let op = newIdentNode(getIdent(x), n.info)
+        n.sons[0] = op
+        orig.sons[0] = op
+        pickBest(op)
+
+      if nfExplicitCall in n.flags:
+        tryOp ".()"
+   
+      if result.state in {csEmpty, csNoMatch}:
+        tryOp "."
+
+    elif nfDotSetter in n.flags:
+      internalAssert f.kind == nkIdent and n.sonsLen == 3
+      let calleeName = newStrNode(nkStrLit, f.ident.s[0.. -2]).withInfo(n.info)
+      let callOp = newIdentNode(getIdent".=", n.info)
+      n.sons[0..1] = [callOp, n[1], calleeName]
+      orig.sons[0..1] = [callOp, orig[1], calleeName]
       pickBest(callOp)
     
     if overloadsState == csEmpty and result.state == csEmpty:

compiler/semdata.nim

@@ -81,6 +81,9 @@ type
     semOverloadedCall*: proc (c: PContext, n, nOrig: PNode,
                               filter: TSymKinds): PNode {.nimcall.}
     semTypeNode*: proc(c: PContext, n: PNode, prev: PType): PType {.nimcall.}
+    semInferredLambda*: proc(c: PContext, pt: TIdTable, n: PNode): PNode
+    semGenerateInstance*: proc (c: PContext, fn: PSym, pt: TIdTable,
+                                info: TLineInfo): PSym
     includedFiles*: TIntSet    # used to detect recursive include files
     userPragmas*: TStrTable
     evalContext*: PEvalContext
@@ -211,7 +214,6 @@ proc makeTypeDesc*(c: PContext, typ: PType): PType =
 
 proc makeTypeSymNode*(c: PContext, typ: PType, info: TLineInfo): PNode =
   let typedesc = makeTypeDesc(c, typ)
-  rawAddSon(typedesc, newTypeS(tyNone, c))
   let sym = newSym(skType, idAnon, getCurrOwner(), info).linkTo(typedesc)
   return newSymNode(sym, info)
 

compiler/semdestruct.nim

@@ -55,7 +55,9 @@ proc doDestructorStuff(c: PContext, s: PSym, n: PNode) =
             useSym(destructableT.destructor),
             n.sons[paramsPos][1][0]]))
 
-proc destroyField(c: PContext, field: PSym, holder: PNode): PNode =
+proc destroyFieldOrFields(c: PContext, field: PNode, holder: PNode): PNode
+
+proc destroySym(c: PContext, field: PSym, holder: PNode): PNode =
   let destructableT = instantiateDestructor(c, field.typ)
   if destructableT != nil:
     result = newNode(nkCall, field.info, @[
@@ -70,56 +72,49 @@ proc destroyCase(c: PContext, n: PNode, holder: PNode): PNode =
   for i in countup(1, n.len - 1):
     # of A, B:
     var caseBranch = newNode(n[i].kind, n[i].info, n[i].sons[0 .. -2])
-    let recList = n[i].lastSon
-    var destroyRecList = newNode(nkStmtList, n[i].info, @[])
-    template addField(f: expr): stmt =
-      let stmt = destroyField(c, f, holder)
-      if stmt != nil:
-        destroyRecList.addSon(stmt)
-        inc nonTrivialFields
     
-    case recList.kind
+    let stmt = destroyFieldOrFields(c, n[i].lastSon, holder)
-    of nkSym:
+    if stmt == nil:
-      addField(recList.sym)
+      caseBranch.addSon(newNode(nkStmtList, n[i].info, @[]))
-    of nkRecList:
-      for j in countup(0, recList.len - 1):
-        addField(recList[j].sym)
     else:
-      internalAssert false
+      caseBranch.addSon(stmt)
+      nonTrivialFields += stmt.len
     
-    caseBranch.addSon(destroyRecList)
     result.addSon(caseBranch)
+  
   # maybe no fields were destroyed?
   if nonTrivialFields == 0:
     result = nil
 
+proc destroyFieldOrFields(c: PContext, field: PNode, holder: PNode): PNode =
+  template maybeAddLine(e: expr): stmt =
+    let stmt = e
+    if stmt != nil:
+      if result == nil: result = newNode(nkStmtList)
+      result.addSon(stmt)
+
+  case field.kind
+  of nkRecCase:
+    maybeAddLine destroyCase(c, field, holder)
+  of nkSym:
+    maybeAddLine destroySym(c, field.sym, holder)
+  of nkRecList:
+    for son in field:
+      maybeAddLine destroyFieldOrFields(c, son, holder)
+  else:
+    internalAssert false
+
 proc generateDestructor(c: PContext, t: PType): PNode =
   ## generate a destructor for a user-defined object or tuple type
   ## returns nil if the destructor turns out to be trivial
   
-  template addLine(e: expr): stmt =
-    if result == nil: result = newNode(nkStmtList)
-    result.addSon(e)
-
   # XXX: This may be true for some C-imported types such as
   # Tposix_spawnattr
   if t.n == nil or t.n.sons == nil: return
   internalAssert t.n.kind == nkRecList
   let destructedObj = newIdentNode(destructorParam, unknownLineInfo())
   # call the destructods of all fields
-  for s in countup(0, t.n.sons.len - 1):
+  result = destroyFieldOrFields(c, t.n, destructedObj)
-    case t.n.sons[s].kind
-    of nkRecCase:
-      let stmt = destroyCase(c, t.n.sons[s], destructedObj)
-      if stmt != nil: addLine(stmt)
-    of nkSym:
-      let stmt = destroyField(c, t.n.sons[s].sym, destructedObj)
-      if stmt != nil: addLine(stmt)
-    else:
-      # XXX just skip it for now so that the compiler doesn't crash, but
-      # please zahary fix it! arbitrary nesting of nkRecList/nkRecCase is
-      # possible. Any thread example seems to trigger this. 
-      discard
   # base classes' destructors will be automatically called by
   # semProcAux for both auto-generated and user-defined destructors
 

compiler/semexprs.nim

@@ -204,7 +204,14 @@ proc semConv(c: PContext, n: PNode): PNode =
   if not isSymChoice(op):
     let status = checkConvertible(c, result.typ, op.typ)
     case status
-    of convOK: discard
+    of convOK:
+      # handle SomeProcType(SomeGenericProc)
+      # XXX: This needs fixing. checkConvertible uses typeRel internally, but
+      # doesn't bother to perform the work done in paramTypeMatchAux/fitNode
+      # so we are redoing the typeRel work here. Why does semConv exist as a
+      # separate proc from fitNode?
+      if op.kind == nkSym and op.sym.isGenericRoutine:
+        result.sons[1] = fitNode(c, result.typ, result.sons[1])
     of convNotNeedeed:
       message(n.info, hintConvFromXtoItselfNotNeeded, result.typ.typeToString)
     of convNotLegal:
@@ -340,21 +347,20 @@ proc semIs(c: PContext, n: PNode): PNode =
   result = n
   n.typ = getSysType(tyBool)
  
-  n.sons[1] = semExprWithType(c, n[1], {efDetermineType})
+  n.sons[1] = semExprWithType(c, n[1], {efDetermineType, efWantIterator})
-  
   if n[2].kind notin {nkStrLit..nkTripleStrLit}:
     let t2 = semTypeNode(c, n[2], nil)
     n.sons[2] = newNodeIT(nkType, n[2].info, t2)
 
-  if n[1].typ.kind != tyTypeDesc:
+  let lhsType = n[1].typ
-    n.sons[1] = makeTypeSymNode(c, n[1].typ, n[1].info)
+  if lhsType.kind != tyTypeDesc:
-  elif n[1].typ.sonsLen == 0:
+    n.sons[1] = makeTypeSymNode(c, lhsType, n[1].info)
+  elif lhsType.base.kind == tyNone:
     # this is a typedesc variable, leave for evals
     return
 
-  let t1 = n[1].typ.sons[0]
   # BUGFIX: don't evaluate this too early: ``T is void``
-  if not containsGenericType(t1): result = isOpImpl(c, n)
+  if not n[1].typ.base.containsGenericType: result = isOpImpl(c, n)
 
 proc semOpAux(c: PContext, n: PNode) =
   const flags = {efDetermineType}
@@ -635,9 +641,11 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
       result = evalStaticExpr(c.module, call, c.p.owner)
       if result.isNil: 
         localError(n.info, errCannotInterpretNodeX, renderTree(call))
+      else: result = fixupTypeAfterEval(c, result, n)
     else:
       result = evalConstExpr(c.module, call)
       if result.isNil: result = n
+      else: result = fixupTypeAfterEval(c, result, n)
     #if result != n:
     #  echo "SUCCESS evaluated at compile time: ", call.renderTree
 
@@ -647,6 +655,8 @@ proc semStaticExpr(c: PContext, n: PNode): PNode =
   if result.isNil:
     localError(n.info, errCannotInterpretNodeX, renderTree(n))
     result = emptyNode
+  else:
+    result = fixupTypeAfterEval(c, result, a)
 
 proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
                                      flags: TExprFlags): PNode =
@@ -687,6 +697,7 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
       # it is a static call!
       result = n.sons[0]
       result.kind = nkCall
+      result.flags.incl nfExplicitCall
       for i in countup(1, sonsLen(n) - 1): addSon(result, n.sons[i])
       return semExpr(c, result, flags)
   else:
@@ -910,8 +921,8 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
   var ty = n.sons[0].typ
   var f: PSym = nil
   result = nil
-  if isTypeExpr(n.sons[0]) or ty.kind == tyTypeDesc and ty.len == 1:
+  if isTypeExpr(n.sons[0]) or ty.kind == tyTypeDesc and ty.base.kind != tyNone:
-    if ty.kind == tyTypeDesc: ty = ty.sons[0]
+    if ty.kind == tyTypeDesc: ty = ty.base
     case ty.kind
     of tyEnum:
       # look up if the identifier belongs to the enum:
@@ -992,7 +1003,7 @@ proc dotTransformation(c: PContext, n: PNode): PNode =
   else:
     var i = considerAcc(n.sons[1])
     result = newNodeI(nkDotCall, n.info)
-    result.flags.incl nfDelegate
+    result.flags.incl nfDotField
     addSon(result, newIdentNode(i, n[1].info))
     addSon(result, copyTree(n[0]))
   
@@ -1075,12 +1086,13 @@ proc semArrayAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
 
 proc propertyWriteAccess(c: PContext, n, nOrig, a: PNode): PNode =
   var id = considerAcc(a[1])
-  let setterId = newIdentNode(getIdent(id.s & '='), n.info)
+  var setterId = newIdentNode(getIdent(id.s & '='), n.info)
   # a[0] is already checked for semantics, that does ``builtinFieldAccess``
   # this is ugly. XXX Semantic checking should use the ``nfSem`` flag for
   # nodes?
   let aOrig = nOrig[0]
   result = newNode(nkCall, n.info, sons = @[setterId, a[0], semExpr(c, n[1])])
+  result.flags.incl nfDotSetter
   let orig = newNode(nkCall, n.info, sons = @[setterId, aOrig[0], nOrig[1]])
   result = semOverloadedCallAnalyseEffects(c, result, orig, {})
   
@@ -1770,22 +1782,6 @@ proc semBlock(c: PContext, n: PNode): PNode =
   closeScope(c)
   dec(c.p.nestedBlockCounter)
 
-proc buildCall(n: PNode): PNode =
-  if n.kind == nkDotExpr and n.len == 2:
-    # x.y --> y(x)
-    result = newNodeI(nkCall, n.info, 2)
-    result.sons[0] = n.sons[1]
-    result.sons[1] = n.sons[0]
-  elif n.kind in nkCallKinds and n.sons[0].kind == nkDotExpr:
-    # x.y(a) -> y(x, a)
-    let a = n.sons[0]
-    result = newNodeI(nkCall, n.info, n.len+1)
-    result.sons[0] = a.sons[1]
-    result.sons[1] = a.sons[0]
-    for i in 1 .. <n.len: result.sons[i+1] = n.sons[i]
-  else:
-    result = n
-
 proc doBlockIsStmtList(n: PNode): bool =
   result = n.kind == nkDo and
            n[paramsPos].sonsLen == 1 and
@@ -1819,6 +1815,10 @@ proc semExport(c: PContext, n: PNode): PNode =
   c.module.ast.add x
   result = n
 
+proc setGenericParams(c: PContext, n: PNode) =
+  for i in 1 .. <n.len:
+    n[i].typ = semTypeNode(c, n[i], nil)
+
 proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = 
   result = n
   if gCmd == cmdIdeTools: suggestExpr(c, n)
@@ -1833,8 +1833,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
       result = symChoice(c, n, s, scClosed)
       if result.kind == nkSym:
         markIndirect(c, result.sym)
-        if isGenericRoutine(result.sym):
+        # if isGenericRoutine(result.sym):
-          localError(n.info, errInstantiateXExplicitely, s.name.s)
+        #   localError(n.info, errInstantiateXExplicitely, s.name.s)
   of nkSym:
     # because of the changed symbol binding, this does not mean that we
     # don't have to check the symbol for semantics here again!
@@ -1890,7 +1890,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   of nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand, nkCallStrLit: 
     # check if it is an expression macro:
     checkMinSonsLen(n, 1)
-    let mode = if nfDelegate in n.flags: {} else: {checkUndeclared}
+    let mode = if nfDotField in n.flags: {} else: {checkUndeclared}
     var s = qualifiedLookUp(c, n.sons[0], mode)
     if s != nil: 
       if gCmd == cmdPretty and n.sons[0].kind == nkDotExpr:
@@ -1924,10 +1924,12 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
       else:
         #liMessage(n.info, warnUser, renderTree(n));
         result = semIndirectOp(c, n, flags)
-    elif isSymChoice(n.sons[0]) or n[0].kind == nkBracketExpr and 
+    elif n[0].kind == nkBracketExpr and isSymChoice(n[0][0]):
-        isSymChoice(n[0][0]):
+      # indirectOp can deal with explicit instantiations; the fixes
+      # the 'newSeq[T](x)' bug
+      setGenericParams(c, n.sons[0])
       result = semDirectOp(c, n, flags)
-    elif nfDelegate in n.flags:
+    elif isSymChoice(n.sons[0]) or nfDotField in n.flags:
       result = semDirectOp(c, n, flags)
     else:
       result = semIndirectOp(c, n, flags)

compiler/semfold.nim

@@ -537,7 +537,7 @@ proc foldArrayAccess(m: PSym, n: PNode): PNode =
       if result.kind == nkExprColonExpr: result = result.sons[1]
     else:
       localError(n.info, errIndexOutOfBounds)
-  of nkBracket, nkMetaNode: 
+  of nkBracket: 
     if (idx >= 0) and (idx < sonsLen(x)): result = x.sons[int(idx)]
     else: localError(n.info, errIndexOutOfBounds)
   of nkStrLit..nkTripleStrLit: 

compiler/semgnrc.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2014 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -215,8 +215,7 @@ proc semGenericStmt(c: PContext, n: PNode,
       if (a.kind != nkIdentDefs) and (a.kind != nkVarTuple): illFormedAst(a)
       checkMinSonsLen(a, 3)
       var L = sonsLen(a)
-      a.sons[L-2] = semGenericStmt(c, a.sons[L-2], flags+{withinTypeDesc}, 
+      a.sons[L-2] = semGenericStmt(c, a.sons[L-2], flags+{withinTypeDesc}, ctx)
-                                   ctx)
       a.sons[L-1] = semGenericStmt(c, a.sons[L-1], flags, ctx)
       for j in countup(0, L-3):
         addPrelimDecl(c, newSymS(skUnknown, getIdentNode(a.sons[j]), c))
@@ -226,8 +225,7 @@ proc semGenericStmt(c: PContext, n: PNode,
       if (a.kind != nkIdentDefs): illFormedAst(a)
       checkMinSonsLen(a, 3)
       var L = sonsLen(a)
-      a.sons[L-2] = semGenericStmt(c, a.sons[L-2], flags+{withinTypeDesc}, 
+      a.sons[L-2] = semGenericStmt(c, a.sons[L-2], flags+{withinTypeDesc}, ctx) 
-                                   ctx) 
       # do not perform symbol lookup for default expressions 
       for j in countup(0, L-3): 
         addPrelimDecl(c, newSymS(skUnknown, getIdentNode(a.sons[j]), c))
@@ -281,8 +279,7 @@ proc semGenericStmt(c: PContext, n: PNode,
       if (a.kind != nkIdentDefs): illFormedAst(a)
       checkMinSonsLen(a, 3)
       var L = sonsLen(a)
-      a.sons[L-2] = semGenericStmt(c, a.sons[L-2], flags+{withinTypeDesc}, 
+      a.sons[L-2] = semGenericStmt(c, a.sons[L-2], flags+{withinTypeDesc}, ctx)
-                                   ctx)
       a.sons[L-1] = semGenericStmt(c, a.sons[L-1], flags, ctx)
       for j in countup(0, L-3): 
         addPrelimDecl(c, newSymS(skUnknown, getIdentNode(a.sons[j]), c))

compiler/sempass2.nim

@@ -84,10 +84,10 @@ proc initVar(a: PEffects, n: PNode) =
 proc initVarViaNew(a: PEffects, n: PNode) =
   if n.kind != nkSym: return
   let s = n.sym
-  if {tfNeedsInit, tfNotNil} * s.typ.flags == {tfNotNil}:
+  if {tfNeedsInit, tfNotNil} * s.typ.flags <= {tfNotNil}:
     # 'x' is not nil, but that doesn't mean it's not nil children
     # are initialized:
-    initVarViaNew(a, n)
+    initVar(a, n)
 
 proc useVar(a: PEffects, n: PNode) =
   let s = n.sym
@@ -466,8 +466,7 @@ proc track(tracked: PEffects, n: PNode) =
         mergeEffects(tracked, effectList.sons[exceptionEffects], n)
         mergeTags(tracked, effectList.sons[tagEffects], n)
     for i in 1 .. <len(n): trackOperand(tracked, n.sons[i], paramType(op, i))
-    if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, 
+    if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, mNewSeq}:
-                                           mNewSeq, mShallowCopy}:
       # may not look like an assignment, but it is:
       initVarViaNew(tracked, n.sons[1])
     for i in 0 .. <safeLen(n):
@@ -477,7 +476,6 @@ proc track(tracked: PEffects, n: PNode) =
     if warnProveField in gNotes: checkFieldAccess(tracked.guards, n)
   of nkTryStmt: trackTryStmt(tracked, n)
   of nkPragma: trackPragmaStmt(tracked, n)
-  of nkMacroDef, nkTemplateDef: discard
   of nkAsgn, nkFastAsgn:
     track(tracked, n.sons[1])
     initVar(tracked, n.sons[0])
@@ -527,7 +525,9 @@ proc track(tracked: PEffects, n: PNode) =
       if sfDiscriminant in x.sons[0].sym.flags:
         addDiscriminantFact(tracked.guards, x)
     setLen(tracked.guards, oldFacts)
-  of nkTypeSection: discard
+  of nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef,
+      nkMacroDef, nkTemplateDef:
+    discard
   else:
     for i in 0 .. <safeLen(n): track(tracked, n.sons[i])
 
@@ -581,22 +581,26 @@ proc setEffectsForProcType*(t: PType, n: PNode) =
     if not isNil(tagsSpec):
       effects.sons[tagEffects] = tagsSpec
 
+proc initEffects(effects: PNode; s: PSym; t: var TEffects) =
+  newSeq(effects.sons, effectListLen)
+  effects.sons[exceptionEffects] = newNodeI(nkArgList, s.info)
+  effects.sons[tagEffects] = newNodeI(nkArgList, s.info)
+  
+  t.exc = effects.sons[exceptionEffects]
+  t.tags = effects.sons[tagEffects]
+  t.owner = s
+  t.init = @[]
+  t.guards = @[]
+  
 proc trackProc*(s: PSym, body: PNode) =
   var effects = s.typ.n.sons[0]
   internalAssert effects.kind == nkEffectList
   # effects already computed?
   if sfForward in s.flags: return
   if effects.len == effectListLen: return
-  newSeq(effects.sons, effectListLen)
-  effects.sons[exceptionEffects] = newNodeI(nkArgList, body.info)
-  effects.sons[tagEffects] = newNodeI(nkArgList, body.info)
   
   var t: TEffects
-  t.exc = effects.sons[exceptionEffects]
+  initEffects(effects, s, t)
-  t.tags = effects.sons[tagEffects]
-  t.owner = s
-  t.init = @[]
-  t.guards = @[]
   track(t, body)
   
   if not isEmptyType(s.typ.sons[0]) and tfNeedsInit in s.typ.sons[0].flags and
@@ -619,3 +623,12 @@ proc trackProc*(s: PSym, body: PNode) =
     # after the check, use the formal spec:
     effects.sons[tagEffects] = tagsSpec
     
+proc trackTopLevelStmt*(module: PSym; n: PNode) =
+  if n.kind in {nkPragma, nkMacroDef, nkTemplateDef, nkProcDef,
+                nkTypeSection, nkConverterDef, nkMethodDef, nkIteratorDef}:
+    return
+  var effects = newNode(nkEffectList, n.info)
+  var t: TEffects
+  initEffects(effects, module, t)
+
+  track(t, n)

compiler/semstmts.nim

@@ -126,7 +126,7 @@ proc implicitlyDiscardable(n: PNode): bool =
 proc fixNilType(n: PNode) =
   if isAtom(n):
     if n.kind != nkNilLit and n.typ != nil:
-      localError(n.info, errDiscardValue)
+      localError(n.info, errDiscardValueX, n.typ.typeToString)
   elif n.kind in {nkStmtList, nkStmtListExpr}:
     n.kind = nkStmtList
     for it in n: fixNilType(it)
@@ -143,7 +143,8 @@ proc discardCheck(c: PContext, result: PNode) =
       while n.kind in skipForDiscardable:
         n = n.lastSon
         n.typ = nil
-    elif c.inTypeClass > 0 and result.typ.kind == tyBool:
+    elif c.inTypeClass > 0:
+      if result.typ.kind == tyBool:
         let verdict = semConstExpr(c, result)
         if verdict.intVal == 0:
           localError(result.info, "type class predicate failed")
@@ -154,7 +155,7 @@ proc discardCheck(c: PContext, result: PNode) =
       else:
         var n = result
         while n.kind in skipForDiscardable: n = n.lastSon
-        localError(n.info, errDiscardValue)
+        localError(n.info, errDiscardValueX, result.typ.typeToString)
 
 proc semIf(c: PContext, n: PNode): PNode = 
   result = n
@@ -331,6 +332,7 @@ proc checkNilable(v: PSym) =
 proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = 
   var b: PNode
   result = copyNode(n)
+  var hasCompileTime = false
   for i in countup(0, sonsLen(n)-1): 
     var a = n.sons[i]
     if gCmd == cmdIdeTools: suggestStmt(c, a)
@@ -349,7 +351,12 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
       # BUGFIX: ``fitNode`` is needed here!
       # check type compability between def.typ and typ:
       if typ != nil: def = fitNode(c, typ, def)
-      else: typ = skipIntLit(def.typ)
+      else:
+        typ = skipIntLit(def.typ)
+        if typ.kind in {tySequence, tyArray, tySet} and
+           typ.lastSon.kind == tyEmpty:
+          localError(def.info, errCannotInferTypeOfTheLiteral,
+                     ($typ.kind).substr(2).toLower)
     else:
       def = ast.emptyNode
       if symkind == skLet: localError(a.info, errLetNeedsInit)
@@ -405,6 +412,8 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
         v.typ = tup.sons[j]
         b.sons[j] = newSymNode(v)
       checkNilable(v)
+      if sfCompileTime in v.flags: hasCompileTime = true
+  if hasCompileTime: vm.setupCompileTimeVar(c.module, result)
 
 proc semConst(c: PContext, n: PNode): PNode = 
   result = copyNode(n)
@@ -764,6 +773,29 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) =
       s.ast = a
       popOwner()
 
+proc checkForMetaFields(n: PNode) =
+  template checkMeta(t) =
+    if t != nil and t.isMetaType and tfGenericTypeParam notin t.flags:
+      localError(n.info, errTIsNotAConcreteType, t.typeToString)
+  
+  case n.kind
+  of nkRecList, nkRecCase:
+    for s in n: checkForMetaFields(s)
+  of nkOfBranch, nkElse:
+    checkForMetaFields(n.lastSon)
+  of nkSym:
+    let t = n.sym.typ
+    case t.kind
+    of tySequence, tySet, tyArray, tyOpenArray, tyVar, tyPtr, tyRef,
+       tyProc, tyGenericInvokation, tyGenericInst:
+      let start = ord(t.kind in {tyGenericInvokation, tyGenericInst})
+      for i in start .. <t.sons.len:
+        checkMeta(t.sons[i])
+    else:
+      checkMeta(t)
+  else:
+    internalAssert false
+
 proc typeSectionFinalPass(c: PContext, n: PNode) = 
   for i in countup(0, sonsLen(n) - 1): 
     var a = n.sons[i]
@@ -780,6 +812,8 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
           assignType(s.typ, t)
           s.typ.id = t.id     # same id
       checkConstructedType(s.info, s.typ)
+      if s.typ.kind in {tyObject, tyTuple}:
+        checkForMetaFields(s.typ.n)
     let aa = a.sons[2]
     if aa.kind in {nkRefTy, nkPtrTy} and aa.len == 1 and
        aa.sons[0].kind == nkObjectTy:
@@ -883,12 +917,19 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
     s = n[namePos].sym
   pushOwner(s)
   openScope(c)
+  var gp: PNode
   if n.sons[genericParamsPos].kind != nkEmpty: 
-    illFormedAst(n)           # process parameters:
+    n.sons[genericParamsPos] = semGenericParamList(c, n.sons[genericParamsPos])
+    gp = n.sons[genericParamsPos]
+  else:
+    gp = newNodeI(nkGenericParams, n.info)
+
   if n.sons[paramsPos].kind != nkEmpty:
-    var gp = newNodeI(nkGenericParams, n.info)
     semParamList(c, n.sons[paramsPos], gp, s)
-    paramsTypeCheck(c, s.typ)
+    # paramsTypeCheck(c, s.typ)
+    if sonsLen(gp) > 0 and n.sons[genericParamsPos].kind == nkEmpty:
+      # we have a list of implicit type parameters:
+      n.sons[genericParamsPos] = gp
   else:
     s.typ = newTypeS(tyProc, c)
     rawAddSon(s.typ, nil)
@@ -900,6 +941,7 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
       localError(n.sons[bodyPos].info, errImplOfXNotAllowed, s.name.s)
     #if efDetermineType notin flags:
     # XXX not good enough; see tnamedparamanonproc.nim
+    if n.sons[genericParamsPos].kind == nkEmpty:
       pushProcCon(c, s)
       addResult(c, s.typ.sons[0], n.info, skProc)
       let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
@@ -913,6 +955,34 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
   popOwner()
   result.typ = s.typ
 
+proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode =
+  var n = n
+  
+  n = replaceTypesInBody(c, pt, n)
+  result = n
+  
+  n.sons[genericParamsPos] = emptyNode
+  n.sons[paramsPos] = n.typ.n
+  
+  openScope(c)
+  var s = n.sons[namePos].sym
+  addParams(c, n.typ.n, skProc)
+  pushProcCon(c, s)
+  addResult(c, n.typ.sons[0], n.info, skProc)
+  let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
+  n.sons[bodyPos] = transformBody(c.module, semBody, n.sons[namePos].sym)
+  addResultNode(c, n)
+  popProcCon(c)
+  closeScope(c)
+  
+  s.ast = result
+
+  # alternative variant (not quite working):
+  # var prc = arg[0].sym
+  # let inferred = c.semGenerateInstance(c, prc, m.bindings, arg.info)
+  # result = inferred.ast
+  # result.kind = arg.kind
+
 proc activate(c: PContext, n: PNode) =
   # XXX: This proc is part of my plan for getting rid of
   # forward declarations. stay tuned.
@@ -1250,7 +1320,7 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
       if n.sons[i].typ == enforceVoidContext or usesResult(n.sons[i]):
         voidContext = true
         n.typ = enforceVoidContext
-      if i == last and efWantValue in flags:
+      if i == last and (length == 1 or efWantValue in flags):
         n.typ = n.sons[i].typ
         if not isEmptyType(n.typ): n.kind = nkStmtListExpr
       elif i != last or voidContext or c.inTypeClass > 0:
@@ -1263,8 +1333,8 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
         let (outer, inner) = insertDestructors(c, n.sons[i])
         if outer != nil:
           n.sons[i] = outer
-          for j in countup(i+1, length-1):
+          var rest = newNode(nkStmtList, n.info, n.sons[i+1 .. length-1])
-            inner.addSon(semStmt(c, n.sons[j]))
+          inner.addSon(semStmtList(c, rest, flags))
           n.sons.setLen(i+1)
           return
       of LastBlockStmts: 

compiler/semthreads.nim

@@ -358,7 +358,7 @@ proc analyse(c: PProcCtx, n: PNode): TThreadOwner =
   of nkConstSection: result = analyseConstSection(c, n)
   of nkTypeSection, nkCommentStmt: result = toVoid
   of nkIfStmt, nkWhileStmt, nkTryStmt, nkCaseStmt, nkStmtList, nkBlockStmt, 
-     nkElifBranch, nkElse, nkExceptBranch, nkOfBranch:
+     nkElifBranch, nkElse, nkExceptBranch, nkOfBranch, nkFinally:
     for i in 0 .. <n.len: discard analyse(c, n[i])
     result = toVoid
   of nkBreakStmt, nkContinueStmt: result = toVoid

compiler/semtypes.nim

@@ -653,6 +653,10 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
   var paramTypId = if not anon and paramType.sym != nil: paramType.sym.name
                    else: nil
 
+  template maybeLift(typ: PType): expr =
+    let lifted = liftingWalk(typ)
+    (if lifted != nil: lifted else: typ)
+
   template addImplicitGeneric(e: expr): expr =
     addImplicitGenericImpl(e, paramTypId)
 
@@ -663,15 +667,19 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
   of tyStatic:
     # proc(a: expr{string}, b: expr{nkLambda})
     # overload on compile time values and AST trees
-    result = addImplicitGeneric(c.newTypeWithSons(tyStatic, paramType.sons))
+    let base = paramType.base.maybeLift
+    if base.isMetaType and procKind == skMacro:
+      localError(info, errMacroBodyDependsOnGenericTypes, paramName)
+    result = addImplicitGeneric(c.newTypeWithSons(tyStatic, @[base]))
     result.flags.incl tfHasStatic
   
   of tyTypeDesc:
     if tfUnresolved notin paramType.flags:
       # naked typedescs are not bindOnce types
-      if paramType.sonsLen == 0 and paramTypId != nil and
+      if paramType.base.kind == tyNone and paramTypId != nil and
          paramTypId.id == typedescId.id: paramTypId = nil
-      result = addImplicitGeneric(c.newTypeWithSons(tyTypeDesc, paramType.sons))
+      result = addImplicitGeneric(
+        c.newTypeWithSons(tyTypeDesc, @[paramType.base]))
   
   of tyDistinct:
     if paramType.sonsLen == 1:
@@ -705,7 +713,6 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
       # result.rawAddSon(copyType(paramType.sons[i], getCurrOwner(), true))
     result = instGenericContainer(c, paramType.sym.info, result,
                                   allowMetaTypes = true)
-    result.lastSon.shouldHaveMeta
     result = newTypeWithSons(c, tyCompositeTypeClass, @[paramType, result])
     result = addImplicitGeneric(result)
   
@@ -1011,7 +1018,8 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     of mOrdinal: result = semOrdinal(c, n, prev)
     of mSeq: result = semContainer(c, n, tySequence, "seq", prev)
     of mVarargs: result = semVarargs(c, n, prev)
-    of mExpr, mTypeDesc:
+    of mTypeDesc: result = makeTypeDesc(c, semTypeNode(c, n[1], nil))
+    of mExpr:
       result = semTypeNode(c, n.sons[0], nil)
       if result != nil:
         result = copyType(result, getCurrOwner(), false)
@@ -1030,9 +1038,8 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
       if s.kind != skError: localError(n.info, errTypeExpected)
       result = newOrPrevType(tyError, prev, c)
     elif s.kind == skParam and s.typ.kind == tyTypeDesc:
-      assert s.typ.len > 0
+      internalAssert s.typ.base.kind != tyNone and prev == nil
-      internalAssert prev == nil
+      result = s.typ.base
-      result = s.typ.sons[0]
     elif prev == nil:
       result = s.typ
     else: 

compiler/semtypinst.nim

@@ -29,6 +29,7 @@ proc checkConstructedType*(info: TLineInfo, typ: PType) =
     localError(info, errVarVarTypeNotAllowed)
   elif computeSize(t) == szIllegalRecursion:
     localError(info, errIllegalRecursionInTypeX, typeToString(t))
+  
   when false:
     if t.kind == tyObject and t.sons[0] != nil:
       if t.sons[0].kind != tyObject or tfFinal in t.sons[0].flags: 
@@ -153,6 +154,9 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode): PNode =
     discard
   of nkSym:
     result.sym = replaceTypeVarsS(cl, n.sym)
+    if result.sym.typ.kind == tyEmpty:
+      # don't add the 'void' field
+      result = newNode(nkRecList, n.info)
   of nkRecWhen:
     var branch: PNode = nil              # the branch to take
     for i in countup(0, sonsLen(n) - 1):
@@ -216,7 +220,7 @@ proc handleGenericInvokation(cl: var TReplTypeVars, t: PType): PType =
   # is difficult to handle: 
   var body = t.sons[0]
   if body.kind != tyGenericBody: internalError(cl.info, "no generic body")
-  var header: PType = nil
+  var header: PType = t
   # search for some instantiation here:
   if cl.allowMetaTypes:
     result = PType(idTableGet(cl.localCache, t))
@@ -228,11 +232,13 @@ proc handleGenericInvokation(cl: var TReplTypeVars, t: PType): PType =
     if x.kind == tyGenericParam:
       x = lookupTypeVar(cl, x)
       if x != nil:
-        if header == nil: header = instCopyType(cl, t)
+        if header == t: header = instCopyType(cl, t)
         header.sons[i] = x
         propagateToOwner(header, x)
+    else:
+      propagateToOwner(header, x)
   
-  if header != nil:
+  if header != t:
     # search again after first pass:
     result = searchInstTypes(header)
     if result != nil: return
@@ -240,6 +246,7 @@ proc handleGenericInvokation(cl: var TReplTypeVars, t: PType): PType =
     header = instCopyType(cl, t)
   
   result = newType(tyGenericInst, t.sons[0].owner)
+  result.flags = header.flags
   # be careful not to propagate unnecessary flags here (don't use rawAddSon)
   result.sons = @[header.sons[0]]
   # ugh need another pass for deeply recursive generic types (e.g. PActor)
@@ -406,14 +413,22 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
       
       else: discard
 
+proc initTypeVars(p: PContext, pt: TIdTable, info: TLineInfo): TReplTypeVars =
+  initIdTable(result.symMap)
+  copyIdTable(result.typeMap, pt)
+  initIdTable(result.localCache)
+  result.info = info
+  result.c = p
+
+proc replaceTypesInBody*(p: PContext, pt: TIdTable, n: PNode): PNode =
+  var cl = initTypeVars(p, pt, n.info)
+  pushInfoContext(n.info)
+  result = replaceTypeVarsN(cl, n)
+  popInfoContext()
+  
 proc generateTypeInstance*(p: PContext, pt: TIdTable, info: TLineInfo,
                            t: PType): PType =
-  var cl: TReplTypeVars
+  var cl = initTypeVars(p, pt, info)
-  initIdTable(cl.symMap)
-  copyIdTable(cl.typeMap, pt)
-  initIdTable(cl.localCache)
-  cl.info = info
-  cl.c = p
   pushInfoContext(info)
   result = replaceTypeVarsT(cl, t)
   popInfoContext()

compiler/sigmatch.nim

@@ -51,6 +51,8 @@ type
     isSubtype,
     isSubrange,              # subrange of the wanted type; no type conversion
                              # but apart from that counts as ``isSubtype``
+    isInferred,              # generic proc was matched against a concrete type
+    isInferredConvertible,   # same as above, but requiring proc CC conversion
     isGeneric,
     isFromIntLit,            # conversion *from* int literal; proven safe
     isEqual
@@ -105,6 +107,7 @@ proc initCandidate*(ctx: PContext, c: var TCandidate, callee: PSym,
       var bound = binding[i].typ
       if bound != nil and formalTypeParam.kind != tyTypeDesc:
         bound = bound.skipTypes({tyTypeDesc})
+      assert bound != nil
       put(c.bindings, formalTypeParam, bound)
 
 proc newCandidate*(ctx: PContext, callee: PSym,
@@ -337,34 +340,60 @@ proc recordRel(c: var TCandidate, f, a: PType): TTypeRelation =
 proc allowsNil(f: PType): TTypeRelation {.inline.} =
   result = if tfNotNil notin f.flags: isSubtype else: isNone
 
-proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
 proc inconsistentVarTypes(f, a: PType): bool {.inline.} =
   result = f.kind != a.kind and (f.kind == tyVar or a.kind == tyVar)
 
+proc procParamTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
+  var f = f
+
+  if a.isMetaType:
+    if f.isMetaType:
+      # We are matching a generic proc (as proc param)
+      # to another generic type appearing in the proc
+      # signature. There is a change that the target
+      # type is already fully-determined, so we are 
+      # going to try resolve it
+      f = generateTypeInstance(c.c, c.bindings, c.call.info, f)
+      if f == nil or f.isMetaType:
+        # no luck resolving the type, so the inference fails
+        return isNone
+    let reverseRel = typeRel(c, a, f)
+    if reverseRel == isGeneric:
+      result = isInferred
+      inc c.genericMatches
+  else:
+    result = typeRel(c, f, a)
+
+  if result <= isSubtype or inconsistentVarTypes(f, a):
+    result = isNone
+ 
+  if result == isEqual:
+    inc c.exactMatches
+    
+proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
   case a.kind
   of tyProc:
     if sonsLen(f) != sonsLen(a): return
-    # Note: We have to do unification for the parameters before the
-    # return type!
     result = isEqual      # start with maximum; also correct for no
                           # params at all
-    for i in countup(1, sonsLen(f)-1):
+    
-      var m = typeRel(c, f.sons[i], a.sons[i])
+    template checkParam(f, a) =
-      if m <= isSubtype or inconsistentVarTypes(f.sons[i], a.sons[i]):
+      result = minRel(result, procParamTypeRel(c, f, a))
-        return isNone
+      if result == isNone: return
-      else: result = minRel(m, result)
+
+    # Note: We have to do unification for the parameters before the
+    # return type!
+    for i in 1 .. <f.sonsLen:
+      checkParam(f.sons[i], a.sons[i])
+    
     if f.sons[0] != nil:
       if a.sons[0] != nil:
-        var m = typeRel(c, f.sons[0], a.sons[0])
+        checkParam(f.sons[0], a.sons[0])
-        # Subtype is sufficient for return types!
-        if m < isSubtype or inconsistentVarTypes(f.sons[0], a.sons[0]):
-          return isNone
-        elif m == isSubtype: result = isConvertible
-        else: result = minRel(m, result)
       else:
         return isNone
     elif a.sons[0] != nil:
       return isNone
+
     if tfNoSideEffect in f.flags and tfNoSideEffect notin a.flags:
       return isNone
     elif tfThread in f.flags and a.flags * {tfThread, tfNoSideEffect} == {}:
@@ -375,7 +404,8 @@ proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
     elif f.callConv != a.callConv:
       # valid to pass a 'nimcall' thingie to 'closure':
       if f.callConv == ccClosure and a.callConv == ccDefault:
-        result = isConvertible
+        result = if result != isInferred: isConvertible
+                 else: isInferredConvertible
       else:
         return isNone
     when useEffectSystem:
@@ -401,18 +431,8 @@ proc typeRangeRel(f, a: PType): TTypeRelation {.noinline.} =
 
 proc matchUserTypeClass*(c: PContext, m: var TCandidate,
                          ff, a: PType): TTypeRelation =
-  #if f.n == nil:
-  #  let r = typeRel(m, f, a)
-  #  return if r == isGeneric: arg else: nil
-
   var body = ff.skipTypes({tyUserTypeClassInst})
 
-  # var prev = PType(idTableGet(m.bindings, f))
-  # if prev != nil:
-  #   if sameType(prev, a): return arg
-  #   else: return nil
-
-  # pushInfoContext(arg.info)
   openScope(c)
   inc c.inTypeClass
 
@@ -461,7 +481,6 @@ proc matchUserTypeClass*(c: PContext, m: var TCandidate,
       else: discard
     
   return isGeneric
-  # put(m.bindings, f, a)
 
 proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
   # typeRel can be used to establish various relationships between types:
@@ -851,7 +870,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
           else:
             internalAssert a.sons != nil and a.sons.len > 0
             c.typedescMatched = true
-            result = typeRel(c, f.sons[0], a.sons[0])
+            result = typeRel(c, f.base, a.base)
         else:
           result = isNone
       else:
@@ -883,20 +902,25 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
   of tyTypeDesc:
     var prev = PType(idTableGet(c.bindings, f))
     if prev == nil:
-      if a.kind == tyTypeDesc:
+      # proc foo(T: typedesc, x: T)
-        if f.sons[0].kind == tyNone:
+      # when `f` is an unresolved typedesc, `a` could be any
+      # type, so we should not perform this check earlier
+      if a.kind != tyTypeDesc: return isNone
+    
+      if f.base.kind == tyNone:
         result = isGeneric
       else:
-          result = typeRel(c, f.sons[0], a.sons[0])
+        result = typeRel(c, f.base, a.base)
+      
       if result != isNone:
         put(c.bindings, f, a)
     else:
-        result = isNone
+      if tfUnresolved in f.flags:
+        result = typeRel(c, prev.base, a)
+      elif a.kind == tyTypeDesc:
+        result = typeRel(c, prev.base, a.base)
       else:
-      internalAssert prev.sonsLen == 1
+        result = isNone
-      let toMatch = if tfUnresolved in f.flags: a
-                    else: a.sons[0]
-      result = typeRel(c, prev.sons[0], toMatch)
   
   of tyStmt:
     result = isGeneric
@@ -1000,11 +1024,33 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType,
       argType = arg.typ
  
   var
-    a = if c.inTypeClass > 0: argType.skipTypes({tyTypeDesc})
+    a = if c.inTypeClass > 0: argType.skipTypes({tyTypeDesc, tyFieldAccessor})
         else: argType
  
     r = typeRel(m, f, a)
 
+  if r != isNone and m.calleeSym != nil and
+     m.calleeSym.kind in {skMacro, skTemplate}:
+    # XXX: duplicating this is ugly, maybe we should move this
+    # directly into typeRel using return-like templates
+    case r
+    of isConvertible, isIntConv: inc(m.convMatches)
+    of isSubtype, isSubrange: inc(m.subtypeMatches)
+    of isGeneric, isInferred: inc(m.genericMatches)
+    of isInferredConvertible: inc(m.genericMatches); inc(m.convMatches)
+    of isFromIntLit: inc(m.intConvMatches, 256)
+    of isEqual: inc(m.exactMatches)
+    of isNone: discard
+
+    if f.kind == tyStmt and argOrig.kind == nkDo:
+      return argOrig[bodyPos]
+    elif f.kind == tyTypeDesc:
+      return arg
+    elif f.kind == tyStatic:
+      return arg.typ.n
+    else:
+      return argOrig
+  
   case r
   of isConvertible:
     inc(m.convMatches)
@@ -1021,18 +1067,18 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType,
     inc(m.subtypeMatches)
     #result = copyTree(arg)
     result = implicitConv(nkHiddenStdConv, f, copyTree(arg), m, c)
+  of isInferred, isInferredConvertible:
+    inc(m.genericMatches)
+    if arg.kind in {nkProcDef, nkIteratorDef} + nkLambdaKinds:
+      result = c.semInferredLambda(c, m.bindings, arg)
+    else:
+      let inferred = c.semGenerateInstance(c, arg.sym, m.bindings, arg.info)
+      result = newSymNode(inferred, arg.info)
+    if r == isInferredConvertible:
+      inc(m.convMatches)
+      result = implicitConv(nkHiddenStdConv, f, result, m, c)
   of isGeneric:
     inc(m.genericMatches)
-    if m.calleeSym != nil and m.calleeSym.kind in {skMacro, skTemplate}:
-      if f.kind == tyStmt and argOrig.kind == nkDo:
-        result = argOrig[bodyPos]
-      elif f.kind == tyTypeDesc:
-        result = arg
-      elif f.kind == tyStatic:
-        result = arg.typ.n
-      else:
-        result = argOrig
-    else:
     result = copyTree(arg)
     result.typ = getInstantiatedType(c, arg, m, f)
     # BUG: f may not be the right key!
@@ -1081,6 +1127,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
     # incorrect to simply use the first fitting match. However, to implement
     # this correctly is inefficient. We have to copy `m` here to be able to
     # roll back the side effects of the unification algorithm.
+
     let c = m.c
     var x, y, z: TCandidate
     initCandidate(c, x, m.callee)

compiler/transf.nim

@@ -754,6 +754,7 @@ proc transformStmt*(module: PSym, n: PNode): PNode =
     result = processTransf(c, n, module)
     result = liftLambdasForTopLevel(module, result)
     incl(result.flags, nfTransf)
+    when useEffectSystem: trackTopLevelStmt(module, result)
 
 proc transformExpr*(module: PSym, n: PNode): PNode =
   if nfTransf in n.flags:

compiler/types.nim

@@ -431,8 +431,8 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
       add(result, typeToString(t.sons[i]))
     add(result, ']')
   of tyTypeDesc:
-    if t.len == 0: result = "typedesc"
+    if t.base.kind == tyNone: result = "typedesc"
-    else: result = "typedesc[" & typeToString(t.sons[0]) & "]"
+    else: result = "typedesc[" & typeToString(t.base) & "]"
   of tyStatic:
     internalAssert t.len > 0
     result = "static[" & typeToString(t.sons[0]) & "]"
@@ -452,7 +452,8 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
       of tyProc: "proc"
       of tyObject: "object"
       of tyTuple: "tuple"
-      else: (internalAssert(false); "")
+      of tyOpenArray: "openarray"
+      else: typeToStr[t.base.kind]
   of tyUserTypeClassInst:
     let body = t.base
     result = body.sym.name.s & "["
@@ -463,7 +464,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
   of tyAnd:
     result = typeToString(t.sons[0]) & " and " & typeToString(t.sons[1])
   of tyOr:
-    result = typeToString(t.sons[0]) & " and " & typeToString(t.sons[1])
+    result = typeToString(t.sons[0]) & " or " & typeToString(t.sons[1])
   of tyNot:
     result = "not " & typeToString(t.sons[0])
   of tyExpr:
@@ -1042,7 +1043,8 @@ proc typeAllowedAux(marker: var TIntSet, typ: PType, kind: TSymKind,
   of tyEmpty:
     result = taField in flags
   of tyTypeClasses:
-    result = true
+    result = tfGenericTypeParam in t.flags or
+             taField notin flags
   of tyGenericBody, tyGenericParam, tyGenericInvokation,
      tyNone, tyForward, tyFromExpr, tyFieldAccessor:
     result = false
@@ -1231,8 +1233,7 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt =
   of tyGenericInst, tyDistinct, tyGenericBody, tyMutable, tyConst, tyIter:
     result = computeSizeAux(lastSon(typ), a)
   of tyTypeDesc:
-    result = if typ.len == 1: computeSizeAux(typ.sons[0], a)
+    result = computeSizeAux(typ.base, a)
-             else: szUnknownSize
   of tyForward: return szIllegalRecursion
   else:
     #internalError("computeSizeAux()")
@@ -1258,7 +1259,7 @@ proc containsGenericTypeIter(t: PType, closure: PObject): bool =
     return true
 
   if t.kind == tyTypeDesc:
-    if t.sons[0].kind == tyNone: return true
+    if t.base.kind == tyNone: return true
     if containsGenericTypeIter(t.base, closure): return true
     return false
   

compiler/vmdef.nim

@@ -8,7 +8,7 @@
 #
 
 ## This module contains the type definitions for the new evaluation engine.
-## An instruction is 1-2 int32s in memory, it is a register based VM.
+## An instruction is 1-3 int32s in memory, it is a register based VM.
 
 import ast, passes, msgs, intsets
 
@@ -32,17 +32,17 @@ type
     opcAsgnFloat,
     opcAsgnRef,
     opcAsgnComplex,
+    opcRegToNode,
+    opcNodeToReg,
 
     opcLdArr,  # a = b[c]
-    opcLdArrRef,
     opcWrArr,  # a[b] = c
-    opcWrArrRef,
     opcLdObj,  # a = b.c
-    opcLdObjRef,
     opcWrObj,  # a.b = c
-    opcWrObjRef,
+    opcAddrReg,
-    opcAddr,
+    opcAddrNode,
-    opcDeref,
+    opcLdDeref,
+    opcWrDeref,
     opcWrStrIdx,
     opcLdStrIdx, # a = b[c]
     
@@ -117,15 +117,13 @@ type
     opcNew,
     opcNewSeq,
     opcLdNull,    # dest = nullvalue(types[Bx])
+    opcLdNullReg,
     opcLdConst,   # dest = constants[Bx]
     opcAsgnConst, # dest = copy(constants[Bx])
     opcLdGlobal,  # dest = globals[Bx]
+
     opcLdImmInt,  # dest = immediate value
     opcNBindSym,
-    opcWrGlobal,
-    opcWrGlobalRef,
-    opcGlobalAlias, # load an alias to a global into a register
-    opcGlobalOnce,  # used to introduce an assignment to a global once
     opcSetType,   # dest.typ = types[Bx]
     opcTypeTrait
 
@@ -159,14 +157,13 @@ type
     slotTempInt,      # some temporary int
     slotTempFloat,    # some temporary float
     slotTempStr,      # some temporary string
-    slotTempComplex   # some complex temporary (n.sons field is used)
+    slotTempComplex   # some complex temporary (s.node field is used)
 
   PProc* = ref object
     blocks*: seq[TBlock]    # blocks; temp data structure
+    sym*: PSym
     slots*: array[TRegister, tuple[inUse: bool, kind: TSlotKind]]
     maxSlots*: int
-    globals*: array[TRegister, int] # hack: to support passing globals byref
-                                    # we map a slot persistently to a global
     
   PCtx* = ref TCtx
   TCtx* = object of passes.TPassContext # code gen context
@@ -183,6 +180,7 @@ type
     callsite*: PNode
     mode*: TEvalMode
     features*: TSandboxFlags
+    traceActive*: bool
 
   TPosition* = distinct int
 

compiler/vmgen.nim

@@ -139,25 +139,12 @@ proc getTemp(c: PCtx; typ: PType): TRegister =
       if not c.slots[i].inUse:
         c.slots[i] = (inUse: true, kind: k)
         return TRegister(i)
+  if c.maxSlots >= high(TRegister):
+    internalError("cannot generate code; too many registers required")
   result = TRegister(c.maxSlots)
   c.slots[c.maxSlots] = (inUse: true, kind: k)
   inc c.maxSlots
 
-proc getGlobalSlot(c: PCtx; n: PNode; s: PSym): TRegister =
-  let p = c.prc
-  for i in 0 .. p.maxSlots-1:
-    if p.globals[i] == s.id: return TRegister(i)
-
-  result = TRegister(p.maxSlots)
-  p.slots[p.maxSlots] = (inUse: true, kind: slotFixedVar)
-  p.globals[p.maxSlots] = s.id
-  inc p.maxSlots
-  # XXX this is still not correct! We need to load the global in a proc init
-  # section, otherwise control flow could lead to a usage before it's been
-  # loaded.
-  c.gABx(n, opcGlobalAlias, result, s.position)
-  # XXX add some internal asserts here
-
 proc freeTemp(c: PCtx; r: TRegister) =
   let c = c.prc
   if c.slots[r].kind >= slotSomeTemp: c.slots[r].inUse = false
@@ -216,10 +203,12 @@ proc genx(c: PCtx; n: PNode; flags: TGenFlags = {}): TRegister =
   internalAssert tmp >= 0
   result = TRegister(tmp)
 
-proc clearDest(n: PNode; dest: var TDest) {.inline.} =
+proc clearDest(c: PCtx; n: PNode; dest: var TDest) {.inline.} =
   # stmt is different from 'void' in meta programming contexts.
   # So we only set dest to -1 if 'void':
-  if n.typ.isNil or n.typ.kind == tyEmpty: dest = -1
+  if dest >= 0 and (n.typ.isNil or n.typ.kind == tyEmpty):
+    c.freeTemp(dest)
+    dest = -1
 
 proc isNotOpr(n: PNode): bool =
   n.kind in nkCallKinds and n.sons[0].kind == nkSym and
@@ -259,7 +248,7 @@ proc genWhile(c: PCtx; n: PNode) =
 proc genBlock(c: PCtx; n: PNode; dest: var TDest) =
   withBlock(n.sons[0].sym):
     c.gen(n.sons[1], dest)
-  clearDest(n, dest)
+  c.clearDest(n, dest)
 
 proc genBreak(c: PCtx; n: PNode) =
   let L1 = c.xjmp(n, opcJmp)
@@ -297,14 +286,16 @@ proc genIf(c: PCtx, n: PNode; dest: var TDest) =
         else:
           c.gen(it.sons[0], tmp)
           elsePos = c.xjmp(it.sons[0], opcFJmp, tmp) # if false
+      c.clearDest(n, dest)
       c.gen(it.sons[1], dest) # then part
       if i < sonsLen(n)-1:
         endings.add(c.xjmp(it.sons[1], opcJmp, 0))
       c.patch(elsePos)
     else:
+      c.clearDest(n, dest)
       c.gen(it.sons[0], dest)
   for endPos in endings: c.patch(endPos)
-  clearDest(n, dest)
+  c.clearDest(n, dest)
 
 proc genAndOr(c: PCtx; n: PNode; opc: TOpcode; dest: var TDest) =
   #   asgn dest, a
@@ -317,9 +308,12 @@ proc genAndOr(c: PCtx; n: PNode; opc: TOpcode; dest: var TDest) =
   c.gen(n.sons[2], dest)
   c.patch(L1)
 
+proc nilLiteral(n: PNode): PNode =
+  result = n
+
 proc rawGenLiteral(c: PCtx; n: PNode): int =
   result = c.constants.len
-  c.constants.add n
+  c.constants.add n.nilLiteral
   internalAssert result < 0x7fff
 
 proc sameConstant*(a, b: PNode): bool =
@@ -385,8 +379,8 @@ proc genCase(c: PCtx; n: PNode; dest: var TDest) =
         if i < sonsLen(n)-1:
           endings.add(c.xjmp(it.lastSon, opcJmp, 0))
         c.patch(elsePos)
+      c.clearDest(n, dest)
   for endPos in endings: c.patch(endPos)
-  clearDest(n, dest)
 
 proc genType(c: PCtx; typ: PType): int =
   for i, t in c.types:
@@ -400,6 +394,7 @@ proc genTry(c: PCtx; n: PNode; dest: var TDest) =
   var endings: seq[TPosition] = @[]
   let elsePos = c.xjmp(n, opcTry, 0)
   c.gen(n.sons[0], dest)
+  c.clearDest(n, dest)
   c.patch(elsePos)
   for i in 1 .. <n.len:
     let it = n.sons[i]
@@ -415,6 +410,7 @@ proc genTry(c: PCtx; n: PNode; dest: var TDest) =
         # general except section:
         c.gABx(it, opcExcept, 0, 0)
       c.gen(it.lastSon, dest)
+      c.clearDest(n, dest)
       if i < sonsLen(n)-1:
         endings.add(c.xjmp(it, opcJmp, 0))
       c.patch(endExcept)
@@ -425,8 +421,8 @@ proc genTry(c: PCtx; n: PNode; dest: var TDest) =
   c.gABx(fin, opcFinally, 0, 0)
   if fin.kind == nkFinally:
     c.gen(fin.sons[0], dest)
+    c.clearDest(n, dest)
   c.gABx(fin, opcFinallyEnd, 0, 0)
-  clearDest(n, dest)
 
 proc genRaise(c: PCtx; n: PNode) =
   let dest = genx(c, n.sons[0])
@@ -455,21 +451,37 @@ proc genCall(c: PCtx; n: PNode; dest: var TDest) =
     c.gABC(n, opcIndCallAsgn, dest, x, n.len)
   c.freeTempRange(x, n.len)
 
+template isGlobal(s: PSym): bool = sfGlobal in s.flags and s.kind != skForVar
+
 proc needsAsgnPatch(n: PNode): bool = 
-  n.kind in {nkBracketExpr, nkDotExpr, nkCheckedFieldExpr}
+  n.kind in {nkBracketExpr, nkDotExpr, nkCheckedFieldExpr,
+             nkDerefExpr, nkHiddenDeref} or (n.kind == nkSym and n.sym.isGlobal)
 
 proc genAsgnPatch(c: PCtx; le: PNode, value: TRegister) =
   case le.kind
   of nkBracketExpr:
-    let dest = c.genx(le.sons[0])
+    let dest = c.genx(le.sons[0], {gfAddrOf})
     let idx = c.genx(le.sons[1])
-    c.gABC(le, opcWrArrRef, dest, idx, value)
+    c.gABC(le, opcWrArr, dest, idx, value)
+    c.freeTemp(dest)
+    c.freeTemp(idx)
   of nkDotExpr, nkCheckedFieldExpr:
     # XXX field checks here
     let left = if le.kind == nkDotExpr: le else: le.sons[0]
-    let dest = c.genx(left.sons[0])
+    let dest = c.genx(left.sons[0], {gfAddrOf})
     let idx = c.genx(left.sons[1])
-    c.gABC(left, opcWrObjRef, dest, idx, value)
+    c.gABC(left, opcWrObj, dest, idx, value)
+    c.freeTemp(dest)
+    c.freeTemp(idx)
+  of nkDerefExpr, nkHiddenDeref:
+    let dest = c.genx(le.sons[0], {gfAddrOf})
+    c.gABC(le, opcWrDeref, dest, value)
+    c.freeTemp(dest)
+  of nkSym:
+    if le.sym.isGlobal:
+      let dest = c.genx(le, {gfAddrOf})
+      c.gABC(le, opcWrDeref, dest, value)
+      c.freeTemp(dest)
   else:
     discard
 
@@ -579,10 +591,10 @@ proc genAddSubInt(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
 
 proc genConv(c: PCtx; n, arg: PNode; dest: var TDest; opc=opcConv) =  
   let tmp = c.genx(arg)
-  c.gABx(n, opcSetType, tmp, genType(c, arg.typ))
   if dest < 0: dest = c.getTemp(n.typ)
   c.gABC(n, opc, dest, tmp)
   c.gABx(n, opc, 0, genType(c, n.typ))
+  c.gABx(n, opc, 0, genType(c, arg.typ))
   c.freeTemp(tmp)
 
 proc genCard(c: PCtx; n: PNode; dest: var TDest) =
@@ -608,6 +620,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
     c.genAddSubInt(n, dest, opcAddInt)
   of mInc, mDec:
     unused(n, dest)
+    # XXX generates inefficient code for globals
     var d = c.genx(n.sons[1]).TDest
     c.genAddSubInt(n, d, if m == mInc: opcAddInt else: opcSubInt)
     c.genAsgnPatch(n.sons[1], d)
@@ -621,6 +634,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
     c.genNewSeq(n)
   of mNewString:
     genUnaryABC(c, n, dest, opcNewStr)
+    # XXX buggy
   of mNewStringOfCap:
     # we ignore the 'cap' argument and translate it as 'newString(0)'.
     # eval n.sons[1] for possible side effects:
@@ -629,6 +643,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
     if dest < 0: dest = c.getTemp(n.typ)
     c.gABC(n, opcNewStr, dest, tmp)
     c.freeTemp(tmp)
+    # XXX buggy
   of mLengthOpenArray, mLengthArray, mLengthSeq:
     genUnaryABI(c, n, dest, opcLenSeq)
   of mLengthStr:
@@ -860,7 +875,6 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
   of mNGenSym: genBinaryABC(c, n, dest, opcGenSym)
   of mMinI, mMaxI, mMinI64, mMaxI64, mAbsF64, mMinF64, mMaxF64, mAbsI, mAbsI64:
     c.genCall(n, dest)
-    clearDest(n, dest)
   of mExpandToAst:
     if n.len != 2:
       globalError(n.info, errGenerated, "expandToAst requires 1 argument")
@@ -891,6 +905,10 @@ const
     tyFloat, tyFloat32, tyFloat64, tyFloat128,
     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64}
 
+proc fitsRegister*(t: PType): bool =
+  t.skipTypes(abstractInst-{tyTypeDesc}).kind in {
+    tyRange, tyEnum, tyBool, tyInt..tyUInt64}
+
 proc requiresCopy(n: PNode): bool =
   if n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind in atomicTypes:
     result = false
@@ -902,22 +920,33 @@ proc requiresCopy(n: PNode): bool =
 proc unneededIndirection(n: PNode): bool =
   n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind == tyRef
 
-proc skipDeref(n: PNode): PNode =
-  if n.kind in {nkDerefExpr, nkHiddenDeref} and unneededIndirection(n.sons[0]):
-    result = n.sons[0]
-  else:
-    result = n
-
 proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
                   flags: TGenFlags) = 
   # a nop for certain types
-  let flags = if opc == opcAddr: flags+{gfAddrOf} else: flags
+  let isAddr = opc in {opcAddrNode, opcAddrReg}
-  if unneededIndirection(n.sons[0]):
+  let newflags = if isAddr: flags+{gfAddrOf} else: flags
-    gen(c, n.sons[0], dest, flags)
+  # consider:
+  # proc foo(f: var ref int) =
+  #   f = new(int)
+  # proc blah() =
+  #   var x: ref int
+  #   foo x
+  #
+  # The type of 'f' is 'var ref int' and of 'x' is 'ref int'. Hence for
+  # nkAddr we must not use 'unneededIndirection', but for deref we use it.
+  if not isAddr and unneededIndirection(n.sons[0]):
+    gen(c, n.sons[0], dest, newflags)
   else:
-    let tmp = c.genx(n.sons[0], flags)
+    let tmp = c.genx(n.sons[0], newflags)
     if dest < 0: dest = c.getTemp(n.typ)
+    if not isAddr:
       gABC(c, n, opc, dest, tmp)
+      if gfAddrOf notin flags and fitsRegister(n.typ):
+        c.gABC(n, opcNodeToReg, dest, dest)
+    elif c.prc.slots[tmp].kind >= slotTempUnknown:
+      gABC(c, n, opcAddrNode, dest, tmp)
+    else:
+      gABC(c, n, opcAddrReg, dest, tmp)
     c.freeTemp(tmp)
 
 proc whichAsgnOpc(n: PNode): TOpcode =
@@ -935,8 +964,7 @@ proc whichAsgnOpc(n: PNode): TOpcode =
 
 proc isRef(t: PType): bool = t.skipTypes(abstractRange-{tyTypeDesc}).kind == tyRef
 
-proc whichAsgnOpc(n: PNode; opc: TOpcode): TOpcode =
+proc whichAsgnOpc(n: PNode; opc: TOpcode): TOpcode = opc
-  if isRef(n.typ): succ(opc) else: opc
 
 proc genAsgn(c: PCtx; dest: TDest; ri: PNode; requiresCopy: bool) =
   let tmp = c.genx(ri)
@@ -944,8 +972,6 @@ proc genAsgn(c: PCtx; dest: TDest; ri: PNode; requiresCopy: bool) =
   gABC(c, ri, whichAsgnOpc(ri), dest, tmp)
   c.freeTemp(tmp)
 
-template isGlobal(s: PSym): bool = sfGlobal in s.flags and s.kind != skForVar
-
 proc setSlot(c: PCtx; v: PSym) =
   # XXX generate type initialization here?
   if v.position == 0:
@@ -954,40 +980,67 @@ proc setSlot(c: PCtx; v: PSym) =
         kind: if v.kind == skLet: slotFixedLet else: slotFixedVar)
     inc c.prc.maxSlots
 
+proc cannotEval(n: PNode) {.noinline.} =
+  globalError(n.info, errGenerated, "cannot evaluate at compile time: " &
+    n.renderTree)
+
+proc isOwnedBy(a, b: PSym): bool =
+  var a = a.owner
+  while a != nil and a.kind != skModule:
+    if a == b: return true
+    a = a.owner
+
+proc checkCanEval(c: PCtx; n: PNode) =
+  # we need to ensure that we don't evaluate 'x' here:
+  # proc foo() = var x ...
+  let s = n.sym
+  if s.position == 0:
+    if s.kind in {skVar, skTemp, skLet, skParam, skResult} and 
+        not s.isOwnedBy(c.prc.sym) and s.owner != c.module:
+      cannotEval(n)
+
 proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) =
   case le.kind
   of nkBracketExpr:
-    let dest = c.genx(le.sons[0])
+    let dest = c.genx(le.sons[0], {gfAddrOf})
     let idx = c.genx(le.sons[1])
     let tmp = c.genx(ri)
     if le.sons[0].typ.skipTypes(abstractVarRange-{tyTypeDesc}).kind in {
         tyString, tyCString}:
       c.gABC(le, opcWrStrIdx, dest, idx, tmp)
     else:
-      c.gABC(le, whichAsgnOpc(le, opcWrArr), dest, idx, tmp)
+      c.gABC(le, opcWrArr, dest, idx, tmp)
     c.freeTemp(tmp)
   of nkDotExpr, nkCheckedFieldExpr:
     # XXX field checks here
     let left = if le.kind == nkDotExpr: le else: le.sons[0]
-    let dest = c.genx(left.sons[0])
+    let dest = c.genx(left.sons[0], {gfAddrOf})
     let idx = c.genx(left.sons[1])
     let tmp = c.genx(ri)
-    c.gABC(left, whichAsgnOpc(left, opcWrObj), dest, idx, tmp)
+    c.gABC(left, opcWrObj, dest, idx, tmp)
+    c.freeTemp(tmp)
+  of nkDerefExpr, nkHiddenDeref:
+    let dest = c.genx(le.sons[0], {gfAddrOf})
+    let tmp = c.genx(ri)
+    c.gABC(le, opcWrDeref, dest, tmp)
     c.freeTemp(tmp)
   of nkSym:
     let s = le.sym
+    checkCanEval(c, le)
     if s.isGlobal:
       withTemp(tmp, le.typ):
-        gen(c, ri, tmp)
+        c.gen(le, tmp, {gfAddrOf})
-        c.gABx(le, whichAsgnOpc(le, opcWrGlobal), tmp, s.position)
+        let val = c.genx(ri)
+        c.gABC(le, opcWrDeref, tmp, val)
+        c.freeTemp(val)
     else:
-      if s.kind == skForVar and c.mode == emRepl: c.setSlot s
+      if s.kind == skForVar: c.setSlot s
       internalAssert s.position > 0 or (s.position == 0 and
                                         s.kind in {skParam,skResult})
       var dest: TRegister = s.position + ord(s.kind == skParam)
       gen(c, ri, dest)
   else:
-    let dest = c.genx(le)
+    let dest = c.genx(le, {gfAddrOf})
     genAsgn(c, dest, ri, requiresCopy)
 
 proc genLit(c: PCtx; n: PNode; dest: var TDest) =
@@ -1013,22 +1066,22 @@ proc importcSym(c: PCtx; info: TLineInfo; s: PSym) =
     localError(info, errGenerated,
                "cannot 'importc' variable at compile time")
 
-proc cannotEval(n: PNode) {.noinline.} =
+proc getNullValue*(typ: PType, info: TLineInfo): PNode
-  globalError(n.info, errGenerated, "cannot evaluate at compile time: " &
-    n.renderTree)
 
 proc genGlobalInit(c: PCtx; n: PNode; s: PSym) =
-  c.globals.add(emptyNode.copyNode)
+  c.globals.add(getNullValue(s.typ, n.info))
   s.position = c.globals.len
   # This is rather hard to support, due to the laziness of the VM code
   # generator. See tests/compile/tmacro2 for why this is necesary:
   #   var decls{.compileTime.}: seq[PNimrodNode] = @[]
-  c.gABx(n, opcGlobalOnce, 0, s.position)
+  let dest = c.getTemp(s.typ)
+  c.gABx(n, opcLdGlobal, dest, s.position)
   let tmp = c.genx(s.ast)
-  c.gABx(n, whichAsgnOpc(n, opcWrGlobal), tmp, s.position)
+  c.gABC(n, opcWrDeref, dest, tmp)
+  c.freeTemp(dest)
   c.freeTemp(tmp)
 
-proc genRdVar(c: PCtx; n: PNode; dest: var TDest) =
+proc genRdVar(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
   let s = n.sym
   if s.isGlobal:
     if sfCompileTime in s.flags or c.mode == emRepl:
@@ -1038,9 +1091,12 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest) =
     if s.position == 0:
       if sfImportc in s.flags: c.importcSym(n.info, s)
       else: genGlobalInit(c, n, s)
-    if dest < 0:
+    if dest < 0: dest = c.getTemp(n.typ)
-      dest = c.getGlobalSlot(n, s)
+    if gfAddrOf notin flags and fitsRegister(s.typ):
-      #c.gABx(n, opcAliasGlobal, dest, s.position)
+      var cc = c.getTemp(n.typ)
+      c.gABx(n, opcLdGlobal, cc, s.position)
+      c.gABC(n, opcNodeToReg, dest, cc)
+      c.freeTemp(cc)
     else:
       c.gABx(n, opcLdGlobal, dest, s.position)
   else:
@@ -1061,7 +1117,13 @@ proc genAccess(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
   let a = c.genx(n.sons[0], flags)
   let b = c.genx(n.sons[1], {})
   if dest < 0: dest = c.getTemp(n.typ)
-  c.gABC(n, (if gfAddrOf in flags: succ(opc) else: opc), dest, a, b)
+  if gfAddrOf notin flags and fitsRegister(n.typ):
+    var cc = c.getTemp(n.typ)
+    c.gABC(n, opc, cc, a, b)
+    c.gABC(n, opcNodeToReg, dest, cc)
+    c.freeTemp(cc)
+  else:
+    c.gABC(n, opc, dest, a, b)
   c.freeTemp(a)
   c.freeTemp(b)
 
@@ -1079,7 +1141,6 @@ proc genArrAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
   else:
     genAccess(c, n, dest, opcLdArr, flags)
 
-proc getNullValue*(typ: PType, info: TLineInfo): PNode
 proc getNullValueAux(obj: PNode, result: PNode) = 
   case obj.kind
   of nkRecList:
@@ -1132,6 +1193,9 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode =
     result = newNodeIT(nkCurly, info, t)
   else: internalError("getNullValue: " & $t.kind)
 
+proc ldNullOpcode(t: PType): TOpcode =
+  if fitsRegister(t): opcLdNullReg else: opcLdNull
+
 proc genVarSection(c: PCtx; n: PNode) =
   for a in n:
     if a.kind == nkCommentStmt: continue
@@ -1142,12 +1206,14 @@ proc genVarSection(c: PCtx; n: PNode) =
         setSlot(c, a[i].sym)
         # v = t[i]
         var v: TDest = -1
-        genRdVar(c, a[i], v)
+        checkCanEval(c, a[i])
-        c.gABC(n, opcLdObj, v, tmp, i)
+        genRdVar(c, a[i], v, {gfAddrOf})
+        c.gABC(n, opcWrObj, v, tmp, i)
         # XXX globals?
       c.freeTemp(tmp)
     elif a.sons[0].kind == nkSym:
       let s = a.sons[0].sym
+      checkCanEval(c, a.sons[0])
       if s.isGlobal:
         if s.position == 0:
           if sfImportc in s.flags: c.importcSym(a.info, s)
@@ -1155,27 +1221,28 @@ proc genVarSection(c: PCtx; n: PNode) =
             let sa = if s.ast.isNil: getNullValue(s.typ, a.info) else: s.ast
             c.globals.add(sa)
             s.position = c.globals.len
-            # "Once support" is unnecessary here
         if a.sons[2].kind == nkEmpty:
           when false:
             withTemp(tmp, s.typ):
               c.gABx(a, opcLdNull, tmp, c.genType(s.typ))
               c.gABx(a, whichAsgnOpc(a.sons[0], opcWrGlobal), tmp, s.position)
         else:
-          let tmp = genx(c, a.sons[2])
+          let tmp = c.genx(a.sons[0], {gfAddrOf})
-          c.gABx(a, whichAsgnOpc(a.sons[0], opcWrGlobal), tmp, s.position)
+          let val = c.genx(a.sons[2])
+          c.gABC(a, opcWrDeref, tmp, val)
+          c.freeTemp(val)
           c.freeTemp(tmp)
       else:
         setSlot(c, s)
         if a.sons[2].kind == nkEmpty:
-          c.gABx(a, opcLdNull, s.position, c.genType(s.typ))
+          c.gABx(a, ldNullOpcode(s.typ), s.position, c.genType(s.typ))
         else:
           gen(c, a.sons[2], s.position.TRegister)
     else:
       # assign to a.sons[0]; happens for closures
       if a.sons[2].kind == nkEmpty:
         let tmp = genx(c, a.sons[0])
-        c.gABx(a, opcLdNull, tmp, c.genType(a.sons[0].typ))
+        c.gABx(a, ldNullOpcode(a[0].typ), tmp, c.genType(a.sons[0].typ))
         c.freeTemp(tmp)
       else:
         genAsgn(c, a.sons[0], a.sons[2], true)
@@ -1183,10 +1250,19 @@ proc genVarSection(c: PCtx; n: PNode) =
 proc genArrayConstr(c: PCtx, n: PNode, dest: var TDest) =
   if dest < 0: dest = c.getTemp(n.typ)
   c.gABx(n, opcLdNull, dest, c.genType(n.typ))
-  if n.len > 0:
+
   let intType = getSysType(tyInt)
+  let seqType = n.typ.skipTypes(abstractVar-{tyTypeDesc})
+  if seqType.kind == tySequence:
+    var tmp = c.getTemp(intType)
+    c.gABx(n, opcLdImmInt, tmp, n.len)
+    c.gABx(n, opcNewSeq, dest, c.genType(seqType))
+    c.gABx(n, opcNewSeq, tmp, 0)
+    c.freeTemp(tmp)
+  
+  if n.len > 0:
     var tmp = getTemp(c, intType)
-    c.gABx(n, opcLdNull, tmp, c.genType(intType))
+    c.gABx(n, opcLdNullReg, tmp, c.genType(intType))
     for x in n:
       let a = c.genx(x)
       c.gABC(n, whichAsgnOpc(x, opcWrArr), dest, tmp, a)
@@ -1250,9 +1326,10 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
   case n.kind
   of nkSym:
     let s = n.sym
+    checkCanEval(c, n)
     case s.kind
     of skVar, skForVar, skTemp, skLet, skParam, skResult:
-      genRdVar(c, n, dest)
+      genRdVar(c, n, dest, flags)
     of skProc, skConverter, skMacro, skTemplate, skMethod, skIterator:
       # 'skTemplate' is only allowed for 'getAst' support:
       if sfImportc in s.flags: c.importcSym(n.info, s)
@@ -1281,7 +1358,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
       genMagic(c, n, dest)
     else:
       genCall(c, n, dest)
-      clearDest(n, dest)
+      clearDest(c, n, dest)
   of nkCharLit..nkInt64Lit:
     if isInt16Lit(n):
       if dest < 0: dest = c.getTemp(n.typ)
@@ -1298,8 +1375,8 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
   of nkDotExpr: genObjAccess(c, n, dest, flags)
   of nkCheckedFieldExpr: genCheckedObjAccess(c, n, dest, flags)
   of nkBracketExpr: genArrAccess(c, n, dest, flags)
-  of nkDerefExpr, nkHiddenDeref: genAddrDeref(c, n, dest, opcDeref, flags)
+  of nkDerefExpr, nkHiddenDeref: genAddrDeref(c, n, dest, opcLdDeref, flags)
-  of nkAddr, nkHiddenAddr: genAddrDeref(c, n, dest, opcAddr, flags)
+  of nkAddr, nkHiddenAddr: genAddrDeref(c, n, dest, opcAddrNode, flags)
   of nkWhenStmt, nkIfStmt, nkIfExpr: genIf(c, n, dest)
   of nkCaseStmt: genCase(c, n, dest)
   of nkWhileStmt:
@@ -1467,7 +1544,7 @@ proc genProc(c: PCtx; s: PSym): int =
     # procs easily:
     let body = s.getBody
     let procStart = c.xjmp(body, opcJmp, 0)
-    var p = PProc(blocks: @[])
+    var p = PProc(blocks: @[], sym: s)
     let oldPrc = c.prc
     c.prc = p
     # iterate over the parameters and allocate space for them:
@@ -1484,9 +1561,9 @@ proc genProc(c: PCtx; s: PSym): int =
     c.gABC(body, opcEof, eofInstr.regA)
     c.optimizeJumps(result)
     s.offset = c.prc.maxSlots
-    #if s.name.s == "concatStyleInterpolation":
+    #if s.name.s == "xmlConstructor":
-    #  c.echoCode(result)
     #  echo renderTree(body)
+    #  c.echoCode(result)
     c.prc = oldPrc
   else:
     c.prc.maxSlots = s.offset

compiler/wordrecg.nim

@@ -62,8 +62,8 @@ type
     wWatchPoint, wSubsChar, 
     wAcyclic, wShallow, wUnroll, wLinearScanEnd, wComputedGoto, wInjectStmt,
     wWrite, wGensym, wInject, wDirty, wInheritable, wThreadVar, wEmit, 
-    wNoStackFrame,
+    wAsmNoStackFrame,
-    wImplicitStatic, wGlobal, wCodegenDecl,
+    wImplicitStatic, wGlobal, wCodegenDecl, wUnchecked,
 
     wAuto, wBool, wCatch, wChar, wClass,
     wConst_cast, wDefault, wDelete, wDouble, wDynamic_cast,
@@ -145,7 +145,7 @@ const
     "subschar", "acyclic", "shallow", "unroll", "linearscanend",
     "computedgoto", "injectstmt",
     "write", "gensym", "inject", "dirty", "inheritable", "threadvar", "emit",
-    "nostackframe", "implicitstatic", "global", "codegendecl",
+    "asmnostackframe", "implicitstatic", "global", "codegendecl", "unchecked",
     
     "auto", "bool", "catch", "char", "class",
     "const_cast", "default", "delete", "double",

doc/manual.txt

@@ -1701,11 +1701,11 @@ algorithm returns true:
     result = isOrdinal(t) or t.kind in {float, float32, float64}
 
   proc isExplicitlyConvertible(a, b: PType): bool =
+    result = false
     if isImplicitlyConvertible(a, b): return true
     if typeEqualsOrDistinct(a, b): return true
     if isIntegralType(a) and isIntegralType(b): return true
     if isSubtype(a, b) or isSubtype(b, a): return true
-    return false
     
 The convertible relation can be relaxed by a user-defined type 
 `converter`:idx:.
@@ -1774,7 +1774,7 @@ Example:
 
 .. code-block:: nimrod
   proc p(x, y: int): int = 
-    return x + y
+    result = x + y
 
   discard p(3, 4) # discard the return value of `p`
 
@@ -1789,7 +1789,7 @@ been declared with the `discardable`:idx: pragma:
 
 .. code-block:: nimrod
   proc p(x, y: int): int {.discardable.} = 
-    return x + y
+    result = x + y
     
   p(3, 4) # now valid
 
@@ -2231,6 +2231,8 @@ Instead of:
 Using statement
 ---------------
 
+**Warning**: The ``using`` statement is highly experimental!
+
 The `using statement`:idx: provides syntactic convenience for procs that
 heavily use a single contextual parameter. When applied to a variable or a
 constant, it will instruct Nimrod to automatically consider the used symbol as
@@ -2438,7 +2440,7 @@ A procedure cannot modify its parameters (unless the parameters have the type
 .. code-block:: nimrod
   proc `$` (x: int): string =
     # converts an integer to a string; this is a prefix operator.
-    return intToStr(x)
+    result = intToStr(x)
 
 Operators with one parameter are prefix operators, operators with two
 parameters are infix operators. (However, the parser distinguishes these from
@@ -2452,7 +2454,7 @@ notation. (Thus an operator can have more than two parameters):
 .. code-block:: nimrod
   proc `*+` (a, b, c: int): int =
     # Multiply and add
-    return a * b + c
+    result = a * b + c
 
   assert `*+`(3, 4, 6) == `*`(a, `+`(b, c))
 
@@ -2498,7 +2500,7 @@ different; for this a special setter syntax is needed:
   
   proc host*(s: TSocket): int {.inline.} =
     ## getter of hostAddr
-    return s.FHost
+    s.FHost
 
   var
     s: TSocket
@@ -2648,11 +2650,12 @@ return values. This can be done in a cleaner way by returning a tuple:
 
 .. code-block:: nimrod
   proc divmod(a, b: int): tuple[res, remainder: int] =
-    return (a div b, a mod b)
+    (a div b, a mod b)
 
   var t = divmod(8, 5)
+
   assert t.res == 1
-  assert t.remainder = 3
+  assert t.remainder == 3
 
 One can use `tuple unpacking`:idx: to access the tuple's fields:
 
@@ -2724,7 +2727,7 @@ dispatch.
 
   method eval(e: ref TPlusExpr): int =
     # watch out: relies on dynamic binding
-    return eval(e.a) + eval(e.b)
+    result = eval(e.a) + eval(e.b)
   
   proc newLit(x: int): ref TLiteral =
     new(result)
@@ -2923,13 +2926,13 @@ parameters of an outer factory proc:
 
 .. code-block:: nimrod
   proc mycount(a, b: int): iterator (): int =
-    return iterator (): int =
+    result = iterator (): int =
       var x = a
       while x <= b:
         yield x
         inc x
 
-  let foo = mycount 1, 4
+  let foo = mycount(1, 4)
 
   for f in foo():
     echo f
@@ -3373,9 +3376,9 @@ module to illustrate this:
     ## requires `x` and `y` to be of the same tuple type
     ## generic ``==`` operator for tuples that is lifted from the components
     ## of `x` and `y`.
+    result = true
     for a, b in fields(x, y):
-      if a != b: return false
+      if a != b: result = false
-    return true
 
 Alternatively, the ``distinct`` type modifier can be applied to the type class 
 to allow each param matching the type class to bind to a different type. 
@@ -3997,9 +4000,9 @@ predicate:
 
   proc re(pattern: semistatic[string]): TRegEx =
     when isStatic(pattern):
-      return precompiledRegex(pattern)
+      result = precompiledRegex(pattern)
     else:
-      return compile(pattern)
+      result = compile(pattern)
 
 Static params can also appear in the signatures of generic types:
 
@@ -4103,6 +4106,59 @@ types that will match the typedesc param:
 
 The constraint can be a concrete type or a type class.
 
+Special Operators
+=================
+
+dot operators
+-------------
+
+Nimrod offers a special family of dot operators that can be used to
+intercept and rewrite proc call and field access attempts, referring
+to previously undeclared symbol names. They can be used to provide a
+fluent interface to objects lying outside the static confines of the
+Nimrod's type system such as values from dynamic scripting languages
+or dynamic file formats such as JSON or XML.
+
+When Nimrod encounters an expression that cannot be resolved by the
+standard overload resolution rules, the current scope will be searched
+for a dot operator that can be matched against a re-written form of
+the expression, where the unknown field or proc name is converted to
+an additional static string parameter:
+
+.. code-block:: nimrod
+  a.b # becomes `.`(a, "b")
+  a.b(c, d) # becomes `.`(a, "b", c, d)
+
+The matched dot operators can be symbols of any callable kind (procs,
+templates and macros), depending on the desired effect:
+
+.. code-block:: nimrod
+  proc `.` (js: PJsonNode, field: string): JSON = js[field]
+
+  var js = parseJson("{ x: 1, y: 2}")
+  echo js.x # outputs 1
+  echo js.y # outputs 2
+
+The following dot operators are available:
+
+operator `.`
+------------ 
+This operator will be matched against both field accesses and method calls.
+
+operator `.()`
+---------------
+This operator will be matched exclusively against method calls. It has higher
+precedence than the `.` operator and this allows you to handle expressions like
+`x.y` and `x.y()` differently if you are interfacing with a scripting language
+for example.
+
+operator `.=`
+-------------
+This operator will be matched against assignments to missing fields.
+
+.. code-block:: nimrod
+  a.b = c # becomes `.=`(a, "b", c)
+
 
 Term rewriting macros
 =====================
@@ -4506,7 +4562,7 @@ This is best illustrated by an example:
   proc p*(x: A.T1): A.T1 =
     # this works because the compiler has already
     # added T1 to A's interface symbol table
-    return x + 1
+    result = x + 1
 
 
 Import statement
@@ -4755,42 +4811,6 @@ This may change in future versions of language, but for now use
 the ``finalizer`` parameter to ``new``.
 
 
-delegator pragma
-----------------
-
-**Note**: The design of the delegator feature is subject to change.
-
-The delegator pragma can be used to intercept and rewrite proc call and field
-access attempts referring to previously undeclared symbol names. It can be used
-to provide a fluent interface to objects lying outside the static confines of
-the Nimrod's type system such as values from dynamic scripting languages or
-dynamic file formats such as JSON or XML.
-
-A delegator is a special form of the `()` operator marked with the delagator
-pragma. When Nimrod encounters an expression that cannot be resolved by the
-standard overload resolution, any delegators in the current scope will be
-matched against a rewritten form of the expression following the standard
-signature matching rules. In the rewritten expression, the name of the unknown
-proc or field name is inserted as an additional static string parameter always
-appearing in the leading position:
-
-.. code-block:: nimrod
-  a.b => delegator("b", a)
-  a.b(c, d) => delegator("b", a, c)
-  a b, c, d => delegator("a", b, c, d)
-
-
-The delegators can be any callable symbol type (procs, templates, macros)
-depending on the desired effect:
-
-.. code-block:: nimrod
-  proc `()` (field: string, js: PJsonNode): JSON {.delegator.} = js[field]
-
-  var js = parseJson("{ x: 1, y: 2}")
-  echo js.x # outputs 1
-  echo js.y # outputs 2
-
-
 procvar pragma
 --------------
 The `procvar`:idx: pragma is used to mark a proc that it can be passed to a
@@ -4873,16 +4893,16 @@ field which is used for runtime type identification is omitted. This is
 necessary for binary compatibility with other compiled languages.
 
 
-NoStackFrame pragma
+AsmNoStackFrame pragma
--------------------
+----------------------
-A proc can be marked with the `noStackFrame`:idx: pragma to tell the compiler
+A proc can be marked with the `AsmNoStackFrame`:idx: pragma to tell the compiler
 it should not generate a stack frame for the proc. There are also no exit
 statements like ``return result;`` generated and the generated C function is
 declared as ``__declspec(naked)`` or ``__attribute__((naked))`` (depending on
 the used C compiler).
 
-**Note**: This pragma should only be used by procs which consist solely of assembler
+**Note**: This pragma should only be used by procs which consist solely of
-statements.
+assembler statements.
 
 error pragma
 ------------
@@ -5134,6 +5154,8 @@ Example:
 .. code-block:: nimrod
   {.deadCodeElim: on.}
 
+
+..
   NoForward pragma
   ----------------
   The `noforward`:idx: pragma can be used to turn on and off a special compilation
@@ -5180,6 +5202,7 @@ Example:
 
     foo(10)
 
+
 Pragma pragma
 -------------
 
@@ -5197,7 +5220,7 @@ Example:
     {.pragma: rtl, importc, dynlib: "client.dll", cdecl.}
     
   proc p*(a, b: int): int {.rtl.} = 
-    return a+b
+    result = a+b
 
 In the example a new pragma named ``rtl`` is introduced that either imports
 a symbol from a dynamic library or exports the symbol for dynamic library
@@ -5297,6 +5320,53 @@ strings automatically:
   printf("hallo %s", "world") # "world" will be passed as C string
 
 
+Union pragma
+------------
+The `union`:idx: pragma can be applied to any ``object`` type. It means all
+of the object's fields are overlaid in memory. This produces a ``union``
+instead of a ``struct`` in the generated C/C++ code. The object declaration
+then must not use inheritance or any GC'ed memory but this is currently not
+checked.
+
+**Future directions**: GC'ed memory should be allowed in unions and the GC
+should scan unions conservatively.
+
+
+Unchecked pragma
+----------------
+The `unchecked`:idx: pragma can be used to mark a named array as ``unchecked``
+meaning its bounds are not checked. This is often useful when one wishes to
+implement his own flexibly sized arrays. Additionally an unchecked array is
+translated into a C array of undetermined size:
+
+.. code-block:: nimrod
+  type
+    ArrayPart{.unchecked.} = array[0..0, int]
+    MySeq = object
+      len, cap: int
+      data: ArrayPart
+
+Produces roughly this C code:
+
+.. code-block:: C
+  typedef struct {
+    NI len;
+    NI cap;
+    NI data[];
+  } MySeq;
+
+The bounds checking done at compile time is not disabled for now, so to access
+``s.data[C]`` (where ``C`` is a constant) the array's index needs needs to
+include ``C``.
+
+The base type of the unchecked array may not contain any GC'ed memory but this
+is currently not checked.
+
+**Future directions**: GC'ed memory should be allowed in unchecked arrays and
+there should be an explicit annotation of how the GC is to determine the
+runtime size of the array.
+
+
 Dynlib pragma for import
 ------------------------
 With the `dynlib`:idx: pragma a procedure or a variable can be imported from

doc/tut1.txt

@@ -690,8 +690,8 @@ Nimrod provides the ability to overload procedures similar to C++:
 .. code-block:: nimrod
   proc toString(x: int): string = ...
   proc toString(x: bool): string =
-    if x: return "true"
+    if x: result = "true"
-    else: return "false"
+    else: result = "false"
 
   echo(toString(13))   # calls the toString(x: int) proc
   echo(toString(true)) # calls the toString(x: bool) proc
@@ -1569,7 +1569,7 @@ This is best illustrated by an example:
   proc p*(x: A.T1): A.T1 =
     # this works because the compiler has already
     # added T1 to A's interface symbol table
-    return x + 1
+    result = x + 1
 
 
 A symbol of a module *can* be *qualified* with the ``module.symbol`` syntax. If
@@ -1600,11 +1600,11 @@ rules apply:
 
 .. code-block:: nimrod
   # Module A
-  proc x*(a: int): string = return $a
+  proc x*(a: int): string = result = $a
 
 .. code-block:: nimrod
   # Module B
-  proc x*(a: string): string = return $a
+  proc x*(a: string): string = result = $a
 
 .. code-block:: nimrod
   # Module C

doc/tut2.txt

@@ -126,7 +126,7 @@ The syntax for type conversions is ``destination_type(expression_to_convert)``
 
 .. code-block:: nimrod
   proc getID(x: TPerson): int =
-    return TStudent(x).id
+    TStudent(x).id
 
 The ``EInvalidObjectConversion`` exception is raised if ``x`` is not a
 ``TStudent``.
@@ -238,7 +238,7 @@ is needed:
   
   proc host*(s: TSocket): int {.inline.} =
     ## getter of hostAddr
-    return s.FHost
+    s.FHost
 
   var
     s: TSocket

examples/htmlrefs.nim

@@ -36,7 +36,7 @@ block mainLoop:
               case x.kind
               of xmlEof: break mainLoop
               of xmlElementClose: break
-              else: nil
+              else: discard
             x.next() # skip ``xmlElementClose``
             # now we have the description for the ``a`` element
             var desc = ""

examples/htmltitle.nim

@@ -7,7 +7,7 @@ import os, streams, parsexml, strutils
 if paramCount() < 1:
   quit("Usage: htmltitle filename[.html]")
 
-var filename = addFileExt(ParamStr(1), "html")
+var filename = addFileExt(paramStr(1), "html")
 var s = newFileStream(filename, fmRead)
 if s == nil: quit("cannot open the file " & filename)
 var x: TXmlParser
@@ -23,13 +23,13 @@ while true:
         title.add(x.charData)
         x.next()
       if x.kind == xmlElementEnd and cmpIgnoreCase(x.elementName, "title") == 0:
-        Echo("Title: " & title)
+        echo("Title: " & title)
         quit(0) # Success!
       else:
         echo(x.errorMsgExpected("/title"))
   
   of xmlEof: break # end of file reached
-  else: nil # ignore other events
+  else: discard # ignore other events
 
 x.close()
 quit("Could not determine title!")

koch.nim

@@ -58,6 +58,15 @@ Boot options:
 
 proc exe(f: string): string = return addFileExt(f, ExeExt)
 
+proc findNim(): string =
+  var nimrod = "nimrod".exe
+  result = "bin" / nimrod
+  if existsFile(result): return
+  for dir in split(getEnv("PATH"), PathSep):
+    if existsFile(dir / nimrod): return dir / nimrod
+  # assume there is a symlink to the exe or something:
+  return nimrod
+
 proc exec(cmd: string) =
   echo(cmd)
   if execShellCmd(cmd) != 0: quit("FAILURE")
@@ -70,15 +79,15 @@ const
   compileNimInst = "-d:useLibzipSrc tools/niminst/niminst"
 
 proc csource(args: string) = 
-  exec("nimrod cc $1 -r $3 --var:version=$2 csource compiler/nimrod.ini $1" %
+  exec("$4 cc $1 -r $3 --var:version=$2 csource compiler/nimrod.ini $1" %
-       [args, NimrodVersion, compileNimInst])
+       [args, NimrodVersion, compileNimInst, findNim()])
 
 proc zip(args: string) = 
-  exec("nimrod cc -r $2 --var:version=$1 zip compiler/nimrod.ini" %
+  exec("$3 cc -r $2 --var:version=$1 zip compiler/nimrod.ini" %
-       [NimrodVersion, compileNimInst])
+       [NimrodVersion, compileNimInst, findNim()])
   
 proc buildTool(toolname, args: string) = 
-  exec("nimrod cc $# $#" % [args, toolname])
+  exec("$# cc $# $#" % [findNim(), args, toolname])
   copyFile(dest="bin"/ splitFile(toolname).name.exe, source=toolname.exe)
 
 proc inno(args: string) =
@@ -90,13 +99,13 @@ proc inno(args: string) =
        NimrodVersion)
 
 proc install(args: string) = 
-  exec("nimrod cc -r $# --var:version=$# scripts compiler/nimrod.ini" %
+  exec("$# cc -r $# --var:version=$# scripts compiler/nimrod.ini" %
-       [compileNimInst, NimrodVersion])
+       [findNim(), compileNimInst, NimrodVersion])
   exec("sh ./install.sh $#" % args)
 
 proc web(args: string) =
-  exec(("nimrod cc -r tools/nimweb.nim web/nimrod --putenv:nimrodversion=$#" &
+  exec("$# cc -r tools/nimweb.nim web/nimrod --putenv:nimrodversion=$#" % 
-        " --path:$#") % [NimrodVersion, getCurrentDir()])
+       [findNim(), NimrodVersion])
 
 # -------------- boot ---------------------------------------------------------
 

lib/core/macros.nim

@@ -516,7 +516,7 @@ proc last*(node: PNimrodNode): PNimrodNode {.compileTime.} = node[node.high]
 
 
 const
-  RoutineNodes* = {nnkProcDef, nnkMethodDef, nnkDo, nnkLambda}
+  RoutineNodes* = {nnkProcDef, nnkMethodDef, nnkDo, nnkLambda, nnkIteratorDef}
   AtomicNodes* = {nnkNone..nnkNilLit}
   CallNodes* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand, 
     nnkCallStrLit, nnkHiddenCallConv}

lib/impure/db_mysql.nim

@@ -87,7 +87,7 @@ proc newRow(L: int): TRow =
   
 proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) =  
   if row != nil:
-    while mysql.FetchRow(sqlres) != nil: nil
+    while mysql.FetchRow(sqlres) != nil: discard
   mysql.FreeResult(sqlres)
   
 iterator fastRows*(db: TDbConn, query: TSqlQuery,

lib/nimbase.h

@@ -285,8 +285,8 @@ static N_INLINE(NI32, float32ToInt32)(float x) {
 
 typedef struct TStringDesc* string;
 
-/* declared size of a sequence: */
+/* declared size of a sequence/variable length array: */
-#if defined(__GNUC__)
+#if defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
 #  define SEQ_DECL_SIZE /* empty is correct! */
 #else
 #  define SEQ_DECL_SIZE 1000000

lib/packages/docutils/docutils.babel

@@ -0,0 +1,6 @@
+[Package]
+name          = "docutils"
+version       = "0.9.0"
+author        = "Andreas Rumpf"
+description   = "Nimrod's reStructuredText processor."
+license       = "MIT"

lib/packages/docutils/highlite.nim

@@ -61,9 +61,8 @@ proc getSourceLanguage*(name: string): TSourceLanguage =
     if cmpIgnoreStyle(name, sourceLanguageToStr[i]) == 0: 
       return i
   result = langNone
-
+proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: cstring) =
-proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: string) = 
+  g.buf = buf
-  g.buf = cstring(buf)
   g.kind = low(TTokenClass)
   g.start = 0
   g.length = 0
@@ -71,6 +70,8 @@ proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: string) =
   var pos = 0                     # skip initial whitespace:
   while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos)
   g.pos = pos
+proc initGeneralTokenizer*(g: var TGeneralTokenizer, buf: string) = 
+  initGeneralTokenizer(g, cstring(buf))
 
 proc deinitGeneralTokenizer*(g: var TGeneralTokenizer) = 
   discard

lib/packages/docutils/rst.nim

@@ -1543,7 +1543,7 @@ proc dirRaw(p: var TRstParser): PRstNode =
     elif cmpIgnoreCase(result.sons[0].sons[0].text, "latex") == 0: 
       dirRawAux(p, result, rnRawLatex, parseLiteralBlock)
     else:
-      rstMessage(p, meInvalidDirective, result.sons[0].text)
+      rstMessage(p, meInvalidDirective, result.sons[0].sons[0].text)
   else:
     dirRawAux(p, result, rnRaw, parseSectionWrapper)
 

lib/posix/epoll.nim

@@ -0,0 +1,90 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2013 Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+from posix import TSocketHandle
+
+const
+  EPOLLIN* = 0x00000001
+  EPOLLPRI* = 0x00000002
+  EPOLLOUT* = 0x00000004
+  EPOLLERR* = 0x00000008
+  EPOLLHUP* = 0x00000010
+  EPOLLRDNORM* = 0x00000040
+  EPOLLRDBAND* = 0x00000080
+  EPOLLWRNORM* = 0x00000100
+  EPOLLWRBAND* = 0x00000200
+  EPOLLMSG* = 0x00000400
+  EPOLLRDHUP* = 0x00002000
+  EPOLLWAKEUP* = 1 shl 29
+  EPOLLONESHOT* = 1 shl 30
+  EPOLLET* = 1 shl 31
+
+# Valid opcodes ( "op" parameter ) to issue to epoll_ctl().
+
+const 
+  EPOLL_CTL_ADD* = 1          # Add a file descriptor to the interface.  
+  EPOLL_CTL_DEL* = 2          # Remove a file descriptor from the interface.  
+  EPOLL_CTL_MOD* = 3          # Change file descriptor epoll_event structure.  
+
+type 
+  epoll_data* {.importc: "union epoll_data", 
+      header: "<sys/epoll.h>", pure, final.} = object # TODO: This is actually a union.
+    #thePtr* {.importc: "ptr".}: pointer
+    fd*: cint # \
+    #u32*: uint32
+    #u64*: uint64
+
+  epoll_event* {.importc: "struct epoll_event", header: "<sys/epoll.h>", pure, final.} = object 
+    events*: uint32 # Epoll events 
+    data*: epoll_data # User data variable 
+
+proc epoll_create*(size: cint): cint {.importc: "epoll_create", 
+    header: "<sys/epoll.h>".}
+  ## Creates an epoll instance.  Returns an fd for the new instance.
+  ##   The "size" parameter is a hint specifying the number of file
+  ##   descriptors to be associated with the new instance.  The fd
+  ##   returned by epoll_create() should be closed with close().  
+
+proc epoll_create1*(flags: cint): cint {.importc: "epoll_create1", 
+    header: "<sys/epoll.h>".}
+  ## Same as epoll_create but with an FLAGS parameter.  The unused SIZE
+  ##   parameter has been dropped.  
+
+proc epoll_ctl*(epfd: cint; op: cint; fd: cint | TSocketHandle; event: ptr epoll_event): cint {.
+    importc: "epoll_ctl", header: "<sys/epoll.h>".}
+  ## Manipulate an epoll instance "epfd". Returns 0 in case of success,
+  ##   -1 in case of error ( the "errno" variable will contain the
+  ##   specific error code ) The "op" parameter is one of the EPOLL_CTL_*
+  ##   constants defined above. The "fd" parameter is the target of the
+  ##   operation. The "event" parameter describes which events the caller
+  ##   is interested in and any associated user data.  
+
+proc epoll_wait*(epfd: cint; events: ptr epoll_event; maxevents: cint; 
+                 timeout: cint): cint {.importc: "epoll_wait", 
+    header: "<sys/epoll.h>".}
+  ## Wait for events on an epoll instance "epfd". Returns the number of
+  ##   triggered events returned in "events" buffer. Or -1 in case of
+  ##   error with the "errno" variable set to the specific error code. The
+  ##   "events" parameter is a buffer that will contain triggered
+  ##   events. The "maxevents" is the maximum number of events to be
+  ##   returned ( usually size of "events" ). The "timeout" parameter
+  ##   specifies the maximum wait time in milliseconds (-1 == infinite).
+  ##
+  ##   This function is a cancellation point and therefore not marked with
+  ##   __THROW.
+
+
+#proc epoll_pwait*(epfd: cint; events: ptr epoll_event; maxevents: cint; 
+#                  timeout: cint; ss: ptr sigset_t): cint {.
+#    importc: "epoll_pwait", header: "<sys/epoll.h>".}
+# Same as epoll_wait, but the thread's signal mask is temporarily
+#   and atomically replaced with the one provided as parameter.
+#
+#   This function is a cancellation point and therefore not marked with
+#   __THROW.  

lib/posix/linux.nim

@@ -0,0 +1,25 @@
+import posix
+
+const
+  CSIGNAL* = 0x000000FF
+  CLONE_VM* = 0x00000100
+  CLONE_FS* = 0x00000200
+  CLONE_FILES* = 0x00000400
+  CLONE_SIGHAND* = 0x00000800
+  CLONE_PTRACE* = 0x00002000
+  CLONE_VFORK* = 0x00004000
+  CLONE_PARENT* = 0x00008000
+  CLONE_THREAD* = 0x00010000
+  CLONE_NEWNS* = 0x00020000
+  CLONE_SYSVSEM* = 0x00040000
+  CLONE_SETTLS* = 0x00080000
+  CLONE_PARENT_SETTID* = 0x00100000
+  CLONE_CHILD_CLEARTID* = 0x00200000
+  CLONE_DETACHED* = 0x00400000
+  CLONE_UNTRACED* = 0x00800000
+  CLONE_CHILD_SETTID* = 0x01000000
+  CLONE_STOPPED* = 0x02000000
+
+# fn should be of type proc (a2: pointer): void {.cdecl.}
+proc clone*(fn: pointer; child_stack: pointer; flags: cint;
+            arg: pointer; ptid: ptr TPid; tls: pointer; ctid: ptr TPid): cint {.importc, header: "<sched.h>".}

lib/posix/posix.nim

@@ -2066,6 +2066,7 @@ proc pthread_spin_unlock*(a1: ptr Tpthread_spinlock): cint {.
 proc pthread_testcancel*() {.importc, header: "<pthread.h>".}
 
 
+proc exitnow*(code: int): void {.importc: "_exit", header: "<unistd.h>".}
 proc access*(a1: cstring, a2: cint): cint {.importc, header: "<unistd.h>".}
 proc alarm*(a1: cint): cint {.importc, header: "<unistd.h>".}
 proc chdir*(a1: cstring): cint {.importc, header: "<unistd.h>".}
@@ -2265,6 +2266,7 @@ proc gmtime_r*(a1: var TTime, a2: var Ttm): ptr Ttm {.importc, header: "<time.h>
 proc localtime*(a1: var TTime): ptr Ttm {.importc, header: "<time.h>".}
 proc localtime_r*(a1: var TTime, a2: var Ttm): ptr Ttm {.importc, header: "<time.h>".}
 proc mktime*(a1: var Ttm): TTime  {.importc, header: "<time.h>".}
+proc timegm*(a1: var Ttm): TTime  {.importc, header: "<time.h>".}
 proc nanosleep*(a1, a2: var Ttimespec): cint {.importc, header: "<time.h>".}
 proc strftime*(a1: cstring, a2: int, a3: cstring,
            a4: var Ttm): int {.importc, header: "<time.h>".}
@@ -2356,7 +2358,7 @@ proc FD_ZERO*(a1: var TFdSet) {.importc, header: "<sys/select.h>".}
 
 proc pselect*(a1: cint, a2, a3, a4: ptr TFdSet, a5: ptr Ttimespec,
          a6: var Tsigset): cint  {.importc, header: "<sys/select.h>".}
-proc select*(a1: cint, a2, a3, a4: ptr TFdSet, a5: ptr Ttimeval): cint {.
+proc select*(a1: cint | TSocketHandle, a2, a3, a4: ptr TFdSet, a5: ptr Ttimeval): cint {.
              importc, header: "<sys/select.h>".}
 
 when hasSpawnH:

lib/pure/asyncio.nim

@@ -167,7 +167,7 @@ proc asyncSocket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM,
   result = newAsyncSocket()
   result.socket = socket(domain, typ, protocol, buffered)
   result.proto = protocol
-  if result.socket == InvalidSocket: OSError(OSLastError())
+  if result.socket == invalidSocket: osError(osLastError())
   result.socket.setBlocking(false)
 
 proc toAsyncSocket*(sock: TSocket, state: TInfo = SockConnected): PAsyncSocket =
@@ -357,7 +357,7 @@ proc acceptAddr*(server: PAsyncSocket, client: var PAsyncSocket,
     client.sslNeedAccept = false
     client.info = SockConnected
 
-  if c == InvalidSocket: SocketError(server.socket)
+  if c == invalidSocket: socketError(server.socket)
   c.setBlocking(false) # TODO: Needs to be tested.
   
   # deleg.open is set in ``toDelegate``.
@@ -481,7 +481,7 @@ proc recvLine*(s: PAsyncSocket, line: var TaintedString): bool {.deprecated.} =
   of RecvDisconnected:
     result = true
   of RecvFail:
-    s.SocketError(async = true)
+    s.socketError(async = true)
     result = false
 {.pop.}
 
@@ -615,11 +615,11 @@ proc poll*(d: PDispatcher, timeout: int = 500): bool =
     if d.hasDataBuffered(d.deleVal):
       hasDataBufferedCount.inc()
       d.handleRead(d.deleVal)
-  if hasDataBufferedCount > 0: return True
+  if hasDataBufferedCount > 0: return true
   
   if readDg.len() == 0 and writeDg.len() == 0:
     ## TODO: Perhaps this shouldn't return if errorDg has something?
-    return False
+    return false
   
   if select(readDg, writeDg, errorDg, timeout) != 0:
     for i in 0..len(d.delegates)-1:

lib/pure/asyncio2.nim

@@ -0,0 +1,922 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2014 Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+import os, oids, tables, strutils, macros
+
+import sockets2, net
+
+## Asyncio2 
+## --------
+##
+## This module implements a brand new asyncio module based on Futures.
+## IOCP is used under the hood on Windows and the selectors module is used for
+## other operating systems.
+
+# -- Futures
+
+type
+  PFutureBase* = ref object of PObject
+    cb: proc () {.closure.}
+    finished: bool
+
+  PFuture*[T] = ref object of PFutureBase
+    value: T
+    error: ref EBase
+
+proc newFuture*[T](): PFuture[T] =
+  ## Creates a new future.
+  new(result)
+  result.finished = false
+
+proc complete*[T](future: PFuture[T], val: T) =
+  ## Completes ``future`` with value ``val``.
+  assert(not future.finished, "Future already finished, cannot finish twice.")
+  assert(future.error == nil)
+  future.value = val
+  future.finished = true
+  if future.cb != nil:
+    future.cb()
+
+proc fail*[T](future: PFuture[T], error: ref EBase) =
+  ## Completes ``future`` with ``error``.
+  assert(not future.finished, "Future already finished, cannot finish twice.")
+  future.finished = true
+  future.error = error
+  if future.cb != nil:
+    future.cb()
+
+proc `callback=`*(future: PFutureBase, cb: proc () {.closure.}) =
+  ## Sets the callback proc to be called when the future completes.
+  ##
+  ## If future has already completed then ``cb`` will be called immediately.
+  ##
+  ## **Note**: You most likely want the other ``callback`` setter which
+  ## passes ``future`` as a param to the callback.
+  future.cb = cb
+  if future.finished:
+    future.cb()
+
+proc `callback=`*[T](future: PFuture[T],
+    cb: proc (future: PFuture[T]) {.closure.}) =
+  ## Sets the callback proc to be called when the future completes.
+  ##
+  ## If future has already completed then ``cb`` will be called immediately.
+  future.callback = proc () = cb(future)
+
+proc read*[T](future: PFuture[T]): T =
+  ## Retrieves the value of ``future``. Future must be finished otherwise
+  ## this function will fail with a ``EInvalidValue`` exception.
+  ##
+  ## If the result of the future is an error then that error will be raised.
+  if future.finished:
+    if future.error != nil: raise future.error
+    return future.value
+  else:
+    # TODO: Make a custom exception type for this?
+    raise newException(EInvalidValue, "Future still in progress.")
+
+proc finished*[T](future: PFuture[T]): bool =
+  ## Determines whether ``future`` has completed.
+  ##
+  ## ``True`` may indicate an error or a value. Use ``hasError`` to distinguish.
+  future.finished
+
+proc failed*[T](future: PFuture[T]): bool =
+  ## Determines whether ``future`` completed with an error.
+  future.error != nil
+
+# TODO: Get rid of register. Do it implicitly.
+
+when defined(windows) or defined(nimdoc):
+  import winlean
+  type
+    TCompletionKey = dword
+
+    TCompletionData* = object
+      sock: TSocketHandle
+      cb: proc (sock: TSocketHandle, bytesTransferred: DWORD,
+                errcode: TOSErrorCode) {.closure.}
+
+    PDispatcher* = ref object
+      ioPort: THandle
+      hasHandles: bool
+
+    TCustomOverlapped = object
+      Internal*: DWORD
+      InternalHigh*: DWORD
+      Offset*: DWORD
+      OffsetHigh*: DWORD
+      hEvent*: THANDLE
+      data*: TCompletionData
+
+    PCustomOverlapped = ptr TCustomOverlapped
+
+  proc newDispatcher*(): PDispatcher =
+    ## Creates a new Dispatcher instance.
+    new result
+    result.ioPort = CreateIOCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1)
+
+  proc register*(p: PDispatcher, sock: TSocketHandle) =
+    ## Registers ``sock`` with the dispatcher ``p``.
+    if CreateIOCompletionPort(sock.THandle, p.ioPort,
+                              cast[TCompletionKey](sock), 1) == 0:
+      OSError(OSLastError())
+    p.hasHandles = true
+
+  proc poll*(p: PDispatcher, timeout = 500) =
+    ## Waits for completion events and processes them.
+    if not p.hasHandles:
+      raise newException(EInvalidValue, "No handles registered in dispatcher.")
+    
+    let llTimeout =
+      if timeout ==  -1: winlean.INFINITE
+      else: timeout.int32
+    var lpNumberOfBytesTransferred: DWORD
+    var lpCompletionKey: ULONG
+    var lpOverlapped: POverlapped
+    let res = GetQueuedCompletionStatus(p.ioPort, addr lpNumberOfBytesTransferred,
+        addr lpCompletionKey, addr lpOverlapped, llTimeout).bool
+
+    # http://stackoverflow.com/a/12277264/492186
+    # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html
+    var customOverlapped = cast[PCustomOverlapped](lpOverlapped)
+    if res:
+      # This is useful for ensuring the reliability of the overlapped struct.
+      assert customOverlapped.data.sock == lpCompletionKey.TSocketHandle
+
+      customOverlapped.data.cb(customOverlapped.data.sock,
+          lpNumberOfBytesTransferred, TOSErrorCode(-1))
+      dealloc(customOverlapped)
+    else:
+      let errCode = OSLastError()
+      if lpOverlapped != nil:
+        assert customOverlapped.data.sock == lpCompletionKey.TSocketHandle
+        customOverlapped.data.cb(customOverlapped.data.sock,
+            lpNumberOfBytesTransferred, errCode)
+        dealloc(customOverlapped)
+      else:
+        if errCode.int32 == WAIT_TIMEOUT:
+          # Timed out
+          discard
+        else: OSError(errCode)
+
+  var connectExPtr: pointer = nil
+  var acceptExPtr: pointer = nil
+  var getAcceptExSockAddrsPtr: pointer = nil
+
+  proc initPointer(s: TSocketHandle, func: var pointer, guid: var TGUID): bool =
+    # Ref: https://github.com/powdahound/twisted/blob/master/twisted/internet/iocpreactor/iocpsupport/winsock_pointers.c
+    var bytesRet: DWord
+    func = nil
+    result = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, addr guid,
+                      sizeof(TGUID).dword, addr func, sizeof(pointer).DWORD,
+                      addr bytesRet, nil, nil) == 0
+
+  proc initAll() =
+    let dummySock = socket()
+    if not initPointer(dummySock, connectExPtr, WSAID_CONNECTEX):
+      OSError(OSLastError())
+    if not initPointer(dummySock, acceptExPtr, WSAID_ACCEPTEX):
+      OSError(OSLastError())
+    if not initPointer(dummySock, getAcceptExSockAddrsPtr, WSAID_GETACCEPTEXSOCKADDRS):
+      OSError(OSLastError())
+
+  proc connectEx(s: TSocketHandle, name: ptr TSockAddr, namelen: cint, 
+                  lpSendBuffer: pointer, dwSendDataLength: dword,
+                  lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool =
+    if connectExPtr.isNil: raise newException(EInvalidValue, "Need to initialise ConnectEx().")
+    let func =
+      cast[proc (s: TSocketHandle, name: ptr TSockAddr, namelen: cint, 
+         lpSendBuffer: pointer, dwSendDataLength: dword,
+         lpdwBytesSent: PDWORD, lpOverlapped: POverlapped): bool {.stdcall.}](connectExPtr)
+
+    result = func(s, name, namelen, lpSendBuffer, dwSendDataLength, lpdwBytesSent,
+         lpOverlapped)
+
+  proc acceptEx(listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer,
+                 dwReceiveDataLength, dwLocalAddressLength,
+                 dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD,
+                 lpOverlapped: POverlapped): bool =
+    if acceptExPtr.isNil: raise newException(EInvalidValue, "Need to initialise AcceptEx().")
+    let func =
+      cast[proc (listenSock, acceptSock: TSocketHandle, lpOutputBuffer: pointer,
+                 dwReceiveDataLength, dwLocalAddressLength,
+                 dwRemoteAddressLength: DWORD, lpdwBytesReceived: PDWORD,
+                 lpOverlapped: POverlapped): bool {.stdcall.}](acceptExPtr)
+    result = func(listenSock, acceptSock, lpOutputBuffer, dwReceiveDataLength,
+        dwLocalAddressLength, dwRemoteAddressLength, lpdwBytesReceived,
+        lpOverlapped)
+
+  proc getAcceptExSockaddrs(lpOutputBuffer: pointer,
+      dwReceiveDataLength, dwLocalAddressLength, dwRemoteAddressLength: DWORD,
+      LocalSockaddr: ptr ptr TSockAddr, LocalSockaddrLength: lpint,
+      RemoteSockaddr: ptr ptr TSockAddr, RemoteSockaddrLength: lpint) =
+    if getAcceptExSockAddrsPtr.isNil:
+      raise newException(EInvalidValue, "Need to initialise getAcceptExSockAddrs().")
+
+    let func =
+      cast[proc (lpOutputBuffer: pointer,
+                 dwReceiveDataLength, dwLocalAddressLength,
+                 dwRemoteAddressLength: DWORD, LocalSockaddr: ptr ptr TSockAddr,
+                 LocalSockaddrLength: lpint, RemoteSockaddr: ptr ptr TSockAddr,
+                RemoteSockaddrLength: lpint) {.stdcall.}](getAcceptExSockAddrsPtr)
+    
+    func(lpOutputBuffer, dwReceiveDataLength, dwLocalAddressLength,
+                  dwRemoteAddressLength, LocalSockaddr, LocalSockaddrLength,
+                  RemoteSockaddr, RemoteSockaddrLength)
+
+  proc connect*(p: PDispatcher, socket: TSocketHandle, address: string, port: TPort,
+    af = AF_INET): PFuture[int] =
+    ## Connects ``socket`` to server at ``address:port``.
+    ##
+    ## Returns a ``PFuture`` which will complete when the connection succeeds
+    ## or an error occurs.
+
+    var retFuture = newFuture[int]()# TODO: Change to void when that regression is fixed.
+    # Apparently ``ConnectEx`` expects the socket to be initially bound:
+    var saddr: Tsockaddr_in
+    saddr.sin_family = int16(toInt(af))
+    saddr.sin_port = 0
+    saddr.sin_addr.s_addr = INADDR_ANY
+    if bindAddr(socket, cast[ptr TSockAddr](addr(saddr)),
+                  sizeof(saddr).TSockLen) < 0'i32:
+      OSError(OSLastError())
+
+    var aiList = getAddrInfo(address, port, af)
+    var success = false
+    var lastError: TOSErrorCode
+    var it = aiList
+    while it != nil:
+      # "the OVERLAPPED structure must remain valid until the I/O completes"
+      # http://blogs.msdn.com/b/oldnewthing/archive/2011/02/02/10123392.aspx
+      var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped)))
+      ol.data = TCompletionData(sock: socket, cb:
+        proc (sock: TSocketHandle, bytesCount: DWord, errcode: TOSErrorCode) =
+          if not retFuture.finished:
+            if errcode == TOSErrorCode(-1):
+              retFuture.complete(0)
+            else:
+              retFuture.fail(newException(EOS, osErrorMsg(errcode)))
+      )
+      
+      var ret = connectEx(socket, it.ai_addr, sizeof(TSockAddrIn).cint,
+                          nil, 0, nil, cast[POverlapped](ol))
+      if ret:
+        # Request to connect completed immediately.
+        success = true
+        retFuture.complete(0)
+        # We don't deallocate ``ol`` here because even though this completed
+        # immediately poll will still be notified about its completion and it will
+        # free ``ol``.
+        break
+      else:
+        lastError = OSLastError()
+        if lastError.int32 == ERROR_IO_PENDING:
+          # In this case ``ol`` will be deallocated in ``poll``.
+          success = true
+          break
+        else:
+          dealloc(ol)
+          success = false
+      it = it.ai_next
+
+    dealloc(aiList)
+    if not success:
+      retFuture.fail(newException(EOS, osErrorMsg(lastError)))
+    return retFuture
+
+  proc recv*(p: PDispatcher, socket: TSocketHandle, size: int,
+             flags: int = 0): PFuture[string] =
+    ## Reads ``size`` bytes from ``socket``. Returned future will complete once
+    ## all of the requested data is read. If socket is disconnected during the
+    ## recv operation then the future may complete with only a part of the
+    ## requested data read. If socket is disconnected and no data is available
+    ## to be read then the future will complete with a value of ``""``.
+
+    var retFuture = newFuture[string]()
+    
+    var dataBuf: TWSABuf
+    dataBuf.buf = newString(size)
+    dataBuf.len = size
+    
+    var bytesReceived: DWord
+    var flagsio = flags.dword
+    var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped)))
+    ol.data = TCompletionData(sock: socket, cb:
+      proc (sock: TSocketHandle, bytesCount: DWord, errcode: TOSErrorCode) =
+        if not retFuture.finished:
+          if errcode == TOSErrorCode(-1):
+            if bytesCount == 0 and dataBuf.buf[0] == '\0':
+              retFuture.complete("")
+            else:
+              var data = newString(size)
+              copyMem(addr data[0], addr dataBuf.buf[0], size)
+              retFuture.complete($data)
+          else:
+            retFuture.fail(newException(EOS, osErrorMsg(errcode)))
+    )
+
+    let ret = WSARecv(socket, addr dataBuf, 1, addr bytesReceived,
+                      addr flagsio, cast[POverlapped](ol), nil)
+    if ret == -1:
+      let err = OSLastError()
+      if err.int32 != ERROR_IO_PENDING:
+        retFuture.fail(newException(EOS, osErrorMsg(err)))
+        dealloc(ol)
+    elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0':
+      # We have to ensure that the buffer is empty because WSARecv will tell
+      # us immediatelly when it was disconnected, even when there is still
+      # data in the buffer.
+      # We want to give the user as much data as we can. So we only return
+      # the empty string (which signals a disconnection) when there is
+      # nothing left to read.
+      retFuture.complete("")
+      # TODO: "For message-oriented sockets, where a zero byte message is often 
+      # allowable, a failure with an error code of WSAEDISCON is used to 
+      # indicate graceful closure." 
+      # ~ http://msdn.microsoft.com/en-us/library/ms741688%28v=vs.85%29.aspx
+    else:
+      # Request to read completed immediately.
+      var data = newString(size)
+      copyMem(addr data[0], addr dataBuf.buf[0], size)
+      retFuture.complete($data)
+      # We don't deallocate ``ol`` here because even though this completed
+      # immediately poll will still be notified about its completion and it will
+      # free ``ol``.
+    return retFuture
+
+  proc send*(p: PDispatcher, socket: TSocketHandle, data: string): PFuture[int] =
+    ## Sends ``data`` to ``socket``. The returned future will complete once all
+    ## data has been sent.
+    var retFuture = newFuture[int]()
+
+    var dataBuf: TWSABuf
+    dataBuf.buf = data
+    dataBuf.len = data.len
+
+    var bytesReceived, flags: DWord
+    var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped)))
+    ol.data = TCompletionData(sock: socket, cb:
+      proc (sock: TSocketHandle, bytesCount: DWord, errcode: TOSErrorCode) =
+        if not retFuture.finished:
+          if errcode == TOSErrorCode(-1):
+            retFuture.complete(0)
+          else:
+            retFuture.fail(newException(EOS, osErrorMsg(errcode)))
+    )
+
+    let ret = WSASend(socket, addr dataBuf, 1, addr bytesReceived,
+                      flags, cast[POverlapped](ol), nil)
+    if ret == -1:
+      let err = osLastError()
+      if err.int32 != ERROR_IO_PENDING:
+        retFuture.fail(newException(EOS, osErrorMsg(err)))
+        dealloc(ol)
+    else:
+      retFuture.complete(0)
+      # We don't deallocate ``ol`` here because even though this completed
+      # immediately poll will still be notified about its completion and it will
+      # free ``ol``.
+    return retFuture
+
+  proc acceptAddr*(p: PDispatcher, socket: TSocketHandle): 
+      PFuture[tuple[address: string, client: TSocketHandle]] =
+    ## Accepts a new connection. Returns a future containing the client socket
+    ## corresponding to that connection and the remote address of the client.
+    ## The future will complete when the connection is successfully accepted.
+    
+    var retFuture = newFuture[tuple[address: string, client: TSocketHandle]]()
+
+    var clientSock = socket()
+    if clientSock == OSInvalidSocket: osError(osLastError())
+
+    const lpOutputLen = 1024
+    var lpOutputBuf = newString(lpOutputLen)
+    var dwBytesReceived: DWORD
+    let dwReceiveDataLength = 0.DWORD # We don't want any data to be read.
+    let dwLocalAddressLength = DWORD(sizeof (TSockaddr_in) + 16)
+    let dwRemoteAddressLength = DWORD(sizeof(TSockaddr_in) + 16)
+
+    template completeAccept(): stmt {.immediate, dirty.} =
+      var listenSock = socket
+      let setoptRet = setsockopt(clientSock, SOL_SOCKET,
+          SO_UPDATE_ACCEPT_CONTEXT, addr listenSock,
+          sizeof(listenSock).TSockLen)
+      if setoptRet != 0: osError(osLastError())
+
+      var LocalSockaddr, RemoteSockaddr: ptr TSockAddr
+      var localLen, remoteLen: int32
+      getAcceptExSockaddrs(addr lpOutputBuf[0], dwReceiveDataLength,
+                           dwLocalAddressLength, dwRemoteAddressLength,
+                           addr LocalSockaddr, addr localLen,
+                           addr RemoteSockaddr, addr remoteLen)
+      # TODO: IPv6. Check ``sa_family``. http://stackoverflow.com/a/9212542/492186
+      retFuture.complete(
+        (address: $inet_ntoa(cast[ptr Tsockaddr_in](remoteSockAddr).sin_addr),
+         client: clientSock)
+      )
+
+    var ol = cast[PCustomOverlapped](alloc0(sizeof(TCustomOverlapped)))
+    ol.data = TCompletionData(sock: socket, cb:
+      proc (sock: TSocketHandle, bytesCount: DWord, errcode: TOSErrorCode) =
+        if not retFuture.finished:
+          if errcode == TOSErrorCode(-1):
+            completeAccept()
+          else:
+            retFuture.fail(newException(EOS, osErrorMsg(errcode)))
+    )
+
+    # http://msdn.microsoft.com/en-us/library/windows/desktop/ms737524%28v=vs.85%29.aspx
+    let ret = acceptEx(socket, clientSock, addr lpOutputBuf[0],
+                       dwReceiveDataLength, 
+                       dwLocalAddressLength,
+                       dwRemoteAddressLength,
+                       addr dwBytesReceived, cast[POverlapped](ol))
+
+    if not ret:
+      let err = osLastError()
+      if err.int32 != ERROR_IO_PENDING:
+        retFuture.fail(newException(EOS, osErrorMsg(err)))
+        dealloc(ol)
+    else:
+      completeAccept()
+      # We don't deallocate ``ol`` here because even though this completed
+      # immediately poll will still be notified about its completion and it will
+      # free ``ol``.
+
+    return retFuture
+
+  initAll()
+else:
+  import selectors
+  from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK
+  type
+    TCallback = proc (sock: TSocketHandle): bool {.closure.}
+
+    PData* = ref object of PObject
+      sock: TSocketHandle
+      readCBs: seq[TCallback]
+      writeCBs: seq[TCallback]
+
+    PDispatcher* = ref object
+      selector: PSelector
+
+  proc newDispatcher*(): PDispatcher =
+    new result
+    result.selector = newSelector()
+
+  proc update(p: PDispatcher, sock: TSocketHandle, events: set[TEvent]) =
+    assert sock in p.selector
+    echo("Update: ", events)
+    if events == {}:
+      discard p.selector.unregister(sock)
+    else:
+      discard p.selector.update(sock, events)
+  
+  proc addRead(p: PDispatcher, sock: TSocketHandle, cb: TCallback) =
+    if sock notin p.selector:
+      var data = PData(sock: sock, readCBs: @[cb], writeCBs: @[])
+      p.selector.register(sock, {EvRead}, data.PObject)
+    else:
+      p.selector[sock].data.PData.readCBs.add(cb)
+      p.update(sock, p.selector[sock].events + {EvRead})
+  
+  proc addWrite(p: PDispatcher, sock: TSocketHandle, cb: TCallback) =
+    if sock notin p.selector:
+      var data = PData(sock: sock, readCBs: @[], writeCBs: @[cb])
+      p.selector.register(sock, {EvWrite}, data.PObject)
+    else:
+      p.selector[sock].data.PData.writeCBs.add(cb)
+      p.update(sock, p.selector[sock].events + {EvWrite})
+  
+  proc poll*(p: PDispatcher, timeout = 500) =
+    for info in p.selector.select(timeout):
+      let data = PData(info.key.data)
+      assert data.sock == info.key.fd
+      echo("R: ", data.readCBs.len, " W: ", data.writeCBs.len, ". ", info.events)
+      
+      if EvRead in info.events:
+        var newReadCBs: seq[TCallback] = @[]
+        for cb in data.readCBs:
+          if not cb(data.sock):
+            # Callback wants to be called again.
+            newReadCBs.add(cb)
+        data.readCBs = newReadCBs
+      
+      if EvWrite in info.events:
+        var newWriteCBs: seq[TCallback] = @[]
+        for cb in data.writeCBs:
+          if not cb(data.sock):
+            # Callback wants to be called again.
+            newWriteCBs.add(cb)
+        data.writeCBs = newWriteCBs
+  
+      var newEvents: set[TEvent]
+      if data.readCBs.len != 0: newEvents = {EvRead}
+      if data.writeCBs.len != 0: newEvents = newEvents + {EvWrite}
+      p.update(data.sock, newEvents)
+  
+  proc connect*(p: PDispatcher, socket: TSocketHandle, address: string, port: TPort,
+    af = AF_INET): PFuture[int] =
+    var retFuture = newFuture[int]()
+    
+    proc cb(sock: TSocketHandle): bool =
+      # We have connected.
+      retFuture.complete(0)
+      return true
+    
+    var aiList = getAddrInfo(address, port, af)
+    var success = false
+    var lastError: TOSErrorCode
+    var it = aiList
+    while it != nil:
+      var ret = connect(socket, it.ai_addr, it.ai_addrlen.TSocklen)
+      if ret == 0:
+        # Request to connect completed immediately.
+        success = true
+        retFuture.complete(0)
+        break
+      else:
+        lastError = osLastError()
+        if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS:
+          success = true
+          addWrite(p, socket, cb)
+          break
+        else:
+          success = false
+      it = it.ai_next
+
+    dealloc(aiList)
+    if not success:
+      retFuture.fail(newException(EOS, osErrorMsg(lastError)))
+    return retFuture
+
+  proc recv*(p: PDispatcher, socket: TSocketHandle, size: int,
+             flags: int = 0): PFuture[string] =
+    var retFuture = newFuture[string]()
+    
+    var readBuffer = newString(size)
+    var sizeRead = 0
+    
+    proc cb(sock: TSocketHandle): bool =
+      result = true
+      let netSize = size - sizeRead
+      let res = recv(sock, addr readBuffer[sizeRead], netSize, flags.cint)
+      if res < 0:
+        let lastError = osLastError()
+        if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: 
+          retFuture.fail(newException(EOS, osErrorMsg(lastError)))
+        else:
+          result = false # We still want this callback to be called.
+      elif res == 0:
+        # Disconnected
+        if sizeRead == 0:
+          retFuture.complete("")
+        else:
+          readBuffer.setLen(sizeRead)
+          retFuture.complete(readBuffer)
+      else:
+        sizeRead.inc(res)
+        if res != netSize:
+          result = false # We want to read all the data requested.
+        else:
+          retFuture.complete(readBuffer)
+  
+    addRead(p, socket, cb)
+    return retFuture
+
+  proc send*(p: PDispatcher, socket: TSocketHandle, data: string): PFuture[int] =
+    var retFuture = newFuture[int]()
+    
+    var written = 0
+    
+    proc cb(sock: TSocketHandle): bool =
+      result = true
+      let netSize = data.len-written
+      var d = data.cstring
+      let res = send(sock, addr d[written], netSize, 0.cint)
+      if res < 0:
+        let lastError = osLastError()
+        if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}:
+          retFuture.fail(newException(EOS, osErrorMsg(lastError)))
+        else:
+          result = false # We still want this callback to be called.
+      else:
+        written.inc(res)
+        if res != netSize:
+          result = false # We still have data to send.
+        else:
+          retFuture.complete(0)
+    addWrite(p, socket, cb)
+    return retFuture
+        
+
+  proc acceptAddr*(p: PDispatcher, socket: TSocketHandle): 
+      PFuture[tuple[address: string, client: TSocketHandle]] =
+    var retFuture = newFuture[tuple[address: string, client: TSocketHandle]]()
+    proc cb(sock: TSocketHandle): bool =
+      result = true
+      var sockAddress: Tsockaddr_in
+      var addrLen = sizeof(sockAddress).TSocklen
+      var client = accept(sock, cast[ptr TSockAddr](addr(sockAddress)),
+                          addr(addrLen))
+      if client == osInvalidSocket:
+        let lastError = osLastError()
+        assert lastError.int32 notin {EWOULDBLOCK, EAGAIN}
+        if lastError.int32 == EINTR:
+          return false
+        else:
+          retFuture.fail(newException(EOS, osErrorMsg(lastError)))
+      else:
+        retFuture.complete(($inet_ntoa(sockAddress.sin_addr), client))
+    addRead(p, socket, cb)
+    return retFuture
+
+proc accept*(p: PDispatcher, socket: TSocketHandle): PFuture[TSocketHandle] =
+  ## Accepts a new connection. Returns a future containing the client socket
+  ## corresponding to that connection.
+  ## The future will complete when the connection is successfully accepted.
+  var retFut = newFuture[TSocketHandle]()
+  var fut = p.acceptAddr(socket)
+  fut.callback =
+    proc (future: PFuture[tuple[address: string, client: TSocketHandle]]) =
+      assert future.finished
+      if future.failed:
+        retFut.fail(future.error)
+      else:
+        retFut.complete(future.read.client)
+  return retFut
+
+# -- Await Macro
+
+template createCb*(cbName, varNameIterSym, retFutureSym: expr): stmt {.immediate, dirty.} =
+  proc cbName {.closure.} =
+    if not varNameIterSym.finished:
+      var next = varNameIterSym()
+      if next == nil:
+        assert retFutureSym.finished, "Async procedure's return Future was not finished."
+      else:
+        next.callback = cbName
+
+template createVar(futSymName: string, asyncProc: PNimrodNode,
+                   valueReceiver: expr) {.immediate, dirty.} =
+  # TODO: Used template here due to bug #926
+  result = newNimNode(nnkStmtList)
+  var futSym = newIdentNode(futSymName) #genSym(nskVar, "future")
+  result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y
+  result.add newNimNode(nnkYieldStmt).add(futSym) # -> yield future<x>
+  valueReceiver = newDotExpr(futSym, newIdentNode("read")) # -> future<x>.read
+
+proc processBody(node, retFutureSym: PNimrodNode): PNimrodNode {.compileTime.} =
+  result = node
+  case node.kind
+  of nnkReturnStmt:
+    result = newNimNode(nnkStmtList)
+    result.add newCall(newIdentNode("complete"), retFutureSym,
+      if node[0].kind == nnkEmpty: newIdentNode("result") else: node[0])
+    result.add newNimNode(nnkYieldStmt).add(newNilLit())
+  of nnkCommand:
+    if node[0].ident == !"await":
+      case node[1].kind
+      of nnkIdent:
+        # await x
+        result = newNimNode(nnkYieldStmt).add(node[1]) # -> yield x
+      of nnkCall:
+        # await foo(p, x)
+        var futureValue: PNimrodNode
+        createVar("future" & $node[1][0].toStrLit, node[1], futureValue)
+        result.add futureValue
+      else:
+        error("Invalid node kind in 'await', got: " & $node[1].kind)
+    elif node[1].kind == nnkCommand and node[1][0].kind == nnkIdent and
+         node[1][0].ident == !"await":
+      # foo await x
+      var newCommand = node
+      createVar("future" & $node[0].ident, node[1][0], newCommand[1])
+      result.add newCommand
+
+  of nnkVarSection, nnkLetSection:
+    case node[0][2].kind
+    of nnkCommand:
+      if node[0][2][0].ident == !"await":
+        # var x = await y
+        var newVarSection = node # TODO: Should this use copyNimNode?
+        createVar("future" & $node[0][0].ident, node[0][2][1],
+          newVarSection[0][2])
+        result.add newVarSection
+    else: discard
+  of nnkAsgn:
+    case node[1].kind
+    of nnkCommand:
+      if node[1][0].ident == !"await":
+        # x = await y
+        var newAsgn = node
+        createVar("future" & $node[0].ident, node[1][1], newAsgn[1])
+        result.add newAsgn
+    else: discard
+  of nnkDiscardStmt:
+    # discard await x
+    if node[0][0].ident == !"await":
+      var dummy = newNimNode(nnkStmtList)
+      createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], dummy)
+  else: discard
+  
+  for i in 0 .. <result.len:
+    result[i] = processBody(result[i], retFutureSym)
+  #echo(treeRepr(result))
+
+proc getName(node: PNimrodNode): string {.compileTime.} =
+  case node.kind
+  of nnkPostfix:
+    return $node[1].ident
+  of nnkIdent:
+    return $node.ident
+  else:
+    assert false
+
+macro async*(prc: stmt): stmt {.immediate.} =
+  expectKind(prc, nnkProcDef)
+
+  hint("Processing " & prc[0].getName & " as an async proc.")
+
+  # Verify that the return type is a PFuture[T]
+  if prc[3][0].kind == nnkIdent:
+    error("Expected return type of 'PFuture' got '" & $prc[3][0] & "'")
+  elif prc[3][0].kind == nnkBracketExpr:
+    if $prc[3][0][0] != "PFuture":
+      error("Expected return type of 'PFuture' got '" & $prc[3][0][0] & "'")
+  
+  # TODO: Why can't I use genSym? I get illegal capture errors for Syms.
+  # TODO: It seems genSym is broken. Change all usages back to genSym when fixed
+
+  var outerProcBody = newNimNode(nnkStmtList)
+
+  # -> var retFuture = newFuture[T]()
+  var retFutureSym = newIdentNode("retFuture") #genSym(nskVar, "retFuture")
+  outerProcBody.add(
+    newVarStmt(retFutureSym, 
+      newCall(
+        newNimNode(nnkBracketExpr).add(
+          newIdentNode("newFuture"),
+          prc[3][0][1])))) # Get type from return type of this proc.
+
+  # -> iterator nameIter(): PFutureBase {.closure.} = 
+  # ->   var result: T
+  # ->   <proc_body>
+  # ->   complete(retFuture, result)
+  var iteratorNameSym = newIdentNode($prc[0].getName & "Iter") #genSym(nskIterator, $prc[0].ident & "Iter")
+  var procBody = prc[6].processBody(retFutureSym)
+  procBody.insert(0, newNimNode(nnkVarSection).add(
+    newIdentDefs(newIdentNode("result"), prc[3][0][1]))) # -> var result: T
+  procBody.add(
+    newCall(newIdentNode("complete"),
+      retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result)
+  
+  var closureIterator = newProc(iteratorNameSym, [newIdentNode("PFutureBase")],
+                                procBody, nnkIteratorDef)
+  closureIterator[4] = newNimNode(nnkPragma).add(newIdentNode("closure"))
+  outerProcBody.add(closureIterator)
+
+  # -> var nameIterVar = nameIter
+  # -> var first = nameIterVar()
+  var varNameIterSym = newIdentNode($prc[0].getName & "IterVar") #genSym(nskVar, $prc[0].ident & "IterVar")
+  var varNameIter = newVarStmt(varNameIterSym, iteratorNameSym)
+  outerProcBody.add varNameIter
+  var varFirstSym = genSym(nskVar, "first")
+  var varFirst = newVarStmt(varFirstSym, newCall(varNameIterSym))
+  outerProcBody.add varFirst
+
+  # -> createCb(cb, nameIter, retFuture)
+  var cbName = newIdentNode("cb")
+  var procCb = newCall("createCb", cbName, varNameIterSym, retFutureSym)
+  outerProcBody.add procCb
+
+  # -> first.callback = cb
+  outerProcBody.add newAssignment(
+    newDotExpr(varFirstSym, newIdentNode("callback")),
+    cbName)
+
+  # -> return retFuture
+  outerProcBody.add newNimNode(nnkReturnStmt).add(retFutureSym)
+  
+  result = prc
+
+  # Remove the 'async' pragma.
+  for i in 0 .. <result[4].len:
+    if result[4][i].ident == !"async":
+      result[4].del(i)
+
+  result[6] = outerProcBody
+
+  echo(toStrLit(result))
+
+proc recvLine*(p: PDispatcher, socket: TSocketHandle): PFuture[string] {.async.} =
+  ## Reads a line of data from ``socket``. Returned future will complete once
+  ## a full line is read or an error occurs.
+  ##
+  ## If a full line is read ``\r\L`` is not
+  ## added to ``line``, however if solely ``\r\L`` is read then ``line``
+  ## will be set to it.
+  ## 
+  ## If the socket is disconnected, ``line`` will be set to ``""``.
+  
+  template addNLIfEmpty(): stmt =
+    if result.len == 0:
+      result.add("\c\L")
+
+  result = ""
+  var c = ""
+  while true:
+    c = await p.recv(socket, 1)
+    if c.len == 0:
+      return
+    if c == "\r":
+      c = await p.recv(socket, 1, MSG_PEEK)
+      if c.len > 0 and c == "\L":
+        discard await p.recv(socket, 1)
+      addNLIfEmpty()
+      return
+    elif c == "\L":
+      addNLIfEmpty()
+      return
+    add(result.string, c)
+
+when isMainModule:
+  
+  var p = newDispatcher()
+  var sock = socket()
+  sock.setBlocking false
+
+
+  when false:
+    # Await tests
+    proc main(p: PDispatcher): PFuture[int] {.async.} =
+      discard await p.connect(sock, "irc.freenode.net", TPort(6667))
+      while true:
+        var line = await p.recvLine(sock)
+        echo("Line is: ", line.repr)
+        if line == "":
+          echo "Disconnected"
+          break
+
+    proc peekTest(p: PDispatcher): PFuture[int] {.async.} =
+      discard await p.connect(sock, "localhost", TPort(6667))
+      while true:
+        var line = await p.recv(sock, 1, MSG_PEEK)
+        var line2 = await p.recv(sock, 1)
+        echo(line.repr)
+        echo(line2.repr)
+        echo("---")
+        if line2 == "": break
+        sleep(500)
+
+    var f = main(p)
+    
+
+  else:
+    when false:
+
+      var f = p.connect(sock, "irc.freenode.org", TPort(6667))
+      f.callback =
+        proc (future: PFuture[int]) =
+          echo("Connected in future!")
+          echo(future.read)
+          for i in 0 .. 50:
+            var recvF = p.recv(sock, 10)
+            recvF.callback =
+              proc (future: PFuture[string]) =
+                echo("Read ", future.read.len, ": ", future.read.repr)
+
+    else:
+
+      sock.bindAddr(TPort(6667))
+      sock.listen()
+      proc onAccept(future: PFuture[TSocketHandle]) =
+        echo "Accepted"
+        var t = p.send(future.read, "test\c\L")
+        t.callback =
+          proc (future: PFuture[int]) =
+            echo(future.read)
+        
+        var f = p.accept(sock)
+        f.callback = onAccept
+        
+      var f = p.accept(sock)
+      f.callback = onAccept
+  
+  while true:
+    p.poll()
+
+
+
+
+
+  
+
+  

lib/pure/collections/sets.nim

@@ -241,3 +241,7 @@ proc `<=`*[A](s, t: TSet[A]): bool =
       
 proc `==`*[A](s, t: TSet[A]): bool =
   s.counter == t.counter and s <= t
+
+proc map*[A, B](data: TSet[A], op: proc (x: A): B {.closure.}): TSet[B] =
+  result = initSet[B]()
+  for item in data: result.incl(op(item))

lib/pure/dynlib.nim

@@ -14,7 +14,7 @@
 type
   TLibHandle* = pointer ## a handle to a dynamically loaded library
 
-proc loadLib*(path: string): TLibHandle
+proc loadLib*(path: string, global_symbols=false): TLibHandle
   ## loads a library from `path`. Returns nil if the library could not 
   ## be loaded.
 
@@ -53,6 +53,7 @@ when defined(posix):
   #
   var
     RTLD_NOW {.importc: "RTLD_NOW", header: "<dlfcn.h>".}: int
+    RTLD_GLOBAL {.importc: "RTLD_GLOBAL", header: "<dlfcn.h>".}: int
 
   proc dlclose(lib: TLibHandle) {.importc, header: "<dlfcn.h>".}
   proc dlopen(path: CString, mode: int): TLibHandle {.
@@ -60,7 +61,10 @@ when defined(posix):
   proc dlsym(lib: TLibHandle, name: cstring): pointer {.
       importc, header: "<dlfcn.h>".}
 
-  proc loadLib(path: string): TLibHandle = return dlopen(path, RTLD_NOW)
+  proc loadLib(path: string, global_symbols=false): TLibHandle = 
+    var flags = RTLD_NOW
+    if global_symbols: flags = flags or RTLD_GLOBAL
+    return dlopen(path, flags)
   proc loadLib(): TLibHandle = return dlopen(nil, RTLD_NOW)
   proc unloadLib(lib: TLibHandle) = dlclose(lib)
   proc symAddr(lib: TLibHandle, name: cstring): pointer = 
@@ -81,7 +85,7 @@ elif defined(windows) or defined(dos):
   proc getProcAddress(lib: THINSTANCE, name: cstring): pointer {.
       importc: "GetProcAddress", header: "<windows.h>", stdcall.}
 
-  proc loadLib(path: string): TLibHandle =
+  proc loadLib(path: string, global_symbols=false): TLibHandle =
     result = cast[TLibHandle](winLoadLibrary(path))
   proc loadLib(): TLibHandle =
     result = cast[TLibHandle](winLoadLibrary(nil))

lib/pure/htmlgen.nim

@@ -1,12 +1,17 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2014 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 #
 
+## **Warning**: This module uses ``immediate`` macros which are known to
+## cause problems. Do yourself a favor and import the module
+## as ``from htmlgen import nil`` and then fully qualify the macros.
+##
+##
 ## This module implements a simple `XML`:idx: and `HTML`:idx: code 
 ## generator. Each commonly used HTML tag has a corresponding macro
 ## that generates a string with its HTML representation.
@@ -15,11 +20,11 @@
 ##
 ## .. code-block:: nimrod
 ##   var nim = "Nimrod"
-##   echo h1(a(href="http://nimrod-code.org", nim))
+##   echo h1(a(href="http://nimrod-lang.org", nim))
 ##  
 ## Writes the string::
 ##   
-##   <h1><a href="http://nimrod-code.org">Nimrod</a></h1>
+##   <h1><a href="http://nimrod-lang.org">Nimrod</a></h1>
 ##
 
 import

lib/pure/net.nim

@@ -0,0 +1,55 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2014 Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements a high-level cross-platform sockets interface.
+
+import sockets2, os
+
+type
+  TSocket* = TSocketHandle
+
+proc bindAddr*(socket: TSocket, port = TPort(0), address = "") {.
+  tags: [FReadIO].} =
+
+  ## binds an address/port number to a socket.
+  ## Use address string in dotted decimal form like "a.b.c.d"
+  ## or leave "" for any address.
+
+  if address == "":
+    var name: TSockaddr_in
+    when defined(windows):
+      name.sin_family = toInt(AF_INET).int16
+    else:
+      name.sin_family = toInt(AF_INET)
+    name.sin_port = htons(int16(port))
+    name.sin_addr.s_addr = htonl(INADDR_ANY)
+    if bindAddr(socket, cast[ptr TSockAddr](addr(name)),
+                  sizeof(name).TSocklen) < 0'i32:
+      osError(osLastError())
+  else:
+    var aiList = getAddrInfo(address, port, AF_INET)
+    if bindAddr(socket, aiList.ai_addr, aiList.ai_addrlen.TSocklen) < 0'i32:
+      dealloc(aiList)
+      osError(osLastError())
+    dealloc(aiList)
+
+proc setBlocking*(s: TSocket, blocking: bool) {.tags: [].} =
+  ## Sets blocking mode on socket
+  when defined(Windows):
+    var mode = clong(ord(not blocking)) # 1 for non-blocking, 0 for blocking
+    if ioctlsocket(s, FIONBIO, addr(mode)) == -1:
+      osError(osLastError())
+  else: # BSD sockets
+    var x: int = fcntl(s, F_GETFL, 0)
+    if x == -1:
+      osError(osLastError())
+    else:
+      var mode = if blocking: x and not O_NONBLOCK else: x or O_NONBLOCK
+      if fcntl(s, F_SETFL, mode) == -1:
+        osError(osLastError())

lib/pure/nimprof.nim

@@ -67,7 +67,7 @@ when withThreads:
 
 proc hookAux(st: TStackTrace, costs: int) =
   # this is quite performance sensitive!
-  when withThreads: Acquire profilingLock
+  when withThreads: acquire profilingLock
   inc totalCalls
   var last = high(st)
   while last > 0 and isNil(st[last]): dec last
@@ -106,7 +106,7 @@ proc hookAux(st: TStackTrace, costs: int) =
       h = ((5 * h) + 1) and high(profileData)
       inc chain
     maxChainLen = max(maxChainLen, chain)
-  when withThreads: Release profilingLock
+  when withThreads: release profilingLock
 
 when defined(memProfiler):
   const

lib/pure/os.nim

@@ -1037,7 +1037,10 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1",
   ## the process has finished. To execute a program without having a
   ## shell involved, use the `execProcess` proc of the `osproc`
   ## module.
+  when defined(linux):
     result = c_system(command) shr 8
+  else:
+    result = c_system(command)
 
 # Environment handling cannot be put into RTL, because the ``envPairs``
 # iterator depends on ``environment``.
@@ -1189,7 +1192,8 @@ iterator walkFiles*(pattern: string): string {.tags: [FReadDir].} =
     res = findFirstFile(pattern, f)
     if res != -1:
       while true:
-        if not skipFindData(f):
+        if not skipFindData(f) and
+            (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) == 0'i32:
           yield splitFile(pattern).dir / extractFilename(getFilename(f))
         if findNextFile(res, f) == 0'i32: break
       findClose(res)

lib/pure/osproc.nim

@@ -13,13 +13,16 @@
 include "system/inclrtl"
 
 import
-  strutils, os, strtabs, streams, sequtils
+  strutils, os, strtabs, streams
 
 when defined(windows):
   import winlean
 else:
   import posix
 
+when defined(linux):
+  import linux
+
 type
   TProcess = object of TObject
     when defined(windows):
@@ -44,7 +47,7 @@ type
     poStdErrToStdOut,    ## merge stdout and stderr to the stdout stream
     poParentStreams      ## use the parent's streams
 
-template poUseShell*: TProcessOption {.deprecated.} = poUsePath
+const poUseShell* {.deprecated.} = poUsePath
   ## Deprecated alias for poUsePath.
 
 proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} =
@@ -165,6 +168,9 @@ proc processID*(p: PProcess): int {.rtl, extern: "nosp$1".} =
 proc waitForExit*(p: PProcess, timeout: int = -1): int {.rtl,
   extern: "nosp$1", tags: [].}
   ## waits for the process to finish and returns `p`'s error code.
+  ##
+  ## **Warning**: Be careful when using waitForExit for processes created without
+  ## poParentStreams because they may fill output buffers, causing deadlock.
 
 proc peekExitCode*(p: PProcess): int {.tags: [].}
   ## return -1 if the process is still running. Otherwise the process' exit code
@@ -590,6 +596,23 @@ elif not defined(useNimRtl):
       copyMem(result[i], addr(x[0]), x.len+1)
       inc(i)
 
+  type TStartProcessData = object
+    sysCommand: cstring
+    sysArgs: cstringArray
+    sysEnv: cstringArray
+    workingDir: cstring
+    pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint]
+    optionPoUsePath: bool
+    optionPoParentStreams: bool
+    optionPoStdErrToStdOut: bool
+
+  proc startProcessAuxSpawn(data: TStartProcessData): TPid {.tags: [FExecIO, FReadEnv].}
+  proc startProcessAuxFork(data: TStartProcessData): TPid {.tags: [FExecIO, FReadEnv].}
+  {.push stacktrace: off, profiler: off.}
+  proc startProcessAfterFork(data: ptr TStartProcessData) {.
+    tags: [FExecIO, FReadEnv], cdecl.}
+  {.pop.}
+
   proc startProcess(command: string,
                  workingDir: string = "",
                  args: openArray[string] = [],
@@ -604,100 +627,48 @@ elif not defined(useNimRtl):
          pipe(pStderr) != 0'i32:
         osError(osLastError())
 
-    var sys_command: string
+    var sysCommand: string
-    var sys_args_raw: seq[string]
+    var sysArgsRaw: seq[string]
     if poEvalCommand in options:
-      sys_command = "/bin/sh"
+      sysCommand = "/bin/sh"
-      sys_args_raw = @[sys_command, "-c", command]
+      sysArgsRaw = @[sysCommand, "-c", command]
       assert args.len == 0
     else:
-      sys_command = command
+      sysCommand = command
-      sys_args_raw = @[command]
+      sysArgsRaw = @[command]
       for arg in args.items:
-        sys_args_raw.add arg
+        sysArgsRaw.add arg
-
-    var sys_args = allocCStringArray(sys_args_raw)
-    finally: deallocCStringArray(sys_args)
 
     var pid: TPid
-    when defined(posix_spawn) and not defined(useFork):
-      var attr: Tposix_spawnattr
-      var fops: Tposix_spawn_file_actions
 
-      template chck(e: expr) =
+    var sysArgs = allocCStringArray(sysArgsRaw)
-        if e != 0'i32: osError(osLastError())
+    finally: deallocCStringArray(sysArgs)
 
-      chck posix_spawn_file_actions_init(fops)
+    var sysEnv = if env == nil:
-      chck posix_spawnattr_init(attr)
+        envToCStringArray()
-
-      var mask: Tsigset
-      chck sigemptyset(mask)
-      chck posix_spawnattr_setsigmask(attr, mask)
-      chck posix_spawnattr_setpgroup(attr, 0'i32)
-
-      chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or
-                                          POSIX_SPAWN_SETSIGMASK or
-                                          POSIX_SPAWN_SETPGROUP)
-
-      if poParentStreams notin options:
-        chck posix_spawn_file_actions_addclose(fops, pStdin[writeIdx])
-        chck posix_spawn_file_actions_adddup2(fops, pStdin[readIdx], readIdx)
-        chck posix_spawn_file_actions_addclose(fops, pStdout[readIdx])
-        chck posix_spawn_file_actions_adddup2(fops, pStdout[writeIdx], writeIdx)
-        chck posix_spawn_file_actions_addclose(fops, pStderr[readIdx])
-        if poStdErrToStdOut in options:
-          chck posix_spawn_file_actions_adddup2(fops, pStdout[writeIdx], 2)
       else:
-          chck posix_spawn_file_actions_adddup2(fops, p_stderr[writeIdx], 2)
+        envToCStringArray(env)
 
-      var sys_env = if env == nil: envToCStringArray() else: envToCStringArray(env)
+    finally: deallocCStringArray(sysEnv)
-      var res: cint
-      # This is incorrect!
-      if workingDir.len > 0: os.setCurrentDir(workingDir)
-      if poUsePath in options:
-        res = posix_spawnp(pid, sys_command, fops, attr, sys_args, sys_env)
-      else:
-        res = posix_spawn(pid, sys_command, fops, attr, sys_args, sys_env)
-      deallocCStringArray(sys_env)
-      discard posix_spawn_file_actions_destroy(fops)
-      discard posix_spawnattr_destroy(attr)
-      chck res
 
-    else:
+    var data: TStartProcessData
-      pid = fork()
+    data.sysCommand = sysCommand
-      if pid < 0: osError(osLastError())
+    data.sysArgs = sysArgs
-      if pid == 0:
+    data.sysEnv = sysEnv
-        ## child process:
+    data.pStdin = pStdin
+    data.pStdout = pStdout
+    data.pStderr = pStderr
+    data.optionPoParentStreams = poParentStreams in options
+    data.optionPoUsePath = poUsePath in options
+    data.optionPoStdErrToStdOut = poStdErrToStdOut in options
+    data.workingDir = workingDir
 
-        if poParentStreams notin options:
-          discard close(p_stdin[writeIdx])
-          if dup2(p_stdin[readIdx], readIdx) < 0: osError(osLastError())
-          discard close(p_stdout[readIdx])
-          if dup2(p_stdout[writeIdx], writeIdx) < 0: osError(osLastError())
-          discard close(p_stderr[readIdx])
-          if poStdErrToStdOut in options:
-            if dup2(p_stdout[writeIdx], 2) < 0: osError(osLastError())
-          else:
-            if dup2(p_stderr[writeIdx], 2) < 0: osError(osLastError())
 
-        # Create a new process group
+    when defined(posix_spawn) and not defined(useFork) and not defined(useClone) and not defined(linux):
-        if setpgid(0, 0) == -1: quit("setpgid call failed: " & $strerror(errno))
+      pid = startProcessAuxSpawn(data)
+    else:
+      pid = startProcessAuxFork(data)
 
-        if workingDir.len > 0: os.setCurrentDir(workingDir)
-
-        if env == nil:
-          if poUsePath in options:
-            discard execvp(sys_command, sys_args)
-          else:
-            discard execv(sys_command, sys_args)
-        else:
-          var c_env = envToCStringArray(env)
-          if poUsePath in options:
-            discard execvpe(sys_command, sys_args, c_env)
-          else:
-            discard execve(sys_command, sys_args, c_env)
-        # too risky to raise an exception here:
-        quit("execve call failed: " & $strerror(errno))
     # Parent process. Copy process information.
     if poEchoCmd in options:
       echo(command, " ", join(args, " "))
@@ -723,6 +694,137 @@ elif not defined(useNimRtl):
       discard close(pStdin[readIdx])
       discard close(pStdout[writeIdx])
 
+  proc startProcessAuxSpawn(data: TStartProcessData): TPid =
+    var attr: Tposix_spawnattr
+    var fops: Tposix_spawn_file_actions
+
+    template chck(e: expr) =
+      if e != 0'i32: osError(osLastError())
+
+    chck posix_spawn_file_actions_init(fops)
+    chck posix_spawnattr_init(attr)
+
+    var mask: Tsigset
+    chck sigemptyset(mask)
+    chck posix_spawnattr_setsigmask(attr, mask)
+    chck posix_spawnattr_setpgroup(attr, 0'i32)
+
+    chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or
+                                        POSIX_SPAWN_SETSIGMASK or
+                                        POSIX_SPAWN_SETPGROUP)
+
+    if not data.optionPoParentStreams:
+      chck posix_spawn_file_actions_addclose(fops, data.pStdin[writeIdx])
+      chck posix_spawn_file_actions_adddup2(fops, data.pStdin[readIdx], readIdx)
+      chck posix_spawn_file_actions_addclose(fops, data.pStdout[readIdx])
+      chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], writeIdx)
+      chck posix_spawn_file_actions_addclose(fops, data.pStderr[readIdx])
+      if data.optionPoStdErrToStdOut:
+        chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], 2)
+      else:
+        chck posix_spawn_file_actions_adddup2(fops, data.pStderr[writeIdx], 2)
+
+    var res: cint
+    # FIXME: chdir is global to process
+    if data.workingDir.len > 0:
+      setCurrentDir($data.workingDir)
+    var pid: TPid
+
+    if data.optionPoUsePath:
+      res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv)
+    else:
+      res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv)
+
+    discard posix_spawn_file_actions_destroy(fops)
+    discard posix_spawnattr_destroy(attr)
+    chck res
+    return pid
+
+  proc startProcessAuxFork(data: TStartProcessData): TPid =
+    if pipe(data.pErrorPipe) != 0:
+        osError(osLastError())
+
+    finally:
+      discard close(data.pErrorPipe[readIdx])
+
+    var pid: TPid
+    var dataCopy = data
+
+    when defined(useClone):
+      const stackSize = 65536
+      let stackEnd = cast[clong](alloc(stackSize))
+      let stack = cast[pointer](stackEnd + stackSize)
+      let fn: pointer = startProcessAfterFork
+      pid = clone(fn, stack,
+                  cint(CLONE_VM or CLONE_VFORK or SIGCHLD),
+                  pointer(addr dataCopy), nil, nil, nil)
+      discard close(data.pErrorPipe[writeIdx])
+      dealloc(stack)
+    else:
+      pid = fork()
+      if pid == 0:
+        startProcessAfterFork(addr(dataCopy))
+        exitnow(1)
+
+    discard close(data.pErrorPipe[writeIdx])
+    if pid < 0: osError(osLastError())
+
+    var error: cint
+    let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error))
+    if sizeRead == sizeof(error):
+      osError($strerror(error))
+
+    return pid
+
+  {.push stacktrace: off, profiler: off.}
+  proc startProcessFail(data: ptr TStartProcessData) =
+    var error: cint = errno
+    discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error))
+    exitnow(1)
+
+  when defined(macosx):
+    var environ {.importc.}: cstringArray
+
+  proc startProcessAfterFork(data: ptr TStartProcessData) =
+    # Warning: no GC here!
+    # Or anythink that touches global structures - all called nimrod procs
+    # must be marked with noStackFrame. Inspect C code after making changes.
+    if not data.optionPoParentStreams:
+      discard close(data.pStdin[writeIdx])
+      if dup2(data.pStdin[readIdx], readIdx) < 0:
+        startProcessFail(data)
+      discard close(data.pStdout[readIdx])
+      if dup2(data.pStdout[writeIdx], writeIdx) < 0:
+        startProcessFail(data)
+      discard close(data.pStderr[readIdx])
+      if data.optionPoStdErrToStdOut:
+        if dup2(data.pStdout[writeIdx], 2) < 0:
+          startProcessFail(data)
+      else:
+        if dup2(data.pStderr[writeIdx], 2) < 0:
+          startProcessFail(data)
+
+    if data.workingDir.len > 0:
+      if chdir(data.workingDir) < 0:
+        startProcessFail(data)
+
+    discard close(data.pErrorPipe[readIdx])
+    discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC)
+
+    if data.optionPoUsePath:
+      when defined(macosx):
+        # MacOSX doesn't have execvpe, so we need workaround.
+        # On MacOSX we can arrive here only from fork, so this is safe:
+        environ = data.sysEnv
+        discard execvp(data.sysCommand, data.sysArgs)
+      else:
+        discard execvpe(data.sysCommand, data.sysArgs, data.sysEnv)
+    else:
+      discard execve(data.sysCommand, data.sysArgs, data.sysEnv)
+
+    startProcessFail(data)
+  {.pop}
+
   proc close(p: PProcess) =
     if p.inStream != nil: close(p.inStream)
     if p.outStream != nil: close(p.outStream)
@@ -791,7 +893,10 @@ elif not defined(useNimRtl):
   proc csystem(cmd: cstring): cint {.nodecl, importc: "system".}
 
   proc execCmd(command: string): int =
+    when defined(linux):
       result = csystem(command) shr 8
+    else:
+      result = csystem(command)
 
   proc createFdSet(fd: var TFdSet, s: seq[PProcess], m: var int) =
     FD_ZERO(fd)

lib/pure/pegs.nim

@@ -836,7 +836,9 @@ iterator findAll*(s: string, pattern: TPeg, start = 0): string =
   while i < s.len:
     c.ml = 0
     var L = rawMatch(s, pattern, i, c)
-    if L < 0: break
+    if L < 0:
+      inc(i, 1)
+    else:
       yield substr(s, i, i+L-1)
       inc(i, L)
     

lib/pure/selectors.nim

@@ -0,0 +1,250 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2014 Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# TODO: Docs.
+
+import tables, os, unsigned, hashes
+
+when defined(linux): import posix, epoll
+elif defined(windows): import winlean
+
+proc hash*(x: TSocketHandle): THash {.borrow.}
+
+type
+  TEvent* = enum
+    EvRead, EvWrite
+
+  PSelectorKey* = ref object
+    fd*: TSocketHandle
+    events*: set[TEvent] ## The events which ``fd`` listens for.
+    data*: PObject ## User object.
+
+  TReadyInfo* = tuple[key: PSelectorKey, events: set[TEvent]]
+
+when defined(linux) or defined(nimdoc):
+  type
+    PSelector* = ref object
+      epollFD: cint
+      events: array[64, ptr epoll_event]
+      fds: TTable[TSocketHandle, PSelectorKey]
+  
+  proc createEventStruct(events: set[TEvent], fd: TSocketHandle): epoll_event =
+    if EvRead in events:
+      result.events = EPOLLIN
+    if EvWrite in events:
+      result.events = result.events or EPOLLOUT
+    result.data.fd = fd.cint
+  
+  proc register*(s: PSelector, fd: TSocketHandle, events: set[TEvent],
+      data: PObject): PSelectorKey {.discardable.} =
+    ## Registers file descriptor ``fd`` to selector ``s`` with a set of TEvent
+    ## ``events``.
+    if s.fds.hasKey(fd):
+      raise newException(EInvalidValue, "File descriptor already exists.")
+    
+    var event = createEventStruct(events, fd)
+  
+    if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fd, addr(event)) != 0:
+      OSError(OSLastError())
+  
+    var key = PSelectorKey(fd: fd, events: events, data: data)
+  
+    s.fds[fd] = key
+    result = key
+  
+  proc update*(s: PSelector, fd: TSocketHandle,
+      events: set[TEvent]): PSelectorKey {.discardable.} =
+    ## Updates the events which ``fd`` wants notifications for.
+    if not s.fds.hasKey(fd):
+      raise newException(EInvalidValue, "File descriptor not found.")
+    var event = createEventStruct(events, fd)
+    
+    s.fds[fd].events = events
+    echo("About to update")
+    if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fd, addr(event)) != 0:
+      OSError(OSLastError())
+    echo("finished updating")
+    result = s.fds[fd]
+  
+  proc unregister*(s: PSelector, fd: TSocketHandle): PSelectorKey {.discardable.} =
+    if not s.fds.hasKey(fd):
+      raise newException(EInvalidValue, "File descriptor not found.")
+    if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fd, nil) != 0:
+      OSError(OSLastError())
+    result = s.fds[fd]
+    s.fds.del(fd)
+
+  proc close*(s: PSelector) =
+    if s.epollFD.close() != 0: OSError(OSLastError())
+    dealloc(addr s.events) # TODO: Test this
+  
+  proc select*(s: PSelector, timeout: int): seq[TReadyInfo] =
+    ##
+    ## The ``events`` field of the returned ``key`` contains the original events
+    ## for which the ``fd`` was bound. This is contrary to the ``events`` field
+    ## of the ``TReadyInfo`` tuple which determines which events are ready
+    ## on the ``fd``.
+    result = @[]
+    
+    let evNum = epoll_wait(s.epollFD, s.events[0], 64.cint, timeout.cint)
+    if evNum < 0: OSError(OSLastError())
+    if evNum == 0: return @[]
+    for i in 0 .. <evNum:
+      var evSet: set[TEvent] = {}
+      if (s.events[i].events and EPOLLIN) != 0: evSet = evSet + {EvRead}
+      if (s.events[i].events and EPOLLOUT) != 0: evSet = evSet + {EvWrite}
+      
+      let selectorKey = s.fds[s.events[i].data.fd.TSocketHandle]
+      result.add((selectorKey, evSet))
+  
+  proc newSelector*(): PSelector =
+    new result
+    result.epollFD = epoll_create(64)
+    result.events = cast[array[64, ptr epoll_event]](alloc0(sizeof(epoll_event)*64))
+    result.fds = initTable[TSocketHandle, PSelectorKey]()
+    if result.epollFD < 0:
+      OSError(OSLastError())
+
+  proc contains*(s: PSelector, fd: TSocketHandle): bool =
+    ## Determines whether selector contains a file descriptor.
+    return s.fds.hasKey(fd)
+
+  proc `[]`*(s: PSelector, fd: TSocketHandle): PSelectorKey =
+    ## Retrieves the selector key for ``fd``.
+    return s.fds[fd]
+
+elif defined(windows):
+  type
+    PSelector* = ref object
+      fds: TTable[TSocketHandle, PSelectorKey]
+
+  proc register*(s: PSelector, fd: TSocketHandle, events: set[TEvent],
+      data: PObject): PSelectorKey {.discardable.} =
+    if s.fds.hasKey(fd):
+      raise newException(EInvalidValue, "File descriptor already exists.")
+    var sk = PSelectorKey(fd: fd, events: events, data: data)
+    s.fds[fd] = sk
+    result = sk
+
+  proc update*(s: PSelector, fd: TSocketHandle,
+      events: set[TEvent]): PSelectorKey {.discardable.} =
+    ## Updates the events which ``fd`` wants notifications for.
+    if not s.fds.hasKey(fd):
+      raise newException(EInvalidValue, "File descriptor not found.")
+
+    s.fds[fd].events = events
+    result = s.fds[fd]
+
+  proc unregister*(s: PSelector, fd: TSocketHandle): PSelectorKey {.discardable.} =
+    result = s.fds[fd]
+    s.fds.del(fd)
+
+  proc close*(s: PSelector) = nil
+
+  proc timeValFromMilliseconds(timeout: int): TTimeVal =
+    if timeout != -1:
+      var seconds = timeout div 1000
+      result.tv_sec = seconds.int32
+      result.tv_usec = ((timeout - seconds * 1000) * 1000).int32
+
+  proc createFdSet(rd, wr: var TFdSet, fds: TTable[TSocketHandle, PSelectorKey],
+      m: var int) =
+    FD_ZERO(rd); FD_ZERO(wr)
+    for k, v in pairs(fds):
+      if EvRead in v.events: 
+        m = max(m, int(k))
+        FD_SET(k, rd)
+      if EvWrite in v.events:
+        m = max(m, int(k))
+        FD_SET(k, wr)
+     
+  proc getReadyFDs(rd, wr: var TFdSet, fds: TTable[TSocketHandle, PSelectorKey]):
+      seq[TReadyInfo] =
+    result = @[]
+    for k, v in pairs(fds):
+      var events: set[TEvent] = {}
+      if FD_ISSET(k, rd) != 0'i32:
+        events = events + {EvRead}
+      if FD_ISSET(k, wr) != 0'i32:
+        events = events + {EvWrite}
+      result.add((v, events))
+
+  proc select(fds: TTable[TSocketHandle, PSelectorKey], timeout = 500):
+    seq[TReadyInfo] =
+    var tv {.noInit.}: TTimeVal = timeValFromMilliseconds(timeout)
+    
+    var rd, wr: TFdSet
+    var m = 0
+    createFdSet(rd, wr, fds, m)
+    
+    var retCode = 0
+    if timeout != -1:
+      retCode = int(select(TSocketHandle(m+1), addr(rd), addr(wr), nil, addr(tv)))
+    else:
+      retCode = int(select(TSocketHandle(m+1), addr(rd), addr(wr), nil, nil))
+    
+    if retCode < 0:
+      OSError(OSLastError())
+    elif retCode == 0:
+      return @[]
+    else:
+      return getReadyFDs(rd, wr, fds)
+
+  proc select*(s: PSelector, timeout: int): seq[TReadyInfo] =
+    result = select(s.fds, timeout)
+
+  proc newSelector*(): PSelector =
+    new result
+    result.fds = initTable[TSocketHandle, PSelectorKey]()
+
+  proc contains*(s: PSelector, fd: TSocketHandle): bool =
+    return s.fds.hasKey(fd)
+
+  proc `[]`*(s: PSelector, fd: TSocketHandle): PSelectorKey =
+    return s.fds[fd]
+
+elif defined(bsd) or defined(macosx):
+  # TODO: kqueue
+  {.error: "Sorry your platform is not supported yet.".}
+else:
+  {.error: "Sorry your platform is not supported.".}
+
+when isMainModule:
+  # Select()
+  import sockets
+  type
+    PSockWrapper = ref object of PObject
+      sock: TSocket
+  
+  var sock = socket()
+  sock.setBlocking(false)
+  sock.connect("irc.freenode.net", TPort(6667))
+  
+  var selector = newSelector()
+  var data = PSockWrapper(sock: sock)
+  let key = selector.register(sock.getFD, {EvWrite}, data)
+  var i = 0
+  while true:
+    let ready = selector.select(1000)
+    echo ready.len
+    if ready.len > 0: echo ready[0].events
+    i.inc
+    if i == 6:
+      assert selector.unregister(sock.getFD).fd == sock.getFD
+      selector.close()
+      break
+  
+  
+  
+  
+  
+  
+  
+  
+  

lib/pure/sockets2.nim

@@ -0,0 +1,213 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2014 Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements a low-level cross-platform sockets interface. Look
+## at the ``net`` module for the higher-level version.
+
+import unsigned, os
+
+when hostos == "solaris":
+  {.passl: "-lsocket -lnsl".}
+
+when defined(Windows):
+  import winlean
+else:
+  import posix
+  export fcntl, F_GETFL, O_NONBLOCK, F_SETFL
+
+export TSocketHandle, TSockaddr_in, TAddrinfo, INADDR_ANY, TSockAddr, TSockLen,
+  inet_ntoa, recv, `==`, connect, send, accept
+
+type
+  
+  TPort* = distinct uint16  ## port type
+  
+  TDomain* = enum   ## domain, which specifies the protocol family of the
+                    ## created socket. Other domains than those that are listed
+                    ## here are unsupported.
+    AF_UNIX,        ## for local socket (using a file). Unsupported on Windows.
+    AF_INET = 2,    ## for network protocol IPv4 or
+    AF_INET6 = 23   ## for network protocol IPv6.
+
+  TType* = enum        ## second argument to `socket` proc
+    SOCK_STREAM = 1,   ## reliable stream-oriented service or Stream Sockets
+    SOCK_DGRAM = 2,    ## datagram service or Datagram Sockets
+    SOCK_RAW = 3,      ## raw protocols atop the network layer.
+    SOCK_SEQPACKET = 5 ## reliable sequenced packet service
+
+  TProtocol* = enum     ## third argument to `socket` proc
+    IPPROTO_TCP = 6,    ## Transmission control protocol. 
+    IPPROTO_UDP = 17,   ## User datagram protocol.
+    IPPROTO_IP,         ## Internet protocol. Unsupported on Windows.
+    IPPROTO_IPV6,       ## Internet Protocol Version 6. Unsupported on Windows.
+    IPPROTO_RAW,        ## Raw IP Packets Protocol. Unsupported on Windows.
+    IPPROTO_ICMP        ## Control message protocol. Unsupported on Windows.
+
+  TServent* {.pure, final.} = object ## information about a service
+    name*: string
+    aliases*: seq[string]
+    port*: TPort
+    proto*: string
+
+  Thostent* {.pure, final.} = object ## information about a given host
+    name*: string
+    aliases*: seq[string]
+    addrtype*: TDomain
+    length*: int
+    addrList*: seq[string]
+
+when defined(windows):
+  let
+    osInvalidSocket* = winlean.INVALID_SOCKET
+
+  const
+    IOCPARM_MASK* = 127
+    IOC_IN* = int(-2147483648)
+    FIONBIO* = IOC_IN.int32 or ((sizeof(int32) and IOCPARM_MASK) shl 16) or 
+                             (102 shl 8) or 126
+
+  proc ioctlsocket*(s: TSocketHandle, cmd: clong, 
+                   argptr: ptr clong): cint {.
+                   stdcall, importc: "ioctlsocket", dynlib: "ws2_32.dll".}
+else:
+  let
+    osInvalidSocket* = posix.INVALID_SOCKET
+
+proc `==`*(a, b: TPort): bool {.borrow.}
+  ## ``==`` for ports.
+
+proc `$`*(p: TPort): string {.borrow.}
+  ## returns the port number as a string
+
+proc toInt*(domain: TDomain): cint
+  ## Converts the TDomain enum to a platform-dependent ``cint``.
+
+proc toInt*(typ: TType): cint
+  ## Converts the TType enum to a platform-dependent ``cint``.
+
+proc toInt*(p: TProtocol): cint
+  ## Converts the TProtocol enum to a platform-dependent ``cint``.
+
+when defined(posix):
+  proc toInt(domain: TDomain): cint =
+    case domain
+    of AF_UNIX:        result = posix.AF_UNIX
+    of AF_INET:        result = posix.AF_INET
+    of AF_INET6:       result = posix.AF_INET6
+    else: discard
+
+  proc toInt(typ: TType): cint =
+    case typ
+    of SOCK_STREAM:    result = posix.SOCK_STREAM
+    of SOCK_DGRAM:     result = posix.SOCK_DGRAM
+    of SOCK_SEQPACKET: result = posix.SOCK_SEQPACKET
+    of SOCK_RAW:       result = posix.SOCK_RAW
+    else: discard
+
+  proc toInt(p: TProtocol): cint =
+    case p
+    of IPPROTO_TCP:    result = posix.IPPROTO_TCP
+    of IPPROTO_UDP:    result = posix.IPPROTO_UDP
+    of IPPROTO_IP:     result = posix.IPPROTO_IP
+    of IPPROTO_IPV6:   result = posix.IPPROTO_IPV6
+    of IPPROTO_RAW:    result = posix.IPPROTO_RAW
+    of IPPROTO_ICMP:   result = posix.IPPROTO_ICMP
+    else: discard
+
+else:
+  proc toInt(domain: TDomain): cint = 
+    result = toU16(ord(domain))
+
+  proc toInt(typ: TType): cint =
+    result = cint(ord(typ))
+  
+  proc toInt(p: TProtocol): cint =
+    result = cint(ord(p))
+
+
+proc socket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM,
+             protocol: TProtocol = IPPROTO_TCP): TSocketHandle =
+  ## Creates a new socket; returns `InvalidSocket` if an error occurs.
+  
+  # TODO: The function which will use this will raise EOS.
+  socket(toInt(domain), toInt(typ), toInt(protocol))
+
+proc close*(socket: TSocketHandle) =
+  ## closes a socket.
+  when defined(windows):
+    discard winlean.closeSocket(socket)
+  else:
+    discard posix.close(socket)
+  # TODO: These values should not be discarded. An EOS should be raised.
+  # http://stackoverflow.com/questions/12463473/what-happens-if-you-call-close-on-a-bsd-socket-multiple-times
+
+proc bindAddr*(socket: TSocketHandle, name: ptr TSockAddr, namelen: TSockLen): cint =
+  result = bindSocket(socket, name, namelen)
+
+proc listen*(socket: TSocketHandle, backlog = SOMAXCONN) {.tags: [FReadIO].} =
+  ## Marks ``socket`` as accepting connections. 
+  ## ``Backlog`` specifies the maximum length of the 
+  ## queue of pending connections.
+  when defined(windows):
+    if winlean.listen(socket, cint(backlog)) < 0'i32: osError(osLastError())
+  else:
+    if posix.listen(socket, cint(backlog)) < 0'i32: osError(osLastError())
+
+proc getAddrInfo*(address: string, port: TPort, af: TDomain = AF_INET, typ: TType = SOCK_STREAM,
+                 prot: TProtocol = IPPROTO_TCP): ptr TAddrInfo =
+  ##
+  ##
+  ## **Warning**: The resulting ``ptr TAddrInfo`` must be freed using ``dealloc``!
+  var hints: TAddrInfo
+  result = nil
+  hints.ai_family = toInt(af)
+  hints.ai_socktype = toInt(typ)
+  hints.ai_protocol = toInt(prot)
+  var gaiResult = getAddrInfo(address, $port, addr(hints), result)
+  if gaiResult != 0'i32:
+    when defined(windows):
+      OSError(OSLastError())
+    else:
+      raise newException(EOS, $gai_strerror(gaiResult))
+
+proc dealloc*(ai: ptr TAddrInfo) =
+  freeaddrinfo(ai)
+
+proc ntohl*(x: int32): int32 = 
+  ## Converts 32-bit integers from network to host byte order.
+  ## On machines where the host byte order is the same as network byte order,
+  ## this is a no-op; otherwise, it performs a 4-byte swap operation.
+  when cpuEndian == bigEndian: result = x
+  else: result = (x shr 24'i32) or
+                 (x shr 8'i32 and 0xff00'i32) or
+                 (x shl 8'i32 and 0xff0000'i32) or
+                 (x shl 24'i32)
+
+proc ntohs*(x: int16): int16 =
+  ## Converts 16-bit integers from network to host byte order. On machines
+  ## where the host byte order is the same as network byte order, this is
+  ## a no-op; otherwise, it performs a 2-byte swap operation.
+  when cpuEndian == bigEndian: result = x
+  else: result = (x shr 8'i16) or (x shl 8'i16)
+
+proc htonl*(x: int32): int32 =
+  ## Converts 32-bit integers from host to network byte order. On machines
+  ## where the host byte order is the same as network byte order, this is
+  ## a no-op; otherwise, it performs a 4-byte swap operation.
+  result = sockets2.ntohl(x)
+
+proc htons*(x: int16): int16 =
+  ## Converts 16-bit positive integers from host to network byte order.
+  ## On machines where the host byte order is the same as network byte
+  ## order, this is a no-op; otherwise, it performs a 2-byte swap operation.
+  result = sockets2.ntohs(x)
+
+when defined(Windows):
+  var wsa: TWSADATA
+  if WSAStartup(0x0101'i16, addr wsa) != 0: OSError(OSLastError())

lib/pure/times.nim

@@ -211,7 +211,9 @@ proc initInterval*(miliseconds, seconds, minutes, hours, days, months,
   result.months = months
   result.years = years
 
-proc isLeapYear(year: int): bool =
+proc isLeapYear*(year: int): bool =
+  ## returns true if ``year`` is a leap year
+
   if year mod 400 == 0:
     return true
   elif year mod 100 == 0: 
@@ -221,7 +223,9 @@ proc isLeapYear(year: int): bool =
   else:
     return false
 
-proc getDaysInMonth(month: TMonth, year: int): int =
+proc getDaysInMonth*(month: TMonth, year: int): int =
+  ## gets the amount of days in a ``month`` of a ``year``
+
   # http://www.dispersiondesign.com/articles/time/number_of_days_in_a_month
   case month 
   of mFeb: result = if isLeapYear(year): 29 else: 28
@@ -553,6 +557,119 @@ proc `$`*(m: TMonth): string =
       "November", "December"]
   return lookup[m]
 
+proc format_token(info: TTimeInfo, token: string, buf: var string) =
+  ## Helper of the format proc to parse individual tokens.
+  ##
+  ## Pass the found token in the user input string, and the buffer where the
+  ## final string is being built. This has to be a var value because certain
+  ## formatting tokens require modifying the previous characters.
+  case token
+  of "d":
+    buf.add($info.monthday)
+  of "dd":
+    if info.monthday < 10:
+      buf.add("0")
+    buf.add($info.monthday)
+  of "ddd":
+    buf.add(($info.weekday)[0 .. 2])
+  of "dddd":
+    buf.add($info.weekday)
+  of "h":
+    buf.add($(if info.hour > 12: info.hour - 12 else: info.hour))
+  of "hh":
+    let amerHour = if info.hour > 12: info.hour - 12 else: info.hour
+    if amerHour < 10:
+      buf.add('0')
+    buf.add($amerHour)
+  of "H":
+    buf.add($info.hour)
+  of "HH":
+    if info.hour < 10:
+      buf.add('0')
+    buf.add($info.hour)
+  of "m":
+    buf.add($info.minute)
+  of "mm":
+    if info.minute < 10:
+      buf.add('0')
+    buf.add($info.minute)
+  of "M":
+    buf.add($(int(info.month)+1))
+  of "MM":
+    if info.month < mOct:
+      buf.add('0')
+    buf.add($(int(info.month)+1))
+  of "MMM":
+    buf.add(($info.month)[0..2])
+  of "MMMM":
+    buf.add($info.month)
+  of "s":
+    buf.add($info.second)
+  of "ss":
+    if info.second < 10:
+      buf.add('0')
+    buf.add($info.second)
+  of "t":
+    if info.hour >= 12:
+      buf.add('P')
+    else: buf.add('A')
+  of "tt":
+    if info.hour >= 12:
+      buf.add("PM")
+    else: buf.add("AM")
+  of "y":
+    var fr = ($info.year).len()-1
+    if fr < 0: fr = 0
+    buf.add(($info.year)[fr .. ($info.year).len()-1])
+  of "yy":
+    var fr = ($info.year).len()-2
+    if fr < 0: fr = 0
+    var fyear = ($info.year)[fr .. ($info.year).len()-1]
+    if fyear.len != 2: fyear = repeatChar(2-fyear.len(), '0') & fyear
+    buf.add(fyear)
+  of "yyy":
+    var fr = ($info.year).len()-3
+    if fr < 0: fr = 0
+    var fyear = ($info.year)[fr .. ($info.year).len()-1]
+    if fyear.len != 3: fyear = repeatChar(3-fyear.len(), '0') & fyear
+    buf.add(fyear)
+  of "yyyy":
+    var fr = ($info.year).len()-4
+    if fr < 0: fr = 0
+    var fyear = ($info.year)[fr .. ($info.year).len()-1]
+    if fyear.len != 4: fyear = repeatChar(4-fyear.len(), '0') & fyear
+    buf.add(fyear)
+  of "yyyyy":
+    var fr = ($info.year).len()-5
+    if fr < 0: fr = 0
+    var fyear = ($info.year)[fr .. ($info.year).len()-1]
+    if fyear.len != 5: fyear = repeatChar(5-fyear.len(), '0') & fyear
+    buf.add(fyear)
+  of "z":
+    let hrs = (info.timezone div 60) div 60
+    buf.add($hrs)
+  of "zz":
+    let hrs = (info.timezone div 60) div 60
+
+    buf.add($hrs)
+    if hrs.abs < 10:
+      var atIndex = buf.len-(($hrs).len-(if hrs < 0: 1 else: 0))
+      buf.insert("0", atIndex)
+  of "zzz":
+    let hrs = (info.timezone div 60) div 60
+
+    buf.add($hrs & ":00")
+    if hrs.abs < 10:
+      var atIndex = buf.len-(($hrs & ":00").len-(if hrs < 0: 1 else: 0))
+      buf.insert("0", atIndex)
+  of "ZZZ":
+    buf.add(info.tzname)
+  of "":
+    discard
+  else:
+    raise newException(EInvalidValue, "Invalid format string: " & token)
+
+
 proc format*(info: TTimeInfo, f: string): string =
   ## This function formats `info` as specified by `f`. The following format
   ## specifiers are available:
@@ -587,8 +704,11 @@ proc format*(info: TTimeInfo, f: string): string =
   ##    ZZZ      Displays the name of the timezone.                                                 ``GMT -> GMT``, ``EST -> EST``
   ## ==========  =================================================================================  ================================================
   ##
-  ## Other strings can be inserted by putting them in ``''``. For example ``hh'->'mm`` will give ``01->56``.
+  ## Other strings can be inserted by putting them in ``''``. For example
-  ## The following characters can be inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]`` ``,``
+  ## ``hh'->'mm`` will give ``01->56``.  The following characters can be
+  ## inserted without quoting them: ``:`` ``-`` ``(`` ``)`` ``/`` ``[`` ``]``
+  ## ``,``. However you don't need to necessarily separate format specifiers, a
+  ## unambiguous format string like ``yyyyMMddhhmmss`` is valid too.
 
   result = ""
   var i = 0
@@ -596,111 +716,7 @@ proc format*(info: TTimeInfo, f: string): string =
   while true:
     case f[i]
     of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',':
-      case currentF
+      format_token(info, currentF, result)
-      of "d":
-        result.add($info.monthday)
-      of "dd":
-        if info.monthday < 10:
-          result.add("0")
-        result.add($info.monthday)
-      of "ddd":
-        result.add(($info.weekday)[0 .. 2])
-      of "dddd":
-        result.add($info.weekday)
-      of "h":
-        result.add($(if info.hour > 12: info.hour - 12 else: info.hour))
-      of "hh":
-        let amerHour = if info.hour > 12: info.hour - 12 else: info.hour
-        if amerHour < 10:
-          result.add('0')
-        result.add($amerHour)
-      of "H":
-        result.add($info.hour)
-      of "HH":
-        if info.hour < 10:
-          result.add('0')
-        result.add($info.hour)
-      of "m":
-        result.add($info.minute)
-      of "mm":
-        if info.minute < 10:
-          result.add('0')
-        result.add($info.minute)
-      of "M":
-        result.add($(int(info.month)+1))
-      of "MM":
-        if info.month < mOct:
-          result.add('0')
-        result.add($(int(info.month)+1))
-      of "MMM":
-        result.add(($info.month)[0..2])
-      of "MMMM":
-        result.add($info.month)
-      of "s":
-        result.add($info.second)
-      of "ss":
-        if info.second < 10:
-          result.add('0')
-        result.add($info.second)
-      of "t":
-        if info.hour >= 12:
-          result.add('P')
-        else: result.add('A')
-      of "tt":
-        if info.hour >= 12:
-          result.add("PM")
-        else: result.add("AM")
-      of "y":
-        var fr = ($info.year).len()-1
-        if fr < 0: fr = 0
-        result.add(($info.year)[fr .. ($info.year).len()-1])
-      of "yy":
-        var fr = ($info.year).len()-2
-        if fr < 0: fr = 0
-        var fyear = ($info.year)[fr .. ($info.year).len()-1]
-        if fyear.len != 2: fyear = repeatChar(2-fyear.len(), '0') & fyear
-        result.add(fyear)
-      of "yyy":
-        var fr = ($info.year).len()-3
-        if fr < 0: fr = 0
-        var fyear = ($info.year)[fr .. ($info.year).len()-1]
-        if fyear.len != 3: fyear = repeatChar(3-fyear.len(), '0') & fyear
-        result.add(fyear)
-      of "yyyy":
-        var fr = ($info.year).len()-4
-        if fr < 0: fr = 0
-        var fyear = ($info.year)[fr .. ($info.year).len()-1]
-        if fyear.len != 4: fyear = repeatChar(4-fyear.len(), '0') & fyear
-        result.add(fyear)
-      of "yyyyy":
-        var fr = ($info.year).len()-5
-        if fr < 0: fr = 0
-        var fyear = ($info.year)[fr .. ($info.year).len()-1]
-        if fyear.len != 5: fyear = repeatChar(5-fyear.len(), '0') & fyear
-        result.add(fyear)
-      of "z":
-        let hrs = (info.timezone div 60) div 60
-        result.add($hrs)
-      of "zz":
-        let hrs = (info.timezone div 60) div 60
-        
-        result.add($hrs)
-        if hrs.abs < 10:
-          var atIndex = result.len-(($hrs).len-(if hrs < 0: 1 else: 0))
-          result.insert("0", atIndex)
-      of "zzz":
-        let hrs = (info.timezone div 60) div 60
-        
-        result.add($hrs & ":00")
-        if hrs.abs < 10:
-          var atIndex = result.len-(($hrs & ":00").len-(if hrs < 0: 1 else: 0))
-          result.insert("0", atIndex)
-      of "ZZZ":
-        result.add(info.tzname)
-      of "":
-        discard
-      else:
-        raise newException(EInvalidValue, "Invalid format string: " & currentF)
 
       currentF = ""
       if f[i] == '\0': break
@@ -712,7 +728,15 @@ proc format*(info: TTimeInfo, f: string): string =
           inc(i)
       else: result.add(f[i])
       
-    else: currentF.add(f[i])
+    else:
+      # Check if the letter being added matches previous accumulated buffer.
+      if currentF.len < 1 or currentF[high(currentF)] == f[i]:
+        currentF.add(f[i])
+      else:
+        format_token(info, currentF, result)
+        dec(i) # Move position back to re-process the character separately.
+        currentF = ""
+
     inc(i)
 
 {.pop.}
@@ -723,12 +747,16 @@ when isMainModule:
 
   var t = getGMTime(fromSeconds(2147483647))
   echo t.format("ddd dd MMM hh:mm:ss ZZZ yyyy")
+  echo t.format("ddd ddMMMhhmmssZZZyyyy")
+  assert t.format("ddd dd MMM hh:mm:ss ZZZ yyyy") == "Tue 19 Jan 03:14:07 UTC 2038"
   assert t.format("ddd ddMMMhh:mm:ssZZZyyyy") == "Tue 19Jan03:14:07UTC2038"
   
   assert t.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" &
     " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == 
     "19 19 Tue Tuesday 3 03 3 03 14 14 1 01 Jan January 7 07 A AM 8 38 038 2038 02038 0 00 00:00 UTC"
 
+  assert t.format("yyyyMMddhhmmss") == "20380119031407"
+  
   var t2 = getGMTime(fromSeconds(160070789)) # Mon 27 Jan 16:06:29 GMT 1975
   assert t2.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" &
     " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") ==

lib/stdlib.babel

@@ -0,0 +1,6 @@
+[Package]
+name          = "stdlib"
+version       = "0.9.0"
+author        = "Dominik Picheta"
+description   = "Nimrod's standard library."
+license       = "MIT"

lib/system.nim

@@ -185,6 +185,8 @@ proc `..`*[T](b: T): TSlice[T] {.noSideEffect, inline.} =
 
 when not defined(niminheritable):
   {.pragma: inheritable.}
+when not defined(nimunion):
+  {.pragma: unchecked.}
 
 const NoFakeVars* = defined(NimrodVM) ## true if the backend doesn't support \
   ## "fake variables" like 'var EBADF {.importc.}: cint'.
@@ -194,9 +196,10 @@ when not defined(JS):
     TGenericSeq {.compilerproc, pure, inheritable.} = object
       len, reserved: int
     PGenericSeq {.exportc.} = ptr TGenericSeq
+    UncheckedCharArray {.unchecked.} = array[0..100_000_000, char]
     # len and space without counting the terminating zero:
     NimStringDesc {.compilerproc, final.} = object of TGenericSeq
-      data: array[0..100_000_000, char]
+      data: UncheckedCharArray
     NimString = ptr NimStringDesc
 
 when not defined(JS) and not defined(NimrodVM):
@@ -1559,7 +1562,7 @@ when not defined(NimrodVM):
     proc seqToPtr[T](x: seq[T]): pointer {.inline, nosideeffect.} =
       result = cast[pointer](x)
   else:
-    proc seqToPtr[T](x: seq[T]): pointer {.noStackFrame, nosideeffect.} =
+    proc seqToPtr[T](x: seq[T]): pointer {.asmNoStackFrame, nosideeffect.} =
       asm """return `x`"""
   
   proc `==` *[T](x, y: seq[T]): bool {.noSideEffect.} =
@@ -1842,7 +1845,7 @@ type
     len*: int           ## length of the inspectable slots
 
 when defined(JS):
-  proc add*(x: var string, y: cstring) {.noStackFrame.} =
+  proc add*(x: var string, y: cstring) {.asmNoStackFrame.} =
     asm """
       var len = `x`[0].length-1;
       for (var i = 0; i < `y`.length; ++i) {
@@ -2059,8 +2062,10 @@ when not defined(JS): #and not defined(NimrodVM):
       ## Flushes `f`'s buffer.
 
     proc readAll*(file: TFile): TaintedString {.tags: [FReadIO].}
-      ## Reads all data from the stream `file`. Raises an IO exception
+      ## Reads all data from the stream `file`.
-      ## in case of an error
+      ##
+      ## Raises an IO exception in case of an error. It is an error if the
+      ## current file position is not at the beginning of the file.
     
     proc readFile*(filename: string): TaintedString {.tags: [FReadIO].}
       ## Opens a file named `filename` for reading. Then calls `readAll`

lib/system/arithm.nim

@@ -111,7 +111,7 @@ const
 when asmVersion and not defined(gcc) and not defined(llvm_gcc):
   # assembler optimized versions for compilers that
   # have an intel syntax assembler:
-  proc addInt(a, b: int): int {.compilerProc, noStackFrame.} =
+  proc addInt(a, b: int): int {.compilerProc, asmNoStackFrame.} =
     # a in eax, and b in edx
     asm """
         mov eax, `a`
@@ -121,7 +121,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc):
       theEnd:
     """
 
-  proc subInt(a, b: int): int {.compilerProc, noStackFrame.} =
+  proc subInt(a, b: int): int {.compilerProc, asmNoStackFrame.} =
     asm """
         mov eax, `a`
         sub eax, `b`
@@ -130,7 +130,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc):
       theEnd:
     """
 
-  proc negInt(a: int): int {.compilerProc, noStackFrame.} =
+  proc negInt(a: int): int {.compilerProc, asmNoStackFrame.} =
     asm """
         mov eax, `a`
         neg eax
@@ -139,7 +139,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc):
       theEnd:
     """
 
-  proc divInt(a, b: int): int {.compilerProc, noStackFrame.} =
+  proc divInt(a, b: int): int {.compilerProc, asmNoStackFrame.} =
     asm """
         mov eax, `a`
         mov ecx, `b`
@@ -150,7 +150,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc):
       theEnd:
     """
 
-  proc modInt(a, b: int): int {.compilerProc, noStackFrame.} =
+  proc modInt(a, b: int): int {.compilerProc, asmNoStackFrame.} =
     asm """
         mov eax, `a`
         mov ecx, `b`
@@ -162,7 +162,7 @@ when asmVersion and not defined(gcc) and not defined(llvm_gcc):
         mov eax, edx
     """
 
-  proc mulInt(a, b: int): int {.compilerProc, noStackFrame.} =
+  proc mulInt(a, b: int): int {.compilerProc, asmNoStackFrame.} =
     asm """
         mov eax, `a`
         mov ecx, `b`

lib/system/gc.nim

@@ -802,7 +802,7 @@ when defined(sparc): # For SPARC architecture.
     # Addresses decrease as the stack grows.
     while sp <= max:
       gcMark(gch, sp[])
-      sp = cast[ppointer](cast[TAddress](sp) +% sizeof(pointer))
+      sp = cast[PPointer](cast[TAddress](sp) +% sizeof(pointer))
 
 elif defined(ELATE):
   {.error: "stack marking code is to be written for this architecture".}

lib/system/jssys.nim

@@ -23,9 +23,9 @@ type
   PCallFrame = ptr TCallFrame
   TCallFrame {.importc, nodecl, final.} = object
     prev: PCallFrame
-    procname: CString
+    procname: cstring
     line: int # current line number
-    filename: CString
+    filename: cstring
 
 var
   framePtr {.importc, nodecl, volatile.}: PCallFrame
@@ -48,7 +48,7 @@ proc getCurrentExceptionMsg*(): string =
 
 proc auxWriteStackTrace(f: PCallFrame): string =
   type
-    TTempFrame = tuple[procname: CString, line: int]
+    TTempFrame = tuple[procname: cstring, line: int]
   var
     it = f
     i = 0
@@ -84,7 +84,7 @@ proc rawWriteStackTrace(): string =
     framePtr = nil
 
 proc raiseException(e: ref E_Base, ename: cstring) {.
-    compilerproc, noStackFrame.} =
+    compilerproc, asmNoStackFrame.} =
   e.name = ename
   if excHandler != nil:
     excHandler.exc = e
@@ -104,7 +104,7 @@ proc raiseException(e: ref E_Base, ename: cstring) {.
     alert(buf)
   asm """throw `e`;"""
 
-proc reraiseException() {.compilerproc, noStackFrame.} =
+proc reraiseException() {.compilerproc, asmNoStackFrame.} =
   if excHandler == nil:
     raise newException(ENoExceptionToReraise, "no exception to reraise")
   else:
@@ -125,7 +125,7 @@ proc raiseIndexError() {.compilerproc, noreturn.} =
 proc raiseFieldError(f: string) {.compilerproc, noreturn.} =
   raise newException(EInvalidField, f & " is not accessible")
 
-proc SetConstr() {.varargs, noStackFrame, compilerproc.} =
+proc SetConstr() {.varargs, asmNoStackFrame, compilerproc.} =
   asm """
     var result = {};
     for (var i = 0; i < arguments.length; ++i) {
@@ -141,7 +141,7 @@ proc SetConstr() {.varargs, noStackFrame, compilerproc.} =
     return result;
   """
 
-proc cstrToNimstr(c: cstring): string {.noStackFrame, compilerproc.} =
+proc cstrToNimstr(c: cstring): string {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = [];
     for (var i = 0; i < `c`.length; ++i) {
@@ -151,7 +151,7 @@ proc cstrToNimstr(c: cstring): string {.noStackFrame, compilerproc.} =
     return result;
   """
 
-proc toJSStr(s: string): cstring {.noStackFrame, compilerproc.} =
+proc toJSStr(s: string): cstring {.asmNoStackFrame, compilerproc.} =
   asm """
     var len = `s`.length-1;
     var result = new Array(len);
@@ -162,7 +162,7 @@ proc toJSStr(s: string): cstring {.noStackFrame, compilerproc.} =
     return result.join("");
   """
 
-proc mnewString(len: int): string {.noStackFrame, compilerproc.} =
+proc mnewString(len: int): string {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = new Array(`len`+1);
     result[0] = 0;
@@ -170,7 +170,7 @@ proc mnewString(len: int): string {.noStackFrame, compilerproc.} =
     return result;
   """
 
-proc SetCard(a: int): int {.compilerproc, noStackFrame.} =
+proc SetCard(a: int): int {.compilerproc, asmNoStackFrame.} =
   # argument type is a fake
   asm """
     var result = 0;
@@ -178,14 +178,14 @@ proc SetCard(a: int): int {.compilerproc, noStackFrame.} =
     return result;
   """
 
-proc SetEq(a, b: int): bool {.compilerproc, noStackFrame.} =
+proc SetEq(a, b: int): bool {.compilerproc, asmNoStackFrame.} =
   asm """
     for (var elem in `a`) { if (!`b`[elem]) return false; }
     for (var elem in `b`) { if (!`a`[elem]) return false; }
     return true;
   """
 
-proc SetLe(a, b: int): bool {.compilerproc, noStackFrame.} =
+proc SetLe(a, b: int): bool {.compilerproc, asmNoStackFrame.} =
   asm """
     for (var elem in `a`) { if (!`b`[elem]) return false; }
     return true;
@@ -194,7 +194,7 @@ proc SetLe(a, b: int): bool {.compilerproc, noStackFrame.} =
 proc SetLt(a, b: int): bool {.compilerproc.} =
   result = SetLe(a, b) and not SetEq(a, b)
 
-proc SetMul(a, b: int): int {.compilerproc, noStackFrame.} =
+proc SetMul(a, b: int): int {.compilerproc, asmNoStackFrame.} =
   asm """
     var result = {};
     for (var elem in `a`) {
@@ -203,7 +203,7 @@ proc SetMul(a, b: int): int {.compilerproc, noStackFrame.} =
     return result;
   """
 
-proc SetPlus(a, b: int): int {.compilerproc, noStackFrame.} =
+proc SetPlus(a, b: int): int {.compilerproc, asmNoStackFrame.} =
   asm """
     var result = {};
     for (var elem in `a`) { result[elem] = true; }
@@ -211,7 +211,7 @@ proc SetPlus(a, b: int): int {.compilerproc, noStackFrame.} =
     return result;
   """
 
-proc SetMinus(a, b: int): int {.compilerproc, noStackFrame.} =
+proc SetMinus(a, b: int): int {.compilerproc, asmNoStackFrame.} =
   asm """
     var result = {};
     for (var elem in `a`) {
@@ -220,7 +220,7 @@ proc SetMinus(a, b: int): int {.compilerproc, noStackFrame.} =
     return result;
   """
 
-proc cmpStrings(a, b: string): int {.noStackFrame, compilerProc.} =
+proc cmpStrings(a, b: string): int {.asmNoStackFrame, compilerProc.} =
   asm """
     if (`a` == `b`) return 0;
     if (!`a`) return -1;
@@ -234,7 +234,7 @@ proc cmpStrings(a, b: string): int {.noStackFrame, compilerProc.} =
 
 proc cmp(x, y: string): int = return cmpStrings(x, y)
 
-proc eqStrings(a, b: string): bool {.noStackFrame, compilerProc.} =
+proc eqStrings(a, b: string): bool {.asmNoStackFrame, compilerProc.} =
   asm """
     if (`a` == `b`) return true;
     if ((!`a`) || (!`b`)) return false;
@@ -300,7 +300,7 @@ type
     setAttributeNode*: proc (attr: ref TNode) {.nimcall.}
 
 when defined(kwin):
-  proc rawEcho {.compilerproc, nostackframe.} =
+  proc rawEcho {.compilerproc, asmNoStackFrame.} =
     asm """
       var buf = "";
       for (var i = 0; i < arguments.length; ++i) {
@@ -312,7 +312,7 @@ when defined(kwin):
 elif defined(nodejs):
   proc ewriteln(x: cstring) = log(x)
   
-  proc rawEcho {.compilerproc, nostackframe.} =
+  proc rawEcho {.compilerproc, asmNoStackFrame.} =
     asm """
       var buf = "";
       for (var i = 0; i < arguments.length; ++i) {
@@ -345,42 +345,42 @@ else:
     node.appendChild(document.createElement("br"))
 
 # Arithmetic:
-proc addInt(a, b: int): int {.noStackFrame, compilerproc.} =
+proc addInt(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = `a` + `b`;
     if (result > 2147483647 || result < -2147483648) `raiseOverflow`();
     return result;
   """
 
-proc subInt(a, b: int): int {.noStackFrame, compilerproc.} =
+proc subInt(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = `a` - `b`;
     if (result > 2147483647 || result < -2147483648) `raiseOverflow`();
     return result;
   """
 
-proc mulInt(a, b: int): int {.noStackFrame, compilerproc.} =
+proc mulInt(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = `a` * `b`;
     if (result > 2147483647 || result < -2147483648) `raiseOverflow`();
     return result;
   """
 
-proc divInt(a, b: int): int {.noStackFrame, compilerproc.} =
+proc divInt(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     if (`b` == 0) `raiseDivByZero`();
     if (`b` == -1 && `a` == 2147483647) `raiseOverflow`();
     return Math.floor(`a` / `b`);
   """
 
-proc modInt(a, b: int): int {.noStackFrame, compilerproc.} =
+proc modInt(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     if (`b` == 0) `raiseDivByZero`();
     if (`b` == -1 && `a` == 2147483647) `raiseOverflow`();
     return Math.floor(`a` % `b`);
   """
 
-proc addInt64(a, b: int): int {.noStackFrame, compilerproc.} =
+proc addInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = `a` + `b`;
     if (result > 9223372036854775807
@@ -388,7 +388,7 @@ proc addInt64(a, b: int): int {.noStackFrame, compilerproc.} =
     return result;
   """
 
-proc subInt64(a, b: int): int {.noStackFrame, compilerproc.} =
+proc subInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = `a` - `b`;
     if (result > 9223372036854775807
@@ -396,7 +396,7 @@ proc subInt64(a, b: int): int {.noStackFrame, compilerproc.} =
     return result;
   """
 
-proc mulInt64(a, b: int): int {.noStackFrame, compilerproc.} =
+proc mulInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     var result = `a` * `b`;
     if (result > 9223372036854775807
@@ -404,90 +404,89 @@ proc mulInt64(a, b: int): int {.noStackFrame, compilerproc.} =
     return result;
   """
 
-proc divInt64(a, b: int): int {.noStackFrame, compilerproc.} =
+proc divInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     if (`b` == 0) `raiseDivByZero`();
     if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`();
     return Math.floor(`a` / `b`);
   """
 
-proc modInt64(a, b: int): int {.noStackFrame, compilerproc.} =
+proc modInt64(a, b: int): int {.asmNoStackFrame, compilerproc.} =
   asm """
     if (`b` == 0) `raiseDivByZero`();
     if (`b` == -1 && `a` == 9223372036854775807) `raiseOverflow`();
     return Math.floor(`a` % `b`);
   """
 
-proc NegInt(a: int): int {.compilerproc.} =
+proc negInt(a: int): int {.compilerproc.} =
   result = a*(-1)
 
-proc NegInt64(a: int64): int64 {.compilerproc.} =
+proc negInt64(a: int64): int64 {.compilerproc.} =
   result = a*(-1)
 
-proc AbsInt(a: int): int {.compilerproc.} =
+proc absInt(a: int): int {.compilerproc.} =
   result = if a < 0: a*(-1) else: a
 
-proc AbsInt64(a: int64): int64 {.compilerproc.} =
+proc absInt64(a: int64): int64 {.compilerproc.} =
   result = if a < 0: a*(-1) else: a
 
-proc LeU(a, b: int): bool {.compilerproc.} =
+proc leU(a, b: int): bool {.compilerproc.} =
   result = abs(a) <= abs(b)
 
-proc LtU(a, b: int): bool {.compilerproc.} =
+proc ltU(a, b: int): bool {.compilerproc.} =
   result = abs(a) < abs(b)
 
-proc LeU64(a, b: int64): bool {.compilerproc.} =
+proc leU64(a, b: int64): bool {.compilerproc.} =
   result = abs(a) <= abs(b)
-
+proc ltU64(a, b: int64): bool {.compilerproc.} =
-proc LtU64(a, b: int64): bool {.compilerproc.} =
   result = abs(a) < abs(b)
 
-proc AddU(a, b: int): int {.compilerproc.} =
+proc addU(a, b: int): int {.compilerproc.} =
   result = abs(a) + abs(b)
-proc AddU64(a, b: int64): int64 {.compilerproc.} =
+proc addU64(a, b: int64): int64 {.compilerproc.} =
   result = abs(a) + abs(b)
 
-proc SubU(a, b: int): int {.compilerproc.} =
+proc subU(a, b: int): int {.compilerproc.} =
   result = abs(a) - abs(b)
-proc SubU64(a, b: int64): int64 {.compilerproc.} =
+proc subU64(a, b: int64): int64 {.compilerproc.} =
   result = abs(a) - abs(b)
 
-proc MulU(a, b: int): int {.compilerproc.} =
+proc mulU(a, b: int): int {.compilerproc.} =
   result = abs(a) * abs(b)
-proc MulU64(a, b: int64): int64 {.compilerproc.} =
+proc mulU64(a, b: int64): int64 {.compilerproc.} =
   result = abs(a) * abs(b)
 
-proc DivU(a, b: int): int {.compilerproc.} =
+proc divU(a, b: int): int {.compilerproc.} =
   result = abs(a) div abs(b)
-proc DivU64(a, b: int64): int64 {.compilerproc.} =
+proc divU64(a, b: int64): int64 {.compilerproc.} =
   result = abs(a) div abs(b)
 
-proc ModU(a, b: int): int {.compilerproc.} =
+proc modU(a, b: int): int {.compilerproc.} =
   result = abs(a) mod abs(b)
-proc ModU64(a, b: int64): int64 {.compilerproc.} =
+proc modU64(a, b: int64): int64 {.compilerproc.} =
   result = abs(a) mod abs(b)
 
-proc Ze(a: int): int {.compilerproc.} =
+proc ze*(a: int): int {.compilerproc.} =
-  result = a
-proc Ze64(a: int64): int64 {.compilerproc.} =
   result = a
 
-proc ToU8(a: int): int8 {.noStackFrame, compilerproc.} =
+proc ze64*(a: int64): int64 {.compilerproc.} =
+  result = a
+
+proc ToU8(a: int): int8 {.asmNoStackFrame, compilerproc.} =
   asm """
     return `a`;
   """
 
-proc ToU16(a: int): int16 {.noStackFrame, compilerproc.} =
+proc ToU16(a: int): int16 {.asmNoStackFrame, compilerproc.} =
   asm """
     return `a`;
   """
 
-proc ToU32(a: int): int32 {.noStackFrame, compilerproc.} =
+proc ToU32(a: int): int32 {.asmNoStackFrame, compilerproc.} =
   asm """
     return `a`;
   """
 
-
 proc nimMin(a, b: int): int {.compilerproc.} = return if a <= b: a else: b
 proc nimMax(a, b: int): int {.compilerproc.} = return if a >= b: a else: b
 
@@ -500,9 +499,9 @@ proc isFatPointer(ti: PNimType): bool =
     tyArray, tyArrayConstr, tyTuple,
     tyOpenArray, tySet, tyVar, tyRef, tyPtr}
 
-proc NimCopy(x: pointer, ti: PNimType): pointer {.compilerproc.}
+proc nimCopy(x: pointer, ti: PNimType): pointer {.compilerproc.}
 
-proc NimCopyAux(dest, src: Pointer, n: ptr TNimNode) {.compilerproc.} =
+proc nimCopyAux(dest, src: Pointer, n: ptr TNimNode) {.compilerproc.} =
   case n.kind
   of nkNone: sysAssert(false, "NimCopyAux")
   of nkSlot:
@@ -518,7 +517,7 @@ proc NimCopyAux(dest, src: Pointer, n: ptr TNimNode) {.compilerproc.} =
       }
     """
 
-proc NimCopy(x: pointer, ti: PNimType): pointer =
+proc nimCopy(x: pointer, ti: PNimType): pointer =
   case ti.kind
   of tyPtr, tyRef, tyVar, tyNil:
     if not isFatPointer(ti):
@@ -586,7 +585,7 @@ proc genericReset(x: Pointer, ti: PNimType): pointer {.compilerproc.} =
     result = nil
 
 proc ArrayConstr(len: int, value: pointer, typ: PNimType): pointer {.
-                 noStackFrame, compilerproc.} =
+                 asmNoStackFrame, compilerproc.} =
   # types are fake
   asm """
     var result = new Array(`len`);
@@ -620,7 +619,7 @@ proc isObj(obj, subclass: PNimType): bool {.compilerproc.} =
     x = x.base
   return true
 
-proc addChar(x: string, c: char) {.compilerproc, noStackFrame.} =
+proc addChar(x: string, c: char) {.compilerproc, asmNoStackFrame.} =
   asm """
     `x`[`x`.length-1] = `c`; `x`.push(0);
   """

lib/windows/winlean.nim

@@ -16,8 +16,12 @@ const
 type
   THandle* = int
   LONG* = int32
+  ULONG* = int
+  PULONG* = ptr int
   WINBOOL* = int32
   DWORD* = int32
+  PDWORD* = ptr DWORD
+  LPINT* = ptr int32
   HDC* = THandle
   HGLRC* = THandle
 
@@ -195,14 +199,14 @@ else:
     importc: "GetCurrentDirectoryA", dynlib: "kernel32", stdcall.}
   proc setCurrentDirectoryA*(lpPathName: cstring): int32 {.
     importc: "SetCurrentDirectoryA", dynlib: "kernel32", stdcall.}
-  proc createDirectoryA*(pathName: cstring, security: pointer=nil): int32 {.
+  proc createDirectoryA*(pathName: cstring, security: Pointer=nil): int32 {.
     importc: "CreateDirectoryA", dynlib: "kernel32", stdcall.}
   proc removeDirectoryA*(lpPathName: cstring): int32 {.
     importc: "RemoveDirectoryA", dynlib: "kernel32", stdcall.}
   proc setEnvironmentVariableA*(lpName, lpValue: cstring): int32 {.
     stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableA".}
 
-  proc getModuleFileNameA*(handle: THandle, buf: cstring, size: int32): int32 {.
+  proc getModuleFileNameA*(handle: THandle, buf: CString, size: int32): int32 {.
     importc: "GetModuleFileNameA", dynlib: "kernel32", stdcall.}
 
 when useWinUnicode:
@@ -300,7 +304,7 @@ else:
                            dwFileAttributes: int32): WINBOOL {.
       stdcall, dynlib: "kernel32", importc: "SetFileAttributesA".}
 
-  proc copyFileA*(lpExistingFileName, lpNewFileName: cstring,
+  proc copyFileA*(lpExistingFileName, lpNewFileName: CString,
                  bFailIfExists: cint): cint {.
     importc: "CopyFileA", stdcall, dynlib: "kernel32".}
 
@@ -632,3 +636,76 @@ when not useWinUnicode:
 proc unmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall,
     dynlib: "kernel32", importc: "UnmapViewOfFile".}
 
+type
+  TOVERLAPPED* {.final, pure.} = object
+    Internal*: DWORD
+    InternalHigh*: DWORD
+    Offset*: DWORD
+    OffsetHigh*: DWORD
+    hEvent*: THANDLE
+
+  POVERLAPPED* = ptr TOVERLAPPED
+
+  POVERLAPPED_COMPLETION_ROUTINE* = proc (para1: DWORD, para2: DWORD,
+      para3: POVERLAPPED){.stdcall.}
+
+  TGUID* {.final, pure.} = object
+    D1*: int32
+    D2*: int16
+    D3*: int16
+    D4*: array [0..7, int8]
+
+const
+  ERROR_IO_PENDING* = 997
+
+proc CreateIoCompletionPort*(FileHandle: THANDLE, ExistingCompletionPort: THANDLE,
+                             CompletionKey: DWORD,
+                             NumberOfConcurrentThreads: DWORD): THANDLE{.stdcall,
+    dynlib: "kernel32", importc: "CreateIoCompletionPort".}
+
+proc GetQueuedCompletionStatus*(CompletionPort: THandle,
+    lpNumberOfBytesTransferred: PDWORD, lpCompletionKey: PULONG,
+                                lpOverlapped: ptr POverlapped,
+                                dwMilliseconds: DWORD): WINBOOL{.stdcall,
+    dynlib: "kernel32", importc: "GetQueuedCompletionStatus".}
+
+const 
+ IOC_OUT* = 0x40000000
+ IOC_IN*  = 0x80000000
+ IOC_WS2* = 0x08000000
+ IOC_INOUT* = IOC_IN or IOC_OUT
+
+template WSAIORW*(x,y): expr = (IOC_INOUT or x or y)
+
+const
+  SIO_GET_EXTENSION_FUNCTION_POINTER* = WSAIORW(IOC_WS2,6).DWORD
+  SO_UPDATE_ACCEPT_CONTEXT* = 0x700B
+
+var
+  WSAID_CONNECTEX*: TGUID = TGUID(D1: 0x25a207b9, D2: 0xddf3'i16, D3: 0x4660, D4: [
+    0x8e'i8, 0xe9'i8, 0x76'i8, 0xe5'i8, 0x8c'i8, 0x74'i8, 0x06'i8, 0x3e'i8])
+  WSAID_ACCEPTEX*: TGUID = TGUID(D1: 0xb5367df1'i32, D2: 0xcbac'i16, D3: 0x11cf, D4: [
+    0x95'i8, 0xca'i8, 0x00'i8, 0x80'i8, 0x5f'i8, 0x48'i8, 0xa1'i8, 0x92'i8])
+  WSAID_GETACCEPTEXSOCKADDRS*: TGUID = TGUID(D1: 0xb5367df2'i32, D2: 0xcbac'i16, D3: 0x11cf, D4: [
+    0x95'i8, 0xca'i8, 0x00'i8, 0x80'i8, 0x5f'i8, 0x48'i8, 0xa1'i8, 0x92'i8])
+
+proc WSAIoctl*(s: TSocketHandle, dwIoControlCode: DWORD, lpvInBuffer: pointer,
+  cbInBuffer: DWORD, lpvOutBuffer: pointer, cbOutBuffer: DWORD,
+  lpcbBytesReturned: PDword, lpOverlapped: POVERLAPPED,
+  lpCompletionRoutine: POVERLAPPED_COMPLETION_ROUTINE): cint 
+  {.stdcall, importc: "WSAIoctl", dynlib: "Ws2_32.dll".}
+
+type
+  TWSABuf* {.importc: "WSABUF", header: "winsock2.h".} = object
+    len*: ULONG
+    buf*: cstring
+
+proc WSARecv*(s: TSocketHandle, buf: ptr TWSABuf, bufCount: DWORD,
+  bytesReceived, flags: PDWORD, lpOverlapped: POverlapped,
+  completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {.
+  stdcall, importc: "WSARecv", dynlib: "Ws2_32.dll".}
+
+proc WSASend*(s: TSocketHandle, buf: ptr TWSABuf, bufCount: DWORD,
+  bytesSent: PDWord, flags: DWORD, lpOverlapped: POverlapped,
+  completionProc: POVERLAPPED_COMPLETION_ROUTINE): cint {.
+  stdcall, importc: "WSASend", dynlib: "Ws2_32.dll".}

lib/wrappers/zip/zlib.nim

@@ -134,6 +134,7 @@ proc gzerror*(thefile: gzFile, errnum: var int32): pbytef{.cdecl, dynlib: libz,
     importc: "gzerror".}
 proc adler32*(adler: uLong, buf: pbytef, length: uInt): uLong{.cdecl, 
     dynlib: libz, importc: "adler32".}
+  ## **Warning**: Adler-32 requires at least a few hundred bytes to get rolling.
 proc crc32*(crc: uLong, buf: pbytef, length: uInt): uLong{.cdecl, dynlib: libz, 
     importc: "crc32".}
 proc deflateInitu*(strm: var TZStream, level: int32, version: cstring, 

readme.md

@@ -33,7 +33,7 @@ If you are on a fairly modern *nix system, the following steps should work:
 $ git clone git://github.com/Araq/Nimrod.git
 $ cd Nimrod
 $ git clone --depth 1 git://github.com/nimrod-code/csources
-$ cd csources && ./build.sh
+$ cd csources && sh build.sh
 $ cd ..
 $ bin/nimrod c koch
 $ ./koch boot -d:release

tests/actiontable/tactiontable2.nim

@@ -1,6 +1,6 @@
 discard """
   line: 21
-  errormsg: "invalid type: 'TTable'"
+  errormsg: "invalid type: 'TTable[string, proc (string)]'"
 """
 
 import tables

tests/ambsym/mambsym1.nim

@@ -7,4 +7,4 @@ type
 proc ha() =
   var
     x: TExport # no error
-  nil
+  discard

tests/ambsym/mambsys1.nim

@@ -4,4 +4,4 @@ type
   TExport* = enum x, y, z
 
 proc foo*(x: int) =
-  nil
+  discard

tests/ambsym/mambsys2.nim

@@ -1,4 +1,4 @@
 type
   TExport* = enum x, y, z # exactly the same type!
 
-proc foo*(x: int) = nil
+proc foo*(x: int) = discard

tests/async/tasyncawait.nim

@@ -0,0 +1,64 @@
+discard """
+  file: "tasyncawait.nim"
+  cmd: "nimrod cc --hints:on $# $#"
+  output: "5000"
+"""
+import asyncio2, sockets2, net, strutils
+
+var disp = newDispatcher()
+var msgCount = 0
+
+const
+  swarmSize = 50
+  messagesToSend = 100
+
+var clientCount = 0
+
+proc sendMessages(disp: PDispatcher, client: TSocketHandle): PFuture[int] {.async.} =
+  for i in 0 .. <messagesToSend:
+    discard await disp.send(client, "Message " & $i & "\c\L") 
+
+proc launchSwarm(disp: PDispatcher, port: TPort): PFuture[int] {.async.} =
+  for i in 0 .. <swarmSize:
+    var sock = socket()
+    #disp.register(sock)
+    discard await disp.connect(sock, "localhost", port)
+    when true:
+      discard await sendMessages(disp, sock)
+      sock.close()
+    else:
+      # Issue #932: https://github.com/Araq/Nimrod/issues/932
+      var msgFut = sendMessages(disp, sock)
+      msgFut.callback =
+        proc () =
+          sock.close()
+
+proc readMessages(disp: PDispatcher, client: TSocketHandle): PFuture[int] {.async.} =
+  while true:
+    var line = await disp.recvLine(client)
+    if line == "":
+      client.close()
+      clientCount.inc
+      break
+    else:
+      if line.startswith("Message "):
+        msgCount.inc
+      else:
+        doAssert false
+
+proc createServer(disp: PDispatcher, port: TPort): PFuture[int] {.async.} =
+  var server = socket()
+  #disp.register(server)
+  server.bindAddr(port)
+  server.listen()
+  while true:
+    discard readMessages(disp, await disp.accept(server))
+
+discard disp.createServer(TPort(10335))
+discard disp.launchSwarm(TPort(10335))
+while true:
+  disp.poll()
+  if clientCount == swarmSize: break
+
+assert msgCount == swarmSize * messagesToSend
+echo msgCount

tests/casestmt/tcasestm.nim

@@ -19,8 +19,8 @@ of eB, eC: write(stdout, "b or c")
 case x
 of "Andreas", "Rumpf": write(stdout, "Hallo Meister!")
 of "aa", "bb": write(stdout, "Du bist nicht mein Meister")
-of "cc", "hash", "when": nil
+of "cc", "hash", "when": discard
-of "will", "it", "finally", "be", "generated": nil
+of "will", "it", "finally", "be", "generated": discard
 
 var z = case i
   of 1..5, 8, 9: "aa"

tests/ccgbugs/tcgbug.nim

@@ -19,5 +19,18 @@ var
 new(a)
 q(a)
 
+# bug #914
+var x = newWideCString("Hello")
+
 echo "success"
 
+
+# bug #833
+
+type
+  PFuture*[T] = ref object
+    value*: T
+    finished*: bool
+    cb: proc (future: PFuture[T]) {.closure.}
+
+var k = PFuture[void]()

tests/collections/tsets.nim

@@ -0,0 +1,17 @@
+discard """
+  output: '''true
+true'''
+"""
+
+import sets
+var
+  a = initSet[int]()
+  b = initSet[int]()
+  c = initSet[string]()
+
+for i in 0..5: a.incl(i)
+for i in 1..6: b.incl(i)
+for i in 0..5: c.incl($i)
+
+echo map(a, proc(x: int): int = x + 1) == b
+echo map(a, proc(x: int): string = $x) == c

tests/compiles/tcompiles.nim

@@ -24,3 +24,5 @@ ok supports(`+`, 34)
 
 no compiles(4+5.0 * "hallo")
 
+no compiles(undeclaredIdentifier)
+no compiles(undeclaredIdentifier)

tests/concurrency/tnodeadlocks.nim

@@ -12,7 +12,7 @@ var
   thr: array [0..5, TThread[tuple[a, b: int]]]
   L, M, N: TLock
 
-proc doNothing() = nil
+proc doNothing() = discard
 
 proc threadFunc(interval: tuple[a, b: int]) {.thread.} = 
   doNothing()

tests/destructor/tdestructor.nim

@@ -12,6 +12,13 @@ myobj destroyed
 ----
 mygeneric3 constructed
 mygeneric1 destroyed
+----
+mygeneric1 destroyed
+----
+myobj destroyed
+----
+----
+myobj destroyed
 '''
 """
 
@@ -32,6 +39,22 @@ type
     y: B
     z: C
   
+  TObjKind = enum A, B, C, D
+
+  TCaseObj = object
+    case kind: TObjKind
+    of A:
+      x: TMyGeneric1[int]
+    of B, C:
+      y: TMyObj
+    else:
+      case innerKind: TObjKind
+      of A, B, C:
+        p: TMyGeneric3[int, float, string]
+      of D:
+        q: TMyGeneric3[TMyObj, int, int]
+      r: string
+
 proc destruct(o: var TMyObj) {.destructor.} =
   if o.p != nil: dealloc o.p
   echo "myobj destroyed"
@@ -57,13 +80,13 @@ proc mygeneric1() =
   echo "mygeneric1 constructed"
 
 proc mygeneric2[T](val: T) =
-  var
+  var a = open()
-    a = open()
-    b = TMyGeneric2[int, T](x: 10, y: val)
-    c = TMyGeneric3[int, int, string](x: 10, y: 20, z: "test")
   
+  var b = TMyGeneric2[int, T](x: 10, y: val)
   echo "mygeneric2 constructed"
 
+  var c = TMyGeneric3[int, int, string](x: 10, y: 20, z: "test")
+  
 proc mygeneric3 =
   var x = TMyGeneric3[int, string, TMyGeneric1[int]](
     x: 10, y: "test", z: TMyGeneric1[int](x: 10))
@@ -82,3 +105,24 @@ mygeneric2[int](10)
 echo "----"
 mygeneric3()
 
+proc caseobj =
+  block:
+    echo "----"
+    var o1 = TCaseObj(kind: A, x: TMyGeneric1[int](x: 10))
+  
+  block:
+    echo "----"
+    var o2 = TCaseObj(kind: B, y: open())
+  
+  block:
+    echo "----"
+    var o3 = TCaseObj(kind: D, innerKind: B, r: "test",
+                      p: TMyGeneric3[int, float, string](x: 10, y: 1.0, z: "test"))
+
+  block:
+    echo "----"
+    var o4 = TCaseObj(kind: D, innerKind: D, r: "test",
+                      q: TMyGeneric3[TMyObj, int, int](x: open(), y: 1, z: 0))
+
+caseobj()
+

tests/destructor/tdictdestruct.nim

@@ -6,7 +6,7 @@ type
   PDict[TK, TV] = ref TDict[TK, TV]
 
 proc fakeNew[T](x: var ref T, destroy: proc (a: ref T) {.nimcall.}) =
-  nil
+  discard
 
 proc destroyDict[TK, TV](a: PDict[TK, TV]) =
     return

tests/discard/tdiscardable.nim

@@ -11,3 +11,19 @@ proc q[T](x, y: T): T {.discardable.} =
 p(8, 2)
 q[float](0.8, 0.2)
 
+# bug #942
+
+template maybeMod(x: Tinteger, module:Natural):expr =
+  if module > 0: x mod module
+  else: x
+
+proc foo(b: int):int =
+  var x = 1
+  result = x.maybeMod(b) # Works fine
+
+proc bar(b: int):int =
+  result = 1
+  result = result.maybeMod(b) # Error: value returned by statement has to be discarded
+
+echo foo(0)
+echo bar(0)

tests/discard/tneedsdiscard.nim

@@ -1,6 +1,6 @@
 discard """
   line: 10
-  errormsg: "value returned by statement has to be discarded"
+  errormsg: "value of type 'bool' has to be discarded"
 """
 
 proc p =

tests/effects/teffects6.nim

@@ -4,7 +4,7 @@ type
   PMenuItem = ref object
 
 proc createMenuItem*(menu: PMenu, label: string, 
-                     action: proc (i: PMenuItem, p: pointer) {.cdecl.}) = nil
+                    action: proc (i: PMenuItem, p: pointer) {.cdecl.}) = discard
 
 var s: PMenu
 createMenuItem(s, "Go to definition...",

tests/exception/texceptionbreak.nim

@@ -0,0 +1,45 @@
+discard """
+  file: "tnestedbreak.nim"
+  output: "1\n2\n3\n4"
+"""
+
+# First variety
+try:
+  raise newException(EOS, "Problem")
+except EOS:
+  for y in [1, 2, 3]:
+    discard
+  try:
+    discard
+  except EOS:
+    discard
+echo "1"
+
+# Second Variety
+try:
+  raise newException(EOS, "Problem")
+except EOS:
+  for y in [1, 2, 3]:
+    discard
+  for y in [1, 2, 3]:
+    discard
+
+echo "2"
+
+# Third Variety
+try:
+  raise newException(EOS, "Problem")
+except EOS:
+  block:
+    break
+
+echo "3"
+
+# Fourth Variety
+block:
+  try:
+    raise newException(EOS, "Problem")
+  except EOS:
+    break
+
+echo "4"