Merge remote-tracking branch 'origin/devel' into initallocator-fix

.gitignore

@@ -1,7 +1,11 @@
 *
 !**/
 !*.*
+
+# Cache
 nimcache/
+rnimcache/
+dnimcache/
 
 *.o
 !/icons/*.o

.gitlab-ci.yml

@@ -0,0 +1,61 @@
+image: ubuntu:16.04
+
+stages:
+  - pre-build
+  - build
+  - deploy
+  - test
+
+.linux_set_path: &linux_set_path_def
+  before_script:
+    - export PATH=$(pwd)/bin:$PATH
+  tags:
+    - linux
+
+.windows_set_path: &win_set_path_def
+  before_script:
+    - set PATH=%CD%\bin;%PATH%
+  tags:
+    - windows
+
+
+build-windows:
+  stage: build
+  script:
+    - ci\build.bat
+  artifacts:
+    paths:
+      - bin\nim.exe
+      - bin\nimd.exe
+      - compiler\nim.exe
+      - koch.exe
+    expire_in: 1 week
+  tags:
+    - windows
+
+deploy-windows:
+  stage: deploy
+  script:
+    - koch.exe winrelease
+  artifacts:
+    paths:
+      - build/*.exe
+      - build/*.zip
+    expire_in: 1 week
+  tags:
+    - windows
+    - fast
+
+
+
+test-windows:
+  stage: test
+  <<: *win_set_path_def
+  script:
+    - call ci\deps.bat
+    - nim c --taintMode:on tests\testament\tester
+    - tests\testament\tester.exe --pedantic all
+  tags:
+    - windows
+    - fast
+

.travis.yml

@@ -28,7 +28,7 @@ script:
   - nimble install zip
   - nimble install opengl
   - nimble install sdl1
-  - nimble install jester
+  - nimble install jester@#head
   - nimble install niminst
   - nim c --taintMode:on tests/testament/tester
   - tests/testament/tester --pedantic all

bootstrap.sh

@@ -3,7 +3,7 @@ set -e
 set -x
 
 if [ ! -e csources/.git ]; then
-	git clone --depth 1 git://github.com/nim-lang/csources.git csources
+	git clone --depth 1 https://github.com/nim-lang/csources.git csources
 fi
 
 cd "csources"

build_tools.sh

@@ -0,0 +1,3 @@
+./bin/nim c --noNimblePath -p:compiler -o:./bin/nimble dist/nimble/src/nimble.nim
+./bin/nim c --noNimblePath -p:compiler -o:./bin/nimsuggest dist/nimsuggest/nimsuggest.nim
+./bin/nim c -o:./bin/nimgrep tools/nimgrep.nim

ci/build.bat

@@ -0,0 +1,14 @@
+REM Some debug info
+echo "Running on %CI_RUNNER_ID% (%CI_RUNNER_DESCRIPTION%) with tags %CI_RUNNER_TAGS%."
+gcc -v
+
+git clone --depth 1 https://github.com/nim-lang/csources.git
+cd csources
+call build64.bat
+cd ..
+set PATH=%CD%\bin;%PATH%
+nim -v
+nim c koch
+koch.exe boot
+copy bin/nim bin/nimd
+koch.exe boot -d:release

ci/build.sh

@@ -0,0 +1,15 @@
+sh ci/deps.sh
+
+# Build from C sources.
+git clone --depth 1 https://github.com/nim-lang/csources.git
+cd csources
+sh build.sh
+cd ..
+# Add Nim to the PATH
+export PATH=$(pwd)/bin:$PATH
+# Bootstrap.
+nim -v
+nim c koch
+./koch boot
+cp bin/nim bin/nimd
+./koch boot -d:release

ci/deps.bat

@@ -0,0 +1,4 @@
+nim e install_nimble.nims
+nim e tests/test_nimscript.nims
+nimble update
+nimble install -y zip opengl sdl1 jester@#head niminst

ci/deps.sh

@@ -0,0 +1,17 @@
+# Some debug info
+echo "Running on $CI_RUNNER_ID ($CI_RUNNER_DESCRIPTION) with tags $CI_RUNNER_TAGS."
+
+# Packages
+apt-get update -qq
+apt-get install -y -qq build-essential git libcurl4-openssl-dev libsdl1.2-dev libgc-dev nodejs fasm
+
+gcc -v
+
+fasm -v
+export PATH=$(pwd)/bin:$PATH
+
+# Nimble deps
+nim e install_nimble.nims
+nim e tests/test_nimscript.nims
+nimble update
+nimble install zip opengl sdl1 jester@#head niminst

ci/nsis_build.bat

@@ -0,0 +1,62 @@
+REM - Run the full testsuite;  tests\testament\tester all
+
+REM - Uncomment the list of changes in news.txt
+REM - write a news ticker entry
+REM - Update the version
+
+REM - Generate the full docs;  koch web0
+REM - Generate the installers;
+REM - Update the version in system.nim
+REM - Test the installers
+REM - Tag the release
+REM - Merge devel into master
+REM - Update csources
+
+set NIMVER=%1
+
+Rem Build -docs file:
+koch web0
+cd web\upload
+7z a -tzip docs-%NIMVER%.zip *.html
+move /y docs-%NIMVER%.zip download
+cd ..\..
+
+Rem Build .zip file:
+rem koch csources -d:release
+rem koch xz -d:release
+rem move /y build\nim-%NIMVER%.zip web\upload\download
+
+
+rem Grab C sources and nimsuggest
+git clone --depth 1 https://github.com/nim-lang/csources.git
+
+set PATH=%CD%\bin;%PATH%
+
+ReM Build Win32 version:
+
+set PATH=C:\Users\araq\projects\mingw32\bin;%PATH%
+cd csources
+call build.bat
+cd ..
+ReM Rebuilding koch is necessary because it uses its pointer size to determine
+ReM which mingw link to put in the NSIS installer.
+nim c --out:koch_temp koch || exit /b
+koch_temp boot -d:release || exit /b
+koch_temp nsis -d:release || exit /b
+koch_temp zip -d:release || exit /b
+dir build
+move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x32.exe || exit /b
+move /y build\nim_%NIMVER%.zip build\nim-%NIMVER%_x32.zip || exit /b
+
+
+ReM Build Win64 version:
+set PATH=C:\Users\araq\projects\mingw64\bin;%PATH%
+cd csources
+call build64.bat
+cd ..
+nim c --out:koch_temp koch || exit /b
+koch_temp boot -d:release || exit /b
+koch_temp nsis -d:release || exit /b
+koch_temp zip -d:release || exit /b
+move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x64.exe || exit /b
+move /y build\nim_%NIMVER%.zip build\nim-%NIMVER%_x64.zip || exit /b

compiler.nimble

@@ -1,6 +1,6 @@
 [Package]
 name = "compiler"
-version = "0.14.3"
+version = "0.15.1"
 author = "Andreas Rumpf"
 description = "Compiler package providing the compiler sources as a library."
 license = "MIT"
@@ -8,4 +8,4 @@ license = "MIT"
 InstallDirs = "doc, compiler"
 
 [Deps]
-Requires: "nim >= 0.13.0"
+Requires: "nim >= 0.14.0"

compiler/aliases.nim

@@ -46,7 +46,8 @@ proc isPartOfAux(a, b: PType, marker: var IntSet): TAnalysisResult =
   if compareTypes(a, b, dcEqIgnoreDistinct): return arYes
   case a.kind
   of tyObject:
-    result = isPartOfAux(a.sons[0], b, marker)
+    if a.sons[0] != nil:
+      result = isPartOfAux(a.sons[0].skipTypes(skipPtrs), b, marker)
     if result == arNo: result = isPartOfAux(a.n, b, marker)
   of tyGenericInst, tyDistinct:
     result = isPartOfAux(lastSon(a), b, marker)

compiler/ast.nim

@@ -437,7 +437,7 @@ type
     nfExplicitCall # x.y() was used instead of x.y
     nfExprCall  # this is an attempt to call a regular expression
     nfIsRef     # this node is a 'ref' node; used for the VM
-    nfIsCursor  # this node is attached a cursor; used for idetools
+    nfPreventCg # this node should be ignored by the codegen
 
   TNodeFlags* = set[TNodeFlag]
   TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 28)
@@ -725,10 +725,7 @@ type
     s*: TStorageLoc
     flags*: TLocFlags         # location's flags
     t*: PType                 # type of location
-    r*: Rope                 # rope value of location (code generators)
+    r*: Rope                  # rope value of location (code generators)
-    heapRoot*: Rope          # keeps track of the enclosing heap object that
-                              # owns this location (required by GC algorithms
-                              # employing heap snapshots or sliding views)
 
   # ---------------- end of backend information ------------------------------
 
@@ -746,10 +743,6 @@ type
   TInstantiation* = object
     sym*: PSym
     concreteTypes*: seq[PType]
-    usedBy*: seq[int32]       # list of modules using the generic
-                              # needed in caas mode for purging the cache
-                              # XXX: it's possible to switch to a
-                              # simple ref count here
     compilesId*: CompilesId
 
   PInstantiation* = ref TInstantiation
@@ -767,7 +760,6 @@ type
     case kind*: TSymKind
     of skType, skGenericParam:
       typeInstCache*: seq[PType]
-      typScope*: PScope
     of routineKinds:
       procInstCache*: seq[PInstantiation]
       gcUnsafetyReason*: PSym  # for better error messages wrt gcsafe
@@ -862,9 +854,6 @@ type
     key*, val*: RootRef
 
   TPairSeq* = seq[TPair]
-  TTable* = object   # the same as table[PObject] of PObject
-    counter*: int
-    data*: TPairSeq
 
   TIdPair* = object
     key*: PIdObj
@@ -936,7 +925,7 @@ const
     skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
                                       nfDotSetter, nfDotField,
-                                      nfIsRef, nfIsCursor, nfLL}
+                                      nfIsRef, nfPreventCg, nfLL}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2
@@ -1108,12 +1097,6 @@ proc copyIdTable*(dest: var TIdTable, src: TIdTable) =
   newSeq(dest.data, len(src.data))
   for i in countup(0, high(src.data)): dest.data[i] = src.data[i]
 
-proc copyTable*(dest: var TTable, src: TTable) =
-  dest.counter = src.counter
-  if isNil(src.data): return
-  setLen(dest.data, len(src.data))
-  for i in countup(0, high(src.data)): dest.data[i] = src.data[i]
-
 proc copyObjectSet*(dest: var TObjectSet, src: TObjectSet) =
   dest.counter = src.counter
   if isNil(src.data): return
@@ -1327,10 +1310,6 @@ proc initStrTable*(x: var TStrTable) =
 proc newStrTable*: TStrTable =
   initStrTable(result)
 
-proc initTable(x: var TTable) =
-  x.counter = 0
-  newSeq(x.data, StartSize)
-
 proc initIdTable*(x: var TIdTable) =
   x.counter = 0
   newSeq(x.data, StartSize)
@@ -1429,6 +1408,7 @@ proc copyNode*(src: PNode): PNode =
   result.info = src.info
   result.typ = src.typ
   result.flags = src.flags * PersistentNodeFlags
+  result.comment = src.comment
   when defined(useNodeIds):
     if result.id == nodeIdToDebug:
       echo "COMES FROM ", src.id
@@ -1447,6 +1427,7 @@ proc shallowCopy*(src: PNode): PNode =
   result.info = src.info
   result.typ = src.typ
   result.flags = src.flags * PersistentNodeFlags
+  result.comment = src.comment
   when defined(useNodeIds):
     if result.id == nodeIdToDebug:
       echo "COMES FROM ", src.id
@@ -1466,6 +1447,7 @@ proc copyTree*(src: PNode): PNode =
   result.info = src.info
   result.typ = src.typ
   result.flags = src.flags * PersistentNodeFlags
+  result.comment = src.comment
   when defined(useNodeIds):
     if result.id == nodeIdToDebug:
       echo "COMES FROM ", src.id
@@ -1511,19 +1493,9 @@ proc hasSubnodeWith*(n: PNode, kind: TNodeKind): bool =
         return true
     result = false
 
-proc replaceSons(n: PNode, oldKind, newKind: TNodeKind) =
-  for i in countup(0, sonsLen(n) - 1):
-    if n.sons[i].kind == oldKind: n.sons[i].kind = newKind
-
-proc sonsNotNil(n: PNode): bool =
-  for i in countup(0, sonsLen(n) - 1):
-    if n.sons[i] == nil:
-      return false
-  result = true
-
 proc getInt*(a: PNode): BiggestInt =
   case a.kind
-  of nkIntLit..nkUInt64Lit: result = a.intVal
+  of nkCharLit..nkUInt64Lit: result = a.intVal
   else:
     internalError(a.info, "getInt")
     result = 0

compiler/astalgo.nim

@@ -31,17 +31,6 @@ proc objectSetIncl*(t: var TObjectSet, obj: RootRef)
 proc objectSetContainsOrIncl*(t: var TObjectSet, obj: RootRef): bool
   # more are not needed ...
 
-# ----------------------- (key, val)-Hashtables ----------------------------
-proc tablePut*(t: var TTable, key, val: RootRef)
-proc tableGet*(t: TTable, key: RootRef): RootRef
-type
-  TCmpProc* = proc (key, closure: RootRef): bool {.nimcall.} # true if found
-
-proc tableSearch*(t: TTable, key, closure: RootRef,
-                  comparator: TCmpProc): RootRef
-  # return val as soon as comparator returns true; if this never happens,
-  # nil is returned
-
 # ----------------------- str table -----------------------------------------
 proc strTableContains*(t: TStrTable, n: PSym): bool
 proc strTableAdd*(t: var TStrTable, n: PSym)
@@ -251,20 +240,6 @@ proc symToYamlAux(n: PSym, marker: var IntSet,
                   indent, maxRecDepth: int): Rope
 proc typeToYamlAux(n: PType, marker: var IntSet,
                    indent, maxRecDepth: int): Rope
-proc strTableToYaml(n: TStrTable, marker: var IntSet, indent: int,
-                    maxRecDepth: int): Rope =
-  var istr = rspaces(indent + 2)
-  result = rope("[")
-  var mycount = 0
-  for i in countup(0, high(n.data)):
-    if n.data[i] != nil:
-      if mycount > 0: add(result, ",")
-      addf(result, "$N$1$2",
-           [istr, symToYamlAux(n.data[i], marker, indent + 2, maxRecDepth - 1)])
-      inc(mycount)
-  if mycount > 0: addf(result, "$N$1", [rspaces(indent)])
-  add(result, "]")
-  assert(mycount == n.counter)
 
 proc ropeConstr(indent: int, c: openArray[Rope]): Rope =
   # array of (name, value) pairs
@@ -463,9 +438,6 @@ proc debug(n: PType) =
 proc debug(n: PNode) =
   echo($debugTree(n, 0, 100))
 
-const
-  EmptySeq = @[]
-
 proc nextTry(h, maxHash: Hash): Hash =
   result = ((5 * h) + 1) and maxHash
   # For any initial h in range(maxHash), repeating that maxHash times
@@ -519,55 +491,6 @@ proc objectSetContainsOrIncl(t: var TObjectSet, obj: RootRef): bool =
   inc(t.counter)
   result = false
 
-proc tableRawGet(t: TTable, key: RootRef): int =
-  var h: Hash = hashNode(key) and high(t.data) # start with real hash value
-  while t.data[h].key != nil:
-    if t.data[h].key == key:
-      return h
-    h = nextTry(h, high(t.data))
-  result = -1
-
-proc tableSearch(t: TTable, key, closure: RootRef,
-                 comparator: TCmpProc): RootRef =
-  var h: Hash = hashNode(key) and high(t.data) # start with real hash value
-  while t.data[h].key != nil:
-    if t.data[h].key == key:
-      if comparator(t.data[h].val, closure):
-        # BUGFIX 1
-        return t.data[h].val
-    h = nextTry(h, high(t.data))
-  result = nil
-
-proc tableGet(t: TTable, key: RootRef): RootRef =
-  var index = tableRawGet(t, key)
-  if index >= 0: result = t.data[index].val
-  else: result = nil
-
-proc tableRawInsert(data: var TPairSeq, key, val: RootRef) =
-  var h: Hash = hashNode(key) and high(data)
-  while data[h].key != nil:
-    assert(data[h].key != key)
-    h = nextTry(h, high(data))
-  assert(data[h].key == nil)
-  data[h].key = key
-  data[h].val = val
-
-proc tableEnlarge(t: var TTable) =
-  var n: TPairSeq
-  newSeq(n, len(t.data) * GrowthFactor)
-  for i in countup(0, high(t.data)):
-    if t.data[i].key != nil: tableRawInsert(n, t.data[i].key, t.data[i].val)
-  swap(t.data, n)
-
-proc tablePut(t: var TTable, key, val: RootRef) =
-  var index = tableRawGet(t, key)
-  if index >= 0:
-    t.data[index].val = val
-  else:
-    if mustRehash(len(t.data), t.counter): tableEnlarge(t)
-    tableRawInsert(t.data, key, val)
-    inc(t.counter)
-
 proc strTableContains(t: TStrTable, n: PSym): bool =
   var h: Hash = n.name.h and high(t.data) # start with real hash value
   while t.data[h] != nil:

compiler/canonicalizer.nim

@@ -383,18 +383,28 @@ proc createDb() =
       interfHash varchar(256) not null,
       fullHash varchar(256) not null,
 
-      created timestamp not null default (DATETIME('now')),
+      created timestamp not null default (DATETIME('now'))
     );""")
 
   db.exec(sql"""
+    create table if not exists Backend(
+      id integer primary key,
+      strongdeps varchar(max) not null,
+      weakdeps varchar(max) not null,
+      header varchar(max) not null,
+      code varchar(max) not null
+    )
+
     create table if not exists Symbol(
       id integer primary key,
       module integer not null,
+      backend integer not null,
       name varchar(max) not null,
       data varchar(max) not null,
       created timestamp not null default (DATETIME('now')),
 
-      foreign key (module) references module(id)
+      foreign key (module) references Module(id),
+      foreign key (backend) references Backend(id)
     );""")
 
   db.exec(sql"""
@@ -409,7 +419,3 @@ proc createDb() =
     );""")
 
 
-  #db.exec(sql"""
-  #  --create unique index if not exists TsstNameIx on TestResult(name);
-  #  """, [])
-

compiler/ccgcalls.nim

@@ -411,7 +411,7 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
         add(result, substr(pat, start, i - 1))
 
 proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
-  var op, a: TLoc
+  var op: TLoc
   initLocExpr(p, ri.sons[0], op)
   # getUniqueType() is too expensive here:
   var typ = skipTypes(ri.sons[0].typ, abstractInst)
@@ -458,7 +458,7 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
 
 proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
   # generates a crappy ObjC call
-  var op, a: TLoc
+  var op: TLoc
   initLocExpr(p, ri.sons[0], op)
   var pl = ~"["
   # getUniqueType() is too expensive here:
@@ -536,8 +536,6 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) =
   else:
     genPrefixCall(p, nil, e, d)
   postStmtActions(p)
-  when false:
-    if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d)
 
 proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
   if ri.sons[0].typ.skipTypes({tyGenericInst}).callConv == ccClosure:
@@ -549,6 +547,3 @@ proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
   else:
     genPrefixCall(p, le, ri, d)
   postStmtActions(p)
-  when false:
-    if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d)
-

compiler/ccgexprs.nim

@@ -30,19 +30,6 @@ proc intLiteral(i: BiggestInt): Rope =
   else:
     result = ~"(IL64(-9223372036854775807) - IL64(1))"
 
-proc int32Literal(i: int): Rope =
-  if i == int(low(int32)):
-    result = ~"(-2147483647 -1)"
-  else:
-    result = rope(i)
-
-proc genHexLiteral(v: PNode): Rope =
-  # hex literals are unsigned in C
-  # so we don't generate hex literals any longer.
-  if v.kind notin {nkIntLit..nkUInt64Lit}:
-    internalError(v.info, "genHexLiteral")
-  result = intLiteral(v.intVal)
-
 proc getStrLit(m: BModule, s: string): Rope =
   discard cgsym(m, "TGenericSeq")
   result = getTempName(m)
@@ -171,7 +158,6 @@ proc getStorageLoc(n: PNode): TStorageLoc =
 proc genRefAssign(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
   if dest.s == OnStack or not usesNativeGC():
     linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src))
-    if needToKeepAlive in flags: keepAlive(p, dest)
   elif dest.s == OnHeap:
     # location is on heap
     # now the writer barrier is inlined for performance:
@@ -198,7 +184,6 @@ proc genRefAssign(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
   else:
     linefmt(p, cpsStmts, "#unsureAsgnRef((void**) $1, $2);$n",
             addrLoc(dest), rdLoc(src))
-    if needToKeepAlive in flags: keepAlive(p, dest)
 
 proc asgnComplexity(n: PNode): int =
   if n != nil:
@@ -218,7 +203,6 @@ proc optAsgnLoc(a: TLoc, t: PType, field: Rope): TLoc =
   result.s = a.s
   result.t = t
   result.r = rdLoc(a) & "." & field
-  result.heapRoot = a.heapRoot
 
 proc genOptAsgnTuple(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
   let newflags =
@@ -268,7 +252,6 @@ proc genGenericAsgn(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
       linefmt(p, cpsStmts,
            "memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n",
            addrLoc(dest), addrLoc(src), rdLoc(dest))
-      if needToKeepAlive in flags: keepAlive(p, dest)
     else:
       linefmt(p, cpsStmts, "#genericShallowAssign((void*)$1, (void*)$2, $3);$n",
               addrLoc(dest), addrLoc(src), genTypeInfo(p.module, dest.t))
@@ -299,7 +282,6 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
     else:
       if dest.s == OnStack or not usesNativeGC():
         linefmt(p, cpsStmts, "$1 = #copyString($2);$n", dest.rdLoc, src.rdLoc)
-        if needToKeepAlive in flags: keepAlive(p, dest)
       elif dest.s == OnHeap:
         # we use a temporary to care for the dreaded self assignment:
         var tmp: TLoc
@@ -310,7 +292,6 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
       else:
         linefmt(p, cpsStmts, "#unsureAsgnRef((void**) $1, #copyString($2));$n",
                addrLoc(dest), rdLoc(src))
-        if needToKeepAlive in flags: keepAlive(p, dest)
   of tyProc:
     if needsComplexAssignment(dest.t):
       # optimize closure assignment:
@@ -400,9 +381,6 @@ proc genDeepCopy(p: BProc; dest, src: TLoc) =
     linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src))
   else: internalError("genDeepCopy: " & $ty.kind)
 
-proc getDestLoc(p: BProc, d: var TLoc, typ: PType) =
-  if d.k == locNone: getTemp(p, typ, d)
-
 proc putLocIntoDest(p: BProc, d: var TLoc, s: TLoc) =
   if d.k != locNone:
     if lfNoDeepCopy in d.flags: genAssignment(p, d, s, {})
@@ -453,13 +431,6 @@ proc unaryStmt(p: BProc, e: PNode, d: var TLoc, frmt: string) =
   initLocExpr(p, e.sons[1], a)
   lineCg(p, cpsStmts, frmt, [rdLoc(a)])
 
-proc binaryStmtChar(p: BProc, e: PNode, d: var TLoc, frmt: string) =
-  var a, b: TLoc
-  if (d.k != locNone): internalError(e.info, "binaryStmtChar")
-  initLocExpr(p, e.sons[1], a)
-  initLocExpr(p, e.sons[2], b)
-  lineCg(p, cpsStmts, frmt, [rdCharLoc(a), rdCharLoc(b)])
-
 proc binaryExpr(p: BProc, e: PNode, d: var TLoc, frmt: string) =
   var a, b: TLoc
   assert(e.sons[1].typ != nil)
@@ -728,8 +699,6 @@ proc genAddr(p: BProc, e: PNode, d: var TLoc) =
 
 template inheritLocation(d: var TLoc, a: TLoc) =
   if d.k == locNone: d.s = a.s
-  if d.heapRoot == nil:
-    d.heapRoot = if a.heapRoot != nil: a.heapRoot else: a.r
 
 proc genRecordFieldAux(p: BProc, e: PNode, d, a: var TLoc): PType =
   initLocExpr(p, e.sons[0], a)
@@ -757,6 +726,7 @@ proc lookupFieldAgain(p: BProc, ty: PType; field: PSym; r: var Rope): PSym =
   var ty = ty
   assert r != nil
   while ty != nil:
+    ty = ty.skipTypes(skipPtrs)
     assert(ty.kind in {tyTuple, tyObject})
     result = lookupInRecord(ty.n, field.name)
     if result != nil: break
@@ -893,7 +863,6 @@ proc genSeqElem(p: BProc, x, y: PNode, d: var TLoc) =
            "if ((NU)($1) >= (NU)($2->$3)) #raiseIndexError();$n",
            rdLoc(b), rdLoc(a), lenField(p))
   if d.k == locNone: d.s = OnHeap
-  d.heapRoot = a.r
   if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}:
     a.r = rfmt(nil, "(*$1)", a.r)
   putIntoDest(p, d, elemType(skipTypes(a.t, abstractVar)),
@@ -1008,9 +977,8 @@ proc genStrConcat(p: BProc, e: PNode, d: var TLoc) =
   add(p.s(cpsStmts), appends)
   if d.k == locNone:
     d = tmp
-    keepAlive(p, tmp)
   else:
-    genAssignment(p, d, tmp, {needToKeepAlive}) # no need for deep copying
+    genAssignment(p, d, tmp, {}) # no need for deep copying
   gcUsage(e)
 
 proc genStrAppend(p: BProc, e: PNode, d: var TLoc) =
@@ -1047,7 +1015,6 @@ proc genStrAppend(p: BProc, e: PNode, d: var TLoc) =
                         rdLoc(dest), rdLoc(a)))
   linefmt(p, cpsStmts, "$1 = #resizeString($1, $2$3);$n",
           rdLoc(dest), lens, rope(L))
-  keepAlive(p, dest)
   add(p.s(cpsStmts), appends)
   gcUsage(e)
 
@@ -1067,7 +1034,6 @@ proc genSeqElemAppend(p: BProc, e: PNode, d: var TLoc) =
       rdLoc(a),
       getTypeDesc(p.module, skipTypes(e.sons[1].typ, abstractVar)),
       getTypeDesc(p.module, bt)])
-  keepAlive(p, a)
   #if bt != b.t:
   #  echo "YES ", e.info, " new: ", typeToString(bt), " old: ", typeToString(b.t)
   initLoc(dest, locExpr, bt, OnHeap)
@@ -1094,7 +1060,7 @@ proc rawGenNew(p: BProc, a: TLoc, sizeExpr: Rope) =
               genTypeInfo(p.module, refType),
               sizeExpr]
   if a.s == OnHeap and usesNativeGC():
-    # use newObjRC1 as an optimization; and we don't need 'keepAlive' either
+    # use newObjRC1 as an optimization
     if canFormAcycle(a.t):
       linefmt(p, cpsStmts, "if ($1) #nimGCunref($1);$n", a.rdLoc)
     else:
@@ -1103,7 +1069,7 @@ proc rawGenNew(p: BProc, a: TLoc, sizeExpr: Rope) =
     linefmt(p, cpsStmts, "$1 = $2;$n", a.rdLoc, b.rdLoc)
   else:
     b.r = ropecg(p.module, "($1) #newObj($2, $3)", args)
-    genAssignment(p, a, b, {needToKeepAlive})  # set the object type:
+    genAssignment(p, a, b, {})  # set the object type:
   let bt = skipTypes(refType.sons[0], abstractRange)
   genObjectInit(p, cpsStmts, bt, a, false)
 
@@ -1134,7 +1100,7 @@ proc genNewSeqAux(p: BProc, dest: TLoc, length: Rope) =
     linefmt(p, cpsStmts, "$1 = $2;$n", dest.rdLoc, call.rdLoc)
   else:
     call.r = ropecg(p.module, "($1) #newSeq($2, $3)", args)
-    genAssignment(p, dest, call, {needToKeepAlive})
+    genAssignment(p, dest, call, {})
 
 proc genNewSeq(p: BProc, e: PNode) =
   var a, b: TLoc
@@ -1199,7 +1165,6 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) =
     tmp2.k = locTemp
     tmp2.t = field.loc.t
     tmp2.s = if isRef: OnHeap else: OnStack
-    tmp2.heapRoot = tmp.r
     expr(p, it.sons[1], tmp2)
 
   if d.k == locNone:
@@ -1246,7 +1211,6 @@ proc genNewFinalize(p: BProc, e: PNode) =
     a, b, f: TLoc
     refType, bt: PType
     ti: Rope
-    oldModule: BModule
   refType = skipTypes(e.sons[1].typ, abstractVarRange)
   initLocExpr(p, e.sons[1], a)
   initLocExpr(p, e.sons[2], f)
@@ -1256,7 +1220,7 @@ proc genNewFinalize(p: BProc, e: PNode) =
   b.r = ropecg(p.module, "($1) #newObj($2, sizeof($3))", [
       getTypeDesc(p.module, refType),
       ti, getTypeDesc(p.module, skipTypes(refType.lastSon, abstractRange))])
-  genAssignment(p, a, b, {needToKeepAlive})  # set the object type:
+  genAssignment(p, a, b, {})  # set the object type:
   bt = skipTypes(refType.lastSon, abstractRange)
   genObjectInit(p, cpsStmts, bt, a, false)
   gcUsage(e)
@@ -1294,7 +1258,7 @@ proc genOf(p: BProc, x: PNode, typ: PType, d: var TLoc) =
   if not p.module.compileToCpp:
     while t.kind == tyObject and t.sons[0] != nil:
       add(r, ~".Sup")
-      t = skipTypes(t.sons[0], typedescInst)
+      t = skipTypes(t.sons[0], skipPtrs)
   if isObjLackingTypeField(t):
     globalError(x.info, errGenerated,
       "no 'of' operator available for pure objects")
@@ -1366,7 +1330,7 @@ proc genDollar(p: BProc, n: PNode, d: var TLoc, frmt: string) =
   initLocExpr(p, n.sons[1], a)
   a.r = ropecg(p.module, frmt, [rdLoc(a)])
   if d.k == locNone: getTemp(p, n.typ, d)
-  genAssignment(p, d, a, {needToKeepAlive})
+  genAssignment(p, d, a, {})
   gcUsage(n)
 
 proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
@@ -1408,12 +1372,10 @@ proc genSetLengthSeq(p: BProc, e: PNode, d: var TLoc) =
   lineCg(p, cpsStmts, setLenPattern, [
       rdLoc(a), rdLoc(b), getTypeDesc(p.module, t),
       getTypeDesc(p.module, t.sons[0])])
-  keepAlive(p, a)
   gcUsage(e)
 
 proc genSetLengthStr(p: BProc, e: PNode, d: var TLoc) =
   binaryStmt(p, e, d, "$1 = #setLengthStr($1, $2);$n")
-  keepAlive(p, d)
   gcUsage(e)
 
 proc genSwap(p: BProc, e: PNode, d: var TLoc) =
@@ -1683,7 +1645,6 @@ proc binaryFloatArith(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
     binaryArith(p, e, d, m)
 
 proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
-  var line, filen: Rope
   case op
   of mOr, mAnd: genAndOr(p, e, d, op)
   of mNot..mToBiggestInt: unaryArith(p, e, d, op)
@@ -1721,10 +1682,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
         getTypeDesc(p.module, ranged), res])
 
   of mConStrStr: genStrConcat(p, e, d)
-  of mAppendStrCh:
+  of mAppendStrCh: binaryStmt(p, e, d, "$1 = #addChar($1, $2);$n")
-    binaryStmt(p, e, d, "$1 = #addChar($1, $2);$n")
-    # strictly speaking we need to generate "keepAlive" here too, but this
-    # very likely not needed and would slow down the code too much I fear
   of mAppendStrStr: genStrAppend(p, e, d)
   of mAppendSeqElem: genSeqElemAppend(p, e, d)
   of mEqStr: genStrEquals(p, e, d)
@@ -1918,7 +1876,7 @@ proc upConv(p: BProc, n: PNode, d: var TLoc) =
     if not p.module.compileToCpp:
       while t.kind == tyObject and t.sons[0] != nil:
         add(r, ".Sup")
-        t = skipTypes(t.sons[0], abstractInst)
+        t = skipTypes(t.sons[0], skipPtrs)
     if nilCheck != nil:
       linefmt(p, cpsStmts, "if ($1) #chckObj($2.m_type, $3);$n",
               nilCheck, r, genTypeInfo(p.module, dest))
@@ -2018,7 +1976,8 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
     of skEnumField:
       putIntoDest(p, d, n.typ, rope(sym.position))
     of skVar, skForVar, skResult, skLet:
-      if sfGlobal in sym.flags: genVarPrototype(p.module, sym)
+      if {sfGlobal, sfThread} * sym.flags != {}:
+        genVarPrototype(p.module, sym)
       if sym.loc.r == nil or sym.loc.t == nil:
         #echo "FAILED FOR PRCO ", p.prc.name.s
         #echo renderTree(p.prc.ast, {renderIds})
@@ -2126,11 +2085,14 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
   of nkCaseStmt: genCase(p, n, d)
   of nkReturnStmt: genReturnStmt(p, n)
   of nkBreakStmt: genBreakStmt(p, n)
-  of nkAsgn: genAsgn(p, n, fastAsgn=false)
+  of nkAsgn:
+    if nfPreventCg notin n.flags:
+      genAsgn(p, n, fastAsgn=false)
   of nkFastAsgn:
-    # transf is overly aggressive with 'nkFastAsgn', so we work around here.
+    if nfPreventCg notin n.flags:
-    # See tests/run/tcnstseq3 for an example that would fail otherwise.
+      # transf is overly aggressive with 'nkFastAsgn', so we work around here.
-    genAsgn(p, n, fastAsgn=p.prc != nil)
+      # See tests/run/tcnstseq3 for an example that would fail otherwise.
+      genAsgn(p, n, fastAsgn=p.prc != nil)
   of nkDiscardStmt:
     if n.sons[0].kind != nkEmpty:
       genLineDir(p, n)

compiler/ccgstmts.nim

@@ -21,7 +21,7 @@ proc registerGcRoot(p: BProc, v: PSym) =
     # we register a specialized marked proc here; this has the advantage
     # that it works out of the box for thread local storage then :-)
     let prc = genTraverseProcForGlobal(p.module, v)
-    appcg(p.module, p.module.initProc.procSec(cpsStmts),
+    appcg(p.module, p.module.initProc.procSec(cpsInit),
       "#nimRegisterGlobalMarker($1);$n", [prc])
 
 proc isAssignedImmediately(n: PNode): bool {.inline.} =
@@ -293,6 +293,8 @@ proc genIf(p: BProc, n: PNode, d: var TLoc) =
   genLineDir(p, n)
   let lend = getLabel(p)
   for i in countup(0, sonsLen(n) - 1):
+    # bug #4230: avoid false sharing between branches:
+    if d.k == locTemp and isEmptyType(n.typ): d.k = locNone
     let it = n.sons[i]
     if it.len == 2:
       when newScopeForIf: startBlock(p)
@@ -359,6 +361,7 @@ proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
       linefmt(p, cpsStmts, "#popCurrentException();$n")
 
 proc genReturnStmt(p: BProc, t: PNode) =
+  if nfPreventCg in t.flags: return
   p.beforeRetNeeded = true
   genLineDir(p, t)
   if (t.sons[0].kind != nkEmpty): genStmts(p, t.sons[0])
@@ -459,7 +462,6 @@ proc genWhileStmt(p: BProc, t: PNode) =
   # significantly worse code
   var
     a: TLoc
-    labl: TLabel
   assert(sonsLen(t) == 2)
   inc(p.withinLoop)
   genLineDir(p, t)
@@ -488,16 +490,20 @@ proc genWhileStmt(p: BProc, t: PNode) =
 
   dec(p.withinLoop)
 
-proc genBlock(p: BProc, t: PNode, d: var TLoc) =
+proc genBlock(p: BProc, n: PNode, d: var TLoc) =
+  # bug #4505: allocate the temp in the outer scope
+  # so that it can escape the generated {}:
+  if not isEmptyType(n.typ) and d.k == locNone:
+    getTemp(p, n.typ, d)
   preserveBreakIdx:
     p.breakIdx = startBlock(p)
-    if t.sons[0].kind != nkEmpty:
+    if n.sons[0].kind != nkEmpty:
       # named block?
-      assert(t.sons[0].kind == nkSym)
+      assert(n.sons[0].kind == nkSym)
-      var sym = t.sons[0].sym
+      var sym = n.sons[0].sym
       sym.loc.k = locOther
       sym.position = p.breakIdx+1
-    expr(p, t.sons[1], d)
+    expr(p, n.sons[1], d)
     endBlock(p)
 
 proc genParForStmt(p: BProc, t: PNode) =
@@ -592,6 +598,8 @@ proc genCaseSecondPass(p: BProc, t: PNode, d: var TLoc,
                        labId, until: int): TLabel =
   var lend = getLabel(p)
   for i in 1..until:
+    # bug #4230: avoid false sharing between branches:
+    if d.k == locTemp and isEmptyType(t.typ): d.k = locNone
     lineF(p, cpsStmts, "LA$1: ;$n", [rope(labId + i)])
     if t.sons[i].kind == nkOfBranch:
       var length = sonsLen(t.sons[i])
@@ -727,6 +735,8 @@ proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) =
     lineF(p, cpsStmts, "switch ($1) {$n", [rdCharLoc(a)])
     var hasDefault = false
     for i in splitPoint+1 .. < n.len:
+      # bug #4230: avoid false sharing between branches:
+      if d.k == locTemp and isEmptyType(n.typ): d.k = locNone
       var branch = n[i]
       if branch.kind == nkOfBranch:
         genCaseRange(p, branch)
@@ -757,16 +767,6 @@ proc genCase(p: BProc, t: PNode, d: var TLoc) =
     else:
       genOrdinalCase(p, t, d)
 
-proc hasGeneralExceptSection(t: PNode): bool =
-  var length = sonsLen(t)
-  var i = 1
-  while (i < length) and (t.sons[i].kind == nkExceptBranch):
-    var blen = sonsLen(t.sons[i])
-    if blen == 1:
-      return true
-    inc(i)
-  result = false
-
 proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   # code to generate:
   #
@@ -807,6 +807,8 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   var i = 1
   var catchAllPresent = false
   while (i < length) and (t.sons[i].kind == nkExceptBranch):
+    # bug #4230: avoid false sharing between branches:
+    if d.k == locTemp and isEmptyType(t.typ): d.k = locNone
     let blen = sonsLen(t.sons[i])
     if i > 1: addf(p.s(cpsStmts), "else ", [])
     if blen == 1:
@@ -912,6 +914,8 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
   inc p.inExceptBlock
   var i = 1
   while (i < length) and (t.sons[i].kind == nkExceptBranch):
+    # bug #4230: avoid false sharing between branches:
+    if d.k == locTemp and isEmptyType(t.typ): d.k = locNone
     var blen = sonsLen(t.sons[i])
     if blen == 1:
       # general except section:
@@ -1089,7 +1093,6 @@ proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
 proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   var a, tmp: TLoc
   var dotExpr = e.sons[0]
-  var d: PSym
   if dotExpr.kind == nkCheckedFieldExpr: dotExpr = dotExpr.sons[0]
   initLocExpr(p, e.sons[0], a)
   getTemp(p, a.t, tmp)

compiler/ccgtrav.nim

@@ -73,7 +73,9 @@ proc genTraverseProc(c: var TTraversalClosure, accessor: Rope, typ: PType) =
     lineF(p, cpsStmts, "}$n", [])
   of tyObject:
     for i in countup(0, sonsLen(typ) - 1):
-      genTraverseProc(c, accessor.parentObj(c.p.module), typ.sons[i])
+      var x = typ.sons[i]
+      if x != nil: x = x.skipTypes(skipPtrs)
+      genTraverseProc(c, accessor.parentObj(c.p.module), x)
     if typ.n != nil: genTraverseProc(c, accessor, typ.n)
   of tyTuple:
     let typ = getUniqueType(typ)

compiler/ccgtypes.nim

@@ -475,11 +475,11 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
         hasField = true
     elif m.compileToCpp:
       appcg(m, result, " : public $1 {$n",
-                      [getTypeDescAux(m, typ.sons[0], check)])
+                      [getTypeDescAux(m, typ.sons[0].skipTypes(skipPtrs), check)])
       hasField = true
     else:
       appcg(m, result, " {$n  $1 Sup;$n",
-                      [getTypeDescAux(m, typ.sons[0], check)])
+                      [getTypeDescAux(m, typ.sons[0].skipTypes(skipPtrs), check)])
       hasField = true
   else:
     addf(result, " {$n", [name])
@@ -546,7 +546,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope =
   of tyRef, tyPtr, tyVar:
     var star = if t.kind == tyVar and tfVarIsPtr notin typ.flags and
                     compileToCpp(m): "&" else: "*"
-    var et = t.lastSon
+    var et = typ.skipTypes(abstractInst).lastSon
     var etB = et.skipTypes(abstractInst)
     if etB.kind in {tyArrayConstr, tyArray, tyOpenArray, tyVarargs}:
       # this is correct! sets have no proper base type, so we treat
@@ -744,14 +744,6 @@ proc getClosureType(m: BModule, t: PType, kind: TClosureTypeKind): Rope =
           "void* ClEnv;$n} $1;$n",
            [result, rettype, desc])
 
-proc getTypeDesc(m: BModule, magic: string): Rope =
-  var sym = magicsys.getCompilerProc(magic)
-  if sym != nil:
-    result = getTypeDesc(m, sym.typ)
-  else:
-    rawMessage(errSystemNeeds, magic)
-    result = nil
-
 proc finishTypeDescriptions(m: BModule) =
   var i = 0
   while i < len(m.typeStack):
@@ -820,7 +812,9 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope) =
 proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope) =
   var base: Rope
   if (sonsLen(typ) > 0) and (typ.sons[0] != nil):
-    base = genTypeInfo(m, typ.sons[0])
+    var x = typ.sons[0]
+    if typ.kind == tyObject: x = x.skipTypes(skipPtrs)
+    base = genTypeInfo(m, x)
   else:
     base = rope("0")
   genTypeInfoAuxBase(m, typ, origType, name, base)
@@ -894,10 +888,11 @@ proc genObjectFields(m: BModule, typ: PType, n: PNode, expr: Rope) =
       else: internalError(n.info, "genObjectFields(nkRecCase)")
   of nkSym:
     var field = n.sym
-    addf(m.s[cfsTypeInit3], "$1.kind = 1;$n" &
+    if field.bitsize == 0:
-        "$1.offset = offsetof($2, $3);$n" & "$1.typ = $4;$n" &
+      addf(m.s[cfsTypeInit3], "$1.kind = 1;$n" &
-        "$1.name = $5;$n", [expr, getTypeDesc(m, typ),
+          "$1.offset = offsetof($2, $3);$n" & "$1.typ = $4;$n" &
-        field.loc.r, genTypeInfo(m, field.typ), makeCString(field.name.s)])
+          "$1.name = $5;$n", [expr, getTypeDesc(m, typ),
+          field.loc.r, genTypeInfo(m, field.typ), makeCString(field.name.s)])
   else: internalError(n.info, "genObjectFields")
 
 proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope) =
@@ -909,7 +904,7 @@ proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope) =
   addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, tmp])
   var t = typ.sons[0]
   while t != nil:
-    t = t.skipTypes(abstractInst)
+    t = t.skipTypes(skipPtrs)
     t.flags.incl tfObjHasKids
     t = t.sons[0]
 
@@ -1000,9 +995,6 @@ proc fakeClosureType(owner: PSym): PType =
 type
   TTypeInfoReason = enum  ## for what do we need the type info?
     tiNew,                ## for 'new'
-    tiNewSeq,             ## for 'newSeq'
-    tiNonVariantAsgn,     ## for generic assignment without variants
-    tiVariantAsgn         ## for generic assignment with variants
 
 include ccgtrav
 

compiler/cgen.nim

@@ -167,10 +167,6 @@ proc linefmt(p: BProc, s: TCProcSection, frmt: FormatStr,
              args: varargs[Rope]) =
   add(p.s(s), indentLine(p, ropecg(p.module, frmt, args)))
 
-proc appLineCg(p: BProc, r: var Rope, frmt: FormatStr,
-               args: varargs[Rope]) =
-  add(r, indentLine(p, ropecg(p.module, frmt, args)))
-
 proc safeLineNm(info: TLineInfo): int =
   result = toLinenumber(info)
   if result < 0: result = 0 # negative numbers are not allowed in #line
@@ -250,7 +246,7 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: TLoc,
     if not p.module.compileToCpp:
       while (s.kind == tyObject) and (s.sons[0] != nil):
         add(r, ".Sup")
-        s = skipTypes(s.sons[0], abstractInst)
+        s = skipTypes(s.sons[0], skipPtrs)
     linefmt(p, section, "$1.m_type = $2;$n", r, genTypeInfo(p.module, t))
   of frEmbedded:
     # worst case for performance:
@@ -259,8 +255,7 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: TLoc,
 
 type
   TAssignmentFlag = enum
-    needToCopy, needForSubtypeCheck, afDestIsNil, afDestIsNotNil, afSrcIsNil,
+    needToCopy, afDestIsNil, afDestIsNotNil, afSrcIsNil, afSrcIsNotNil
-    afSrcIsNotNil, needToKeepAlive
   TAssignmentFlags = set[TAssignmentFlag]
 
 proc genRefAssign(p: BProc, dest, src: TLoc, flags: TAssignmentFlags)
@@ -328,8 +323,9 @@ proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) =
 proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) =
   inc(p.labels)
   result.r = "LOC" & rope(p.labels)
-  addf(p.blocks[0].sections[cpsLocals],
+  #addf(p.blocks[0].sections[cpsLocals],
-     "$1 $2;$n", [getTypeDesc(p.module, t), result.r])
+  #   "$1 $2;$n", [getTypeDesc(p.module, t), result.r])
+  linefmt(p, cpsLocals, "$1 $2;$n", getTypeDesc(p.module, t), result.r)
   result.k = locTemp
   #result.a = - 1
   result.t = t
@@ -338,31 +334,6 @@ proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) =
   result.flags = {}
   constructLoc(p, result, not needsInit)
 
-proc keepAlive(p: BProc, toKeepAlive: TLoc) =
-  when false:
-    # deactivated because of the huge slowdown this causes; GC will take care
-    # of interior pointers instead
-    if optRefcGC notin gGlobalOptions: return
-    var result: TLoc
-    var fid = rope(p.gcFrameId)
-    result.r = "GCFRAME.F" & fid
-    addf(p.gcFrameType, "  $1 F$2;$n",
-        [getTypeDesc(p.module, toKeepAlive.t), fid])
-    inc(p.gcFrameId)
-    result.k = locTemp
-    #result.a = -1
-    result.t = toKeepAlive.t
-    result.s = OnStack
-    result.flags = {}
-
-    if not isComplexValueType(skipTypes(toKeepAlive.t, abstractVarRange)):
-      linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(result), rdLoc(toKeepAlive))
-    else:
-      useStringh(p.module)
-      linefmt(p, cpsStmts,
-           "memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n",
-           addrLoc(result), addrLoc(toKeepAlive), rdLoc(result))
-
 proc initGCFrame(p: BProc): Rope =
   if p.gcFrameId > 0: result = "struct {$1} GCFRAME;$n" % [p.gcFrameType]
 
@@ -620,9 +591,6 @@ proc generateHeaders(m: BModule) =
       addf(m.s[cfsHeaders], "#include $1$N", [rope(it.data)])
     it = PStrEntry(it.next)
 
-proc retIsNotVoid(s: PSym): bool =
-  result = (s.typ.sons[0] != nil) and not isInvalidReturnType(s.typ.sons[0])
-
 proc initFrame(p: BProc, procname, filename: Rope): Rope =
   discard cgsym(p.module, "nimFrame")
   if p.maxFrameLen > 0:
@@ -649,6 +617,24 @@ proc closureSetup(p: BProc, prc: PSym) =
   linefmt(p, cpsStmts, "$1 = ($2) ClEnv;$n",
           rdLoc(env.loc), getTypeDesc(p.module, env.typ))
 
+proc easyResultAsgn(n: PNode): PNode =
+  const harmless = {nkConstSection, nkTypeSection, nkEmpty, nkCommentStmt} +
+                    declarativeDefs
+  case n.kind
+  of nkStmtList, nkStmtListExpr:
+    var i = 0
+    while i < n.len and n[i].kind in harmless: inc i
+    if i < n.len: result = easyResultAsgn(n[i])
+  of nkAsgn, nkFastAsgn:
+    if n[0].kind == nkSym and skResult == n[0].sym.kind:
+      incl n.flags, nfPreventCg
+      return n[1]
+  of nkReturnStmt:
+    if n.len > 0:
+      result = easyResultAsgn(n[0])
+      if result != nil: incl n.flags, nfPreventCg
+  else: discard
+
 proc genProcAux(m: BModule, prc: PSym) =
   var p = newProc(prc, m)
   var header = genProcHeader(m, prc)
@@ -660,11 +646,17 @@ proc genProcAux(m: BModule, prc: PSym) =
     var res = prc.ast.sons[resultPos].sym # get result symbol
     if not isInvalidReturnType(prc.typ.sons[0]):
       if sfNoInit in prc.flags: incl(res.flags, sfNoInit)
-      # declare the result symbol:
+      if sfNoInit in prc.flags and p.module.compileToCpp and (let val = easyResultAsgn(prc.getBody); val != nil):
-      assignLocalVar(p, res)
+        var decl = localVarDecl(p, res)
-      assert(res.loc.r != nil)
+        var a: TLoc
+        initLocExprSingleUse(p, val, a)
+        linefmt(p, cpsStmts, "$1 = $2;$n", decl, rdLoc(a))
+      else:
+        # declare the result symbol:
+        assignLocalVar(p, res)
+        assert(res.loc.r != nil)
+        initLocalVar(p, res, immediateAsgn=false)
       returnStmt = rfmt(nil, "\treturn $1;$n", rdLoc(res.loc))
-      initLocalVar(p, res, immediateAsgn=false)
     else:
       fillResult(res)
       assignParam(p, res)
@@ -796,7 +788,7 @@ proc genProc(m: BModule, prc: PSym) =
           genProcAux(generatedHeader, prc)
 
 proc genVarPrototypeAux(m: BModule, sym: PSym) =
-  assert(sfGlobal in sym.flags)
+  #assert(sfGlobal in sym.flags)
   useHeader(m, sym)
   fillLoc(sym.loc, locGlobalVar, sym.typ, mangleName(sym), OnHeap)
   if (lfNoDecl in sym.loc.flags) or containsOrIncl(m.declaredThings, sym.id):
@@ -824,12 +816,12 @@ proc addIntTypes(result: var Rope) {.inline.} =
 proc getCopyright(cfile: string): Rope =
   if optCompileOnly in gGlobalOptions:
     result = ("/* Generated by Nim Compiler v$1 */$N" &
-        "/*   (c) 2015 Andreas Rumpf */$N" &
+        "/*   (c) 2016 Andreas Rumpf */$N" &
         "/* The generated code is subject to the original license. */$N") %
         [rope(VersionAsString)]
   else:
     result = ("/* Generated by Nim Compiler v$1 */$N" &
-        "/*   (c) 2015 Andreas Rumpf */$N" &
+        "/*   (c) 2016 Andreas Rumpf */$N" &
         "/* The generated code is subject to the original license. */$N" &
         "/* Compiled for: $2, $3, $4 */$N" &
         "/* Command for C compiler:$n   $5 */$N") %
@@ -875,10 +867,11 @@ proc genMainProc(m: BModule) =
     MainProcsWithResult =
       MainProcs & "\treturn nim_program_result;$N"
 
-    NimMainBody =
+    NimMainInner = "N_CDECL(void, NimMainInner)(void) {$N" &
-      "N_CDECL(void, NimMainInner)(void) {$N" &
         "$1" &
-      "}$N$N" &
+      "}$N$N"
+      
+    NimMainProc =
       "N_CDECL(void, NimMain)(void) {$N" &
         "\tvoid (*volatile inner)();$N" &
         "\tPreMain();$N" &
@@ -887,6 +880,8 @@ proc genMainProc(m: BModule) =
         "\t(*inner)();$N" &
       "}$N$N"
 
+    NimMainBody = NimMainInner & NimMainProc
+
     PosixNimMain =
       "int cmdCount;$N" &
       "char** cmdLine;$N" &
@@ -914,7 +909,7 @@ proc genMainProc(m: BModule) =
       "                        LPSTR lpCmdLine, int nCmdShow) {$N" &
       MainProcsWithResult & "}$N$N"
 
-    WinNimDllMain = "N_LIB_EXPORT " & NimMainBody
+    WinNimDllMain = NimMainInner & "N_LIB_EXPORT " & NimMainProc
 
     WinCDllMain =
       "BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fwdreason, $N" &
@@ -1020,7 +1015,7 @@ proc genInitCode(m: BModule) =
       var procname = makeCString(m.module.name.s)
       add(prc, initFrame(m.initProc, procname, m.module.info.quotedFilename))
     else:
-      add(prc, ~"\tTFrame F; FR.len = 0;$N")
+      add(prc, ~"\tTFrame FR; FR.len = 0;$N")
 
   add(prc, genSectionStart(cpsInit))
   add(prc, m.preInitProc.s(cpsInit))

compiler/cgmeth.nim

@@ -145,15 +145,16 @@ proc fixupDispatcher(meth, disp: PSym) =
         disp.typ.lockLevel = meth.typ.lockLevel
 
 proc methodDef*(s: PSym, fromCache: bool) =
-  var L = len(gMethods)
+  let L = len(gMethods)
   var witness: PSym
   for i in countup(0, L - 1):
-    var disp = gMethods[i].dispatcher
+    let disp = gMethods[i].dispatcher
     case sameMethodBucket(disp, s)
     of Yes:
       add(gMethods[i].methods, s)
       attachDispatcher(s, lastSon(disp.ast))
       fixupDispatcher(s, disp)
+      #echo "fixup ", disp.name.s, " ", disp.id
       when useEffectSystem: checkMethodEffects(disp, s)
       if sfBase in s.flags and gMethods[i].methods[0] != s:
         # already exists due to forwarding definition?

compiler/docgen.nim

@@ -27,6 +27,7 @@ type
     seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML.
     jArray: JsonNode
     types: TStrTable
+    isPureRst: bool
 
   PDoc* = ref TDocumentor ## Alias to type less.
 
@@ -440,11 +441,6 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
       dispA(result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}",
             [rope(esc(d.target, literal))])
 
-  if k in routineKinds and nameNode.kind == nkSym:
-    let att = attachToType(d, nameNode.sym)
-    if att != nil:
-      dispA(result, """<span class="attachedType" style="visibility:hidden">$1</span>""", "",
-        [rope esc(d.target, att.name.s)])
   inc(d.id)
   let
     plainNameRope = rope(xmltree.escape(plainName.strip))
@@ -459,25 +455,35 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
 
   var seeSrcRope: Rope = nil
   let docItemSeeSrc = getConfigVar("doc.item.seesrc")
-  if docItemSeeSrc.len > 0 and options.docSeeSrcUrl.len > 0:
+  if docItemSeeSrc.len > 0:
-    let path = n.info.toFilename.extractFilename.rope
+    let cwd = getCurrentDir().canonicalizePath()
-    let urlRope = ropeFormatNamedVars(options.docSeeSrcUrl,
+    var path = n.info.toFullPath
-      ["path", "line"], [path, rope($n.info.line)])
+    if path.startsWith(cwd):
+      path = path[cwd.len+1 .. ^1].replace('\\', '/')
+    var commit = getConfigVar("git.commit")
+    if commit.len == 0: commit = "master"
     dispA(seeSrcRope, "$1", "", [ropeFormatNamedVars(docItemSeeSrc,
-        ["path", "line", "url"], [path,
+        ["path", "line", "url", "commit"], [rope path,
-        rope($n.info.line), urlRope])])
+        rope($n.info.line), rope getConfigVar("git.url"),
+        rope commit])])
 
   add(d.section[k], ropeFormatNamedVars(getConfigVar("doc.item"),
     ["name", "header", "desc", "itemID", "header_plain", "itemSym",
       "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "seeSrc"],
     [nameRope, result, comm, itemIDRope, plainNameRope, plainSymbolRope,
       symbolOrIdRope, plainSymbolEncRope, symbolOrIdEncRope, seeSrcRope]))
+
+  var attype: Rope
+  if k in routineKinds and nameNode.kind == nkSym:
+    let att = attachToType(d, nameNode.sym)
+    if att != nil:
+      attype = rope esc(d.target, att.name.s)
   add(d.toc[k], ropeFormatNamedVars(getConfigVar("doc.item.toc"),
     ["name", "header", "desc", "itemID", "header_plain", "itemSym",
-      "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc"],
+      "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "attype"],
     [rope(getName(d, nameNode, d.splitAfter)), result, comm,
       itemIDRope, plainNameRope, plainSymbolRope, symbolOrIdRope,
-      plainSymbolEncRope, symbolOrIdEncRope]))
+      plainSymbolEncRope, symbolOrIdEncRope, attype]))
 
   # Ironically for types the complexSymbol is *cleaner* than the plainName
   # because it doesn't include object fields or documentation comments. So we
@@ -629,7 +635,9 @@ proc genOutFile(d: PDoc): Rope =
     # Modules get an automatic title for the HTML, but no entry in the index.
     title = "Module " & extractFilename(changeFileExt(d.filename, ""))
 
-  let bodyname = if d.hasToc: "doc.body_toc" else: "doc.body_no_toc"
+  let bodyname = if d.hasToc and not d.isPureRst: "doc.body_toc_group"
+                 elif d.hasToc: "doc.body_toc"
+                 else: "doc.body_no_toc"
   content = ropeFormatNamedVars(getConfigVar(bodyname), ["title",
       "tableofcontents", "moduledesc", "date", "time", "content"],
       [title.rope, toc, d.modDesc, rope(getDateStr()),
@@ -694,6 +702,7 @@ proc commandDoc*() =
 proc commandRstAux(filename, outExt: string) =
   var filen = addFileExt(filename, "txt")
   var d = newDocumentor(filen, options.gConfigVars)
+  d.isPureRst = true
   var rst = parseRst(readFile(filen), filen, 0, 1, d.hasToc,
                      {roSupportRawDirective})
   var modDesc = newStringOfCap(30_000)

compiler/extccomp.nim

@@ -505,10 +505,6 @@ proc noAbsolutePaths: bool {.inline.} =
   # `optGenMapping` is included here for niminst.
   result = gGlobalOptions * {optGenScript, optGenMapping} != {}
 
-const
-  specialFileA = 42
-  specialFileB = 42
-
 var fileCounter: int
 
 proc add(s: var string, many: openArray[string]) =

compiler/idgen.nim

@@ -54,6 +54,6 @@ proc loadMaxIds*(project: string) =
     if f.readLine(line):
       var frontEndId = parseInt(line)
       if f.readLine(line):
-        var backEndId = parseInt(line)
+        # var backEndId = parseInt(line)
         gFrontEndId = max(gFrontEndId, frontEndId)
     f.close()

compiler/importer.nim

@@ -163,7 +163,10 @@ proc myImportModule(c: PContext, n: PNode): PSym =
   var f = checkModuleName(n)
   if f != InvalidFileIDX:
     result = importModuleAs(n, gImportModule(c.module, f))
-    if result.info.fileIndex == n.info.fileIndex:
+    # we cannot perform this check reliably because of
+    # test: modules/import_in_config)
+    if result.info.fileIndex == c.module.info.fileIndex and
+        result.info.fileIndex == n.info.fileIndex:
       localError(n.info, errGenerated, "A module cannot import itself")
     if sfDeprecated in result.flags:
       message(n.info, warnDeprecated, result.name.s)

compiler/installer.ini

@@ -50,6 +50,7 @@ Files: "readme.txt;install.txt;contributors.txt;copying.txt"
 Files: "makefile"
 Files: "koch.nim"
 Files: "install_nimble.nims"
+Files: "install_tools.nims"
 
 Files: "icons/nim.ico"
 Files: "icons/nim.rc"
@@ -77,7 +78,8 @@ Files: "lib"
 
 [Other]
 Files: "examples"
-#Files: "dist/nimble"
+Files: "dist/nimble"
+Files: "dist/nimsuggest"
 
 Files: "tests"
 
@@ -87,10 +89,11 @@ Files: "bin/c2nim.exe"
 Files: "bin/nimgrep.exe"
 Files: "bin/nimsuggest.exe"
 Files: "bin/nimble.exe"
+Files: "bin/makelink.exe"
 Files: "bin/*.dll"
 
-Files: "dist/*.dll"
 Files: "koch.exe"
+Files: "finish.exe"
 ; Files: "dist/mingw"
 Files: "start.bat"
 BinPath: r"bin;dist\mingw\bin;dist"
@@ -98,8 +101,8 @@ BinPath: r"bin;dist\mingw\bin;dist"
 ;           Section | dir | zipFile | size hint (in KB) | url | exe start menu entry
 Download: r"Documentation|doc|docs.zip|13824|http://nim-lang.org/download/docs-${version}.zip|overview.html"
 Download: r"C Compiler (MingW)|dist|mingw.zip|82944|http://nim-lang.org/download/${mingw}.zip"
-Download: r"Support DLL's|bin|nim_dlls.zip|479|http://nim-lang.org/download/dlls.zip"
+Download: r"Support DLLs|bin|nim_dlls.zip|479|http://nim-lang.org/download/dlls.zip"
-Download: r"Aporia IDE|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe"
+Download: r"Aporia Text Editor|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe"
 ; for now only NSIS supports optional downloads
 
 [UnixBin]
@@ -116,7 +119,6 @@ path = r"c:\Program Files (x86)\Inno Setup 5\iscc.exe"
 flags = "/Q"
 
 [NSIS]
-path = r"c:\Program Files (x86)\NSIS\makensis.exe"
 flags = "/V0"
 
 [C_Compiler]

compiler/jsgen.nim

@@ -512,6 +512,10 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
     arithAux(p, n, r, op, jsOps)
   r.kind = resExpr
 
+proc hasFrameInfo(p: PProc): bool =
+  ({optLineTrace, optStackTrace} * p.options == {optLineTrace, optStackTrace}) and
+      ((p.prc == nil) or not (sfPure in p.prc.flags))
+
 proc genLineDir(p: PProc, n: PNode) =
   let line = toLinenumber(n.info)
   if optLineDir in p.options:
@@ -521,9 +525,7 @@ proc genLineDir(p: PProc, n: PNode) =
       ((p.prc == nil) or sfPure notin p.prc.flags):
     useMagic(p, "endb")
     addf(p.body, "endb($1);$n", [rope(line)])
-  elif ({optLineTrace, optStackTrace} * p.options ==
+  elif hasFrameInfo(p):
-      {optLineTrace, optStackTrace}) and
-      ((p.prc == nil) or not (sfPure in p.prc.flags)):
     addf(p.body, "F.line = $1;$n" | "$$F['line'] = $1;$n", [rope(line)])
 
 proc genWhileStmt(p: PProc, n: PNode) =
@@ -558,10 +560,13 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   # code to generate:
   #
   #  ++excHandler;
+  #  var tmpFramePtr = framePtr;
   #  try {
   #    stmts;
+  #    --excHandler;
   #  } catch (EXC) {
   #    var prevJSError = lastJSError; lastJSError = EXC;
+  #    framePtr = tmpFramePtr;
   #    --excHandler;
   #    if (e.typ && e.typ == NTI433 || e.typ == NTI2321) {
   #      stmts;
@@ -572,6 +577,7 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   #    }
   #    lastJSError = prevJSError;
   #  } finally {
+  #    framePtr = tmpFramePtr;
   #    stmts;
   #  }
   genLineDir(p, n)
@@ -584,8 +590,10 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   var catchBranchesExist = length > 1 and n.sons[i].kind == nkExceptBranch
   if catchBranchesExist and p.target == targetJS:
     add(p.body, "++excHandler;" & tnl)
-  var safePoint = "Tmp$1" % [rope(p.unique)]
+  var tmpFramePtr = rope"F"
-  if optStackTrace in p.options: add(p.body, "framePtr = F;" & tnl)
+  if optStackTrace notin p.options:
+    tmpFramePtr = p.getTemp(true)
+    add(p.body, tmpFramePtr & " = framePtr;" & tnl)
   addf(p.body, "try {$n", [])
   if p.target == targetPHP and p.globals == nil:
       p.globals = "global $lastJSError; global $prevJSError;".rope
@@ -595,8 +603,9 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   var generalCatchBranchExists = false
   let dollar = rope(if p.target == targetJS: "" else: "$")
   if p.target == targetJS and catchBranchesExist:
-    addf(p.body, "} catch (EXC) {$n var prevJSError = lastJSError;$n" &
+    addf(p.body, "--excHandler;$n} catch (EXC) {$n var prevJSError = lastJSError;$n" &
         " lastJSError = EXC;$n --excHandler;$n", [])
+    add(p.body, "framePtr = $1;$n" % [tmpFramePtr])
   elif p.target == targetPHP:
     addf(p.body, "} catch (Exception $$EXC) {$n $$prevJSError = $$lastJSError;$n $$lastJSError = $$EXC;$n", [])
   while i < length and n.sons[i].kind == nkExceptBranch:
@@ -618,8 +627,7 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
         addf(orExpr, "isObj($2lastJSError.m_type, $1)",
              [genTypeInfo(p, n.sons[i].sons[j].typ), dollar])
       if i > 1: add(p.body, "else ")
-      addf(p.body, "if ($3lastJSError && ($2)) {$n",
+      addf(p.body, "if ($1lastJSError && ($2)) {$n", [dollar, orExpr])
-        [safePoint, orExpr, dollar])
       gen(p, n.sons[i].sons[blen - 1], a)
       moveInto(p, a, r)
       addf(p.body, "}$n", [])
@@ -631,6 +639,7 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
     addf(p.body, "$1lastJSError = $1prevJSError;$n", [dollar])
   if p.target == targetJS:
     add(p.body, "} finally {" & tnl)
+    add(p.body, "framePtr = $1;$n" % [tmpFramePtr])
   if p.target == targetPHP:
     # XXX ugly hack for PHP codegen
     add(p.body, "}" & tnl)
@@ -1355,7 +1364,7 @@ proc createObjInitList(p: PProc, typ: PType, excludedFieldIDs: IntSet, output: v
     if output.len > 0: output.add(", ")
     addf(output, "m_type: $1" | "'m_type' => $#", [genTypeInfo(p, t)])
   while t != nil:
-    createRecordVarAux(p, t.n, excludedFieldIDs, output)
+    createRecordVarAux(p, t.skipTypes(skipPtrs).n, excludedFieldIDs, output)
     t = t.sons[0]
 
 proc arrayTypeForElemType(typ: PType): string =
@@ -1462,13 +1471,21 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
         useMagic(p, "nimCopy")
         s = "nimCopy(null, $1, $2)" % [a.res, genTypeInfo(p, n.typ)]
     of etyBaseIndex:
-      if (a.typ != etyBaseIndex): internalError(n.info, "genVarInit")
+      let targetBaseIndex = {sfAddrTaken, sfGlobal} * v.flags == {}
-      if {sfAddrTaken, sfGlobal} * v.flags != {}:
+      if a.typ == etyBaseIndex:
-        addf(p.body, "var $1 = [$2, $3];$n",
+        if targetBaseIndex:
-            [v.loc.r, a.address, a.res])
+          addf(p.body, "var $1 = $2, $1_Idx = $3;$n", [
+              v.loc.r, a.address, a.res])
+        else:
+          addf(p.body, "var $1 = [$2, $3];$n",
+              [v.loc.r, a.address, a.res])
       else:
-        addf(p.body, "var $1 = $2; var $1_Idx = $3;$n", [
+        if targetBaseIndex:
-             v.loc.r, a.address, a.res])
+          let tmp = p.getTemp
+          addf(p.body, "var $1 = $2, $3 = $1[0], $3_Idx = $1[1];$n",
+              [tmp, a.res, v.loc.r])
+        else:
+          addf(p.body, "var $1 = $2;$n", [v.loc.r, a.res])
       return
     else:
       s = a.res
@@ -1918,10 +1935,10 @@ proc frameCreate(p: PProc; procname, filename: Rope): Rope =
             procname, filename]
 
 proc frameDestroy(p: PProc): Rope =
-  result = rope(("framePtr = framePtr.prev;" | "$framePtr = $framePtr['prev'];") & tnl)
+  result = rope(("framePtr = F.prev;" | "$framePtr = $F['prev'];") & tnl)
 
 proc genProcBody(p: PProc, prc: PSym): Rope =
-  if optStackTrace in prc.options:
+  if hasFrameInfo(p):
     result = frameCreate(p,
               makeJSString(prc.owner.name.s & '.' & prc.name.s),
               makeJSString(toFilename(prc.info)))
@@ -1935,7 +1952,7 @@ proc genProcBody(p: PProc, prc: PSym): Rope =
   if prc.typ.callConv == ccSysCall and p.target == targetJS:
     result = ("try {$n$1} catch (e) {$n" &
       " alert(\"Unhandled exception:\\n\" + e.message + \"\\n\"$n}") % [result]
-  if optStackTrace in prc.options:
+  if hasFrameInfo(p):
     add(result, frameDestroy(p))
 
 proc genProc(oldProc: PProc, prc: PSym): Rope =

compiler/jstypes.nim

@@ -76,7 +76,7 @@ proc genObjectInfo(p: PProc, typ: PType, name: Rope) =
   addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
   if (typ.kind == tyObject) and (typ.sons[0] != nil):
     addf(p.g.typeInfo, "$1.base = $2;$n",
-         [name, genTypeInfo(p, typ.sons[0])])
+         [name, genTypeInfo(p, typ.sons[0].skipTypes(skipPtrs))])
 
 proc genTupleFields(p: PProc, typ: PType): Rope =
   var s: Rope = nil

compiler/lambdalifting.nim

@@ -226,8 +226,14 @@ proc interestingIterVar(s: PSym): bool {.inline.} =
 template isIterator*(owner: PSym): bool =
   owner.kind == skIterator and owner.typ.callConv == ccClosure
 
+proc liftingHarmful(owner: PSym): bool {.inline.} =
+  ## lambda lifting can be harmful for JS-like code generators.
+  let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro
+  result = gCmd in {cmdCompileToPHP, cmdCompileToJS} and not isCompileTime
+
 proc liftIterSym*(n: PNode; owner: PSym): PNode =
   # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env))
+  if liftingHarmful(owner): return n
   let iter = n.sym
   assert iter.isIterator
 
@@ -838,6 +844,7 @@ proc liftForLoop*(body: PNode; owner: PSym): PNode =
         nkBreakState(cl.state)
         ...
     """
+  if liftingHarmful(owner): return body
   var L = body.len
   if not (body.kind == nkForStmt and body[L-2].kind in nkCallKinds):
     localError(body.info, "ignored invalid for loop")

compiler/lexer.nim

@@ -211,9 +211,6 @@ proc closeLexer*(lex: var TLexer) =
   inc(gLinesCompiled, lex.lineNumber)
   closeBaseLexer(lex)
 
-proc getColumn(L: TLexer): int =
-  result = getColNumber(L, L.bufpos)
-
 proc getLineInfo(L: TLexer): TLineInfo =
   result = newLineInfo(L.fileIdx, L.lineNumber, getColNumber(L, L.bufpos))
 
@@ -237,12 +234,6 @@ proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
 proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
   result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)
 
-proc isFloatLiteral(s: string): bool =
-  for i in countup(0, len(s) - 1):
-    if s[i] in {'.', 'e', 'E'}:
-      return true
-  result = false
-
 {.push overflowChecks: off.}
 # We need to parse the largest uint literal without overflow checks
 proc unsafeParseUInt(s: string, b: var BiggestInt, start = 0): int =

compiler/lowerings.nim

@@ -172,7 +172,7 @@ proc indirectAccess*(a: PNode, b: string, info: TLineInfo): PNode =
     if field != nil: break
     t = t.sons[0]
     if t == nil: break
-    t = t.skipTypes(abstractInst)
+    t = t.skipTypes(skipPtrs)
   #if field == nil:
   #  echo "FIELD ", b
   #  debug deref.typ
@@ -193,7 +193,7 @@ proc getFieldFromObj*(t: PType; v: PSym): PSym =
     if result != nil: break
     t = t.sons[0]
     if t == nil: break
-    t = t.skipTypes(abstractInst)
+    t = t.skipTypes(skipPtrs)
 
 proc indirectAccess*(a: PNode, b: PSym, info: TLineInfo): PNode =
   # returns a[].b as a node

compiler/modules.nim

@@ -38,9 +38,6 @@ proc getModule*(fileIdx: int32): PSym =
   if fileIdx >= 0 and fileIdx < gCompiledModules.len:
     result = gCompiledModules[fileIdx]
 
-template hash(x: PSym): untyped =
-  gMemCacheData[x.position].hash
-
 proc hashChanged(fileIdx: int32): bool =
   internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len
 
@@ -189,14 +186,11 @@ proc compileModule*(fileIdx: int32, flags: TSymFlags): PSym =
         return
     else:
       result.id = getID()
-    if sfMainModule in flags and gProjectIsStdin:
+    let validFile = processModule(result, if sfMainModule in flags and gProjectIsStdin: llStreamOpen(stdin) else: nil, rd)
-      processModule(result, llStreamOpen(stdin), rd)
-    else:
-      processModule(result, nil, rd)
     if optCaasEnabled in gGlobalOptions:
       gMemCacheData[fileIdx].compiledAt = gLastCmdTime
       gMemCacheData[fileIdx].needsRecompile = Recompiled
-      doHash fileIdx
+      if validFile: doHash fileIdx
   else:
     if checkDepMem(fileIdx) == Yes:
       result = compileModule(fileIdx, flags)
@@ -220,12 +214,6 @@ proc includeModule*(s: PSym, fileIdx: int32): PNode {.procvar.} =
     addDep(s, fileIdx)
     doHash(fileIdx)
 
-proc `==^`(a, b: string): bool =
-  try:
-    result = sameFile(a, b)
-  except OSError:
-    result = false
-
 proc compileSystemModule* =
   if magicsys.systemModule == nil:
     systemFileIdx = fileInfoIdx(options.libpath/"system.nim")

compiler/msgs.nim

@@ -65,7 +65,7 @@ type
     errPureTypeMismatch, errTypeMismatch, errButExpected, errButExpectedX,
     errAmbiguousCallXYZ, errWrongNumberOfArguments,
     errXCannotBePassedToProcVar,
-    errXCannotBeInParamDecl, errPragmaOnlyInHeaderOfProc, errImplOfXNotAllowed,
+    errXCannotBeInParamDecl, errPragmaOnlyInHeaderOfProcX, errImplOfXNotAllowed,
     errImplOfXexpected, errNoSymbolToBorrowFromFound, errDiscardValueX,
     errInvalidDiscard, errIllegalConvFromXtoY, errCannotBindXTwice,
     errInvalidOrderInArrayConstructor,
@@ -274,7 +274,7 @@ const
     errWrongNumberOfArguments: "wrong number of arguments",
     errXCannotBePassedToProcVar: "\'$1\' cannot be passed to a procvar",
     errXCannotBeInParamDecl: "$1 cannot be declared in parameter declaration",
-    errPragmaOnlyInHeaderOfProc: "pragmas are only allowed in the header of a proc",
+    errPragmaOnlyInHeaderOfProcX: "pragmas are only allowed in the header of a proc; redefinition of $1",
     errImplOfXNotAllowed: "implementation of \'$1\' is not allowed",
     errImplOfXexpected: "implementation of \'$1\' expected",
     errNoSymbolToBorrowFromFound: "no symbol to borrow from found",
@@ -521,7 +521,7 @@ const
                                          hintCodeBegin, hintCodeEnd,
                                          hintSource, hintStackTrace,
                                          hintGCStats},
-    {low(TNoteKind)..high(TNoteKind)} - {hintStackTrace},
+    {low(TNoteKind)..high(TNoteKind)} - {hintStackTrace, warnUninit},
     {low(TNoteKind)..high(TNoteKind)}]
 
 const
@@ -660,8 +660,6 @@ const
   WarningColor = fgYellow
   HintTitle    = "Hint: "
   HintColor    = fgGreen
-  InfoTitle    = "Info: "
-  InfoColor    = fgCyan
 
 proc getInfoContextLen*(): int = return msgContext.len
 proc setInfoContextLen*(L: int) = setLen(msgContext, L)

compiler/nimblecmd.nim

@@ -63,18 +63,6 @@ proc addNimblePath(p: string, info: TLineInfo) =
     message(info, hintPath, p)
     lists.prependStr(options.lazyPaths, p)
 
-proc addPathWithNimFiles(p: string, info: TLineInfo) =
-  proc hasNimFile(dir: string): bool =
-    for kind, path in walkDir(dir):
-      if kind == pcFile and path.endsWith(".nim"):
-        result = true
-        break
-  if hasNimFile(p):
-    addNimblePath(p, info)
-  else:
-    for kind, p2 in walkDir(p):
-      if hasNimFile(p2): addNimblePath(p2, info)
-
 proc addPathRec(dir: string, info: TLineInfo) =
   var packages = newStringTable(modeStyleInsensitive)
   var pos = dir.len-1

compiler/nversion.nim

@@ -13,5 +13,5 @@
 const
   MaxSetElements* = 1 shl 16  # (2^16) to support unicode character sets?
   VersionAsString* = system.NimVersion
-  RodFileVersion* = "1215"       # modify this if the rod-format changes!
+  RodFileVersion* = "1221"       # modify this if the rod-format changes!
 

compiler/options.nim

@@ -70,6 +70,12 @@ type                          # please make sure we have under 32 options
     optExcessiveStackTrace    # fully qualified module filenames
 
   TGlobalOptions* = set[TGlobalOption]
+
+const
+  harmlessOptions* = {optForceFullMake, optNoLinking, optReportConceptFailures,
+    optRun, optUseColors, optStdout}
+
+type
   TCommands* = enum           # Nim's commands
                               # **keep binary compatible**
     cmdNone, cmdCompileToC, cmdCompileToCpp, cmdCompileToOC,

compiler/parser.nim

@@ -340,26 +340,6 @@ proc parseSymbol(p: var TParser, allowNil = false): PNode =
       if not isKeyword(p.tok.tokType): getTok(p)
       result = ast.emptyNode
 
-proc indexExpr(p: var TParser): PNode =
-  #| indexExpr = expr
-  result = parseExpr(p)
-
-proc indexExprList(p: var TParser, first: PNode, k: TNodeKind,
-                   endToken: TTokType): PNode =
-  #| indexExprList = indexExpr ^+ comma
-  result = newNodeP(k, p)
-  addSon(result, first)
-  getTok(p)
-  optInd(p, result)
-  while p.tok.tokType notin {endToken, tkEof}:
-    var a = indexExpr(p)
-    addSon(result, a)
-    if p.tok.tokType != tkComma: break
-    getTok(p)
-    skipComment(p, a)
-  optPar(p)
-  eat(p, endToken)
-
 proc colonOrEquals(p: var TParser, a: PNode): PNode =
   if p.tok.tokType == tkColon:
     result = newNodeP(nkExprColonExpr, p)

compiler/passes.nim

@@ -46,12 +46,6 @@ proc makePass*(open: TPassOpen = nil,
   result.close = close
   result.process = process
 
-  # This implements a memory preserving scheme: Top level statements are
-  # processed in a pipeline. The compiler never looks at a whole module
-  # any longer. However, this is simple to change, as new passes may perform
-  # whole program optimizations. For now, we avoid it to save a lot of memory.
-proc processModule*(module: PSym, stream: PLLStream, rd: PRodReader)
-
 # the semantic checker needs these:
 var
   gImportModule*: proc (m: PSym, fileIdx: int32): PSym {.nimcall.}
@@ -160,7 +154,8 @@ proc processImplicits(implicits: seq[string], nodeKind: TNodeKind,
     importStmt.addSon str
     if not processTopLevelStmt(importStmt, a): break
 
-proc processModule(module: PSym, stream: PLLStream, rd: PRodReader) =
+proc processModule*(module: PSym, stream: PLLStream,
+                    rd: PRodReader): bool {.discardable.} =
   var
     p: TParsers
     a: TPassContextArray
@@ -173,7 +168,7 @@ proc processModule(module: PSym, stream: PLLStream, rd: PRodReader) =
       s = llStreamOpen(filename, fmRead)
       if s == nil:
         rawMessage(errCannotOpenFile, filename)
-        return
+        return false
     else:
       s = stream
     while true:
@@ -211,4 +206,4 @@ proc processModule(module: PSym, stream: PLLStream, rd: PRodReader) =
     var n = loadInitSection(rd)
     for i in countup(0, sonsLen(n) - 1): processTopLevelStmtCached(n.sons[i], a)
     closePassesCached(a)
-
+  result = true

compiler/pragmas.nim

@@ -567,11 +567,13 @@ proc deprecatedStmt(c: PContext; pragma: PNode) =
   for n in pragma:
     if n.kind in {nkExprColonExpr, nkExprEqExpr}:
       let dest = qualifiedLookUp(c, n[1], {checkUndeclared})
+      assert dest != nil
       let src = considerQuotedIdent(n[0])
       let alias = newSym(skAlias, src, dest, n[0].info)
       incl(alias.flags, sfExported)
       if sfCompilerProc in dest.flags: markCompilerProc(alias)
       addInterfaceDecl(c, alias)
+      n.sons[1] = newSymNode(dest)
     else:
       localError(n.info, "key:value pair expected")
 
@@ -749,7 +751,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         noVal(it)
         incl(sym.flags, sfThread)
         incl(sym.flags, sfProcvar)
-        if sym.typ != nil: incl(sym.typ.flags, tfThread)
+        if sym.typ != nil:
+          incl(sym.typ.flags, tfThread)
+          if sym.typ.callConv == ccClosure: sym.typ.callConv = ccDefault
       of wGcSafe:
         noVal(it)
         if sym.kind != skType: incl(sym.flags, sfThread)

compiler/renderer.nim

@@ -146,12 +146,6 @@ proc put(g: var TSrcGen, kind: TTokType, s: string) =
   else:
     g.pendingWhitespace = s.len
 
-proc putLong(g: var TSrcGen, kind: TTokType, s: string, lineLen: int) =
-  # use this for tokens over multiple lines.
-  addPendingNL(g)
-  addTok(g, kind, s)
-  g.lineLen = lineLen
-
 proc toNimChar(c: char): string =
   case c
   of '\0': result = "\\0"
@@ -264,9 +258,6 @@ proc pushCom(g: var TSrcGen, n: PNode) =
 proc popAllComs(g: var TSrcGen) =
   setLen(g.comStack, 0)
 
-proc popCom(g: var TSrcGen) =
-  setLen(g.comStack, len(g.comStack) - 1)
-
 const
   Space = " "
 
@@ -492,7 +483,7 @@ proc fits(g: TSrcGen, x: int): bool =
 
 type
   TSubFlag = enum
-    rfLongMode, rfNoIndent, rfInConstExpr
+    rfLongMode, rfInConstExpr
   TSubFlags = set[TSubFlag]
   TContext = tuple[spacing: int, flags: TSubFlags]
 
@@ -675,16 +666,6 @@ proc gfor(g: var TSrcGen, n: PNode) =
   gcoms(g)
   gstmts(g, n.sons[length - 1], c)
 
-proc gmacro(g: var TSrcGen, n: PNode) =
-  var c: TContext
-  initContext(c)
-  gsub(g, n.sons[0])
-  putWithSpace(g, tkColon, ":")
-  if longMode(n) or (lsub(n.sons[1]) + g.lineLen > MaxLineLen):
-    incl(c.flags, rfLongMode)
-  gcoms(g)
-  gsons(g, n, c, 1)
-
 proc gcase(g: var TSrcGen, n: PNode) =
   var c: TContext
   initContext(c)

compiler/rodread.nim

@@ -145,13 +145,7 @@ type
 
   PRodReader* = ref TRodReader
 
-var rodCompilerprocs*: TStrTable
+var rodCompilerprocs*: TStrTable # global because this is needed by magicsys
-
-proc handleSymbolFile*(module: PSym): PRodReader
-# global because this is needed by magicsys
-proc loadInitSection*(r: PRodReader): PNode
-
-# implementation
 
 proc rawLoadStub(s: PSym)
 
@@ -206,7 +200,7 @@ proc decodeNodeLazyBody(r: PRodReader, fInfo: TLineInfo,
       var id = decodeVInt(r.s, r.pos)
       result.typ = rrGetType(r, id, result.info)
     case result.kind
-    of nkCharLit..nkInt64Lit:
+    of nkCharLit..nkUInt64Lit:
       if r.s[r.pos] == '!':
         inc(r.pos)
         result.intVal = decodeVBiggestInt(r.s, r.pos)
@@ -324,6 +318,29 @@ proc decodeType(r: PRodReader, info: TLineInfo): PType =
     result.align = decodeVInt(r.s, r.pos).int16
   else:
     result.align = 2
+
+  if r.s[r.pos] == '\14':
+    inc(r.pos)
+    result.lockLevel = decodeVInt(r.s, r.pos).TLockLevel
+  else:
+    result.lockLevel = UnspecifiedLockLevel
+
+  if r.s[r.pos] == '\15':
+    inc(r.pos)
+    result.destructor = rrGetSym(r, decodeVInt(r.s, r.pos), info)
+  if r.s[r.pos] == '\16':
+    inc(r.pos)
+    result.deepCopy = rrGetSym(r, decodeVInt(r.s, r.pos), info)
+  if r.s[r.pos] == '\17':
+    inc(r.pos)
+    result.assignment = rrGetSym(r, decodeVInt(r.s, r.pos), info)
+  while r.s[r.pos] == '\18':
+    inc(r.pos)
+    let x = decodeVInt(r.s, r.pos)
+    doAssert r.s[r.pos] == '\19'
+    inc(r.pos)
+    let y = rrGetSym(r, decodeVInt(r.s, r.pos), info)
+    result.methods.safeAdd((x, y))
   decodeLoc(r, result.loc, info)
   while r.s[r.pos] == '^':
     inc(r.pos)
@@ -349,6 +366,22 @@ proc decodeLib(r: PRodReader, info: TLineInfo): PLib =
     inc(r.pos)
     result.path = decodeNode(r, info)
 
+proc decodeInstantiations(r: PRodReader; info: TLineInfo;
+                          s: var seq[PInstantiation]) =
+  while r.s[r.pos] == '\15':
+    inc(r.pos)
+    var ii: PInstantiation
+    new ii
+    ii.sym = rrGetSym(r, decodeVInt(r.s, r.pos), info)
+    ii.concreteTypes = @[]
+    while r.s[r.pos] == '\17':
+      inc(r.pos)
+      ii.concreteTypes.add rrGetType(r, decodeVInt(r.s, r.pos), info)
+    if r.s[r.pos] == '\20':
+      inc(r.pos)
+      ii.compilesId = decodeVInt(r.s, r.pos)
+    s.safeAdd ii
+
 proc decodeSym(r: PRodReader, info: TLineInfo): PSym =
   var
     id: int
@@ -423,6 +456,27 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym =
   if r.s[r.pos] == '#':
     inc(r.pos)
     result.constraint = decodeNode(r, unknownLineInfo())
+  case result.kind
+  of skType, skGenericParam:
+    while r.s[r.pos] == '\14':
+      inc(r.pos)
+      result.typeInstCache.safeAdd rrGetType(r, decodeVInt(r.s, r.pos), result.info)
+  of routineKinds:
+    decodeInstantiations(r, result.info, result.procInstCache)
+    if r.s[r.pos] == '\16':
+      inc(r.pos)
+      result.gcUnsafetyReason = rrGetSym(r, decodeVInt(r.s, r.pos), result.info)
+  of skModule, skPackage:
+    decodeInstantiations(r, result.info, result.usedGenerics)
+  of skLet, skVar, skField, skForVar:
+    if r.s[r.pos] == '\18':
+      inc(r.pos)
+      result.guard = rrGetSym(r, decodeVInt(r.s, r.pos), result.info)
+    if r.s[r.pos] == '\19':
+      inc(r.pos)
+      result.bitsize = decodeVInt(r.s, r.pos).int16
+  else: discard
+
   if r.s[r.pos] == '(':
     if result.kind in routineKinds:
       result.ast = decodeNodeLazyBody(r, result.info, result)
@@ -566,7 +620,8 @@ proc processRodFile(r: PRodReader, hash: SecureHash) =
     of "GOPTIONS":
       inc(r.pos)              # skip ':'
       var dep = cast[TGlobalOptions](int32(decodeVInt(r.s, r.pos)))
-      if gGlobalOptions != dep: r.reason = rrOptions
+      if gGlobalOptions-harmlessOptions != dep-harmlessOptions:
+        r.reason = rrOptions
     of "CMD":
       inc(r.pos)              # skip ':'
       var dep = cast[TCommands](int32(decodeVInt(r.s, r.pos)))
@@ -580,14 +635,14 @@ proc processRodFile(r: PRodReader, hash: SecureHash) =
         if not condsyms.isDefined(getIdent(w)):
           r.reason = rrDefines #MessageOut('not defined, but should: ' + w);
         if r.s[r.pos] == ' ': inc(r.pos)
-      if (d != countDefinedSymbols()): r.reason = rrDefines
+      if d != countDefinedSymbols(): r.reason = rrDefines
     of "FILES":
       inc(r.pos, 2)           # skip "(\10"
       inc(r.line)
       while r.s[r.pos] != ')':
-        let relativePath = decodeStr(r.s, r.pos)
+        let finalPath = decodeStr(r.s, r.pos)
-        let resolvedPath = relativePath.findModule(r.origFile)
+        #let resolvedPath = relativePath.findModule(r.origFile)
-        let finalPath = if resolvedPath.len > 0: resolvedPath else: relativePath
+        #let finalPath = if resolvedPath.len > 0: resolvedPath else: relativePath
         r.files.add(finalPath.fileInfoIdx)
         inc(r.pos)            # skip #10
         inc(r.line)
@@ -683,8 +738,11 @@ proc newRodReader(modfilename: string, hash: SecureHash,
     if version != RodFileVersion:
       # since ROD files are only for caching, no backwards compatibility is
       # needed
+      #echo "expected version ", version, " ", RodFileVersion
+      result.memfile.close
       result = nil
   else:
+    result.memfile.close
     result = nil
 
 proc rrGetType(r: PRodReader, id: int, info: TLineInfo): PType =
@@ -762,7 +820,7 @@ proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym =
       result = decodeSymSafePos(r, d, info)
   if result != nil and result.kind == skStub: rawLoadStub(result)
 
-proc loadInitSection(r: PRodReader): PNode =
+proc loadInitSection*(r: PRodReader): PNode =
   if r.initIdx == 0 or r.dataIdx == 0: internalError("loadInitSection")
   var oldPos = r.pos
   r.pos = r.initIdx
@@ -818,9 +876,8 @@ proc checkDep(fileIdx: int32): TReasonForRecompile =
   var hash = getHash(fileIdx)
   gMods[fileIdx].reason = rrNone  # we need to set it here to avoid cycles
   result = rrNone
-  var r: PRodReader = nil
   var rodfile = toGeneratedFile(filename.withPackageName, RodExt)
-  r = newRodReader(rodfile, hash, fileIdx)
+  var r = newRodReader(rodfile, hash, fileIdx)
   if r == nil:
     result = (if existsFile(rodfile): rrRodInvalid else: rrRodDoesNotExist)
   else:
@@ -847,7 +904,7 @@ proc checkDep(fileIdx: int32): TReasonForRecompile =
   gMods[fileIdx].rd = r
   gMods[fileIdx].reason = result  # now we know better
 
-proc handleSymbolFile(module: PSym): PRodReader =
+proc handleSymbolFile*(module: PSym): PRodReader =
   let fileIdx = module.fileIdx
   if optSymbolFiles notin gGlobalOptions:
     module.id = getID()
@@ -923,7 +980,7 @@ proc writeNode(f: File; n: PNode) =
       f.write('^')
       f.write(n.typ.id)
     case n.kind
-    of nkCharLit..nkInt64Lit:
+    of nkCharLit..nkUInt64Lit:
       if n.intVal != 0:
         f.write('!')
         f.write(n.intVal)

compiler/rodwrite.nim

@@ -16,8 +16,6 @@ import
   condsyms, ropes, idents, securehash, rodread, passes, importer, idgen,
   rodutils
 
-# implementation
-
 type
   TRodWriter = object of TPassContext
     module: PSym
@@ -39,13 +37,6 @@ type
 
   PRodWriter = ref TRodWriter
 
-proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter
-proc addModDep(w: PRodWriter, dep: string)
-proc addInclDep(w: PRodWriter, dep: string)
-proc addInterfaceSym(w: PRodWriter, s: PSym)
-proc addStmt(w: PRodWriter, n: PNode)
-proc writeRod(w: PRodWriter)
-
 proc getDefines(): string =
   result = ""
   for d in definedSymbolNames():
@@ -83,19 +74,20 @@ proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter =
   result.converters = ""
   result.methods = ""
   result.init = ""
-  result.origFile = module.info.toFilename
+  result.origFile = module.info.toFullPath
   result.data = newStringOfCap(12_000)
 
-proc addModDep(w: PRodWriter, dep: string) =
+proc addModDep(w: PRodWriter, dep: string; info: TLineInfo) =
   if w.modDeps.len != 0: add(w.modDeps, ' ')
-  encodeVInt(fileIdx(w, dep), w.modDeps)
+  let resolved = dep.findModule(info.toFullPath)
+  encodeVInt(fileIdx(w, resolved), w.modDeps)
 
 const
   rodNL = "\x0A"
 
-proc addInclDep(w: PRodWriter, dep: string) =
+proc addInclDep(w: PRodWriter, dep: string; info: TLineInfo) =
-  var resolved = dep.findModule(w.module.info.toFullPath)
+  let resolved = dep.findModule(info.toFullPath)
-  encodeVInt(fileIdx(w, dep), w.inclDeps)
+  encodeVInt(fileIdx(w, resolved), w.inclDeps)
   add(w.inclDeps, " ")
   encodeStr($secureHashFile(resolved), w.inclDeps)
   add(w.inclDeps, rodNL)
@@ -108,6 +100,10 @@ proc pushType(w: PRodWriter, t: PType) =
 proc pushSym(w: PRodWriter, s: PSym) =
   # check so that the stack does not grow too large:
   if iiTableGet(w.index.tab, s.id) == InvalidKey:
+    when false:
+      if s.kind == skMethod:
+        echo "encoding ", s.id, " ", s.name.s
+        writeStackTrace()
     w.sstack.add(s)
 
 proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
@@ -127,7 +123,7 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
     result.add(',')
     encodeVInt(n.info.line, result)
     result.add(',')
-    encodeVInt(fileIdx(w, toFilename(n.info)), result)
+    encodeVInt(fileIdx(w, toFullPath(n.info)), result)
   elif fInfo.line != n.info.line:
     result.add('?')
     encodeVInt(n.info.col, result)
@@ -147,7 +143,7 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
     encodeVInt(n.typ.id, result)
     pushType(w, n.typ)
   case n.kind
-  of nkCharLit..nkInt64Lit:
+  of nkCharLit..nkUInt64Lit:
     if n.intVal != 0:
       result.add('!')
       encodeVBiggestInt(n.intVal, result)
@@ -229,6 +225,27 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t.align != 2:
     add(result, '=')
     encodeVInt(t.align, result)
+  if t.lockLevel.ord != UnspecifiedLockLevel.ord:
+    add(result, '\14')
+    encodeVInt(t.lockLevel.int16, result)
+  if t.destructor != nil and t.destructor.id != 0:
+    add(result, '\15')
+    encodeVInt(t.destructor.id, result)
+    pushSym(w, t.destructor)
+  if t.deepCopy != nil:
+    add(result, '\16')
+    encodeVInt(t.deepcopy.id, result)
+    pushSym(w, t.deepcopy)
+  if t.assignment != nil:
+    add(result, '\17')
+    encodeVInt(t.assignment.id, result)
+    pushSym(w, t.assignment)
+  for i, s in items(t.methods):
+    add(result, '\18')
+    encodeVInt(i, result)
+    add(result, '\19')
+    encodeVInt(s.id, result)
+    pushSym(w, s)
   encodeLoc(w, t.loc, result)
   for i in countup(0, sonsLen(t) - 1):
     if t.sons[i] == nil:
@@ -246,6 +263,19 @@ proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) =
   add(result, '|')
   encodeNode(w, info, lib.path, result)
 
+proc encodeInstantiations(w: PRodWriter; s: seq[PInstantiation];
+                          result: var string) =
+  for t in s:
+    result.add('\15')
+    encodeVInt(t.sym.id, result)
+    pushSym(w, t.sym)
+    for tt in t.concreteTypes:
+      result.add('\17')
+      encodeVInt(tt.id, result)
+      pushType(w, tt)
+    result.add('\20')
+    encodeVInt(t.compilesId, result)
+
 proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
   if s == nil:
     # nil nodes have to be stored too:
@@ -266,7 +296,7 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
   result.add(',')
   if s.info.line != -1'i16: encodeVInt(s.info.line, result)
   result.add(',')
-  encodeVInt(fileIdx(w, toFilename(s.info)), result)
+  encodeVInt(fileIdx(w, toFullPath(s.info)), result)
   if s.owner != nil:
     result.add('*')
     encodeVInt(s.owner.id, result)
@@ -291,6 +321,31 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
   if s.constraint != nil:
     add(result, '#')
     encodeNode(w, unknownLineInfo(), s.constraint, result)
+  case s.kind
+  of skType, skGenericParam:
+    for t in s.typeInstCache:
+      result.add('\14')
+      encodeVInt(t.id, result)
+      pushType(w, t)
+  of routineKinds:
+    encodeInstantiations(w, s.procInstCache, result)
+    if s.gcUnsafetyReason != nil:
+      result.add('\16')
+      encodeVInt(s.gcUnsafetyReason.id, result)
+      pushSym(w, s.gcUnsafetyReason)
+  of skModule, skPackage:
+    encodeInstantiations(w, s.usedGenerics, result)
+    # we don't serialize:
+    #tab*: TStrTable         # interface table for modules
+  of skLet, skVar, skField, skForVar:
+    if s.guard != nil:
+      result.add('\18')
+      encodeVInt(s.guard.id, result)
+      pushSym(w, s.guard)
+    if s.bitsize != 0:
+      result.add('\19')
+      encodeVInt(s.bitsize, result)
+  else: discard
   # lazy loading will soon reload the ast lazily, so the ast needs to be
   # the last entry of a symbol:
   if s.ast != nil:
@@ -298,16 +353,6 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
     # it is not necessary, but Nim's heavy compile-time evaluation features
     # make that unfeasible nowadays:
     encodeNode(w, s.info, s.ast, result)
-    when false:
-      var codeAst: PNode = nil
-      if not astNeeded(s):
-        codeAst = s.ast.sons[codePos]
-        # ugly hack to not store the AST:
-        s.ast.sons[codePos] = ast.emptyNode
-      encodeNode(w, s.info, s.ast, result)
-      if codeAst != nil:
-        # resore the AST:
-        s.ast.sons[codePos] = codeAst
 
 proc addToIndex(w: var TIndex, key, val: int) =
   if key - w.lastIdxKey == 1:
@@ -562,13 +607,15 @@ proc process(c: PPassContext, n: PNode): PNode =
       #            addInterfaceSym(w, a.sons[j].sym);
       #        end
   of nkImportStmt:
-    for i in countup(0, sonsLen(n) - 1): addModDep(w, getModuleName(n.sons[i]))
+    for i in countup(0, sonsLen(n) - 1):
+      addModDep(w, getModuleName(n.sons[i]), n.info)
     addStmt(w, n)
-  of nkFromStmt:
+  of nkFromStmt, nkImportExceptStmt:
-    addModDep(w, getModuleName(n.sons[0]))
+    addModDep(w, getModuleName(n.sons[0]), n.info)
     addStmt(w, n)
   of nkIncludeStmt:
-    for i in countup(0, sonsLen(n) - 1): addInclDep(w, getModuleName(n.sons[i]))
+    for i in countup(0, sonsLen(n) - 1):
+      addInclDep(w, getModuleName(n.sons[i]), n.info)
   of nkPragma:
     addStmt(w, n)
   else:

compiler/ropes.nim

@@ -310,15 +310,19 @@ proc equalsFile*(r: Rope, f: File): bool =
     buf: array[bufSize, char]
     bpos = buf.len
     blen = buf.len
+    btotal = 0
+    rtotal = 0
 
   for s in leaves(r):
     var spos = 0
     let slen = s.len
+    rtotal += slen
     while spos < slen:
       if bpos == blen:
         # Read more data
         bpos = 0
         blen = readBuffer(f, addr(buf[0]), buf.len)
+        btotal += blen
         if blen == 0:  # no more data in file
           result = false
           return
@@ -330,7 +334,8 @@ proc equalsFile*(r: Rope, f: File): bool =
       spos += n
       bpos += n
 
-  result = readBuffer(f, addr(buf[0]), 1) == 0  # check that we've read all
+  result = readBuffer(f, addr(buf[0]), 1) == 0 and
+      btotal == rtotal # check that we've read all
 
 proc equalsFile*(r: Rope, filename: string): bool =
   ## returns true if the contents of the file `f` equal `r`. If `f` does not

compiler/scriptconfig.nim

@@ -43,6 +43,7 @@ proc setupVM*(module: PSym; scriptName: string): PEvalContext =
   template cbos(name, body) {.dirty.} =
     result.registerCallback "stdlib.system." & astToStr(name),
       proc (a: VmArgs) =
+        errorMsg = nil
         try:
           body
         except OSError:
@@ -150,7 +151,7 @@ proc runNimScript*(scriptName: string; freshDefines=true) =
   vm.globalCtx = setupVM(m, scriptName)
 
   compileSystemModule()
-  processModule(m, llStreamOpen(scriptName, fmRead), nil)
+  discard processModule(m, llStreamOpen(scriptName, fmRead), nil)
 
   # ensure we load 'system.nim' again for the real non-config stuff!
   resetAllModulesHard()

compiler/sem.nim

@@ -39,7 +39,6 @@ proc semStmt(c: PContext, n: PNode): PNode
 proc semParamList(c: PContext, n, genericParams: PNode, s: PSym)
 proc addParams(c: PContext, n: PNode, kind: TSymKind)
 proc maybeAddResult(c: PContext, s: PSym, n: PNode)
-proc instGenericContainer(c: PContext, n: PNode, header: PType): PType
 proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode
 proc activate(c: PContext, n: PNode)
 proc semQuoteAst(c: PContext, n: PNode): PNode
@@ -47,6 +46,10 @@ proc finishMethod(c: PContext, s: PSym)
 
 proc indexTypesMatch(c: PContext, f, a: PType, arg: PNode): PNode
 
+proc isArrayConstr(n: PNode): bool {.inline.} =
+  result = n.kind == nkBracket and
+    n.typ.skipTypes(abstractInst).kind == tyArray
+
 template semIdeForTemplateOrGenericCheck(n, requiresCheck) =
   # we check quickly if the node is where the cursor is
   when defined(nimsuggest):
@@ -187,7 +190,6 @@ proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
   # identifier with visibility
 proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode,
                         allowed: TSymFlags): PSym
-proc semStmtScope(c: PContext, n: PNode): PNode
 
 proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind) =
   let t = typeAllowed(typ, kind)
@@ -252,7 +254,7 @@ proc fixupTypeAfterEval(c: PContext, evaluated, eOrig: PNode): PNode =
       result = arg
       # for 'tcnstseq' we support [] to become 'seq'
       if eOrig.typ.skipTypes(abstractInst).kind == tySequence and
-         arg.typ.skipTypes(abstractInst).kind == tyArrayConstr:
+         isArrayConstr(arg):
         arg.typ = eOrig.typ
 
 proc tryConstExpr(c: PContext, n: PNode): PNode =
@@ -445,6 +447,8 @@ proc isImportSystemStmt(n: PNode): bool =
   else: discard
 
 proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =
+  if n.kind == nkDefer:
+    localError(n.info, "defer statement not supported at top level")
   if c.topStmts == 0 and not isImportSystemStmt(n):
     if sfSystemModule notin c.module.flags and
         n.kind notin {nkEmpty, nkCommentStmt}:

compiler/semasgn.nim

@@ -203,7 +203,8 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       defaultOp(c, t, body, x, y)
   of tyObject, tyDistinct:
     if not considerOverloadedOp(c, t, body, x, y):
-      if t.sons[0] != nil: liftBodyAux(c, t.sons[0], body, x, y)
+      if t.sons[0] != nil:
+        liftBodyAux(c, t.sons[0].skipTypes(skipPtrs), body, x, y)
       if t.kind == tyObject: liftBodyObj(c, t.n, body, x, y)
   of tyTuple:
     liftBodyTup(c, t, body, x, y)

compiler/semcall.nim

@@ -34,6 +34,22 @@ proc sameMethodDispatcher(a, b: PSym): bool =
 
 proc determineType(c: PContext, s: PSym)
 
+proc initCandidateSymbols(c: PContext, headSymbol: PNode,
+                       initialBinding: PNode,
+                       filter: TSymKinds,
+                       best, alt: var TCandidate,
+                       o: var TOverloadIter): seq[tuple[s: PSym, scope: int]] =
+  result = @[]
+  var symx = initOverloadIter(o, c, headSymbol)
+  while symx != nil:
+    if symx.kind in filter:
+      result.add((symx, o.lastOverloadScope))
+      symx = nextOverloadIter(o, c, headSymbol)
+  if result.len > 0:
+    initCandidate(c, best, result[0].s, initialBinding, result[0].scope)
+    initCandidate(c, alt, result[0].s, initialBinding, result[0].scope)
+    best.state = csNoMatch
+
 proc pickBestCandidate(c: PContext, headSymbol: PNode,
                        n, orig: PNode,
                        initialBinding: PNode,
@@ -41,66 +57,62 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
                        best, alt: var TCandidate,
                        errors: var CandidateErrors) =
   var o: TOverloadIter
-  # thanks to the lazy semchecking for operands, we need to iterate over the
+  var sym = initOverloadIter(o, c, headSymbol)
-  # symbol table *before* any call to 'initCandidate' which might invoke
+  var scope = o.lastOverloadScope
-  # semExpr which might modify the symbol table in cases like
+  # Thanks to the lazy semchecking for operands, we need to check whether
-  # 'init(a, 1, (var b = new(Type2); b))'.
+  # 'initCandidate' modifies the symbol table (via semExpr).
-  var symx = initOverloadIter(o, c, headSymbol)
+  # This can occur in cases like 'init(a, 1, (var b = new(Type2); b))'
-  let symScope = o.lastOverloadScope
+  let counterInitial = c.currentScope.symbols.counter
-
+  var syms: seq[tuple[s: PSym, scope: int]]
-  var syms: seq[tuple[a: PSym, b: int]] = @[]
+  var nextSymIndex = 0
-  while symx != nil:
+  while sym != nil:
-    if symx.kind in filter:
+    if sym.kind in filter:
-      syms.add((symx, o.lastOverloadScope))
+      # Initialise 'best' and 'alt' with the first available symbol
-    symx = nextOverloadIter(o, c, headSymbol)
+      initCandidate(c, best, sym, initialBinding, scope)
-  if syms.len == 0:
+      initCandidate(c, alt, sym, initialBinding, scope)
-    when false:
+      best.state = csNoMatch
-      if skIterator notin filter:
+      break
-        # also try iterators, but these are 2nd class:
-        symx = initOverloadIter(o, c, headSymbol)
-        while symx != nil:
-          if symx.kind == skIterator:
-            syms.add((symx, 100))
-          symx = nextOverloadIter(o, c, headSymbol)
-        if syms.len == 0: return
     else:
-      return
+      sym = nextOverloadIter(o, c, headSymbol)
-
+      scope = o.lastOverloadScope
   var z: TCandidate
-  initCandidate(c, best, syms[0][0], initialBinding, symScope)
+  while sym != nil:
-  initCandidate(c, alt, syms[0][0], initialBinding, symScope)
+    if sym.kind notin filter:
-  best.state = csNoMatch
+      sym = nextOverloadIter(o, c, headSymbol)
-
+      scope = o.lastOverloadScope
-  for i in 0 .. <syms.len:
+      continue
-    let sym = syms[i][0]
     determineType(c, sym)
-    initCandidate(c, z, sym, initialBinding, syms[i][1])
+    initCandidate(c, z, sym, initialBinding, scope)
-
+    if c.currentScope.symbols.counter == counterInitial or syms != nil:
-    #if sym.name.s == "*" and (n.info ?? "temp5.nim") and n.info.line == 140:
+      matches(c, n, orig, z)
-    #  gDebug = true
+      if errors != nil:
-    matches(c, n, orig, z)
+        errors.safeAdd((sym, int z.mutabilityProblem))
-    if errors != nil:
+        if z.errors != nil:
-      errors.safeAdd((sym, int z.mutabilityProblem))
+          for err in z.errors:
-      if z.errors != nil:
+            errors.add(err)
-        for err in z.errors:
+      if z.state == csMatch:
-          errors.add(err)
+        # little hack so that iterators are preferred over everything else:
-    if z.state == csMatch:
+        if sym.kind == skIterator: inc(z.exactMatches, 200)
-      # little hack so that iterators are preferred over everything else:
+        case best.state
-      if sym.kind == skIterator: inc(z.exactMatches, 200)
+        of csEmpty, csNoMatch: best = z
-      case best.state
+        of csMatch:
-      of csEmpty, csNoMatch: best = z
+          var cmp = cmpCandidates(best, z)
-      of csMatch:
+          if cmp < 0: best = z   # x is better than the best so far
-        var cmp = cmpCandidates(best, z)
+          elif cmp == 0: alt = z # x is as good as the best so far
-        if cmp < 0: best = z   # x is better than the best so far
+    else:
-        elif cmp == 0: alt = z # x is as good as the best so far
+      # Symbol table has been modified. Restart and pre-calculate all syms
-        else: discard
+      # before any further candidate init and compare. SLOW, but rare case.
-      #if sym.name.s == "cmp" and (n.info ?? "rstgen.nim") and n.info.line == 516:
+      syms = initCandidateSymbols(c, headSymbol, initialBinding, filter, best, alt, o)
-      #  echo "Matches ", n.info, " ", typeToString(sym.typ)
+    if syms == nil:
-      #  debug sym
+      sym = nextOverloadIter(o, c, headSymbol)
-      #  writeMatches(z)
+      scope = o.lastOverloadScope
-      #  for i in 1 .. <len(z.call):
+    elif nextSymIndex < syms.len:
-      #    z.call[i].typ.debug
+      # rare case: retrieve the next pre-calculated symbol
-      #  quit 1
+      sym = syms[nextSymIndex].s
+      scope = syms[nextSymIndex].scope
+      nextSymIndex += 1
+    else:
+      break
 
 proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) =
   # Gives a detailed error message; this is separated from semOverloadedCall,
@@ -203,6 +215,7 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
       if result.state != csMatch:
         n.sons.delete(1)
         orig.sons.delete(1)
+        excl n.flags, nfExprCall
       else: return
 
     if nfDotField in n.flags:
@@ -462,3 +475,9 @@ proc searchForBorrowProc(c: PContext, startScope: PScope, fn: PSym): PSym =
     var resolved = semOverloadedCall(c, call, call, {fn.kind})
     if resolved != nil:
       result = resolved.sons[0].sym
+      if not compareTypes(result.typ.sons[0], fn.typ.sons[0], dcEqIgnoreDistinct):
+        result = nil
+      elif result.magic in {mArrPut, mArrGet}:
+        # cannot borrow these magics for now
+        result = nil
+

compiler/semexprs.nim

@@ -136,8 +136,10 @@ proc isCastable(dst, src: PType): bool =
   #                       tyProc, tySet, tyEnum, tyBool, tyChar}
   if skipTypes(dst, abstractInst-{tyOpenArray}).kind == tyOpenArray:
     return false
-  var dstSize, srcSize: BiggestInt
+  if skipTypes(src, abstractInst-{tyTypeDesc}).kind == tyTypeDesc:
+    return false
 
+  var dstSize, srcSize: BiggestInt
   dstSize = computeSize(dst)
   srcSize = computeSize(src)
   if dstSize < 0:
@@ -409,13 +411,11 @@ proc changeType(n: PNode, newType: PType, check: bool) =
   n.typ = newType
 
 proc arrayConstrType(c: PContext, n: PNode): PType =
-  var typ = newTypeS(tyArrayConstr, c)
+  var typ = newTypeS(tyArray, c)
   rawAddSon(typ, nil)     # index type
   if sonsLen(n) == 0:
     rawAddSon(typ, newTypeS(tyEmpty, c)) # needs an empty basetype!
   else:
-    var x = n.sons[0]
-    var lastIndex: BiggestInt = sonsLen(n) - 1
     var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyOrdinal})
     addSonSkipIntLit(typ, t)
   typ.sons[0] = makeRangeType(c, 0, sonsLen(n) - 1, n.info)
@@ -423,7 +423,7 @@ proc arrayConstrType(c: PContext, n: PNode): PType =
 
 proc semArrayConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
   result = newNodeI(nkBracket, n.info)
-  result.typ = newTypeS(tyArrayConstr, c)
+  result.typ = newTypeS(tyArray, c)
   rawAddSon(result.typ, nil)     # index type
   if sonsLen(n) == 0:
     rawAddSon(result.typ, newTypeS(tyEmpty, c)) # needs an empty basetype!
@@ -468,59 +468,11 @@ proc fixAbstractType(c: PContext, n: PNode) =
     if it.kind == nkHiddenSubConv and
         skipTypes(it.typ, abstractVar).kind notin {tyOpenArray, tyVarargs}:
       if skipTypes(it.sons[1].typ, abstractVar).kind in
-            {tyNil, tyArrayConstr, tyTuple, tySet}:
+            {tyNil, tyTuple, tySet} or it[1].isArrayConstr:
         var s = skipTypes(it.typ, abstractVar)
         if s.kind != tyExpr:
           changeType(it.sons[1], s, check=true)
         n.sons[i] = it.sons[1]
-  when false:
-    # XXX finally rewrite that crap!
-    for i in countup(1, sonsLen(n) - 1):
-      var it = n.sons[i]
-      case it.kind
-      of nkHiddenStdConv, nkHiddenSubConv:
-        if it.sons[1].kind == nkBracket:
-          it.sons[1].typ = arrayConstrType(c, it.sons[1])
-          #it.sons[1] = semArrayConstr(c, it.sons[1])
-        if skipTypes(it.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
-          #if n.sons[0].kind == nkSym and IdentEq(n.sons[0].sym.name, "[]="):
-          #  debug(n)
-
-          var s = skipTypes(it.sons[1].typ, abstractVar)
-          if s.kind == tyArrayConstr and s.sons[1].kind == tyEmpty:
-            s = copyType(s, getCurrOwner(), false)
-            skipTypes(s, abstractVar).sons[1] = elemType(
-                skipTypes(it.typ, abstractVar))
-            it.sons[1].typ = s
-          elif s.kind == tySequence and s.sons[0].kind == tyEmpty:
-            s = copyType(s, getCurrOwner(), false)
-            skipTypes(s, abstractVar).sons[0] = elemType(
-                skipTypes(it.typ, abstractVar))
-            it.sons[1].typ = s
-
-        elif skipTypes(it.sons[1].typ, abstractVar).kind in
-            {tyNil, tyArrayConstr, tyTuple, tySet}:
-          var s = skipTypes(it.typ, abstractVar)
-          if s.kind != tyExpr:
-            changeType(it.sons[1], s, check=true)
-          n.sons[i] = it.sons[1]
-      of nkBracket:
-        # an implicitly constructed array (passed to an open array):
-        n.sons[i] = semArrayConstr(c, it, {})
-      else:
-        discard
-        #if (it.typ == nil):
-        #  InternalError(it.info, "fixAbstractType: " & renderTree(it))
-
-proc skipObjConv(n: PNode): PNode =
-  case n.kind
-  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-    if skipTypes(n.sons[1].typ, abstractPtrs).kind in {tyTuple, tyObject}:
-      result = n.sons[1]
-    else:
-      result = n
-  of nkObjUpConv, nkObjDownConv: result = n.sons[0]
-  else: result = n
 
 proc isAssignable(c: PContext, n: PNode; isUnsafeAddr=false): TAssignableResult =
   result = parampatterns.isAssignable(c.p.owner, n, isUnsafeAddr)
@@ -701,9 +653,6 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode,
         # error correction, prevents endless for loop elimination in transf.
         # See bug #2051:
         result.sons[0] = newSymNode(errorSym(c, n))
-      if sfNoSideEffect notin callee.flags:
-        if {sfImportc, sfSideEffect} * callee.flags != {}:
-          incl(c.p.owner.flags, sfSideEffect)
 
 proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode
 
@@ -804,9 +753,6 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
     else:
       result = m.call
       instGenericConvertersSons(c, result, m)
-    # we assume that a procedure that calls something indirectly
-    # has side-effects:
-    if tfNoSideEffect notin t.flags: incl(c.p.owner.flags, sfSideEffect)
   elif t != nil and t.kind == tyTypeDesc:
     if n.len == 1: return semObjConstr(c, n, flags)
     return semConv(c, n)
@@ -986,7 +932,7 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
         tyTuple, tySet, tyUInt..tyUInt64:
       if s.magic == mNone: result = inlineConst(n, s)
       else: result = newSymNode(s, n.info)
-    of tyArrayConstr, tySequence:
+    of tyArray, tySequence:
       # Consider::
       #     const x = []
       #     proc p(a: openarray[int])
@@ -1040,9 +986,6 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
 
     markUsed(n.info, s)
     styleCheckUse(n.info, s)
-    # if a proc accesses a global variable, it is not side effect free:
-    if sfGlobal in s.flags:
-      incl(c.p.owner.flags, sfSideEffect)
     result = newSymNode(s, n.info)
     # We cannot check for access to outer vars for example because it's still
     # not sure the symbol really ends up being used:
@@ -1091,7 +1034,7 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
               result = check
             return result
           if ty.sons[0] == nil: break
-          ty = skipTypes(ty.sons[0], {tyGenericInst})
+          ty = skipTypes(ty.sons[0], skipPtrs)
     # old code, not sure if it's live code:
     markUsed(n.info, s)
     styleCheckUse(n.info, s)
@@ -1169,7 +1112,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
       f = lookupInRecordAndBuildCheck(c, n, ty.n, i, check)
       if f != nil: break
       if ty.sons[0] == nil: break
-      ty = skipTypes(ty.sons[0], {tyGenericInst})
+      ty = skipTypes(ty.sons[0], skipPtrs)
     if f != nil:
       if fieldVisible(c, f):
         # is the access to a public field or in the same module or in a friend?
@@ -1421,8 +1364,9 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
   # a = b # both are vars, means: a[] = b[]
   # a = b # b no 'var T' means: a = addr(b)
   var le = a.typ
-  if skipTypes(le, {tyGenericInst}).kind != tyVar and
+  if (skipTypes(le, {tyGenericInst}).kind != tyVar and
-      isAssignable(c, a) == arNone:
+        isAssignable(c, a) == arNone) or
+      skipTypes(le, abstractVar).kind in {tyOpenArray, tyVarargs}:
     # Direct assignment to a discriminant is allowed!
     localError(a.info, errXCannotBeAssignedTo,
                renderTree(a, {renderNoComments}))
@@ -2128,7 +2072,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
       f = lookupInRecordAndBuildCheck(c, it, t.n, id, check)
       if f != nil: break
       if t.sons[0] == nil: break
-      t = skipTypes(t.sons[0], {tyGenericInst})
+      t = skipTypes(t.sons[0], skipPtrs)
     if f != nil and fieldVisible(c, f):
       it.sons[0] = newSymNode(f)
       e = fitNode(c, f.typ, e)
@@ -2145,7 +2089,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
     while true:
       checkInitialized(objType.n, ids, n.info)
       if objType.sons[0] == nil: break
-      objType = skipTypes(objType.sons[0], {tyGenericInst})
+      objType = skipTypes(objType.sons[0], skipPtrs)
 
 proc semBlock(c: PContext, n: PNode): PNode =
   result = n

compiler/semfields.nim

@@ -152,7 +152,7 @@ proc semForFields(c: PContext, n: PNode, m: TMagic): PNode =
     while t.kind == tyObject:
       semForObjectFields(fc, t.n, n, stmts)
       if t.sons[0] == nil: break
-      t = skipTypes(t.sons[0], abstractPtrs)
+      t = skipTypes(t.sons[0], skipPtrs)
   dec(c.p.nestedLoopCounter)
   # for TR macros this 'while true: ...; break' loop is pretty bad, so
   # we avoid it now if we can:

compiler/semfold.nim

@@ -458,32 +458,6 @@ proc getConstIfExpr(c: PSym, n: PNode): PNode =
       if result == nil: result = getConstExpr(c, it.sons[0])
     else: internalError(it.info, "getConstIfExpr()")
 
-proc partialAndExpr(c: PSym, n: PNode): PNode =
-  # partial evaluation
-  result = n
-  var a = getConstExpr(c, n.sons[1])
-  var b = getConstExpr(c, n.sons[2])
-  if a != nil:
-    if getInt(a) == 0: result = a
-    elif b != nil: result = b
-    else: result = n.sons[2]
-  elif b != nil:
-    if getInt(b) == 0: result = b
-    else: result = n.sons[1]
-
-proc partialOrExpr(c: PSym, n: PNode): PNode =
-  # partial evaluation
-  result = n
-  var a = getConstExpr(c, n.sons[1])
-  var b = getConstExpr(c, n.sons[2])
-  if a != nil:
-    if getInt(a) != 0: result = a
-    elif b != nil: result = b
-    else: result = n.sons[2]
-  elif b != nil:
-    if getInt(b) != 0: result = b
-    else: result = n.sons[1]
-
 proc leValueConv(a, b: PNode): bool =
   result = false
   case a.kind
@@ -540,7 +514,8 @@ proc foldConv*(n, a: PNode; check = false): PNode =
     else:
       result = a
       result.typ = n.typ
-    if check: rangeCheck(n, result.intVal)
+    if check and result.kind in {nkCharLit..nkUInt64Lit}:
+      rangeCheck(n, result.intVal)
   of tyFloat..tyFloat64:
     case skipTypes(a.typ, abstractRange).kind
     of tyInt..tyInt64, tyEnum, tyBool, tyChar:

compiler/seminst.nim

@@ -47,7 +47,7 @@ proc rawHandleSelf(c: PContext; owner: PSym) =
         while t.kind == tyObject:
           addObjFieldsToLocalScope(c, t.n)
           if t.sons[0] == nil: break
-          t = t.sons[0].skipTypes(abstractPtrs)
+          t = t.sons[0].skipTypes(skipPtrs)
 
 proc pushProcCon*(c: PContext; owner: PSym) =
   rawPushProcCon(c, owner)
@@ -97,22 +97,6 @@ proc genericCacheGet(genericSym: PSym, entry: TInstantiation;
       if inst.compilesId == id and sameInstantiation(entry, inst[]):
         return inst.sym
 
-proc removeDefaultParamValues(n: PNode) =
-  # we remove default params, because they cannot be instantiated properly
-  # and they are not needed anyway for instantiation (each param is already
-  # provided).
-  when false:
-    for i in countup(1, sonsLen(n)-1):
-      var a = n.sons[i]
-      if a.kind != nkIdentDefs: IllFormedAst(a)
-      var L = a.len
-      if a.sons[L-1].kind != nkEmpty and a.sons[L-2].kind != nkEmpty:
-        # ``param: typ = defaultVal``.
-        # We don't need defaultVal for semantic checking and it's wrong for
-        # ``cmp: proc (a, b: T): int = cmp``. Hm, for ``cmp = cmp`` that is
-        # not possible... XXX We don't solve this issue here.
-        a.sons[L-1] = ast.emptyNode
-
 proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) =
   # we need to create a fresh set of gensym'ed symbols:
   if n.kind == nkSym and sfGenSym in n.sym.flags and n.sym.owner == orig:
@@ -128,17 +112,6 @@ proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) =
 
 proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind)
 
-proc addProcDecls(c: PContext, fn: PSym) =
-  # get the proc itself in scope (e.g. for recursion)
-  addDecl(c, fn)
-
-  for i in 1 .. <fn.typ.n.len:
-    var param = fn.typ.n.sons[i].sym
-    param.owner = fn
-    addParamOrResult(c, param, fn.kind)
-
-  maybeAddResult(c, fn, fn.ast)
-
 proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) =
   if n.sons[bodyPos].kind != nkEmpty:
     inc c.inGenericInst
@@ -172,9 +145,10 @@ proc fixupInstantiatedSymbols(c: PContext, s: PSym) =
       popInfoContext()
 
 proc sideEffectsCheck(c: PContext, s: PSym) =
-  if {sfNoSideEffect, sfSideEffect} * s.flags ==
+  when false:
-      {sfNoSideEffect, sfSideEffect}:
+    if {sfNoSideEffect, sfSideEffect} * s.flags ==
-    localError(s.info, errXhasSideEffects, s.name.s)
+        {sfNoSideEffect, sfSideEffect}:
+      localError(s.info, errXhasSideEffects, s.name.s)
 
 proc instGenericContainer(c: PContext, info: TLineInfo, header: PType,
                           allowMetaTypes = false): PType =
@@ -187,9 +161,6 @@ proc instGenericContainer(c: PContext, info: TLineInfo, header: PType,
   cl.allowMetaTypes = allowMetaTypes
   result = replaceTypeVarsT(cl, header)
 
-proc instGenericContainer(c: PContext, n: PNode, header: PType): PType =
-  result = instGenericContainer(c, n.info, header)
-
 proc instantiateProcType(c: PContext, pt: TIdTable,
                           prc: PSym, info: TLineInfo) =
   # XXX: Instantiates a generic proc signature, while at the same

compiler/semmagic.nim

@@ -151,13 +151,6 @@ proc isStrangeArray(t: PType): bool =
   let t = t.skipTypes(abstractInst)
   result = t.kind == tyArray and t.firstOrd != 0
 
-proc isNegative(n: PNode): bool =
-  let n = n.skipConv
-  if n.kind in {nkCharLit..nkUInt64Lit}:
-    result = n.intVal < 0
-  elif n.kind in nkCallKinds and n.sons[0].kind == nkSym:
-    result = n.sons[0].sym.magic in {mUnaryMinusI, mUnaryMinusI64}
-
 proc magicsAfterOverloadResolution(c: PContext, n: PNode,
                                    flags: TExprFlags): PNode =
   case n[0].sym.magic

compiler/sempass2.nim

@@ -59,7 +59,7 @@ type
     init: seq[int] # list of initialized variables
     guards: TModel # nested guards
     locked: seq[PNode] # locked locations
-    gcUnsafe, isRecursive, isToplevel: bool
+    gcUnsafe, isRecursive, isToplevel, hasSideEffect: bool
     maxLockLevel, currLockLevel: TLockLevel
   PEffects = var TEffects
 
@@ -131,7 +131,7 @@ proc guardDotAccess(a: PEffects; n: PNode) =
         if field != nil: break
         ty = ty.sons[0]
         if ty == nil: break
-        ty = ty.skipTypes(abstractPtrs)
+        ty = ty.skipTypes(skipPtrs)
     if field == nil:
       localError(n.info, errGenerated, "invalid guard field: " & g.name.s)
       return
@@ -192,6 +192,14 @@ proc markGcUnsafe(a: PEffects; reason: PNode) =
       a.owner.gcUnsafetyReason = newSym(skUnknown, getIdent("<unknown>"),
                                         a.owner, reason.info)
 
+when true:
+  template markSideEffect(a: PEffects; reason: typed) =
+    a.hasSideEffect = true
+else:
+  template markSideEffect(a: PEffects; reason: typed) =
+    a.hasSideEffect = true
+    markGcUnsafe(a, reason)
+
 proc listGcUnsafety(s: PSym; onlyWarning: bool; cycleCheck: var IntSet) =
   let u = s.gcUnsafetyReason
   if u != nil and not cycleCheck.containsOrIncl(u.id):
@@ -226,12 +234,16 @@ proc useVar(a: PEffects, n: PNode) =
         message(n.info, warnUninit, s.name.s)
       # prevent superfluous warnings about the same variable:
       a.init.add s.id
-  if {sfGlobal, sfThread} * s.flags != {} and s.kind in {skVar, skLet}:
+  if {sfGlobal, sfThread} * s.flags != {} and s.kind in {skVar, skLet} and
+      s.magic != mNimVm:
     if s.guard != nil: guardGlobal(a, n, s.guard)
     if {sfGlobal, sfThread} * s.flags == {sfGlobal} and
         (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem):
       #if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n)
       markGcUnsafe(a, s)
+    else:
+      markSideEffect(a, s)
+
 
 type
   TIntersection = seq[tuple[id, count: int]] # a simple count table
@@ -268,10 +280,6 @@ proc createTag(n: PNode): PNode =
     result.typ = sysTypeFromName"TEffect"
   if not n.isNil: result.info = n.info
 
-proc createAnyGlobal(n: PNode): PNode =
-  result = newSymNode(anyGlobal)
-  result.info = n.info
-
 proc addEffect(a: PEffects, e: PNode, useLineInfo=true) =
   assert e.kind != nkRaiseStmt
   var aa = a.exc
@@ -495,6 +503,8 @@ proc propagateEffects(tracked: PEffects, n: PNode, s: PSym) =
   if notGcSafe(s.typ) and sfImportc notin s.flags:
     if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n)
     markGcUnsafe(tracked, s)
+  if tfNoSideEffect notin s.typ.flags:
+    markSideEffect(tracked, s)
   mergeLockLevels(tracked, n, s.getLockLevel)
 
 proc notNilCheck(tracked: PEffects, n: PNode, paramType: PType) =
@@ -550,12 +560,16 @@ proc trackOperand(tracked: PEffects, n: PNode, paramType: PType) =
       if notGcSafe(op) and not isOwnedProcVar(a, tracked.owner):
         if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n)
         markGcUnsafe(tracked, a)
+      elif tfNoSideEffect notin op.flags and not isOwnedProcVar(a, tracked.owner):
+        markSideEffect(tracked, a)
     else:
       mergeEffects(tracked, effectList.sons[exceptionEffects], n)
       mergeTags(tracked, effectList.sons[tagEffects], n)
       if notGcSafe(op):
         if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n)
         markGcUnsafe(tracked, a)
+      elif tfNoSideEffect notin op.flags:
+        markSideEffect(tracked, a)
   notNilCheck(tracked, n, paramType)
 
 proc breaksBlock(n: PNode): bool =
@@ -684,6 +698,7 @@ proc track(tracked: PEffects, n: PNode) =
         if a.sym == tracked.owner: tracked.isRecursive = true
         # even for recursive calls we need to check the lock levels (!):
         mergeLockLevels(tracked, n, a.sym.getLockLevel)
+        if sfSideEffect in a.sym.flags: markSideEffect(tracked, a)
       else:
         mergeLockLevels(tracked, n, op.lockLevel)
       var effectList = op.n.sons[0]
@@ -702,6 +717,10 @@ proc track(tracked: PEffects, n: PNode) =
           if not (a.kind == nkSym and a.sym == tracked.owner):
             if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n)
             markGcUnsafe(tracked, a)
+        if tfNoSideEffect notin op.flags and not importedFromC(a):
+          # and it's not a recursive call:
+          if not (a.kind == nkSym and a.sym == tracked.owner):
+            markSideEffect(tracked, a)
     for i in 1 .. <len(n): trackOperand(tracked, n.sons[i], paramType(op, i))
     if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, mNewSeq}:
       # may not look like an assignment, but it is:
@@ -793,9 +812,6 @@ proc track(tracked: PEffects, n: PNode) =
 proc subtypeRelation(spec, real: PNode): bool =
   result = safeInheritanceDiff(real.excType, spec.typ) <= 0
 
-proc symbolPredicate(spec, real: PNode): bool =
-  result = real.sym.id == spec.sym.id
-
 proc checkRaisesSpec(spec, real: PNode, msg: string, hints: bool;
                      effectPredicate: proc (a, b: PNode): bool {.nimcall.}) =
   # check that any real exception is listed in 'spec'; mark those as used;
@@ -912,8 +928,15 @@ proc trackProc*(s: PSym, body: PNode) =
     else:
       listGcUnsafety(s, onlyWarning=true)
       #localError(s.info, warnGcUnsafe2, s.name.s)
+  if sfNoSideEffect in s.flags and t.hasSideEffect:
+    when false:
+      listGcUnsafety(s, onlyWarning=false)
+    else:
+      localError(s.info, errXhasSideEffects, s.name.s)
   if not t.gcUnsafe:
     s.typ.flags.incl tfGcSafe
+  if not t.hasSideEffect and sfSideEffect notin s.flags:
+    s.typ.flags.incl tfNoSideEffect
   if s.typ.lockLevel == UnspecifiedLockLevel:
     s.typ.lockLevel = t.maxLockLevel
   elif t.maxLockLevel > s.typ.lockLevel:

compiler/semstmts.nim

@@ -155,7 +155,10 @@ proc discardCheck(c: PContext, result: PNode) =
       else:
         var n = result
         while n.kind in skipForDiscardable: n = n.lastSon
-        localError(n.info, errDiscardValueX, result.typ.typeToString)
+        if result.typ.kind == tyProc:
+          localError(n.info, "value of type '" & result.typ.typeToString & "' has to be discarded; for a function call use ()")
+        else:
+          localError(n.info, errDiscardValueX, result.typ.typeToString)
 
 proc semIf(c: PContext, n: PNode): PNode =
   result = n
@@ -659,7 +662,7 @@ proc semRaise(c: PContext, n: PNode): PNode =
   if n.sons[0].kind != nkEmpty:
     n.sons[0] = semExprWithType(c, n.sons[0])
     var typ = n.sons[0].typ
-    if typ.kind != tyRef or typ.sons[0].kind != tyObject:
+    if typ.kind != tyRef or typ.lastSon.kind != tyObject:
       localError(n.info, errExprCannotBeRaised)
 
 proc addGenericParamListToScope(c: PContext, n: PNode) =
@@ -1141,11 +1144,6 @@ type
   TProcCompilationSteps = enum
     stepRegisterSymbol,
     stepDetermineType,
-    stepCompileBody
-
-proc isForwardDecl(s: PSym): bool =
-  internalAssert s.kind == skProc
-  result = s.ast[bodyPos].kind != nkEmpty
 
 proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
                 validPragmas: TSpecialWords,
@@ -1183,8 +1181,6 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     s.ast = n
     #s.scope = c.currentScope
 
-    # if typeIsDetermined: assert phase == stepCompileBody
-    # else: assert phase == stepDetermineType
   # before compiling the proc body, set as current the scope
   # where the proc was declared
   let oldScope = c.currentScope
@@ -1234,7 +1230,8 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
       implicitPragmas(c, s, n, validPragmas)
   else:
     if n.sons[pragmasPos].kind != nkEmpty:
-      localError(n.sons[pragmasPos].info, errPragmaOnlyInHeaderOfProc)
+      localError(n.sons[pragmasPos].info, errPragmaOnlyInHeaderOfProcX,
+        "'" & proto.name.s & "' from " & $proto.info)
     if sfForward notin proto.flags:
       wrongRedefinition(n.info, proto.name.s)
     excl(proto.flags, sfForward)
@@ -1403,6 +1400,11 @@ proc semMacroDef(c: PContext, n: PNode): PNode =
   if namePos >= result.safeLen: return result
   var s = result.sons[namePos].sym
   var t = s.typ
+  var allUntyped = true
+  for i in 1 .. t.n.len-1:
+    let param = t.n.sons[i].sym
+    if param.typ.kind != tyExpr: allUntyped = false
+  if allUntyped: incl(s.flags, sfAllUntyped)
   if t.sons[0] == nil: localError(n.info, errXNeedsReturnType, "macro")
   if n.sons[bodyPos].kind == nkEmpty:
     localError(n.info, errImplOfXexpected, s.name.s)
@@ -1566,8 +1568,3 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
 proc semStmt(c: PContext, n: PNode): PNode =
   # now: simply an alias:
   result = semExprNoType(c, n)
-
-proc semStmtScope(c: PContext, n: PNode): PNode =
-  openScope(c)
-  result = semStmt(c, n)
-  closeScope(c)

compiler/semtempl.nim

@@ -287,35 +287,6 @@ proc semTemplBodySons(c: var TemplCtx, n: PNode): PNode =
   for i in 0.. < n.len:
     result.sons[i] = semTemplBody(c, n.sons[i])
 
-proc wrapInBind(c: var TemplCtx; n: PNode; opr: string): PNode =
-  let ident = getIdent(opr)
-  if ident.id in c.toInject: return n
-
-  let s = searchInScopes(c.c, ident)
-  if s != nil:
-    var callee: PNode
-    if contains(c.toBind, s.id):
-      callee = symChoice(c.c, n, s, scClosed)
-    elif contains(c.toMixin, s.name.id):
-      callee = symChoice(c.c, n, s, scForceOpen)
-    elif s.owner == c.owner and sfGenSym in s.flags:
-      # template tmp[T](x: var seq[T]) =
-      # var yz: T
-      incl(s.flags, sfUsed)
-      callee = newSymNode(s, n.info)
-      styleCheckUse(n.info, s)
-    else:
-      callee = semTemplSymbol(c.c, n, s)
-
-    let call = newNodeI(nkCall, n.info)
-    call.add(callee)
-    for i in 0 .. n.len-1: call.add(n[i])
-    result = newNodeI(nkBind, n.info, 2)
-    result.sons[0] = n
-    result.sons[1] = call
-  else:
-    result = n
-
 proc oprIsRoof(n: PNode): bool =
   const roof = "^"
   case n.kind

compiler/semtypes.nim

@@ -110,6 +110,8 @@ proc semContainer(c: PContext, n: PNode, kind: TTypeKind, kindStr: string,
   result = newOrPrevType(kind, prev, c)
   if sonsLen(n) == 2:
     var base = semTypeNode(c, n.sons[1], nil)
+    if base.kind == tyVoid:
+      localError(n.info, errTIsNotAConcreteType, typeToString(base))
     addSonSkipIntLit(result, base)
   else:
     localError(n.info, errXExpectsOneTypeParam, kindStr)
@@ -659,11 +661,12 @@ proc semObjectNode(c: PContext, n: PNode, prev: PType): PType =
   if n.sonsLen == 0: return newConstraint(c, tyObject)
   var check = initIntSet()
   var pos = 0
-  var base: PType = nil
+  var base, realBase: PType = nil
   # n.sons[0] contains the pragmas (if any). We process these later...
   checkSonsLen(n, 3)
   if n.sons[1].kind != nkEmpty:
-    base = skipTypesOrNil(semTypeNode(c, n.sons[1].sons[0], nil), skipPtrs)
+    realBase = semTypeNode(c, n.sons[1].sons[0], nil)
+    base = skipTypesOrNil(realBase, skipPtrs)
     if base.isNil:
       localError(n.info, errIllegalRecursionInTypeX, "object")
     else:
@@ -674,9 +677,10 @@ proc semObjectNode(c: PContext, n: PNode, prev: PType): PType =
         if concreteBase.kind != tyError:
           localError(n.sons[1].info, errInheritanceOnlyWithNonFinalObjects)
         base = nil
+        realBase = nil
   if n.kind != nkObjectTy: internalError(n.info, "semObjectNode")
   result = newOrPrevType(tyObject, prev, c)
-  rawAddSon(result, base)
+  rawAddSon(result, realBase)
   if result.n.isNil:
     result.n = newNodeI(nkRecList, n.info)
   else:
@@ -1137,14 +1141,12 @@ proc semProcTypeWithScope(c: PContext, n: PNode,
   checkSonsLen(n, 2)
   openScope(c)
   result = semProcTypeNode(c, n.sons[0], nil, prev, kind, isType=true)
+  # start with 'ccClosure', but of course pragmas can overwrite this:
+  result.callConv = ccClosure
   # dummy symbol for `pragma`:
   var s = newSymS(kind, newIdentNode(getIdent("dummy"), n.info), c)
   s.typ = result
-  if n.sons[1].kind == nkEmpty or n.sons[1].len == 0:
+  if n.sons[1].kind != nkEmpty and n.sons[1].len > 0:
-    if result.callConv == ccDefault:
-      result.callConv = ccClosure
-      #Message(n.info, warnImplicitClosure, renderTree(n))
-  else:
     pragma(c, s, n.sons[1], procTypePragmas)
     when useEffectSystem: setEffectsForProcType(result, n.sons[1])
   closeScope(c)

compiler/semtypinst.nim

@@ -86,7 +86,7 @@ type
 
 proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType
 proc replaceTypeVarsS(cl: var TReplTypeVars, s: PSym): PSym
-proc replaceTypeVarsN*(cl: var TReplTypeVars, n: PNode): PNode
+proc replaceTypeVarsN*(cl: var TReplTypeVars, n: PNode; start=0): PNode
 
 template checkMetaInvariants(cl: TReplTypeVars, t: PType) =
   when false:
@@ -151,7 +151,7 @@ proc reResolveCallsWithTypedescParams(cl: var TReplTypeVars, n: PNode): PNode =
 
   return n
 
-proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode): PNode =
+proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode; start=0): PNode =
   if n == nil: return
   result = copyNode(n)
   if n.typ != nil:
@@ -195,7 +195,9 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode): PNode =
     var length = sonsLen(n)
     if length > 0:
       newSons(result, length)
-      for i in countup(0, length - 1):
+      if start > 0:
+        result.sons[0] = n.sons[0]
+      for i in countup(start, length - 1):
         result.sons[i] = replaceTypeVarsN(cl, n.sons[i])
 
 proc replaceTypeVarsS(cl: var TReplTypeVars, s: PSym): PSym =
@@ -462,8 +464,8 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
               r = skipTypes(r2, {tyPtr, tyRef})
           result.sons[i] = r
           propagateToOwner(result, r)
-
+      # bug #4677: Do not instantiate effect lists
-      result.n = replaceTypeVarsN(cl, result.n)
+      result.n = replaceTypeVarsN(cl, result.n, ord(result.kind==tyProc))
 
       case result.kind
       of tyArray:

compiler/sigmatch.nim

@@ -352,18 +352,28 @@ proc handleFloatRange(f, a: PType): TTypeRelation =
       else: result = isIntConv
     else: result = isNone
 
-proc isObjectSubtype(a, f: PType): int =
+proc isObjectSubtype(c: var TCandidate; a, f, fGenericOrigin: PType): int =
   var t = a
   assert t.kind == tyObject
   var depth = 0
+  var last = a
   while t != nil and not sameObjectTypes(f, t):
     assert t.kind == tyObject
     t = t.sons[0]
     if t == nil: break
-    t = skipTypes(t, {tyGenericInst})
+    last = t
+    t = skipTypes(t, skipPtrs)
     inc depth
   if t != nil:
+    if fGenericOrigin != nil and last.kind == tyGenericInst and
+        last.len-1 == fGenericOrigin.len:
+      for i in countup(1, sonsLen(fGenericOrigin) - 1):
+        let x = PType(idTableGet(c.bindings, fGenericOrigin.sons[i]))
+        if x == nil:
+          put(c, fGenericOrigin.sons[i], last.sons[i])
     result = depth
+  else:
+    result = -1
 
 type
   SkippedPtr = enum skippedNone, skippedRef, skippedPtr
@@ -840,12 +850,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
     of tyOpenArray, tyVarargs:
       result = typeRel(c, base(f), base(a))
       if result < isGeneric: result = isNone
-    of tyArrayConstr:
+    of tyArray, tyArrayConstr:
-      if (f.sons[0].kind != tyGenericParam) and (a.sons[1].kind == tyEmpty):
-        result = isSubtype    # [] is allowed here
-      elif typeRel(c, base(f), a.sons[1]) >= isGeneric:
-        result = isSubtype
-    of tyArray:
       if (f.sons[0].kind != tyGenericParam) and (a.sons[1].kind == tyEmpty):
         result = isSubtype
       elif typeRel(c, base(f), a.sons[1]) >= isGeneric:
@@ -896,7 +901,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
         result = isEqual
         # elif tfHasMeta in f.flags: result = recordRel(c, f, a)
       else:
-        var depth = isObjectSubtype(a, f)
+        var depth = isObjectSubtype(c, a, f, nil)
         if depth > 0:
           inc(c.inheritancePenalty, depth)
           result = isSubtype
@@ -1012,6 +1017,15 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
       result = isGeneric
     else:
       let genericBody = f.sons[0]
+      var askip = skippedNone
+      var fskip = skippedNone
+      let aobj = x.skipToObject(askip)
+      let fobj = genericBody.lastSon.skipToObject(fskip)
+      if fobj != nil and aobj != nil and askip == fskip:
+        let depth = isObjectSubtype(c, aobj, fobj, f)
+        if depth >= 0:
+          c.inheritancePenalty += depth
+          return if depth == 0: isGeneric else: isSubtype
       result = typeRel(c, genericBody, x)
       if result != isNone:
         # see tests/generics/tgeneric3.nim for an example that triggers this
@@ -1075,7 +1089,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
   of tyBuiltInTypeClass:
     considerPreviousT:
       let targetKind = f.sons[0].kind
-      if targetKind == a.skipTypes({tyRange, tyGenericInst}).kind or
+      if targetKind == a.skipTypes({tyRange, tyGenericInst, tyBuiltInTypeClass}).kind or
          (targetKind in {tyProc, tyPointer} and a.kind == tyNil):
         put(c, f, a)
         return isGeneric
@@ -1310,8 +1324,9 @@ proc localConvMatch(c: PContext, m: var TCandidate, f, a: PType,
   var call = newNodeI(nkCall, arg.info)
   call.add(f.n.copyTree)
   call.add(arg.copyTree)
-  result = c.semOverloadedCall(c, call, call, routineKinds)
+  result = c.semExpr(c, call)
   if result != nil:
+    if result.typ == nil: return nil
     # resulting type must be consistent with the other arguments:
     var r = typeRel(m, f.sons[0], result.typ)
     if r < isGeneric: return nil
@@ -1320,13 +1335,6 @@ proc localConvMatch(c: PContext, m: var TCandidate, f, a: PType,
     if r == isGeneric:
       result.typ = getInstantiatedType(c, arg, m, base(f))
     m.baseTypeMatch = true
-    # bug #4545: allow the call to go through a 'var T':
-    let vt = result.sons[0].typ.sons[1]
-    if vt.kind == tyVar:
-      let x = result.sons[1]
-      let va = newNodeIT(nkHiddenAddr, x.info, vt)
-      va.add x
-      result.sons[1] = va
 
 proc incMatches(m: var TCandidate; r: TTypeRelation; convMatch = 1) =
   case r
@@ -1559,8 +1567,13 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
 
 
 proc setSon(father: PNode, at: int, son: PNode) =
-  if sonsLen(father) <= at: setLen(father.sons, at + 1)
+  let oldLen = father.len
+  if oldLen <= at:
+    setLen(father.sons, at + 1)
   father.sons[at] = son
+  # insert potential 'void' parameters:
+  #for i in oldLen ..< at:
+  #  father.sons[i] = newNodeIT(nkEmpty, son.info, getSysType(tyVoid))
 
 # we are allowed to modify the calling node in the 'prepare*' procs:
 proc prepareOperand(c: PContext; formal: PType; a: PNode): PNode =
@@ -1591,17 +1604,17 @@ proc prepareNamedParam(a: PNode) =
     a.sons[0] = newIdentNode(considerQuotedIdent(a.sons[0]), info)
 
 proc arrayConstr(c: PContext, n: PNode): PType =
-  result = newTypeS(tyArrayConstr, c)
+  result = newTypeS(tyArray, c)
   rawAddSon(result, makeRangeType(c, 0, 0, n.info))
   addSonSkipIntLit(result, skipTypes(n.typ, {tyGenericInst, tyVar, tyOrdinal}))
 
 proc arrayConstr(c: PContext, info: TLineInfo): PType =
-  result = newTypeS(tyArrayConstr, c)
+  result = newTypeS(tyArray, c)
   rawAddSon(result, makeRangeType(c, 0, -1, info))
   rawAddSon(result, newTypeS(tyEmpty, c)) # needs an empty basetype!
 
 proc incrIndexType(t: PType) =
-  assert t.kind == tyArrayConstr
+  assert t.kind == tyArray
   inc t.sons[0].n.sons[1].intVal
 
 template isVarargsUntyped(x): untyped =

compiler/suggest.nim

@@ -273,7 +273,7 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) =
       while true:
         suggestObject(c, t.n, outputs)
         if t.sons[0] == nil: break
-        t = skipTypes(t.sons[0], {tyGenericInst})
+        t = skipTypes(t.sons[0], skipPtrs)
       suggestOperations(c, n, typ, outputs)
     elif typ.kind == tyTuple and typ.n != nil:
       suggestSymList(c, typ.n, outputs)
@@ -413,17 +413,16 @@ proc safeSemExpr*(c: PContext, n: PNode): PNode =
     result = ast.emptyNode
 
 proc suggestExpr*(c: PContext, node: PNode) =
-  if nfIsCursor notin node.flags:
+  if gTrackPos.line < 0: return
-    if gTrackPos.line < 0: return
+  var cp = inCheckpoint(node.info)
-    var cp = inCheckpoint(node.info)
+  if cp == cpNone: return
-    if cp == cpNone: return
   var outputs = 0
   # This keeps semExpr() from coming here recursively:
   if c.compilesContextId > 0: return
   inc(c.compilesContextId)
 
   if gIdeCmd == ideSug:
-    var n = if nfIsCursor in node.flags: node else: findClosestDot(node)
+    var n = findClosestDot(node)
     if n == nil: n = node
     if n.kind == nkDotExpr:
       var obj = safeSemExpr(c, n.sons[0])
@@ -436,7 +435,7 @@ proc suggestExpr*(c: PContext, node: PNode) =
       suggestEverything(c, n, outputs)
 
   elif gIdeCmd == ideCon:
-    var n = if nfIsCursor in node.flags: node else: findClosestCall(node)
+    var n = findClosestCall(node)
     if n == nil: n = node
     if n.kind in nkCallKinds:
       var a = copyNode(n)

compiler/transf.nim

@@ -594,13 +594,6 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
   popTransCon(c)
   # echo "transformed: ", stmtList.PNode.renderTree
 
-proc getMagicOp(call: PNode): TMagic =
-  if call.sons[0].kind == nkSym and
-      call.sons[0].sym.kind in {skProc, skMethod, skConverter}:
-    result = call.sons[0].sym.magic
-  else:
-    result = mNone
-
 proc transformCase(c: PTransf, n: PNode): PTransNode =
   # removes `elif` branches of a case stmt
   # adds ``else: nil`` if needed for the code generator

compiler/trees.nim

@@ -12,32 +12,21 @@
 import
   ast, astalgo, lexer, msgs, strutils, wordrecg
 
-proc hasSon(father, son: PNode): bool =
+proc cyclicTreeAux(n: PNode, visited: var seq[PNode]): bool =
-  for i in countup(0, sonsLen(father) - 1):
+  if n == nil: return
-    if father.sons[i] == son:
+  for v in visited:
-      return true
+    if v == n: return true
-  result = false
-
-proc cyclicTreeAux(n, s: PNode): bool =
-  if n == nil:
-    return false
-  if hasSon(s, n):
-    return true
-  var m = sonsLen(s)
-  addSon(s, n)
   if not (n.kind in {nkEmpty..nkNilLit}):
-    for i in countup(0, sonsLen(n) - 1):
+    visited.add(n)
-      if cyclicTreeAux(n.sons[i], s):
+    for nSon in n.sons:
-        return true
+      if cyclicTreeAux(nSon, visited): return true
-  result = false
+    discard visited.pop()
-  delSon(s, m)
 
 proc cyclicTree*(n: PNode): bool =
-  var s = newNodeI(nkEmpty, n.info)
+  var visited: seq[PNode] = @[]
-  result = cyclicTreeAux(n, s)
+  cyclicTreeAux(n, visited)
 
 proc exprStructuralEquivalent*(a, b: PNode; strictSymEquality=false): bool =
-  result = false
   if a == b:
     result = true
   elif (a != nil) and (b != nil) and (a.kind == b.kind):
@@ -61,7 +50,6 @@ proc exprStructuralEquivalent*(a, b: PNode; strictSymEquality=false): bool =
         result = true
 
 proc sameTree*(a, b: PNode): bool =
-  result = false
   if a == b:
     result = true
   elif a != nil and b != nil and a.kind == b.kind:
@@ -84,17 +72,6 @@ proc sameTree*(a, b: PNode): bool =
           if not sameTree(a.sons[i], b.sons[i]): return
         result = true
 
-proc getProcSym*(call: PNode): PSym =
-  result = call.sons[0].sym
-
-proc getOpSym*(op: PNode): PSym =
-  if op.kind notin {nkCall, nkHiddenCallConv, nkCommand, nkCallStrLit}:
-    result = nil
-  else:
-    if sonsLen(op) <= 0: internalError(op.info, "getOpSym")
-    elif op.sons[0].kind == nkSym: result = op.sons[0].sym
-    else: result = nil
-
 proc getMagic*(op: PNode): TMagic =
   case op.kind
   of nkCallKinds:
@@ -104,9 +81,8 @@ proc getMagic*(op: PNode): TMagic =
   else: result = mNone
 
 proc isConstExpr*(n: PNode): bool =
-  result = (n.kind in
+  const atomKinds = {nkCharLit..nkNilLit} # Char, Int, UInt, Str, Float and Nil literals
-      {nkCharLit..nkInt64Lit, nkStrLit..nkTripleStrLit,
+  n.kind in atomKinds or nfAllConst in n.flags
-       nkFloatLit..nkFloat64Lit, nkNilLit}) or (nfAllConst in n.flags)
 
 proc isCaseObj*(n: PNode): bool =
   if n.kind == nkRecCase: return true
@@ -131,25 +107,6 @@ proc isDeepConstExpr*(n: PNode): bool =
         result = true
   else: discard
 
-proc flattenTreeAux(d, a: PNode, op: TMagic) =
-  if (getMagic(a) == op):     # a is a "leaf", so add it:
-    for i in countup(1, sonsLen(a) - 1): # BUGFIX
-      flattenTreeAux(d, a.sons[i], op)
-  else:
-    addSon(d, copyTree(a))
-
-proc flattenTree*(root: PNode, op: TMagic): PNode =
-  result = copyNode(root)
-  if getMagic(root) == op:
-    # BUGFIX: forget to copy prc
-    addSon(result, copyNode(root.sons[0]))
-    flattenTreeAux(result, root, op)
-
-proc swapOperands*(op: PNode) =
-  var tmp = op.sons[1]
-  op.sons[1] = op.sons[2]
-  op.sons[2] = tmp
-
 proc isRange*(n: PNode): bool {.inline.} =
   if n.kind in nkCallKinds:
     if n[0].kind == nkIdent and n[0].ident.id == ord(wDotDot) or
@@ -168,7 +125,6 @@ proc unnestStmts(n, result: PNode) =
     result.add(n)
 
 proc flattenStmts*(n: PNode): PNode =
-  ## flattens a nested statement list; used for pattern matching
   result = newNodeI(nkStmtList, n.info)
   unnestStmts(n, result)
   if result.len == 1:

compiler/types.nim

@@ -94,7 +94,8 @@ proc invalidGenericInst(f: PType): bool =
 
 proc isPureObject(typ: PType): bool =
   var t = typ
-  while t.kind == tyObject and t.sons[0] != nil: t = t.sons[0]
+  while t.kind == tyObject and t.sons[0] != nil:
+    t = t.sons[0].skipTypes(skipPtrs)
   result = t.sym != nil and sfPure in t.sym.flags
 
 proc getOrdValue(n: PNode): BiggestInt =
@@ -145,10 +146,6 @@ proc elemType*(t: PType): PType =
   else: result = t.lastSon
   assert(result != nil)
 
-proc skipGeneric(t: PType): PType =
-  result = t
-  while result.kind == tyGenericInst: result = lastSon(result)
-
 proc isOrdinalType(t: PType): bool =
   assert(t != nil)
   const
@@ -232,7 +229,8 @@ proc searchTypeForAux(t: PType, predicate: TTypePredicate,
   if result: return
   case t.kind
   of tyObject:
-    result = searchTypeForAux(t.sons[0], predicate, marker)
+    if t.sons[0] != nil:
+      result = searchTypeForAux(t.sons[0].skipTypes(skipPtrs), predicate, marker)
     if not result: result = searchTypeNodeForAux(t.n, predicate, marker)
   of tyGenericInst, tyDistinct:
     result = searchTypeForAux(lastSon(t), predicate, marker)
@@ -269,7 +267,9 @@ proc analyseObjectWithTypeFieldAux(t: PType,
       if searchTypeNodeForAux(t.n, isObjectWithTypeFieldPredicate, marker):
         return frEmbedded
     for i in countup(0, sonsLen(t) - 1):
-      res = analyseObjectWithTypeFieldAux(t.sons[i], marker)
+      var x = t.sons[i]
+      if x != nil: x = x.skipTypes(skipPtrs)
+      res = analyseObjectWithTypeFieldAux(x, marker)
       if res == frEmbedded:
         return frEmbedded
       if res == frHeader: result = frHeader
@@ -578,10 +578,6 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
     result = typeToStr[t.kind]
   result.addTypeFlags(t)
 
-proc resultType(t: PType): PType =
-  assert(t.kind == tyProc)
-  result = t.sons[0]          # nil is allowed
-
 proc base(t: PType): PType =
   result = t.sons[0]
 
@@ -661,7 +657,14 @@ proc lengthOrd(t: PType): BiggestInt =
   case t.kind
   of tyInt64, tyInt32, tyInt: result = lastOrd(t)
   of tyDistinct, tyConst, tyMutable: result = lengthOrd(t.sons[0])
-  else: result = lastOrd(t) - firstOrd(t) + 1
+  else:
+    let last = lastOrd t
+    let first = firstOrd t
+    # XXX use a better overflow check here:
+    if last == high(BiggestInt) and first <= 0:
+      result = last
+    else:
+      result = lastOrd(t) - firstOrd(t) + 1
 
 # -------------- type equality -----------------------------------------------
 
@@ -772,18 +775,6 @@ proc equalParams(a, b: PNode): TParamsEquality =
         result = paramsIncompatible # overloading by different
                                     # result types does not work
 
-proc sameLiteral(x, y: PNode): bool =
-  if x.kind == y.kind:
-    case x.kind
-    of nkCharLit..nkInt64Lit: result = x.intVal == y.intVal
-    of nkFloatLit..nkFloat64Lit: result = x.floatVal == y.floatVal
-    of nkNilLit: result = true
-    else: assert(false)
-
-proc sameRanges(a, b: PNode): bool =
-  result = sameLiteral(a.sons[0], b.sons[0]) and
-           sameLiteral(a.sons[1], b.sons[1])
-
 proc sameTuple(a, b: PType, c: var TSameTypeClosure): bool =
   # two tuples are equivalent iff the names, types and positions are the same;
   # however, both types may not have any field names (t.n may be nil) which
@@ -1070,10 +1061,10 @@ proc typeAllowedNode(marker: var IntSet, n: PNode, kind: TSymKind,
       of nkNone..nkNilLit:
         discard
       else:
+        if n.kind == nkRecCase and kind in {skProc, skConst}:
+          return n[0].typ
         for i in countup(0, sonsLen(n) - 1):
           let it = n.sons[i]
-          if it.kind == nkRecCase and kind in {skProc, skConst}:
-            return n.typ
           result = typeAllowedNode(marker, it, kind, flags)
           if result != nil: break
 
@@ -1306,7 +1297,7 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt =
     a = maxAlign
   of tyObject:
     if typ.sons[0] != nil:
-      result = computeSizeAux(typ.sons[0], a)
+      result = computeSizeAux(typ.sons[0].skipTypes(skipPtrs), a)
       if result < 0: return
       maxAlign = a
     elif isObjectWithTypeFieldPredicate(typ):

compiler/typesrenderer.nim

@@ -19,20 +19,14 @@ proc renderPlainSymbolName*(n: PNode): string =
   ## for the HTML hyperlinks.
   result = ""
   case n.kind
-  of nkPostfix:
+  of nkPostfix, nkAccQuoted:
-    for i in 0 .. <n.len:
+    result = renderPlainSymbolName(n[<n.len])
-      result = renderPlainSymbolName(n[<n.len])
-      if result.len > 0:
-        return
   of nkIdent:
-    if n.ident.s != "*":
+    result = n.ident.s
-      result = n.ident.s
   of nkSym:
     result = n.sym.renderDefinitionName(noQuotes = true)
   of nkPragmaExpr:
     result = renderPlainSymbolName(n[0])
-  of nkAccQuoted:
-    result = renderPlainSymbolName(n[<n.len])
   else:
     internalError(n.info, "renderPlainSymbolName() with " & $n.kind)
   assert(not result.isNil)

compiler/vm.nim

@@ -171,6 +171,7 @@ proc copyValue(src: PNode): PNode =
   result.info = src.info
   result.typ = src.typ
   result.flags = src.flags * PersistentNodeFlags
+  result.comment = src.comment
   when defined(useNodeIds):
     if result.id == nodeIdToDebug:
       echo "COMES FROM ", src.id
@@ -269,6 +270,7 @@ proc cleanUpOnException(c: PCtx; tos: PStackFrame):
         c.currentExceptionA = nil
         # execute the corresponding handler:
         while c.code[pc2].opcode == opcExcept: inc pc2
+        discard f.safePoints.pop
         return (pc2, f)
       inc pc2
       if c.code[pc2].opcode != opcExcept and nextExceptOrFinally >= 0:
@@ -282,7 +284,8 @@ proc cleanUpOnException(c: PCtx; tos: PStackFrame):
       pc2 = nextExceptOrFinally
     if c.code[pc2].opcode == opcFinally:
       # execute the corresponding handler, but don't quit walking the stack:
-      return (pc2, f)
+      discard f.safePoints.pop
+      return (pc2+1, f)
     # not the right one:
     discard f.safePoints.pop
 
@@ -964,7 +967,13 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       # we'll execute in the 'raise' handler
       let rbx = instr.regBx - wordExcess - 1 # -1 for the following 'inc pc'
       inc pc, rbx
-      assert c.code[pc+1].opcode in {opcExcept, opcFinally}
+      while c.code[pc+1].opcode == opcExcept:
+        let rbx = c.code[pc+1].regBx - wordExcess - 1
+        inc pc, rbx
+      #assert c.code[pc+1].opcode in {opcExcept, opcFinally}
+      if c.code[pc+1].opcode != opcFinally:
+        # in an except handler there is no active safe point for the 'try':
+        tos.popSafePoint()
     of opcFinally:
       # just skip it; it's followed by the code we need to execute anyway
       tos.popSafePoint()
@@ -1063,7 +1072,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcRepr:
       decodeB(rkNode)
       createStr regs[ra]
-      regs[ra].node.strVal = renderTree(regs[rb].regToNode, {renderNoComments})
+      regs[ra].node.strVal = renderTree(regs[rb].regToNode, {renderNoComments, renderDocComments})
     of opcQuit:
       if c.mode in {emRepl, emStaticExpr, emStaticStmt}:
         message(c.debug[pc], hintQuitCalled)

compiler/vmdeps.nim

@@ -109,10 +109,6 @@ proc mapTypeToAstX(t: PType; info: TLineInfo;
     mapTypeToBracketX(name, m, t, info, inst)
   template newNodeX(kind): untyped =
     newNodeIT(kind, if t.n.isNil: info else: t.n.info, t)
-  template newIdent(s): untyped =
-    var r = newNodeX(nkIdent)
-    r.add !s
-    r
   template newIdentDefs(n,t): untyped =
     var id = newNodeX(nkIdentDefs)
     id.add n  # name

compiler/vmgen.nim

@@ -748,7 +748,7 @@ proc genConv(c: PCtx; n, arg: PNode; dest: var TDest; opc=opcConv) =
   let tmp = c.genx(arg)
   if dest < 0: dest = c.getTemp(n.typ)
   c.gABC(n, opc, dest, tmp)
-  c.gABx(n, opc, 0, genType(c, n.typ))
+  c.gABx(n, opc, 0, genType(c, n.typ.skipTypes({tyStatic})))
   c.gABx(n, opc, 0, genType(c, arg.typ.skipTypes({tyStatic})))
   c.freeTemp(tmp)
 
@@ -1127,14 +1127,6 @@ proc fitsRegister*(t: PType): bool =
   t.skipTypes(abstractInst-{tyTypeDesc}).kind in {
     tyRange, tyEnum, tyBool, tyInt..tyUInt64, tyChar}
 
-proc requiresCopy(n: PNode): bool =
-  if n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind in atomicTypes:
-    result = false
-  elif n.kind in ({nkCurly, nkBracket, nkPar, nkObjConstr}+nkCallKinds):
-    result = false
-  else:
-    result = true
-
 proc unneededIndirection(n: PNode): bool =
   n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind == tyRef
 
@@ -1215,8 +1207,6 @@ proc whichAsgnOpc(n: PNode): TOpcode =
   else:
     opcAsgnComplex
 
-proc isRef(t: PType): bool = t.skipTypes(abstractRange-{tyTypeDesc}).kind == tyRef
-
 proc whichAsgnOpc(n: PNode; opc: TOpcode): TOpcode = opc
 
 proc genAsgn(c: PCtx; dest: TDest; ri: PNode; requiresCopy: bool) =
@@ -1268,9 +1258,6 @@ proc isTemp(c: PCtx; dest: TDest): bool =
 template needsAdditionalCopy(n): untyped =
   not c.isTemp(dest) and not fitsRegister(n.typ)
 
-proc skipDeref(n: PNode): PNode =
-  result = if n.kind in {nkDerefExpr, nkHiddenDeref}: n.sons[0] else: n
-
 proc preventFalseAlias(c: PCtx; n: PNode; opc: TOpcode;
                        dest, idx, value: TRegister) =
   # opcLdObj et al really means "load address". We sometimes have to create a
@@ -1848,7 +1835,6 @@ proc genParams(c: PCtx; params: PNode) =
   # res.sym.position is already 0
   c.prc.slots[0] = (inUse: true, kind: slotFixedVar)
   for i in 1.. <params.len:
-    let param = params.sons[i].sym
     c.prc.slots[i] = (inUse: true, kind: slotFixedLet)
   c.prc.maxSlots = max(params.len, 1)
 

config/nim.cfg

@@ -66,7 +66,7 @@ path="$lib/pure"
 @end
 
 @if unix:
-  @if not bsd:
+  @if not bsd or haiku:
     # -fopenmp
     gcc.options.linker = "-ldl"
     gcc.cpp.options.linker = "-ldl"
@@ -80,6 +80,14 @@ path="$lib/pure"
     # at least NetBSD has problems with thread local storage:
     tlsEmulation:on
   @end
+  @if haiku:
+    # -fopenmp
+    gcc.options.linker = "-lroot -lnetwork"
+    gcc.cpp.options.linker = "-lroot -lnetwork"
+    clang.options.linker = "-lroot -lnetwork"
+    clang.cpp.options.linker = "-lroot -lnetwork"
+    tcc.options.linker = "-lroot -lnetwork"
+  @end
 @end
 
 # Configuration for the Intel C/C++ compiler:
@@ -122,6 +130,17 @@ clang.objc.options.linker = "-lobjc -lgnustep-base"
   clang.objc.options.linker = "-framework Foundation"
 @end
 
+# Options for FreeBSD, OpenBSD, NetBSD linker to add locations for searching
+# shared libraries.
+@if freebsd or openbsd or netbsd:
+  gcc.options.linker = "-Wl,-rpath=.:/usr/local/lib:/usr/pkg/lib:/usr/X11R6/lib"
+  gcc.cpp.options.linker = "-Wl,-rpath=.:/usr/local/lib:/usr/pkg/lib:/usr/X11R6/lib"
+  llvm_gcc.options.linker = "-Wl,-rpath=.:/usr/local/lib:/usr/pkg/lib:/usr/X11R6/lib"
+  llvm_gcc.cpp.options.linker = "-Wl,-rpath=.:/usr/local/lib:/usr/pkg/lib:/usr/X11R6/lib"
+  clang.options.linker = "-Wl,-rpath=.:/usr/local/lib:/usr/pkg/lib:/usr/X11R6/lib"
+  clang.cpp.options.linker = "-Wl,-rpath=.:/usr/local/lib:/usr/pkg/lib:/usr/X11R6/lib"
+@end
+
 # Configuration for the VxWorks
 # This has been tested with VxWorks 6.9 only
 @if vxworks:

config/nimdoc.cfg

@@ -49,7 +49,7 @@ $seeSrc
 # See doc.item for available substitution variables.
 doc.item.toc = """
   <li><a class="reference" href="#$itemSymOrID"
-    title="$header_plain">$name</a></li>
+    title="$header_plain">$name<span class="attachedType" style="visibility:hidden">$attype</span></a></li>
 """
 
 # HTML rendered for doc.item's seeSrc variable. Note that this will render to
@@ -60,8 +60,10 @@ doc.item.toc = """
 # * $url: whatever you did pass through the --docSeeSrcUrl switch (which also
 #   gets variables path/line replaced!)
 doc.item.seesrc = """&nbsp;&nbsp;<a
-href="${url}/${path}#L${line}"
+href="${url}/tree/${commit}/${path}#L${line}"
-class="link-seesrc" target="_blank">Source</a>"""
+class="link-seesrc" target="_blank">Source</a>
+<a href="${url}/edit/devel/${path}#L${line}" class="link-seesrc" target="_blank" >Edit</a>
+"""
 
 doc.toc = """
 <ul class="simple simple-toc" id="toc-list">
@@ -72,9 +74,38 @@ $content
 doc.body_toc = """
 <div class="row">
   <div class="three columns">
+  <div>
+    Search: <input type="text" id="searchInput"
+      onkeyup="search()" />
+  </div>
   $tableofcontents
   </div>
   <div class="nine columns" id="content">
+  <div id="tocRoot"></div>
+  <p class="module-desc">$moduledesc</p>
+  $content
+  </div>
+</div>
+"""
+
+doc.body_toc_group = """
+<div class="row">
+  <div class="three columns">
+  <div>
+    Search: <input type="text" id="searchInput"
+      onkeyup="search()" />
+  </div>
+  <div>
+    Group by:
+    <select onchange="groupBy(this.value)">
+      <option value="section">Section</option>
+      <option value="type">Type</option>
+    </select>
+  </div>
+  $tableofcontents
+  </div>
+  <div class="nine columns" id="content">
+  <div id="tocRoot"></div>
   <p class="module-desc">$moduledesc</p>
   $content
   </div>
@@ -93,7 +124,7 @@ doc.listing_end = "</pre>"
 doc.file = """<?xml version="1.0" encoding="utf-8" ?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<!--  This file is generated by Nimrod. -->
+<!--  This file is generated by Nim. -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
@@ -1244,8 +1275,16 @@ dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator
   span.pragmaend {
     cursor: pointer;
   }
+
+div.search_results {
+  background-color: antiquewhite;
+  margin: 3em;
+  padding: 1em;
+  border: 1px solid #4d4d4d;
+}
 </style>
 
+<script type="text/javascript" src="../dochack.js"></script>
 
 <script type="text/javascript">
 function togglepragma(d) {

doc/advopt.txt

@@ -3,6 +3,7 @@ Advanced commands:
   //compileToCpp, cpp       compile project to C++ code
   //compileToOC, objc       compile project to Objective C code
   //js                      compile project to Javascript
+  //e                       run a Nimscript file
   //rst2html                convert a reStructuredText file to HTML
   //rst2tex                 convert a reStructuredText file to TeX
   //jsondoc                 extract the documentation to a json file

doc/lib.rst

@@ -582,4 +582,4 @@ Nim programming language.
   nimblepkglist.js or have javascript disabled in your browser.</b></div>
 
   <script type="text/javascript" src="nimblepkglist.js"></script>
-  <script type="text/javascript" src="http://irclogs.nim-lang.org/packages?callback=gotPackageList"></script>
+  <script type="text/javascript" src="http://irclogs.nim-lang.org/packages?callback=gotPackageList" async></script>

doc/manual/lexing.txt

@@ -28,7 +28,7 @@ The parser uses a stack of indentation levels: the stack consists of integers
 counting the spaces. The indentation information is queried at strategic
 places in the parser but ignored otherwise: The pseudo terminal ``IND{>}``
 denotes an indentation that consists of more spaces than the entry at the top
-of the stack; IND{=} an indentation that has the same number of spaces. ``DED``
+of the stack; ``IND{=}`` an indentation that has the same number of spaces. ``DED``
 is another pseudo terminal that describes the *action* of popping a value
 from the stack, ``IND{>}`` then implies to push onto the stack.
 
@@ -399,7 +399,7 @@ These keywords are also operators:
 `=`:tok:, `:`:tok:, `::`:tok: are not available as general operators; they
 are used for other notational purposes.
 
-``*:`` is as a special case the two tokens `*`:tok: and `:`:tok:
+``*:`` is as a special case treated as the two tokens `*`:tok: and `:`:tok:
 (to support ``var v*: T``).
 
 

doc/manual/pragmas.txt

@@ -161,7 +161,7 @@ requires full qualification.
 
 asmNoStackFrame pragma
 ----------------------
-A proc can be marked with the ``AsmNoStackFrame`` pragma to tell the compiler
+A proc can be marked with the ``asmNoStackFrame`` pragma to tell the compiler
 it should not generate a stack frame for the proc. There are also no exit
 statements like ``return result;`` generated and the generated C function is
 declared as ``__declspec(naked)`` or ``__attribute__((naked))`` (depending on

doc/manual/procs.txt

@@ -61,14 +61,14 @@ Calling a procedure can be done in many different ways:
 .. code-block:: nim
   proc callme(x, y: int, s: string = "", c: char, b: bool = false) = ...
 
-  # call with positional arguments # parameter bindings:
+  # call with positional arguments      # parameter bindings:
-  callme(0, 1, "abc", '\t', true)  # (x=0, y=1, s="abc", c='\t', b=true)
+  callme(0, 1, "abc", '\t', true)       # (x=0, y=1, s="abc", c='\t', b=true)
   # call with named and positional arguments:
-  callme(y=1, x=0, "abd", '\t')    # (x=0, y=1, s="abd", c='\t', b=false)
+  callme(y=1, x=0, "abd", '\t')         # (x=0, y=1, s="abd", c='\t', b=false)
   # call with named arguments (order is not relevant):
-  callme(c='\t', y=1, x=0)         # (x=0, y=1, s="", c='\t', b=false)
+  callme(c='\t', y=1, x=0)              # (x=0, y=1, s="", c='\t', b=false)
   # call as a command statement: no () needed:
-  callme 0, 1, "abc", '\t'
+  callme 0, 1, "abc", '\t'              # (x=0, y=1, s="abc", c='\t', b=false)
 
 A procedure may call itself recursively.
 

doc/manual/stmts.txt

@@ -462,10 +462,10 @@ While statement
 Example:
 
 .. code-block:: nim
-  echo "Please tell me your password: \n"
+  echo "Please tell me your password:"
   var pw = readLine(stdin)
   while pw != "12345":
-    echo "Wrong password! Next try: \n"
+    echo "Wrong password! Next try:"
     pw = readLine(stdin)
 
 

doc/manual/syntax.txt

@@ -62,7 +62,8 @@ Precedence level    Operators                                      First charact
 ================  ===============================================  ==================  ===============
 
 
-Whether an operator is used a prefix operator is also affected by preceeding whitespace (this parsing change was introduced with version 0.13.0):
+Whether an operator is used a prefix operator is also affected by preceding
+whitespace (this parsing change was introduced with version 0.13.0):
 
 .. code-block:: nim
   echo $foo

doc/manual/templates.txt

@@ -406,11 +406,11 @@ builtin can be used for that:
 
   macro debug(n: varargs[typed]): untyped =
     result = newNimNode(nnkStmtList, n)
-    for i in 0..n.len-1:
+    for x in n:
       # we can bind symbols in scope via 'bindSym':
-      add(result, newCall(bindSym"write", bindSym"stdout", toStrLit(n[i])))
+      add(result, newCall(bindSym"write", bindSym"stdout", toStrLit(x)))
       add(result, newCall(bindSym"write", bindSym"stdout", newStrLitNode(": ")))
-      add(result, newCall(bindSym"writeLine", bindSym"stdout", n[i]))
+      add(result, newCall(bindSym"writeLine", bindSym"stdout", x))
 
   var
     a: array [0..10, int]

doc/manual/type_sections.txt

@@ -13,7 +13,7 @@ Example:
     Sym = object       # a symbol
       name: string     # the symbol's name
       line: int        # the line the symbol was declared in
-      code: Node      # the symbol's abstract syntax tree
+      code: Node       # the symbol's abstract syntax tree
 
 A type section begins with the ``type`` keyword. It contains multiple
 type definitions. A type definition binds a type to a name. Type definitions

doc/manual/types.txt

@@ -227,8 +227,8 @@ floating pointer values at compile time; this means expressions like
 Boolean type
 ------------
 The boolean type is named `bool`:idx: in Nim and can be one of the two
-pre-defined values ``true`` and ``false``. Conditions in while,
+pre-defined values ``true`` and ``false``. Conditions in ``while``,
-if, elif, when statements need to be of type bool.
+``if``, ``elif``, ``when``-statements need to be of type ``bool``.
 
 This condition holds::
 
@@ -390,7 +390,10 @@ i-th *unichar*. The iterator ``runes`` from the `unicode module
 
 cstring type
 ------------
-The ``cstring`` type represents a pointer to a zero-terminated char array
+
+The ``cstring`` type meaning `compatible string` is the native representation
+of a string for the compilation backend. For the C backend the ``cstring`` type
+represents a pointer to a zero-terminated char array
 compatible to the type ``char*`` in Ansi C. Its primary purpose lies in easy
 interfacing with C. The index operation ``s[i]`` means the i-th *char* of
 ``s``; however no bounds checking for ``cstring`` is performed making the
@@ -421,7 +424,6 @@ string from a cstring:
   var cstr: cstring = str
   var newstr: string = $cstr
 
-
 Structured types
 ----------------
 A variable of a structured type can hold multiple values at the same
@@ -607,7 +609,7 @@ is similar to the ``instanceof`` operator in Java.
       age: int        # no * means that the field is hidden
 
     Student = ref object of Person # a student is a person
-      id: int                  # with an id field
+      id: int                      # with an id field
 
   var
     student: Student
@@ -1260,4 +1262,4 @@ Is the same as:
 
 However later versions of the language might change this to mean "infer the
 parameters' types from the body". Then the above ``foo`` would be rejected as
-the parameters' types can not be infered from an empty ``discard`` statement.
+the parameters' types can not be inferred from an empty ``discard`` statement.

doc/nimsuggest.rst

@@ -80,7 +80,7 @@ a location. A query location consists of:
 
 ``col``
     An integer with the column you are going to query. For the
-    compiler columns start at **1**.
+    compiler columns start at **0**.
 
 
 Definitions
@@ -160,7 +160,7 @@ tab characters (``\t``). The values of each column are:
 6. Line where the symbol is located in the file. Lines start to
    count at **1**.
 7. Column where the symbol is located in the file. Columns start
-   to count at **1**.
+   to count at **0**.
 8. Docstring for the symbol if available or the empty string. To
    differentiate the docstring from end of answer,
    the docstring is always provided enclosed in double quotes, and

doc/sets_fragment.txt

@@ -1,5 +1,5 @@
-The set type models the mathematical notion of a set. The set's
+The set type models the mathematical notion of a set. The set's basetype can
-basetype can only be an ordinal type of a certain size, namely:
+only be an ordinal type of a certain size, namely:
   * ``int8``-``int16``
   * ``uint8``/``byte``-``uint16``
   * ``char``

doc/tut2.rst

@@ -674,17 +674,18 @@ variable number of arguments:
 
   macro debug(n: varargs[expr]): stmt =
     # `n` is a Nim AST that contains a list of expressions;
-    # this macro returns a list of statements:
+    # this macro returns a list of statements (n is passed for proper line
+    # information):
     result = newNimNode(nnkStmtList, n)
     # iterate over any argument that is passed to this macro:
-    for i in 0..n.len-1:
+    for x in n:
       # add a call to the statement list that writes the expression;
       # `toStrLit` converts an AST to its string representation:
-      result.add(newCall("write", newIdentNode("stdout"), toStrLit(n[i])))
+      result.add(newCall("write", newIdentNode("stdout"), toStrLit(x)))
       # add a call to the statement list that writes ": "
       result.add(newCall("write", newIdentNode("stdout"), newStrLitNode(": ")))
       # add a call to the statement list that writes the expressions value:
-      result.add(newCall("writeLine", newIdentNode("stdout"), n[i]))
+      result.add(newCall("writeLine", newIdentNode("stdout"), x))
 
   var
     a: array[0..10, int]

finish.nim

@@ -0,0 +1,158 @@
+
+# -------------- post unzip steps ---------------------------------------------
+
+import strutils, os, osproc
+
+when defined(windows):
+  import registry
+
+  proc askBool(m: string): bool =
+    stdout.write m
+    while true:
+      let answer = stdin.readLine().normalize
+      case answer
+      of "y", "yes":
+        return true
+      of "n", "no":
+        return false
+      else:
+        echo "Please type 'y' or 'n'"
+
+  proc askNumber(m: string; a, b: int): int =
+    stdout.write m
+    stdout.write " [" & $a & ".." & $b & "] "
+    while true:
+      let answer = stdin.readLine()
+      try:
+        result = parseInt answer
+        if result < a or result > b:
+          raise newException(ValueError, "number out of range")
+        break
+      except ValueError:
+        echo "Please type in a number between ", a, " and ", b
+
+  proc patchConfig(mingw: string) =
+    const
+      cfgFile = "config/nim.cfg"
+      lookFor = """#gcc.path = r"$nim\dist\mingw\bin""""
+      replacePattern = """gcc.path = r"$1""""
+    try:
+      let cfg = readFile(cfgFile)
+      let newCfg = cfg.replace(lookFor, replacePattern % mingw)
+      if newCfg == cfg:
+        echo "Could not patch 'config/nim.cfg' [Error]"
+        echo "Reason: patch substring not found:"
+        echo lookFor
+      else:
+        writeFile(cfgFile, newCfg)
+    except IOError:
+      echo "Could not access 'config/nim.cfg' [Error]"
+
+  proc addToPathEnv(e: string) =
+    let p = getUnicodeValue(r"Environment", "Path", HKEY_CURRENT_USER)
+    let x = if e.contains(Whitespace): "\"" & e & "\"" else: e
+    setUnicodeValue(r"Environment", "Path", p & ";" & x, HKEY_CURRENT_USER)
+
+  proc createShortcut(src, dest: string; icon = "") =
+    var cmd = "bin\\makelink.exe \"" & src & "\" \"\" \"" & dest &
+         ".lnk\" \"\" 1 \"" & splitFile(src).dir & "\""
+    if icon.len != 0:
+      cmd.add " \"" & icon & "\" 0"
+    discard execShellCmd(cmd)
+
+  proc createStartMenuEntry() =
+    let appdata = getEnv("APPDATA")
+    if appdata.len == 0: return
+    let dest = appdata & r"\Microsoft\Windows\Start Menu\Programs\Nim-" &
+               NimVersion
+    if dirExists(dest): return
+    if askBool("Would like to add Nim-" & NimVersion &
+               " to your start menu? (y/n) "):
+      createDir(dest)
+      createShortcut(getCurrentDir() / "start.bat", dest / "Nim",
+                     getCurrentDir() / r"icons\nim.ico")
+      if fileExists("doc/overview.html"):
+        createShortcut(getCurrentDir() / "doc" / "overview.html",
+                       dest / "Overview")
+      if dirExists(r"dist\aporia-0.4.0"):
+        createShortcut(getCurrentDir() / r"dist\aporia-0.4.0\bin\aporia.exe",
+                       dest / "Aporia")
+
+  proc checkGccArch(mingw: string): bool =
+    let gccExe = mingw / r"gcc.exe"
+    if fileExists(gccExe):
+      try:
+        let arch = execProcess(gccExe, ["-dumpmachine"], nil, {poStdErrToStdOut,
+                                                               poUsePath})
+        when hostCPU == "i386":
+          result = arch.startsWith("i686-")
+        elif hostCPU == "amd64":
+          result = arch.startsWith("x86_64-")
+        else:
+          {.error: "Unknown CPU for Windows.".}
+      except OSError, IOError:
+        result = false
+
+  proc tryDirs(dirs: varargs[string]): string =
+    let bits = $(sizeof(pointer)*8)
+    for d in dirs:
+      if dirExists d:
+        let x = expandFilename(d / "bin")
+        if checkGccArch(x): return x
+      elif dirExists(d & bits):
+        let x = expandFilename((d & bits) / "bin")
+        if checkGccArch(x): return x
+
+proc main() =
+  when defined(windows):
+    let desiredPath = expandFilename(getCurrentDir() / "bin")
+    let p = getUnicodeValue(r"Environment", "Path",
+      HKEY_CURRENT_USER)
+    var alreadyInPath = false
+    var mingWchoices: seq[string] = @[]
+    for x in p.split(';'):
+      let y = expandFilename(if x[0] == '"' and x[^1] == '"':
+                  substr(x, 1, x.len-2) else: x)
+      if y == desiredPath: alreadyInPath = true
+      if y.toLowerAscii.contains("mingw"):
+        if dirExists(y) and checkGccArch(y):
+          mingWchoices.add y
+
+    if alreadyInPath:
+      echo "bin/nim.exe is already in your PATH [Skipping]"
+    else:
+      if askBool("nim.exe is not in your PATH environment variable.\n" &
+          " Should it be added permanently? (y/n) "):
+        addToPathEnv(desiredPath)
+    if mingWchoices.len == 0:
+      # No mingw in path, so try a few locations:
+      let alternative = tryDirs("dist/mingw", "../mingw", r"C:\mingw")
+      if alternative.len == 0:
+        echo "No MingW found in PATH and no candidate found " &
+            " in the standard locations [Error]"
+      else:
+        if askBool("Found a MingW directory that is not in your PATH.\n" &
+                   alternative &
+                   "\nShould it be added to your PATH permanently? (y/n) "):
+          addToPathEnv(alternative)
+        elif askBool("Do you want to patch Nim's config to use this? (y/n) "):
+          patchConfig(alternative)
+    elif mingWchoices.len == 1:
+      if askBool("MingW installation found at " & mingWchoices[0] & "\n" &
+         "Do you want to patch Nim's config to use this?\n" &
+         "(Not required since it's in your PATH!) (y/n) "):
+        patchConfig(mingWchoices[0])
+    else:
+      echo "Multiple MingW installations found: "
+      for i in 0..high(mingWchoices):
+        echo "[", i, "] ", mingWchoices[i]
+      if askBool("Do you want to patch Nim's config to use one of these? (y/n) "):
+        let idx = askNumber("Which one do you want to use for Nim? ",
+            1, len(mingWchoices))
+        patchConfig(mingWchoices[idx-1])
+    createStartMenuEntry()
+  else:
+    echo("Add ", getCurrentDir(), "/bin to your PATH...")
+
+when isMainModule:
+  main()

install.txt

@@ -30,6 +30,15 @@ There are also ``install.sh`` and ``deinstall.sh`` scripts for distributing
 the files over the UNIX hierarchy. However, updating your Nim installation
 is more cumbersome then.
 
+To complete the installation you should also build Nim's tools like
+``nimsuggest``, ``nimble`` or ``nimgrep`` via::
+
+  nim e install_tools.nims
+
+Note that these tools should also end up in your ``PATH`` so adding
+``$your_install_dir/bin/nim`` to your ``PATH`` is preferred over the symlink
+solution.
+
 
 Installation on the Macintosh
 -----------------------------

install_nimble.nims

@@ -16,4 +16,8 @@ mkDir "bin/nimblepkg"
 for file in listFiles("nimble" & $id & "/src/nimblepkg/"):
   cpFile file, "bin/nimblepkg/" & file.extractFilename
 
-mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe
+try:
+  mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe
+except OSError:
+  cpFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe
+  

install_tools.nims

@@ -0,0 +1,18 @@
+
+import ospaths
+
+mode = ScriptMode.Verbose
+
+if not dirExists"dist/nimble":
+  echo "[Error] This script only works for the tarball."
+else:
+  let nimbleExe = "./bin/nimble".toExe
+  selfExec "c --noNimblePath -p:compiler -o:" & nimbleExe &
+      " dist/nimble/src/nimble.nim"
+
+  let nimsugExe = "./bin/nimsuggest".toExe
+  selfExec "c --noNimblePath -p:compiler -o:" & nimsugExe &
+      " dist/nimsuggest/nimsuggest.nim"
+
+  let nimgrepExe = "./bin/nimgrep".toExe
+  selfExec "c -o:./bin/nimgrep tools/nimgrep.nim"

koch.nim

@@ -20,11 +20,6 @@ import
 
 const VersionAsString = system.NimVersion #"0.10.2"
 
-when defined(withUpdate):
-  import httpclient
-when defined(haveZipLib):
-  import zipfiles
-
 const
   HelpText = """
 +-----------------------------------------------------------------+
@@ -40,7 +35,7 @@ Options:
   --help, -h               shows this help and quits
 Possible Commands:
   boot [options]           bootstraps with given command line options
-  install [bindir]         installs to given directory; Unix only!
+  distrohelper [bindir]    helper for distro packagers
   geninstall               generate ./install.sh; Unix only!
   testinstall              test tar.xz package; Unix only! Only for devs!
   clean                    cleans Nim project; removes generated files
@@ -52,8 +47,6 @@ Possible Commands:
   xz                       builds the installation XZ package
   nsis [options]           builds the NSIS Setup installer (for Windows)
   tests [options]          run the testsuite
-  update                   updates nim to the latest version from github
-                           (compile koch with -d:withUpdate to enable)
   temp options             creates a temporary compiler for testing
   winrelease               creates a release (for coredevs only)
 Boot options:
@@ -69,7 +62,10 @@ Web options:
                            build the official docs, use UA-48159761-1
 """
 
-proc exe(f: string): string = return addFileExt(f, ExeExt)
+proc exe(f: string): string =
+  result = addFileExt(f, ExeExt)
+  when defined(windows):
+    result = result.replace('/','\\')
 
 proc findNim(): string =
   var nim = "nim".exe
@@ -123,6 +119,8 @@ proc testUnixInstall() =
       execCleanPath("./koch boot -d:release", destDir / "bin")
       # check the docs build:
       execCleanPath("./koch web", destDir / "bin")
+      # check nimble builds:
+      execCleanPath("./bin/nim e install_tools.nims")
       # check the tests work:
       execCleanPath("./koch tests", destDir / "bin")
     else:
@@ -143,13 +141,13 @@ proc copyExe(source, dest: string) =
   inclFilePermissions(dest, {fpUserExec})
 
 const
-  compileNimInst = "-d:useLibzipSrc tools/niminst/niminst"
+  compileNimInst = "tools/niminst/niminst"
 
 proc csource(args: string) =
   exec("$4 cc $1 -r $3 --var:version=$2 --var:mingw=none csource --main:compiler/nim.nim compiler/installer.ini $1" %
        [args, VersionAsString, compileNimInst, findNim()])
 
-proc bundleNimble() =
+proc bundleNimbleSrc() =
   if dirExists("dist/nimble/.git"):
     exec("git --git-dir dist/nimble/.git pull")
   else:
@@ -157,20 +155,38 @@ proc bundleNimble() =
   let tags = execProcess("git --git-dir dist/nimble/.git tag -l v*").splitLines
   let tag = tags[^1]
   exec("git --git-dir dist/nimble/.git checkout " & tag)
+
+proc bundleNimbleExe() =
+  bundleNimbleSrc()
   # now compile Nimble and copy it to $nim/bin for the installer.ini
   # to pick it up:
   exec(findNim() & " c dist/nimble/src/nimble.nim")
   copyExe("dist/nimble/src/nimble".exe, "bin/nimble".exe)
 
+proc bundleNimsuggest(buildExe: bool) =
+  if dirExists("dist/nimsuggest/.git"):
+    exec("git --git-dir dist/nimsuggest/.git pull")
+  else:
+    exec("git clone https://github.com/nim-lang/nimsuggest.git dist/nimsuggest")
+  if buildExe:
+    exec(findNim() & " c --noNimblePath -p:compiler dist/nimsuggest/nimsuggest.nim")
+    copyExe("dist/nimsuggest/nimsuggest".exe, "bin/nimsuggest".exe)
+
+proc bundleFinishExe() =
+  exec(findNim() & " c finish.nim")
+
 proc zip(args: string) =
-  bundleNimble()
+  bundleNimbleSrc()
+  bundleNimsuggest(false)
+  bundleFinishExe()
   exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" %
        [VersionAsString, compileNimInst, findNim()])
   exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim zip compiler/installer.ini" %
        ["tools/niminst/niminst".exe, VersionAsString])
 
 proc xz(args: string) =
-  bundleNimble()
+  bundleNimbleSrc()
+  bundleNimsuggest(false)
   exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" %
        [VersionAsString, compileNimInst, findNim()])
   exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim xz compiler/installer.ini" %
@@ -181,7 +197,9 @@ proc buildTool(toolname, args: string) =
   copyFile(dest="bin"/ splitFile(toolname).name.exe, source=toolname.exe)
 
 proc nsis(args: string) =
-  bundleNimble()
+  bundleNimbleExe()
+  bundleNimsuggest(true)
+  bundleFinishExe()
   # make sure we have generated the niminst executables:
   buildTool("tools/niminst/niminst", args)
   #buildTool("tools/nimgrep", args)
@@ -195,11 +213,8 @@ proc geninstall(args="") =
   exec("$# cc -r $# --var:version=$# --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini $#" %
        [findNim(), compileNimInst, VersionAsString, args])
 
-proc install(args: string) =
-  geninstall()
-  exec("sh ./install.sh $#" % args)
-
 proc web(args: string) =
+  exec("$# js tools/dochack/dochack.nim" % findNim())
   exec("$# cc -r tools/nimweb.nim $# web/website.ini --putenv:nimversion=$#" %
        [findNim(), args, VersionAsString])
 
@@ -244,11 +259,13 @@ proc boot(args: string) =
   var finalDest = "bin" / "nim".exe
   # default to use the 'c' command:
   let bootOptions = if args.len == 0 or args.startsWith("-"): "c" else: ""
+  let smartNimcache = if "release" in args: "rnimcache" else: "dnimcache"
 
   copyExe(findStartNim(), 0.thVersion)
   for i in 0..2:
     echo "iteration: ", i+1
-    exec i.thVersion & " $# $# compiler" / "nim.nim" % [bootOptions, args]
+    exec i.thVersion & " $# $# --nimcache:$# compiler" / "nim.nim" % [bootOptions, args,
+        smartNimcache]
     if sameFileContent(output, i.thVersion):
       copyExe(output, finalDest)
       echo "executables are equal: SUCCESS!"
@@ -273,7 +290,7 @@ proc cleanAux(dir: string) =
   for kind, path in walkDir(dir):
     case kind
     of pcFile:
-      var (dir, name, ext) = splitFile(path)
+      var (_, name, ext) = splitFile(path)
       if ext == "" or cleanExt.contains(ext):
         if not ignore.contains(name):
           echo "removing: ", path
@@ -302,99 +319,14 @@ proc clean(args: string) =
       echo "removing dir: ", path
       removeDir(path)
 
-# -------------- update -------------------------------------------------------
-
-when defined(withUpdate):
-  when defined(windows):
-    {.warning: "Windows users: Make sure to run 'koch update' in Bash.".}
-
-  proc update(args: string) =
-    when defined(windows):
-      echo("Windows users: Make sure to be running this in Bash. ",
-           "If you aren't, press CTRL+C now.")
-
-    var thisDir = getAppDir()
-    var git = findExe("git")
-    echo("Checking for git repo and git executable...")
-    if existsDir(thisDir & "/.git") and git != "":
-      echo("Git repo found!")
-      # use git to download latest source
-      echo("Checking for updates...")
-      discard startCmd(git & " fetch origin master")
-      var procs = startCmd(git & " diff origin/master master")
-      var errcode = procs.waitForExit()
-      var output = readLine(procs.outputStream)
-      echo(output)
-      if errcode == 0:
-        if output == "":
-          # No changes
-          echo("No update. Exiting...")
-          return
-        else:
-          echo("Fetching updates from repo...")
-          var pullout = execCmdEx(git & " pull origin master")
-          if pullout[1] != 0:
-            quit("An error has occurred.")
-          else:
-            if pullout[0].startsWith("Already up-to-date."):
-              quit("No new changes fetched from the repo. " &
-                   "Local branch must be ahead of it. Exiting...")
-      else:
-        quit("An error has occurred.")
-
-    else:
-      echo("No repo or executable found!")
-      when defined(haveZipLib):
-        echo("Falling back.. Downloading source code from repo...")
-        # use dom96's httpclient to download zip
-        downloadFile("https://github.com/Araq/Nim/zipball/master",
-                     thisDir / "update.zip")
-        try:
-          echo("Extracting source code from archive...")
-          var zip: TZipArchive
-          discard open(zip, thisDir & "/update.zip", fmRead)
-          extractAll(zip, thisDir & "/")
-        except:
-          quit("Error reading archive.")
-      else:
-        quit("No failback available. Exiting...")
-
-    echo("Starting update...")
-    boot(args)
-    echo("Update complete!")
-
 # -------------- builds a release ---------------------------------------------
 
-proc run7z(platform: string, patterns: varargs[string]) =
-  const tmpDir = "nim-" & VersionAsString
-  createDir tmpDir
-  try:
-    for pattern in patterns:
-      for f in walkFiles(pattern):
-        if "nimcache" notin f:
-          copyFile(f, tmpDir / f)
-    exec("7z a -tzip $1-$2.zip $1" % [tmpDir, platform])
-  finally:
-    removeDir tmpDir
-
 proc winRelease() =
-  boot(" -d:release")
+  exec(r"call ci\nsis_build.bat " & VersionAsString)
-  #buildTool("tools/niminst/niminst", " -d:release")
-  buildTool("tools/nimgrep", " -d:release")
-  buildTool("compiler/nimfix/nimfix", " -d:release")
-  buildTool("compiler/nimsuggest/nimsuggest", " -d:release")
-
-  #run7z("win32", "bin/nim.exe", "bin/c2nim.exe", "bin/nimgrep.exe",
-  #      "bin/nimfix.exe",
-  #      "bin/nimble.exe", "bin/*.dll",
-  #      "config", "dist/*.dll", "examples", "lib",
-  #      "readme.txt", "contributors.txt", "copying.txt")
-
-  # second step: XXX build 64 bit version
 
 # -------------- tests --------------------------------------------------------
 
-template `|`(a, b): expr = (if a.len > 0: a else: b)
+template `|`(a, b): string = (if a.len > 0: a else: b)
 
 proc tests(args: string) =
   # we compile the tester with taintMode:on to have a basic
@@ -444,14 +376,9 @@ of cmdArgument:
   of "xz": xz(op.cmdLineRest)
   of "nsis": nsis(op.cmdLineRest)
   of "geninstall": geninstall(op.cmdLineRest)
-  of "install": install(op.cmdLineRest)
+  of "distrohelper": geninstall()
   of "testinstall": testUnixInstall()
   of "test", "tests": tests(op.cmdLineRest)
-  of "update":
-    when defined(withUpdate):
-      update(op.cmdLineRest)
-    else:
-      quit "this Koch has not been compiled with -d:withUpdate"
   of "temp": temp(op.cmdLineRest)
   of "winrelease": winRelease()
   else: showHelp()

lib/arch/arch.nim

@@ -6,6 +6,9 @@
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 #
+# Architecture-specific optimizations and features.
+# arch.nim can be imported by only a subset of the
+# architectures supported by Nim.
 
 when defined(windows):
   const

lib/core/macros.nim

@@ -333,7 +333,7 @@ proc parseStmt*(s: string): NimNode {.noSideEffect, compileTime.} =
   let x = internalErrorFlag()
   if x.len > 0: raise newException(ValueError, x)
 
-proc getAst*(macroOrTemplate: expr): NimNode {.magic: "ExpandToAst", noSideEffect.}
+proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEffect.}
   ## Obtains the AST nodes returned from a macro or template invocation.
   ## Example:
   ##
@@ -342,7 +342,7 @@ proc getAst*(macroOrTemplate: expr): NimNode {.magic: "ExpandToAst", noSideEffec
   ##   macro FooMacro() =
   ##     var ast = getAst(BarTemplate())
 
-proc quote*(bl: stmt, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.}
+proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.}
   ## Quasi-quoting operator.
   ## Accepts an expression or a block and returns the AST that represents it.
   ## Within the quoted AST, you are able to interpolate NimNode expressions
@@ -663,7 +663,7 @@ proc copyChildrenTo*(src, dest: NimNode) {.compileTime.}=
   for i in 0 .. < src.len:
     dest.add src[i].copyNimTree
 
-template expectRoutine(node: NimNode): stmt =
+template expectRoutine(node: NimNode) =
   expectKind(node, RoutineNodes)
 
 proc name*(someProc: NimNode): NimNode {.compileTime.} =
@@ -870,7 +870,7 @@ proc hasArgOfName* (params: NimNode; name: string): bool {.compiletime.}=
   ## Search nnkFormalParams for an argument.
   assert params.kind == nnkFormalParams
   for i in 1 .. <params.len:
-    template node: expr = params[i]
+    template node: untyped = params[i]
     if name.eqIdent( $ node[0]):
       return true
 
@@ -891,7 +891,7 @@ proc boolVal*(n: NimNode): bool {.compileTime, noSideEffect.} =
   else: n == bindSym"true" # hacky solution for now
 
 when not defined(booting):
-  template emit*(e: static[string]): stmt {.deprecated.} =
+  template emit*(e: static[string]): untyped {.deprecated.} =
     ## accepts a single string argument and treats it as nim code
     ## that should be inserted verbatim in the program
     ## Example:
@@ -900,6 +900,6 @@ when not defined(booting):
     ##   emit("echo " & '"' & "hello world".toUpper & '"')
     ##
     ## Deprecated since version 0.15 since it's so rarely useful.
-    macro payload: stmt {.gensym.} =
+    macro payload: untyped {.gensym.} =
       result = parseStmt(e)
     payload()

lib/impure/re.nim

@@ -359,7 +359,7 @@ proc parallelReplace*(s: string, subs: openArray[
 proc transformFile*(infile, outfile: string,
                     subs: openArray[tuple[pattern: Regex, repl: string]]) =
   ## reads in the file `infile`, performs a parallel replacement (calls
-  ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an
+  ## `parallelReplace`) and writes back to `outfile`. Raises ``IOError`` if an
   ## error occurs. This is supposed to be used for quick scripting.
   var x = readFile(infile).string
   writeFile(outfile, x.parallelReplace(subs))

lib/nimbase.h

@@ -415,16 +415,6 @@ struct TFrame {
 
 #define NIM_POSIX_INIT  __attribute__((constructor))
 
-#if defined(_MSCVER) && defined(__i386__)
-__declspec(naked) int __fastcall NimXadd(volatile int* pNum, int val) {
-  __asm {
-    lock xadd dword ptr [ECX], EDX
-    mov EAX, EDX
-    ret
-  }
-}
-#endif
-
 #ifdef __GNUC__
 #  define likely(x) __builtin_expect(x, 1)
 #  define unlikely(x) __builtin_expect(x, 0)

lib/packages/docutils/rstgen.nim

@@ -415,18 +415,28 @@ proc sortIndex(a: var openArray[IndexEntry]) =
       a[j] <- v
     if h == 1: break
 
+proc escapeLink(s: string): string =
+  result = newStringOfCap(s.len + s.len shr 2)
+  for c in items(s):
+    case c
+    of 'a'..'z', '_', 'A'..'Z', '0'..'9', '.', '#', ',', '/':
+      result.add c
+    else:
+      add(result, "%")
+      add(result, toHex(ord(c), 2))
+
 proc generateSymbolIndex(symbols: seq[IndexEntry]): string =
-  result = ""
+  result = "<dl>"
   var i = 0
   while i < symbols.len:
-    let keyword= symbols[i].keyword
+    let keyword = symbols[i].keyword
-    let cleaned_keyword = keyword[1..keyword.high - 1]
+    let cleaned_keyword = keyword.escapeLink
-    result.addf("<dt><a name=\"$2\" href=\"#$2\"><span>$1:</span></a></dt><ul class=\"simple\"><dd>\n",
+    result.addf("<dt><a name=\"$2\" href=\"#$2\"><span>$1:</span></a></dt><dd><ul class=\"simple\">\n",
                 [keyword, cleaned_keyword])
     var j = i
     while j < symbols.len and keyword == symbols[j].keyword:
       let
-        url = symbols[j].link
+        url = symbols[j].link.escapeLink
         text = if not symbols[j].linkTitle.isNil: symbols[j].linkTitle else: url
         desc = if not symbols[j].linkDesc.isNil: symbols[j].linkDesc else: ""
       if desc.len > 0:
@@ -439,6 +449,7 @@ proc generateSymbolIndex(symbols: seq[IndexEntry]): string =
       inc j
     result.add("</ul></dd>\n")
     i = j
+  result.add("</dl>")
 
 proc isDocumentationTitle(hyperlink: string): bool =
   ## Returns true if the hyperlink is actually a documentation title.
@@ -463,9 +474,9 @@ proc indentToLevel(level: var int, newLevel: int): string =
   if level == newLevel:
     return
   if newLevel > level:
-    result = repeat("<ul>", newLevel - level)
+    result = repeat("<li><ul>", newLevel - level)
   else:
-    result = repeat("</ul>", level - newLevel)
+    result = repeat("</ul></li>", level - newLevel)
   level = newLevel
 
 proc generateDocumentationTOC(entries: seq[IndexEntry]): string =
@@ -503,7 +514,7 @@ proc generateDocumentationTOC(entries: seq[IndexEntry]): string =
     else:
       result.add(level.indentToLevel(levels[L].level))
       result.add("<li><a href=\"" & link & "\">" &
-        levels[L].text & "</a>\n")
+        levels[L].text & "</a></li>\n")
     inc L
   result.add(level.indentToLevel(1) & "</ul>\n")
   assert(not titleRef.isNil,
@@ -520,7 +531,7 @@ proc generateDocumentationIndex(docs: IndexedDocs): string =
   for title in titles:
     let tocList = generateDocumentationTOC(docs.getOrDefault(title))
     result.add("<ul><li><a href=\"" &
-      title.link & "\">" & title.keyword & "</a>\n" & tocList & "</ul>\n")
+      title.link & "\">" & title.keyword & "</a>\n" & tocList & "</li></ul>\n")
 
 proc generateDocumentationJumps(docs: IndexedDocs): string =
   ## Returns a plain list of hyperlinks to documentation TOCs in HTML.

lib/posix/posix.nim

@@ -1694,6 +1694,8 @@ var
 
   INADDR_ANY* {.importc, header: "<netinet/in.h>".}: InAddrScalar
     ## IPv4 local host address.
+  INADDR_LOOPBACK* {.importc, header: "<netinet/in.h>".}: InAddrScalar
+    ## IPv4 loopback address.
   INADDR_BROADCAST* {.importc, header: "<netinet/in.h>".}: InAddrScalar
     ## IPv4 broadcast address.
 
@@ -2653,6 +2655,12 @@ proc poll*(a1: ptr TPollfd, a2: Tnfds, a3: int): cint {.
 proc realpath*(name, resolved: cstring): cstring {.
   importc: "realpath", header: "<stdlib.h>".}
 
+proc mkstemp*(tmpl: cstring): cint {.importc, header: "<stdlib.h>".}
+  ## Create a temporary file.
+  ##
+  ## **Warning**: The `tmpl` argument is written to by `mkstemp` and thus
+  ## can't be a string literal. If in doubt copy the string before passing it.
+
 proc utimes*(path: cstring, times: ptr array[2, Timeval]): int {.
   importc: "utimes", header: "<sys/time.h>".}
   ## Sets file access and modification times.

lib/posix/termios.nim

@@ -259,3 +259,13 @@ proc tcFlow*(fd: cint; action: cint): cint {.importc: "tcflow",
 # Get process group ID for session leader for controlling terminal FD.
 
 proc tcGetSid*(fd: cint): Pid {.importc: "tcgetsid", header: "<termios.h>".}
+
+# Window size ioctl.  Should work on on any Unix that xterm has been ported to.
+var TIOCGWINSZ*{.importc, header: "<sys/ioctl.h>".}: culong
+
+type IOctl_WinSize* {.importc: "struct winsize", header: "<sys/ioctl.h>",
+                      final, pure.} = object
+  ws_row*, ws_col*, ws_xpixel*, ws_ypixel*: cushort
+
+proc ioctl*(fd: cint, request: culong, reply: ptr IOctl_WinSize): int {.
+  importc: "ioctl", header: "<stdio.h>", varargs.}

lib/pure/algorithm.nim

@@ -179,7 +179,7 @@ proc sort*[T](a: var openArray[T],
   ##    sort(myStrArray, system.cmp)
   ##
   ## You can inline adhoc comparison procs with the `do notation
-  ## <manual.html#do-notation>`_. Example:
+  ## <manual.html#procedures-do-notation>`_. Example:
   ##
   ## .. code-block:: nim
   ##

lib/pure/asyncdispatch.nim

@@ -9,7 +9,7 @@
 
 include "system/inclrtl"
 
-import os, oids, tables, strutils, macros, times, heapqueue
+import os, oids, tables, strutils, times, heapqueue
 
 import nativesockets, net, queues
 
@@ -105,6 +105,36 @@ export Port, SocketFlag
 ## be used to await a procedure returning a ``Future[void]``:
 ## ``await socket.send("foobar")``.
 ##
+## If an awaited future completes with an error, then ``await`` will re-raise
+## this error. To avoid this, you can use the ``yield`` keyword instead of
+## ``await``. The following section shows different ways that you can handle
+## exceptions in async procs.
+##
+## Handling Exceptions
+## ~~~~~~~~~~~~~~~~~~~
+##
+## The most reliable way to handle exceptions is to use ``yield`` on a future
+## then check the future's ``failed`` property. For example:
+##
+##   .. code-block:: Nim
+##     var future = sock.recv(100)
+##     yield future
+##     if future.failed:
+##       # Handle exception
+##
+## The ``async`` procedures also offer limited support for the try statement.
+##
+##    .. code-block:: Nim
+##      try:
+##        let data = await sock.recv(100)
+##        echo("Received ", data)
+##      except:
+##        # Handle exception
+##
+## Unfortunately the semantics of the try statement may not always be correct,
+## and occassionally the compilation may fail altogether.
+## As such it is better to use the former style when possible.
+##
 ## Discarding futures
 ## ------------------
 ##
@@ -126,279 +156,10 @@ export Port, SocketFlag
 ## * Can't await in a ``except`` body
 ## * Forward declarations for async procs are broken,
 ##   link includes workaround: https://github.com/nim-lang/Nim/issues/3182.
-## * FutureVar[T] needs to be completed manually.
 
 # TODO: Check if yielded future is nil and throw a more meaningful exception
 
-# -- Futures
+include includes/asyncfutures
-
-type
-  FutureBase* = ref object of RootObj ## Untyped future.
-    cb: proc () {.closure,gcsafe.}
-    finished: bool
-    error*: ref Exception ## Stored exception
-    errorStackTrace*: string
-    when not defined(release):
-      stackTrace: string ## For debugging purposes only.
-      id: int
-      fromProc: string
-
-  Future*[T] = ref object of FutureBase ## Typed future.
-    value: T ## Stored value
-
-  FutureVar*[T] = distinct Future[T]
-
-  FutureError* = object of Exception
-    cause*: FutureBase
-
-{.deprecated: [PFutureBase: FutureBase, PFuture: Future].}
-
-when not defined(release):
-  var currentID = 0
-
-proc callSoon*(cbproc: proc ()) {.gcsafe.}
-
-proc newFuture*[T](fromProc: string = "unspecified"): Future[T] =
-  ## Creates a new future.
-  ##
-  ## Specifying ``fromProc``, which is a string specifying the name of the proc
-  ## that this future belongs to, is a good habit as it helps with debugging.
-  new(result)
-  result.finished = false
-  when not defined(release):
-    result.stackTrace = getStackTrace()
-    result.id = currentID
-    result.fromProc = fromProc
-    currentID.inc()
-
-proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] =
-  ## Create a new ``FutureVar``. This Future type is ideally suited for
-  ## situations where you want to avoid unnecessary allocations of Futures.
-  ##
-  ## Specifying ``fromProc``, which is a string specifying the name of the proc
-  ## that this future belongs to, is a good habit as it helps with debugging.
-  result = FutureVar[T](newFuture[T](fromProc))
-
-proc clean*[T](future: FutureVar[T]) =
-  ## Resets the ``finished`` status of ``future``.
-  Future[T](future).finished = false
-  Future[T](future).error = nil
-
-proc checkFinished[T](future: Future[T]) =
-  ## Checks whether `future` is finished. If it is then raises a
-  ## ``FutureError``.
-  when not defined(release):
-    if future.finished:
-      var msg = ""
-      msg.add("An attempt was made to complete a Future more than once. ")
-      msg.add("Details:")
-      msg.add("\n  Future ID: " & $future.id)
-      msg.add("\n  Created in proc: " & future.fromProc)
-      msg.add("\n  Stack trace to moment of creation:")
-      msg.add("\n" & indent(future.stackTrace.strip(), 4))
-      when T is string:
-        msg.add("\n  Contents (string): ")
-        msg.add("\n" & indent(future.value.repr, 4))
-      msg.add("\n  Stack trace to moment of secondary completion:")
-      msg.add("\n" & indent(getStackTrace().strip(), 4))
-      var err = newException(FutureError, msg)
-      err.cause = future
-      raise err
-
-proc complete*[T](future: Future[T], val: T) =
-  ## Completes ``future`` with value ``val``.
-  #assert(not future.finished, "Future already finished, cannot finish twice.")
-  checkFinished(future)
-  assert(future.error == nil)
-  future.value = val
-  future.finished = true
-  if future.cb != nil:
-    future.cb()
-
-proc complete*(future: Future[void]) =
-  ## Completes a void ``future``.
-  #assert(not future.finished, "Future already finished, cannot finish twice.")
-  checkFinished(future)
-  assert(future.error == nil)
-  future.finished = true
-  if future.cb != nil:
-    future.cb()
-
-proc complete*[T](future: FutureVar[T]) =
-  ## Completes a ``FutureVar``.
-  template fut: expr = Future[T](future)
-  checkFinished(fut)
-  assert(fut.error == nil)
-  fut.finished = true
-  if fut.cb != nil:
-    fut.cb()
-
-proc fail*[T](future: Future[T], error: ref Exception) =
-  ## Completes ``future`` with ``error``.
-  #assert(not future.finished, "Future already finished, cannot finish twice.")
-  checkFinished(future)
-  future.finished = true
-  future.error = error
-  future.errorStackTrace =
-    if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error)
-  if future.cb != nil:
-    future.cb()
-  else:
-    # This is to prevent exceptions from being silently ignored when a future
-    # is discarded.
-    # TODO: This may turn out to be a bad idea.
-    # Turns out this is a bad idea.
-    #raise error
-    discard
-
-proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) =
-  ## Sets the callback proc to be called when the future completes.
-  ##
-  ## If future has already completed then ``cb`` will be called immediately.
-  ##
-  ## **Note**: You most likely want the other ``callback`` setter which
-  ## passes ``future`` as a param to the callback.
-  future.cb = cb
-  if future.finished:
-    callSoon(future.cb)
-
-proc `callback=`*[T](future: Future[T],
-    cb: proc (future: Future[T]) {.closure,gcsafe.}) =
-  ## Sets the callback proc to be called when the future completes.
-  ##
-  ## If future has already completed then ``cb`` will be called immediately.
-  future.callback = proc () = cb(future)
-
-proc injectStacktrace[T](future: Future[T]) =
-  # TODO: Come up with something better.
-  when not defined(release):
-    var msg = ""
-    msg.add("\n  " & future.fromProc & "'s lead up to read of failed Future:")
-
-    if not future.errorStackTrace.isNil and future.errorStackTrace != "":
-      msg.add("\n" & indent(future.errorStackTrace.strip(), 4))
-    else:
-      msg.add("\n    Empty or nil stack trace.")
-    future.error.msg.add(msg)
-
-proc read*[T](future: Future[T]): T =
-  ## Retrieves the value of ``future``. Future must be finished otherwise
-  ## this function will fail with a ``ValueError`` exception.
-  ##
-  ## If the result of the future is an error then that error will be raised.
-  if future.finished:
-    if future.error != nil:
-      injectStacktrace(future)
-      raise future.error
-    when T isnot void:
-      return future.value
-  else:
-    # TODO: Make a custom exception type for this?
-    raise newException(ValueError, "Future still in progress.")
-
-proc readError*[T](future: Future[T]): ref Exception =
-  ## Retrieves the exception stored in ``future``.
-  ##
-  ## An ``ValueError`` exception will be thrown if no exception exists
-  ## in the specified Future.
-  if future.error != nil: return future.error
-  else:
-    raise newException(ValueError, "No error in future.")
-
-proc mget*[T](future: FutureVar[T]): var T =
-  ## Returns a mutable value stored in ``future``.
-  ##
-  ## Unlike ``read``, this function will not raise an exception if the
-  ## Future has not been finished.
-  result = Future[T](future).value
-
-proc finished*[T](future: Future[T]): bool =
-  ## Determines whether ``future`` has completed.
-  ##
-  ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish.
-  future.finished
-
-proc failed*(future: FutureBase): bool =
-  ## Determines whether ``future`` completed with an error.
-  return future.error != nil
-
-proc asyncCheck*[T](future: Future[T]) =
-  ## Sets a callback on ``future`` which raises an exception if the future
-  ## finished with an error.
-  ##
-  ## This should be used instead of ``discard`` to discard void futures.
-  future.callback =
-    proc () =
-      if future.failed:
-        injectStacktrace(future)
-        raise future.error
-
-proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] =
-  ## Returns a future which will complete once both ``fut1`` and ``fut2``
-  ## complete.
-  var retFuture = newFuture[void]("asyncdispatch.`and`")
-  fut1.callback =
-    proc () =
-      if fut2.finished: retFuture.complete()
-  fut2.callback =
-    proc () =
-      if fut1.finished: retFuture.complete()
-  return retFuture
-
-proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] =
-  ## Returns a future which will complete once either ``fut1`` or ``fut2``
-  ## complete.
-  var retFuture = newFuture[void]("asyncdispatch.`or`")
-  proc cb() =
-    if not retFuture.finished: retFuture.complete()
-  fut1.callback = cb
-  fut2.callback = cb
-  return retFuture
-
-proc all*[T](futs: varargs[Future[T]]): auto =
-  ## Returns a future which will complete once
-  ## all futures in ``futs`` complete.
-  ##
-  ## If the awaited futures are not ``Future[void]``, the returned future
-  ## will hold the values of all awaited futures in a sequence.
-  ##
-  ## If the awaited futures *are* ``Future[void]``,
-  ## this proc returns ``Future[void]``.
-
-  when T is void:
-    var
-      retFuture = newFuture[void]("asyncdispatch.all")
-      completedFutures = 0
-
-    let totalFutures = len(futs)
-
-    for fut in futs:
-      fut.callback = proc(f: Future[T]) =
-        inc(completedFutures)
-
-        if completedFutures == totalFutures:
-          retFuture.complete()
-
-    return retFuture
-
-  else:
-    var
-      retFuture = newFuture[seq[T]]("asyncdispatch.all")
-      retValues = newSeq[T](len(futs))
-      completedFutures = 0
-
-    for i, fut in futs:
-      proc setCallback(i: int) =
-        fut.callback = proc(f: Future[T]) =
-          retValues[i] = f.read()
-          inc(completedFutures)
-
-          if completedFutures == len(retValues):
-            retFuture.complete(retValues)
-
-      setCallback(i)
-
-    return retFuture
 
 type
   PDispatcherBase = ref object of RootRef
@@ -500,48 +261,49 @@ when defined(windows) or defined(nimdoc):
       raise newException(ValueError,
         "No handles or timers registered in dispatcher.")
 
-    let at = p.adjustedTimeout(timeout)
+    if p.handles.len != 0:
-    var llTimeout =
+      let at = p.adjustedTimeout(timeout)
-      if at == -1: winlean.INFINITE
+      var llTimeout =
-      else: at.int32
+        if at == -1: winlean.INFINITE
+        else: at.int32
 
-    var lpNumberOfBytesTransferred: Dword
+      var lpNumberOfBytesTransferred: Dword
-    var lpCompletionKey: ULONG_PTR
+      var lpCompletionKey: ULONG_PTR
-    var customOverlapped: PCustomOverlapped
+      var customOverlapped: PCustomOverlapped
-    let res = getQueuedCompletionStatus(p.ioPort,
+      let res = getQueuedCompletionStatus(p.ioPort,
-        addr lpNumberOfBytesTransferred, addr lpCompletionKey,
+          addr lpNumberOfBytesTransferred, addr lpCompletionKey,
-        cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool
+          cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool
 
-    # http://stackoverflow.com/a/12277264/492186
+      # http://stackoverflow.com/a/12277264/492186
-    # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html
+      # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html
-    if res:
+      if res:
-      # This is useful for ensuring the reliability of the overlapped struct.
+        # This is useful for ensuring the reliability of the overlapped struct.
-      assert customOverlapped.data.fd == lpCompletionKey.AsyncFD
-
-      customOverlapped.data.cb(customOverlapped.data.fd,
-          lpNumberOfBytesTransferred, OSErrorCode(-1))
-
-      # If cell.data != nil, then system.protect(rawEnv(cb)) was called,
-      # so we need to dispose our `cb` environment, because it is not needed
-      # anymore.
-      if customOverlapped.data.cell.data != nil:
-        system.dispose(customOverlapped.data.cell)
-
-      GC_unref(customOverlapped)
-    else:
-      let errCode = osLastError()
-      if customOverlapped != nil:
         assert customOverlapped.data.fd == lpCompletionKey.AsyncFD
+
         customOverlapped.data.cb(customOverlapped.data.fd,
-            lpNumberOfBytesTransferred, errCode)
+            lpNumberOfBytesTransferred, OSErrorCode(-1))
+
+        # If cell.data != nil, then system.protect(rawEnv(cb)) was called,
+        # so we need to dispose our `cb` environment, because it is not needed
+        # anymore.
         if customOverlapped.data.cell.data != nil:
           system.dispose(customOverlapped.data.cell)
+
         GC_unref(customOverlapped)
       else:
-        if errCode.int32 == WAIT_TIMEOUT:
+        let errCode = osLastError()
-          # Timed out
+        if customOverlapped != nil:
-          discard
+          assert customOverlapped.data.fd == lpCompletionKey.AsyncFD
-        else: raiseOSError(errCode)
+          customOverlapped.data.cb(customOverlapped.data.fd,
+              lpNumberOfBytesTransferred, errCode)
+          if customOverlapped.data.cell.data != nil:
+            system.dispose(customOverlapped.data.cell)
+          GC_unref(customOverlapped)
+        else:
+          if errCode.int32 == WAIT_TIMEOUT:
+            # Timed out
+            discard
+          else: raiseOSError(errCode)
 
     # Timer processing.
     processTimers(p)
@@ -749,8 +511,8 @@ when defined(windows) or defined(nimdoc):
           retFuture.complete("")
     return retFuture
 
-  proc recvInto*(socket: AsyncFD, buf: cstring, size: int,
+  proc recvInto*(socket: AsyncFD, buf: pointer, size: int,
-                flags = {SocketFlag.SafeDisconn}): Future[int] =
+                 flags = {SocketFlag.SafeDisconn}): Future[int] =
     ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must
     ## at least be of that size. Returned future will complete once all the
     ## data requested is read, a part of the data has been read, or the socket
@@ -773,7 +535,7 @@ when defined(windows) or defined(nimdoc):
 
     #buf[] = '\0'
     var dataBuf: TWSABuf
-    dataBuf.buf = buf
+    dataBuf.buf = cast[cstring](buf)
     dataBuf.len = size.ULONG
 
     var bytesReceived: Dword
@@ -819,16 +581,18 @@ when defined(windows) or defined(nimdoc):
           retFuture.complete(bytesReceived)
     return retFuture
 
-  proc send*(socket: AsyncFD, data: string,
+  proc send*(socket: AsyncFD, buf: pointer, size: int,
              flags = {SocketFlag.SafeDisconn}): Future[void] =
-    ## Sends ``data`` to ``socket``. The returned future will complete once all
+    ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future will complete once all
     ## data has been sent.
+    ## **WARNING**: Use it with caution. If ``buf`` refers to GC'ed object, you must use GC_ref/GC_unref calls
+    ## to avoid early freeing of the buffer
     verifyPresence(socket)
     var retFuture = newFuture[void]("send")
 
     var dataBuf: TWSABuf
-    dataBuf.buf = data # since this is not used in a callback, this is fine
+    dataBuf.buf = cast[cstring](buf)
-    dataBuf.len = data.len.ULONG
+    dataBuf.len = size.ULONG
 
     var bytesReceived, lowFlags: Dword
     var ol = PCustomOverlapped()
@@ -1281,43 +1045,45 @@ else:
 
   proc poll*(timeout = 500) =
     let p = getGlobalDispatcher()
-    for info in p.selector.select(p.adjustedTimeout(timeout)):
-      let data = PData(info.key.data)
-      assert data.fd == info.key.fd.AsyncFD
-      #echo("In poll ", data.fd.cint)
-      # There may be EvError here, but we handle them in callbacks,
-      # so that exceptions can be raised from `send(...)` and
-      # `recv(...)` routines.
 
-      if EvRead in info.events:
+    if p.selector.len > 0:
-        # Callback may add items to ``data.readCBs`` which causes issues if
+      for info in p.selector.select(p.adjustedTimeout(timeout)):
-        # we are iterating over ``data.readCBs`` at the same time. We therefore
+        let data = PData(info.key.data)
-        # make a copy to iterate over.
+        assert data.fd == info.key.fd.AsyncFD
-        let currentCBs = data.readCBs
+        #echo("In poll ", data.fd.cint)
-        data.readCBs = @[]
+        # There may be EvError here, but we handle them in callbacks,
-        for cb in currentCBs:
+        # so that exceptions can be raised from `send(...)` and
-          if not cb(data.fd):
+        # `recv(...)` routines.
-            # Callback wants to be called again.
-            data.readCBs.add(cb)
 
-      if EvWrite in info.events:
+        if EvRead in info.events:
-        let currentCBs = data.writeCBs
+          # Callback may add items to ``data.readCBs`` which causes issues if
-        data.writeCBs = @[]
+          # we are iterating over ``data.readCBs`` at the same time. We therefore
-        for cb in currentCBs:
+          # make a copy to iterate over.
-          if not cb(data.fd):
+          let currentCBs = data.readCBs
-            # Callback wants to be called again.
+          data.readCBs = @[]
-            data.writeCBs.add(cb)
+          for cb in currentCBs:
+            if not cb(data.fd):
+              # Callback wants to be called again.
+              data.readCBs.add(cb)
 
-      if info.key in p.selector:
+        if EvWrite in info.events:
-        var newEvents: set[Event]
+          let currentCBs = data.writeCBs
-        if data.readCBs.len != 0: newEvents = {EvRead}
+          data.writeCBs = @[]
-        if data.writeCBs.len != 0: newEvents = newEvents + {EvWrite}
+          for cb in currentCBs:
-        if newEvents != info.key.events:
+            if not cb(data.fd):
-          update(data.fd, newEvents)
+              # Callback wants to be called again.
-      else:
+              data.writeCBs.add(cb)
-        # FD no longer a part of the selector. Likely been closed
+
-        # (e.g. socket disconnected).
+        if info.key in p.selector:
-        discard
+          var newEvents: set[Event]
+          if data.readCBs.len != 0: newEvents = {EvRead}
+          if data.writeCBs.len != 0: newEvents = newEvents + {EvWrite}
+          if newEvents != info.key.events:
+            update(data.fd, newEvents)
+        else:
+          # FD no longer a part of the selector. Likely been closed
+          # (e.g. socket disconnected).
+          discard
 
     # Timer processing.
     processTimers(p)
@@ -1398,7 +1164,7 @@ else:
     addRead(socket, cb)
     return retFuture
 
-  proc recvInto*(socket: AsyncFD, buf: cstring, size: int,
+  proc recvInto*(socket: AsyncFD, buf: pointer, size: int,
                   flags = {SocketFlag.SafeDisconn}): Future[int] =
     var retFuture = newFuture[int]("recvInto")
 
@@ -1422,7 +1188,7 @@ else:
     addRead(socket, cb)
     return retFuture
 
-  proc send*(socket: AsyncFD, data: string,
+  proc send*(socket: AsyncFD, buf: pointer, size: int,
              flags = {SocketFlag.SafeDisconn}): Future[void] =
     var retFuture = newFuture[void]("send")
 
@@ -1430,8 +1196,8 @@ else:
 
     proc cb(sock: AsyncFD): bool =
       result = true
-      let netSize = data.len-written
+      let netSize = size-written
-      var d = data.cstring
+      var d = cast[cstring](buf)
       let res = send(sock.SocketHandle, addr d[written], netSize.cint,
                      MSG_NOSIGNAL)
       if res < 0:
@@ -1576,368 +1342,31 @@ proc accept*(socket: AsyncFD,
         retFut.complete(future.read.client)
   return retFut
 
+proc send*(socket: AsyncFD, data: string,
+           flags = {SocketFlag.SafeDisconn}): Future[void] =
+  ## Sends ``data`` to ``socket``. The returned future will complete once all
+  ## data has been sent.
+  var retFuture = newFuture[void]("send")
+
+  var copiedData = data
+  GC_ref(copiedData) # we need to protect data until send operation is completed
+               # or failed.
+
+  let sendFut = socket.send(addr copiedData[0], data.len, flags)
+  sendFut.callback =
+    proc () =
+      GC_unref(copiedData)
+      if sendFut.failed:
+        retFuture.fail(sendFut.error)
+      else:
+        retFuture.complete()
+
+  return retFuture
+
 # -- Await Macro
+include asyncmacro
 
-proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} =
+proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} =
-  # Skips a nest of StmtList's.
-  result = node
-  if node[0].kind == nnkStmtList:
-    result = skipUntilStmtList(node[0])
-
-proc skipStmtList(node: NimNode): NimNode {.compileTime.} =
-  result = node
-  if node[0].kind == nnkStmtList:
-    result = node[0]
-
-template createCb(retFutureSym, iteratorNameSym,
-                   name: untyped) =
-  var nameIterVar = iteratorNameSym
-  #{.push stackTrace: off.}
-  proc cb {.closure,gcsafe.} =
-    try:
-      if not nameIterVar.finished:
-        var next = nameIterVar()
-        if next == nil:
-          assert retFutureSym.finished, "Async procedure's (" &
-                 name & ") return Future was not finished."
-        else:
-          next.callback = cb
-    except:
-      if retFutureSym.finished:
-        # Take a look at tasyncexceptions for the bug which this fixes.
-        # That test explains it better than I can here.
-        raise
-      else:
-        retFutureSym.fail(getCurrentException())
-  cb()
-  #{.pop.}
-proc generateExceptionCheck(futSym,
-    tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} =
-  if tryStmt.kind == nnkNilLit:
-    result = rootReceiver
-  else:
-    var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[]
-    let errorNode = newDotExpr(futSym, newIdentNode("error"))
-    for i in 1 .. <tryStmt.len:
-      let exceptBranch = tryStmt[i]
-      if exceptBranch[0].kind == nnkStmtList:
-        exceptionChecks.add((newIdentNode("true"), exceptBranch[0]))
-      else:
-        var exceptIdentCount = 0
-        var ifCond: NimNode
-        for i in 0 .. <exceptBranch.len:
-          let child = exceptBranch[i]
-          if child.kind == nnkIdent:
-            let cond = infix(errorNode, "of", child)
-            if exceptIdentCount == 0:
-              ifCond = cond
-            else:
-              ifCond = infix(ifCond, "or", cond)
-          else:
-            break
-          exceptIdentCount.inc
-
-        expectKind(exceptBranch[exceptIdentCount], nnkStmtList)
-        exceptionChecks.add((ifCond, exceptBranch[exceptIdentCount]))
-    # -> -> else: raise futSym.error
-    exceptionChecks.add((newIdentNode("true"),
-        newNimNode(nnkRaiseStmt).add(errorNode)))
-    # Read the future if there is no error.
-    # -> else: futSym.read
-    let elseNode = newNimNode(nnkElse, fromNode)
-    elseNode.add newNimNode(nnkStmtList, fromNode)
-    elseNode[0].add rootReceiver
-
-    let ifBody = newStmtList()
-    ifBody.add newCall(newIdentNode("setCurrentException"), errorNode)
-    ifBody.add newIfStmt(exceptionChecks)
-    ifBody.add newCall(newIdentNode("setCurrentException"), newNilLit())
-
-    result = newIfStmt(
-      (newDotExpr(futSym, newIdentNode("failed")), ifBody)
-    )
-    result.add elseNode
-
-template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver,
-                rootReceiver: expr, fromNode: NimNode) =
-  ## Params:
-  ##    futureVarNode: The NimNode which is a symbol identifying the Future[T]
-  ##                   variable to yield.
-  ##    fromNode: Used for better debug information (to give context).
-  ##    valueReceiver: The node which defines an expression that retrieves the
-  ##                   future's value.
-  ##
-  ##    rootReceiver: ??? TODO
-  # -> yield future<x>
-  result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode)
-  # -> future<x>.read
-  valueReceiver = newDotExpr(futureVarNode, newIdentNode("read"))
-  result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver,
-      fromNode)
-
-template createVar(result: var NimNode, futSymName: string,
-                   asyncProc: NimNode,
-                   valueReceiver, rootReceiver: expr,
-                   fromNode: NimNode) =
-  result = newNimNode(nnkStmtList, fromNode)
-  var futSym = genSym(nskVar, "future")
-  result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y
-  useVar(result, futSym, valueReceiver, rootReceiver, fromNode)
-
-proc processBody(node, retFutureSym: NimNode,
-                 subTypeIsVoid: bool,
-                 tryStmt: NimNode): NimNode {.compileTime.} =
-  #echo(node.treeRepr)
-  result = node
-  case node.kind
-  of nnkReturnStmt:
-    result = newNimNode(nnkStmtList, node)
-    if node[0].kind == nnkEmpty:
-      if not subTypeIsVoid:
-        result.add newCall(newIdentNode("complete"), retFutureSym,
-            newIdentNode("result"))
-      else:
-        result.add newCall(newIdentNode("complete"), retFutureSym)
-    else:
-      let x = node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt)
-      if x.kind == nnkYieldStmt: result.add x
-      else:
-        result.add newCall(newIdentNode("complete"), retFutureSym, x)
-
-    result.add newNimNode(nnkReturnStmt, node).add(newNilLit())
-    return # Don't process the children of this return stmt
-  of nnkCommand, nnkCall:
-    if node[0].kind == nnkIdent and node[0].ident == !"await":
-      case node[1].kind
-      of nnkIdent, nnkInfix, nnkDotExpr:
-        # await x
-        # await x or y
-        result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x
-      of nnkCall, nnkCommand:
-        # await foo(p, x)
-        # await foo p, x
-        var futureValue: NimNode
-        result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue,
-                  futureValue, node)
-      else:
-        error("Invalid node kind in 'await', got: " & $node[1].kind)
-    elif node.len > 1 and node[1].kind == nnkCommand and
-         node[1][0].kind == nnkIdent and node[1][0].ident == !"await":
-      # foo await x
-      var newCommand = node
-      result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1],
-                newCommand, node)
-
-  of nnkVarSection, nnkLetSection:
-    case node[0][2].kind
-    of nnkCommand:
-      if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await":
-        # var x = await y
-        var newVarSection = node # TODO: Should this use copyNimNode?
-        result.createVar("future" & $node[0][0].ident, node[0][2][1],
-          newVarSection[0][2], newVarSection, node)
-    else: discard
-  of nnkAsgn:
-    case node[1].kind
-    of nnkCommand:
-      if node[1][0].ident == !"await":
-        # x = await y
-        var newAsgn = node
-        result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node)
-    else: discard
-  of nnkDiscardStmt:
-    # discard await x
-    if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and
-          node[0][0].ident == !"await":
-      var newDiscard = node
-      result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1],
-                newDiscard[0], newDiscard, node)
-  of nnkTryStmt:
-    # try: await x; except: ...
-    result = newNimNode(nnkStmtList, node)
-    template wrapInTry(n, tryBody: expr) =
-      var temp = n
-      n[0] = tryBody
-      tryBody = temp
-
-      # Transform ``except`` body.
-      # TODO: Could we perform some ``await`` transformation here to get it
-      # working in ``except``?
-      tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, nil)
-
-    proc processForTry(n: NimNode, i: var int,
-                       res: NimNode): bool {.compileTime.} =
-      ## Transforms the body of the tryStmt. Does not transform the
-      ## body in ``except``.
-      ## Returns true if the tryStmt node was transformed into an ifStmt.
-      result = false
-      var skipped = n.skipStmtList()
-      while i < skipped.len:
-        var processed = processBody(skipped[i], retFutureSym,
-                                    subTypeIsVoid, n)
-
-        # Check if we transformed the node into an exception check.
-        # This suggests skipped[i] contains ``await``.
-        if processed.kind != skipped[i].kind or processed.len != skipped[i].len:
-          processed = processed.skipUntilStmtList()
-          expectKind(processed, nnkStmtList)
-          expectKind(processed[2][1], nnkElse)
-          i.inc
-
-          if not processForTry(n, i, processed[2][1][0]):
-            # We need to wrap the nnkElse nodes back into a tryStmt.
-            # As they are executed if an exception does not happen
-            # inside the awaited future.
-            # The following code will wrap the nodes inside the
-            # original tryStmt.
-            wrapInTry(n, processed[2][1][0])
-
-          res.add processed
-          result = true
-        else:
-          res.add skipped[i]
-          i.inc
-    var i = 0
-    if not processForTry(node, i, result):
-      # If the tryStmt hasn't been transformed we can just put the body
-      # back into it.
-      wrapInTry(node, result)
-    return
-  else: discard
-
-  for i in 0 .. <result.len:
-    result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, nil)
-
-proc getName(node: NimNode): string {.compileTime.} =
-  case node.kind
-  of nnkPostfix:
-    return $node[1].ident
-  of nnkIdent:
-    return $node.ident
-  of nnkEmpty:
-    return "anonymous"
-  else:
-    error("Unknown name.")
-
-proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
-  ## This macro transforms a single procedure into a closure iterator.
-  ## The ``async`` macro supports a stmtList holding multiple async procedures.
-  if prc.kind notin {nnkProcDef, nnkLambda}:
-      error("Cannot transform this node kind into an async proc." &
-            " Proc definition or lambda node expected.")
-
-  hint("Processing " & prc[0].getName & " as an async proc.")
-
-  let returnType = prc[3][0]
-  var baseType: NimNode
-  # Verify that the return type is a Future[T]
-  if returnType.kind == nnkBracketExpr:
-    let fut = repr(returnType[0])
-    if fut != "Future":
-      error("Expected return type of 'Future' got '" & fut & "'")
-    baseType = returnType[1]
-  elif returnType.kind in nnkCallKinds and $returnType[0] == "[]":
-    let fut = repr(returnType[1])
-    if fut != "Future":
-      error("Expected return type of 'Future' got '" & fut & "'")
-    baseType = returnType[2]
-  elif returnType.kind == nnkEmpty:
-    baseType = returnType
-  else:
-    error("Expected return type of 'Future' got '" & repr(returnType) & "'")
-
-  let subtypeIsVoid = returnType.kind == nnkEmpty or
-        (baseType.kind == nnkIdent and returnType[1].ident == !"void")
-
-  var outerProcBody = newNimNode(nnkStmtList, prc[6])
-
-  # -> var retFuture = newFuture[T]()
-  var retFutureSym = genSym(nskVar, "retFuture")
-  var subRetType =
-    if returnType.kind == nnkEmpty: newIdentNode("void")
-    else: baseType
-  outerProcBody.add(
-    newVarStmt(retFutureSym,
-      newCall(
-        newNimNode(nnkBracketExpr, prc[6]).add(
-          newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`.
-          subRetType),
-      newLit(prc[0].getName)))) # Get type from return type of this proc
-
-  # -> iterator nameIter(): FutureBase {.closure.} =
-  # ->   {.push warning[resultshadowed]: off.}
-  # ->   var result: T
-  # ->   {.pop.}
-  # ->   <proc_body>
-  # ->   complete(retFuture, result)
-  var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter")
-  var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil)
-  # don't do anything with forward bodies (empty)
-  if procBody.kind != nnkEmpty:
-    if not subtypeIsVoid:
-      procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"),
-        newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add(
-          newIdentNode("warning"), newIdentNode("resultshadowed")),
-        newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.}
-
-      procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add(
-        newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T
-
-      procBody.insert(2, newNimNode(nnkPragma).add(
-        newIdentNode("pop"))) # -> {.pop.})
-
-      procBody.add(
-        newCall(newIdentNode("complete"),
-          retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result)
-    else:
-      # -> complete(retFuture)
-      procBody.add(newCall(newIdentNode("complete"), retFutureSym))
-
-    var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")],
-                                  procBody, nnkIteratorDef)
-    closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure"))
-    outerProcBody.add(closureIterator)
-
-    # -> createCb(retFuture)
-    #var cbName = newIdentNode("cb")
-    var procCb = getAst createCb(retFutureSym, iteratorNameSym,
-                         newStrLitNode(prc[0].getName))
-    outerProcBody.add procCb
-
-    # -> return retFuture
-    outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym)
-
-  result = prc
-
-  # Remove the 'async' pragma.
-  for i in 0 .. <result[4].len:
-    if result[4][i].kind == nnkIdent and result[4][i].ident == !"async":
-      result[4].del(i)
-  result[4] = newEmptyNode()
-  if subtypeIsVoid:
-    # Add discardable pragma.
-    if returnType.kind == nnkEmpty:
-      # Add Future[void]
-      result[3][0] = parseExpr("Future[void]")
-  if procBody.kind != nnkEmpty:
-    result[6] = outerProcBody
-  #echo(treeRepr(result))
-  #if prc[0].getName == "testInfix":
-  #  echo(toStrLit(result))
-
-macro async*(prc: untyped): untyped =
-  ## Macro which processes async procedures into the appropriate
-  ## iterators and yield statements.
-  if prc.kind == nnkStmtList:
-    for oneProc in prc:
-      result = newStmtList()
-      result.add asyncSingleProc(oneProc)
-  else:
-    result = asyncSingleProc(prc)
-  when defined(nimDumpAsync):
-    echo repr result
-
-proc recvLine*(socket: AsyncFD): Future[string] {.async.} =
   ## Reads a line of data from ``socket``. Returned future will complete once
   ## a full line is read or an error occurs.
   ##
@@ -1955,6 +1384,8 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} =
   ##
   ## **Note**: This procedure is mostly used for testing. You likely want to
   ## use ``asyncnet.recvLine`` instead.
+  ##
+  ## **Deprecated since version 0.15.0**: Use ``asyncnet.recvLine()`` instead.
 
   template addNLIfEmpty(): stmt =
     if result.len == 0:

lib/pure/asyncfile.nim

@@ -24,6 +24,8 @@
 
 import asyncdispatch, os
 
+# TODO: Fix duplication introduced by PR #4683.
+
 when defined(windows) or defined(nimdoc):
   import winlean
 else:
@@ -112,6 +114,80 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile =
 
     register(result.fd)
 
+proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] =
+  ## Read ``size`` bytes from the specified file asynchronously starting at
+  ## the current position of the file pointer.
+  ##
+  ## If the file pointer is past the end of the file then an empty string is
+  ## returned.
+  var retFuture = newFuture[int]("asyncfile.readBuffer")
+
+  when defined(windows) or defined(nimdoc):
+    var ol = PCustomOverlapped()
+    GC_ref(ol)
+    ol.data = CompletionData(fd: f.fd, cb:
+      proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) =
+        if not retFuture.finished:
+          if errcode == OSErrorCode(-1):
+            assert bytesCount > 0
+            assert bytesCount <= size
+            f.offset.inc bytesCount
+            retFuture.complete(bytesCount)
+          else:
+            if errcode.int32 == ERROR_HANDLE_EOF:
+              retFuture.complete(0)
+            else:
+              retFuture.fail(newException(OSError, osErrorMsg(errcode)))
+    )
+    ol.offset = DWord(f.offset and 0xffffffff)
+    ol.offsetHigh = DWord(f.offset shr 32)
+
+    # According to MSDN we're supposed to pass nil to lpNumberOfBytesRead.
+    let ret = readFile(f.fd.Handle, buf, size.int32, nil,
+                       cast[POVERLAPPED](ol))
+    if not ret.bool:
+      let err = osLastError()
+      if err.int32 != ERROR_IO_PENDING:
+        GC_unref(ol)
+        retFuture.fail(newException(OSError, osErrorMsg(err)))
+    else:
+      # Request completed immediately.
+      var bytesRead: DWord
+      let overlappedRes = getOverlappedResult(f.fd.Handle,
+          cast[POverlapped](ol), bytesRead, false.WinBool)
+      if not overlappedRes.bool:
+        let err = osLastError()
+        if err.int32 == ERROR_HANDLE_EOF:
+          retFuture.complete(0)
+        else:
+          retFuture.fail(newException(OSError, osErrorMsg(osLastError())))
+      else:
+        assert bytesRead > 0
+        assert bytesRead <= size
+        f.offset.inc bytesRead
+        retFuture.complete(bytesRead)
+  else:
+    proc cb(fd: AsyncFD): bool =
+      result = true
+      let res = read(fd.cint, cast[cstring](buf), size.cint)
+      if res < 0:
+        let lastError = osLastError()
+        if lastError.int32 != EAGAIN:
+          retFuture.fail(newException(OSError, osErrorMsg(lastError)))
+        else:
+          result = false # We still want this callback to be called.
+      elif res == 0:
+        # EOF
+        retFuture.complete(0)
+      else:
+        f.offset.inc(res)
+        retFuture.complete(res)
+
+    if not cb(f.fd):
+      addRead(f.fd, cb)
+
+  return retFuture
+
 proc read*(f: AsyncFile, size: int): Future[string] =
   ## Read ``size`` bytes from the specified file asynchronously starting at
   ## the current position of the file pointer.
@@ -238,6 +314,73 @@ proc readAll*(f: AsyncFile): Future[string] {.async.} =
       return
     result.add data
 
+proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] =
+  ## Writes ``size`` bytes from ``buf`` to the file specified asynchronously.
+  ##
+  ## The returned Future will complete once all data has been written to the
+  ## specified file.
+  var retFuture = newFuture[void]("asyncfile.writeBuffer")
+  when defined(windows) or defined(nimdoc):
+    var ol = PCustomOverlapped()
+    GC_ref(ol)
+    ol.data = CompletionData(fd: f.fd, cb:
+      proc (fd: AsyncFD, bytesCount: DWord, errcode: OSErrorCode) =
+        if not retFuture.finished:
+          if errcode == OSErrorCode(-1):
+            assert bytesCount == size.int32
+            f.offset.inc(size)
+            retFuture.complete()
+          else:
+            retFuture.fail(newException(OSError, osErrorMsg(errcode)))
+    )
+    ol.offset = DWord(f.offset and 0xffffffff)
+    ol.offsetHigh = DWord(f.offset shr 32)
+
+    # According to MSDN we're supposed to pass nil to lpNumberOfBytesWritten.
+    let ret = writeFile(f.fd.Handle, buf, size.int32, nil,
+                       cast[POVERLAPPED](ol))
+    if not ret.bool:
+      let err = osLastError()
+      if err.int32 != ERROR_IO_PENDING:
+        GC_unref(ol)
+        retFuture.fail(newException(OSError, osErrorMsg(err)))
+    else:
+      # Request completed immediately.
+      var bytesWritten: DWord
+      let overlappedRes = getOverlappedResult(f.fd.Handle,
+          cast[POverlapped](ol), bytesWritten, false.WinBool)
+      if not overlappedRes.bool:
+        retFuture.fail(newException(OSError, osErrorMsg(osLastError())))
+      else:
+        assert bytesWritten == size.int32
+        f.offset.inc(size)
+        retFuture.complete()
+  else:
+    var written = 0
+
+    proc cb(fd: AsyncFD): bool =
+      result = true
+      let remainderSize = size-written
+      var cbuf = cast[cstring](buf)
+      let res = write(fd.cint, addr cbuf[written], remainderSize.cint)
+      if res < 0:
+        let lastError = osLastError()
+        if lastError.int32 != EAGAIN:
+          retFuture.fail(newException(OSError, osErrorMsg(lastError)))
+        else:
+          result = false # We still want this callback to be called.
+      else:
+        written.inc res
+        f.offset.inc res
+        if res != remainderSize:
+          result = false # We still have data to write.
+        else:
+          retFuture.complete()
+
+    if not cb(f.fd):
+      addWrite(f.fd, cb)
+  return retFuture
+
 proc write*(f: AsyncFile, data: string): Future[void] =
   ## Writes ``data`` to the file specified asynchronously.
   ##

lib/pure/asyncftpclient.nim

@@ -281,7 +281,7 @@ proc getFile(ftp: AsyncFtpClient, file: File, total: BiggestInt,
   assertReply(await(ftp.expectReply()), "226")
 
 proc defaultOnProgressChanged*(total, progress: BiggestInt,
-    speed: float): Future[void] {.nimcall,gcsafe.} =
+    speed: float): Future[void] {.nimcall,gcsafe,procvar.} =
   ## Default FTP ``onProgressChanged`` handler. Does nothing.
   result = newFuture[void]()
   #echo(total, " ", progress, " ", speed)
@@ -354,6 +354,20 @@ proc store*(ftp: AsyncFtpClient, file, dest: string,
 
   await doUpload(ftp, destFile, onProgressChanged)
 
+proc rename*(ftp: AsyncFtpClient, nameFrom: string, nameTo: string) {.async.} =
+  ## Rename a file or directory on the remote FTP Server from current name
+  ## ``name_from`` to new name ``name_to``
+  assertReply(await ftp.send("RNFR " & name_from), "350")
+  assertReply(await ftp.send("RNTO " & name_to), "250")
+
+proc removeFile*(ftp: AsyncFtpClient, filename: string) {.async.} =
+  ## Delete a file ``filename`` on the remote FTP server
+  assertReply(await ftp.send("DELE " & filename), "250")
+
+proc removeDir*(ftp: AsyncFtpClient, dir: string) {.async.} =
+  ## Delete a directory ``dir`` on the remote FTP server
+  assertReply(await ftp.send("RMD " & dir), "250")
+
 proc newAsyncFtpClient*(address: string, port = Port(21),
     user, pass = ""): AsyncFtpClient =
   ## Creates a new ``AsyncFtpClient`` object.
@@ -373,6 +387,11 @@ when not defined(testing) and isMainModule:
     echo await ftp.listDirs()
     await ftp.store("payload.jpg", "payload.jpg")
     await ftp.retrFile("payload.jpg", "payload2.jpg")
+    await ftp.rename("payload.jpg", "payload_renamed.jpg")
+    await ftp.store("payload.jpg", "payload_remove.jpg")
+    await ftp.removeFile("payload_remove.jpg")
+    await ftp.createDir("deleteme")
+    await ftp.removeDir("deleteme")
     echo("Finished")
 
   waitFor main(ftp)

lib/pure/asynchttpserver.nim

@@ -9,6 +9,12 @@
 
 ## This module implements a high performance asynchronous HTTP server.
 ##
+## This HTTP server has not been designed to be used in production, but
+## for testing applications locally. Because of this, when deploying your
+## application you should use a reverse proxy (for example nginx) instead of
+## allowing users to connect directly to this server.
+##
+##
 ## Examples
 ## --------
 ##
@@ -38,7 +44,7 @@ export httpcore except parseHeader
 type
   Request* = object
     client*: AsyncSocket # TODO: Separate this into a Response object?
-    reqMethod*: string
+    reqMethod*: HttpMethod
     headers*: HttpHeaders
     protocol*: tuple[orig: string, major, minor: int]
     url*: Uri
@@ -127,7 +133,14 @@ proc processClient(client: AsyncSocket, address: string,
     var i = 0
     for linePart in lineFut.mget.split(' '):
       case i
-      of 0: request.reqMethod.shallowCopy(linePart.normalize)
+      of 0:
+        try:
+          # TODO: this is likely slow.
+          request.reqMethod = parseEnum[HttpMethod]("http" & linePart)
+        except ValueError:
+          asyncCheck request.respond(Http400, "Invalid request method. Got: " &
+                                     linePart)
+          continue
       of 1: parseUri(linePart, request.url)
       of 2:
         try:
@@ -159,7 +172,7 @@ proc processClient(client: AsyncSocket, address: string,
         request.client.close()
         return
 
-    if request.reqMethod == "post":
+    if request.reqMethod == HttpPost:
       # Check for Expect header
       if request.headers.hasKey("Expect"):
         if "100-continue" in request.headers["Expect"]:
@@ -178,17 +191,12 @@ proc processClient(client: AsyncSocket, address: string,
       else:
         request.body = await client.recv(contentLength)
         assert request.body.len == contentLength
-    elif request.reqMethod == "post":
+    elif request.reqMethod == HttpPost:
       await request.respond(Http400, "Bad Request. No Content-Length.")
       continue
 
-    case request.reqMethod
+    # Call the user's callback.
-    of "get", "post", "head", "put", "delete", "trace", "options",
+    await callback(request)
-       "connect", "patch":
-      await callback(request)
-    else:
-      await request.respond(Http400, "Invalid request method. Got: " &
-        request.reqMethod)
 
     if "upgrade" in request.headers.getOrDefault("connection"):
       return

lib/pure/asyncmacro.nim

@@ -0,0 +1,515 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2015 Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## AsyncMacro
+## *************
+## `asyncdispatch` module depends on the `asyncmacro` module to work properly.
+
+import macros, strutils
+
+proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} =
+  # Skips a nest of StmtList's.
+  result = node
+  if node[0].kind == nnkStmtList:
+    result = skipUntilStmtList(node[0])
+
+proc skipStmtList(node: NimNode): NimNode {.compileTime.} =
+  result = node
+  if node[0].kind == nnkStmtList:
+    result = node[0]
+
+template createCb(retFutureSym, iteratorNameSym,
+                  name, futureVarCompletions: untyped) =
+  var nameIterVar = iteratorNameSym
+  #{.push stackTrace: off.}
+  proc cb {.closure,gcsafe.} =
+    try:
+      if not nameIterVar.finished:
+        var next = nameIterVar()
+        if next == nil:
+          assert retFutureSym.finished, "Async procedure's (" &
+                 name & ") return Future was not finished."
+        else:
+          next.callback = cb
+    except:
+      if retFutureSym.finished:
+        # Take a look at tasyncexceptions for the bug which this fixes.
+        # That test explains it better than I can here.
+        raise
+      else:
+        retFutureSym.fail(getCurrentException())
+
+      futureVarCompletions
+  cb()
+  #{.pop.}
+proc generateExceptionCheck(futSym,
+    tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} =
+  if tryStmt.kind == nnkNilLit:
+    result = rootReceiver
+  else:
+    var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[]
+    let errorNode = newDotExpr(futSym, newIdentNode("error"))
+    for i in 1 .. <tryStmt.len:
+      let exceptBranch = tryStmt[i]
+      if exceptBranch[0].kind == nnkStmtList:
+        exceptionChecks.add((newIdentNode("true"), exceptBranch[0]))
+      else:
+        var exceptIdentCount = 0
+        var ifCond: NimNode
+        for i in 0 .. <exceptBranch.len:
+          let child = exceptBranch[i]
+          if child.kind == nnkIdent:
+            let cond = infix(errorNode, "of", child)
+            if exceptIdentCount == 0:
+              ifCond = cond
+            else:
+              ifCond = infix(ifCond, "or", cond)
+          else:
+            break
+          exceptIdentCount.inc
+
+        expectKind(exceptBranch[exceptIdentCount], nnkStmtList)
+        exceptionChecks.add((ifCond, exceptBranch[exceptIdentCount]))
+    # -> -> else: raise futSym.error
+    exceptionChecks.add((newIdentNode("true"),
+        newNimNode(nnkRaiseStmt).add(errorNode)))
+    # Read the future if there is no error.
+    # -> else: futSym.read
+    let elseNode = newNimNode(nnkElse, fromNode)
+    elseNode.add newNimNode(nnkStmtList, fromNode)
+    elseNode[0].add rootReceiver
+
+    let ifBody = newStmtList()
+    ifBody.add newCall(newIdentNode("setCurrentException"), errorNode)
+    ifBody.add newIfStmt(exceptionChecks)
+    ifBody.add newCall(newIdentNode("setCurrentException"), newNilLit())
+
+    result = newIfStmt(
+      (newDotExpr(futSym, newIdentNode("failed")), ifBody)
+    )
+    result.add elseNode
+
+template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver,
+                rootReceiver: expr, fromNode: NimNode) =
+  ## Params:
+  ##    futureVarNode: The NimNode which is a symbol identifying the Future[T]
+  ##                   variable to yield.
+  ##    fromNode: Used for better debug information (to give context).
+  ##    valueReceiver: The node which defines an expression that retrieves the
+  ##                   future's value.
+  ##
+  ##    rootReceiver: ??? TODO
+  # -> yield future<x>
+  result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode)
+  # -> future<x>.read
+  valueReceiver = newDotExpr(futureVarNode, newIdentNode("read"))
+  result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver,
+      fromNode)
+
+template createVar(result: var NimNode, futSymName: string,
+                   asyncProc: NimNode,
+                   valueReceiver, rootReceiver: expr,
+                   fromNode: NimNode) =
+  result = newNimNode(nnkStmtList, fromNode)
+  var futSym = genSym(nskVar, "future")
+  result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y
+  useVar(result, futSym, valueReceiver, rootReceiver, fromNode)
+
+proc createFutureVarCompletions(futureVarIdents: seq[NimNode]): NimNode
+                                {.compileTime.} =
+  result = newStmtList()
+  # Add calls to complete each FutureVar parameter.
+  for ident in futureVarIdents:
+    # Only complete them if they have not been completed already by the user.
+    result.add newIfStmt(
+      (
+        newCall(newIdentNode("not"),
+                newDotExpr(ident, newIdentNode("finished"))),
+        newCall(newIdentNode("complete"), ident)
+      )
+    )
+
+proc processBody(node, retFutureSym: NimNode,
+                 subTypeIsVoid: bool, futureVarIdents: seq[NimNode],
+                 tryStmt: NimNode): NimNode {.compileTime.} =
+  #echo(node.treeRepr)
+  result = node
+  case node.kind
+  of nnkReturnStmt:
+    result = newNimNode(nnkStmtList, node)
+    if node[0].kind == nnkEmpty:
+      if not subTypeIsVoid:
+        result.add newCall(newIdentNode("complete"), retFutureSym,
+            newIdentNode("result"))
+      else:
+        result.add newCall(newIdentNode("complete"), retFutureSym)
+    else:
+      let x = node[0].processBody(retFutureSym, subTypeIsVoid,
+                                  futureVarIdents, tryStmt)
+      if x.kind == nnkYieldStmt: result.add x
+      else:
+        result.add newCall(newIdentNode("complete"), retFutureSym, x)
+
+    result.add createFutureVarCompletions(futureVarIdents)
+
+    result.add newNimNode(nnkReturnStmt, node).add(newNilLit())
+    return # Don't process the children of this return stmt
+  of nnkCommand, nnkCall:
+    if node[0].kind == nnkIdent and node[0].ident == !"await":
+      case node[1].kind
+      of nnkIdent, nnkInfix, nnkDotExpr, nnkCall, nnkCommand:
+        # await x
+        # await x or y
+        # await foo(p, x)
+        # await foo p, x
+        var futureValue: NimNode
+        result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue,
+                  futureValue, node)
+      else:
+        error("Invalid node kind in 'await', got: " & $node[1].kind)
+    elif node.len > 1 and node[1].kind == nnkCommand and
+         node[1][0].kind == nnkIdent and node[1][0].ident == !"await":
+      # foo await x
+      var newCommand = node
+      result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1],
+                newCommand, node)
+
+  of nnkVarSection, nnkLetSection:
+    case node[0][2].kind
+    of nnkCommand:
+      if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await":
+        # var x = await y
+        var newVarSection = node # TODO: Should this use copyNimNode?
+        result.createVar("future" & $node[0][0].ident, node[0][2][1],
+          newVarSection[0][2], newVarSection, node)
+    else: discard
+  of nnkAsgn:
+    case node[1].kind
+    of nnkCommand:
+      if node[1][0].ident == !"await":
+        # x = await y
+        var newAsgn = node
+        result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node)
+    else: discard
+  of nnkDiscardStmt:
+    # discard await x
+    if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and
+          node[0][0].ident == !"await":
+      var newDiscard = node
+      result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1],
+                newDiscard[0], newDiscard, node)
+  of nnkTryStmt:
+    # try: await x; except: ...
+    result = newNimNode(nnkStmtList, node)
+    template wrapInTry(n, tryBody: expr) =
+      var temp = n
+      n[0] = tryBody
+      tryBody = temp
+
+      # Transform ``except`` body.
+      # TODO: Could we perform some ``await`` transformation here to get it
+      # working in ``except``?
+      tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid,
+                               futureVarIdents, nil)
+
+    proc processForTry(n: NimNode, i: var int,
+                       res: NimNode): bool {.compileTime.} =
+      ## Transforms the body of the tryStmt. Does not transform the
+      ## body in ``except``.
+      ## Returns true if the tryStmt node was transformed into an ifStmt.
+      result = false
+      var skipped = n.skipStmtList()
+      while i < skipped.len:
+        var processed = processBody(skipped[i], retFutureSym,
+                                    subTypeIsVoid, futureVarIdents, n)
+
+        # Check if we transformed the node into an exception check.
+        # This suggests skipped[i] contains ``await``.
+        if processed.kind != skipped[i].kind or processed.len != skipped[i].len:
+          processed = processed.skipUntilStmtList()
+          expectKind(processed, nnkStmtList)
+          expectKind(processed[2][1], nnkElse)
+          i.inc
+
+          if not processForTry(n, i, processed[2][1][0]):
+            # We need to wrap the nnkElse nodes back into a tryStmt.
+            # As they are executed if an exception does not happen
+            # inside the awaited future.
+            # The following code will wrap the nodes inside the
+            # original tryStmt.
+            wrapInTry(n, processed[2][1][0])
+
+          res.add processed
+          result = true
+        else:
+          res.add skipped[i]
+          i.inc
+    var i = 0
+    if not processForTry(node, i, result):
+      # If the tryStmt hasn't been transformed we can just put the body
+      # back into it.
+      wrapInTry(node, result)
+    return
+  else: discard
+
+  for i in 0 .. <result.len:
+    result[i] = processBody(result[i], retFutureSym, subTypeIsVoid,
+                            futureVarIdents, nil)
+
+proc getName(node: NimNode): string {.compileTime.} =
+  case node.kind
+  of nnkPostfix:
+    return $node[1].ident
+  of nnkIdent:
+    return $node.ident
+  of nnkEmpty:
+    return "anonymous"
+  else:
+    error("Unknown name.")
+
+proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} =
+  result = @[]
+  for i in 1 .. <len(params):
+    expectKind(params[i], nnkIdentDefs)
+    if params[i][1].kind == nnkBracketExpr and
+       ($params[i][1][0].ident).normalize == "futurevar":
+      result.add(params[i][0])
+
+proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
+  ## This macro transforms a single procedure into a closure iterator.
+  ## The ``async`` macro supports a stmtList holding multiple async procedures.
+  if prc.kind notin {nnkProcDef, nnkLambda}:
+      error("Cannot transform this node kind into an async proc." &
+            " Proc definition or lambda node expected.")
+
+  hint("Processing " & prc[0].getName & " as an async proc.")
+
+  let returnType = prc[3][0]
+  var baseType: NimNode
+  # Verify that the return type is a Future[T]
+  if returnType.kind == nnkBracketExpr:
+    let fut = repr(returnType[0])
+    if fut != "Future":
+      error("Expected return type of 'Future' got '" & fut & "'")
+    baseType = returnType[1]
+  elif returnType.kind in nnkCallKinds and $returnType[0] == "[]":
+    let fut = repr(returnType[1])
+    if fut != "Future":
+      error("Expected return type of 'Future' got '" & fut & "'")
+    baseType = returnType[2]
+  elif returnType.kind == nnkEmpty:
+    baseType = returnType
+  else:
+    error("Expected return type of 'Future' got '" & repr(returnType) & "'")
+
+  let subtypeIsVoid = returnType.kind == nnkEmpty or
+        (baseType.kind == nnkIdent and returnType[1].ident == !"void")
+
+  let futureVarIdents = getFutureVarIdents(prc[3])
+
+  var outerProcBody = newNimNode(nnkStmtList, prc[6])
+
+  # -> var retFuture = newFuture[T]()
+  var retFutureSym = genSym(nskVar, "retFuture")
+  var subRetType =
+    if returnType.kind == nnkEmpty: newIdentNode("void")
+    else: baseType
+  outerProcBody.add(
+    newVarStmt(retFutureSym,
+      newCall(
+        newNimNode(nnkBracketExpr, prc[6]).add(
+          newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`.
+          subRetType),
+      newLit(prc[0].getName)))) # Get type from return type of this proc
+
+  # -> iterator nameIter(): FutureBase {.closure.} =
+  # ->   {.push warning[resultshadowed]: off.}
+  # ->   var result: T
+  # ->   {.pop.}
+  # ->   <proc_body>
+  # ->   complete(retFuture, result)
+  var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter")
+  var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid,
+                                    futureVarIdents, nil)
+  # don't do anything with forward bodies (empty)
+  if procBody.kind != nnkEmpty:
+    if not subtypeIsVoid:
+      procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"),
+        newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add(
+          newIdentNode("warning"), newIdentNode("resultshadowed")),
+        newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.}
+
+      procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add(
+        newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T
+
+      procBody.insert(2, newNimNode(nnkPragma).add(
+        newIdentNode("pop"))) # -> {.pop.})
+
+      procBody.add(
+        newCall(newIdentNode("complete"),
+          retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result)
+    else:
+      # -> complete(retFuture)
+      procBody.add(newCall(newIdentNode("complete"), retFutureSym))
+
+    procBody.add(createFutureVarCompletions(futureVarIdents))
+
+    var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")],
+                                  procBody, nnkIteratorDef)
+    closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure"))
+    outerProcBody.add(closureIterator)
+
+    # -> createCb(retFuture)
+    #var cbName = newIdentNode("cb")
+    var procCb = getAst createCb(retFutureSym, iteratorNameSym,
+                         newStrLitNode(prc[0].getName),
+                         createFutureVarCompletions(futureVarIdents))
+    outerProcBody.add procCb
+
+    # -> return retFuture
+    outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym)
+
+  result = prc
+
+  # Remove the 'async' pragma.
+  for i in 0 .. <result[4].len:
+    if result[4][i].kind == nnkIdent and result[4][i].ident == !"async":
+      result[4].del(i)
+  result[4] = newEmptyNode()
+  if subtypeIsVoid:
+    # Add discardable pragma.
+    if returnType.kind == nnkEmpty:
+      # Add Future[void]
+      result[3][0] = parseExpr("Future[void]")
+  if procBody.kind != nnkEmpty:
+    result[6] = outerProcBody
+  #echo(treeRepr(result))
+  #if prc[0].getName == "testInfix":
+  #  echo(toStrLit(result))
+
+macro async*(prc: untyped): untyped =
+  ## Macro which processes async procedures into the appropriate
+  ## iterators and yield statements.
+  if prc.kind == nnkStmtList:
+    for oneProc in prc:
+      result = newStmtList()
+      result.add asyncSingleProc(oneProc)
+  else:
+    result = asyncSingleProc(prc)
+  when defined(nimDumpAsync):
+    echo repr result
+
+
+# Multisync
+proc emptyNoop[T](x: T): T =
+  # The ``await``s are replaced by a call to this for simplicity.
+  when T isnot void:
+    return x
+
+proc stripAwait(node: NimNode): NimNode =
+  ## Strips out all ``await`` commands from a procedure body, replaces them
+  ## with ``emptyNoop`` for simplicity.
+  result = node
+
+  let emptyNoopSym = bindSym("emptyNoop")
+
+  case node.kind
+  of nnkCommand, nnkCall:
+    if node[0].kind == nnkIdent and node[0].ident == !"await":
+      node[0] = emptyNoopSym
+    elif node.len > 1 and node[1].kind == nnkCommand and
+         node[1][0].kind == nnkIdent and node[1][0].ident == !"await":
+      # foo await x
+      node[1][0] = emptyNoopSym
+  of nnkVarSection, nnkLetSection:
+    case node[0][2].kind
+    of nnkCommand:
+      if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await":
+        # var x = await y
+        node[0][2][0] = emptyNoopSym
+    else: discard
+  of nnkAsgn:
+    case node[1].kind
+    of nnkCommand:
+      if node[1][0].ident == !"await":
+        # x = await y
+        node[1][0] = emptyNoopSym
+    else: discard
+  of nnkDiscardStmt:
+    # discard await x
+    if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and
+          node[0][0].ident == !"await":
+      node[0][0] = emptyNoopSym
+  else: discard
+
+  for i in 0 .. <result.len:
+    result[i] = stripAwait(result[i])
+
+proc splitParams(param: NimNode, async: bool): NimNode =
+  expectKind(param, nnkIdentDefs)
+  result = param
+  if param[1].kind == nnkInfix and $param[1][0].ident in ["|", "or"]:
+    let firstType = param[1][1]
+    let firstTypeName = $firstType.ident
+    let secondType = param[1][2]
+    let secondTypeName = $secondType.ident
+
+    # Make sure that at least one has the name `async`, otherwise we shouldn't
+    # touch it.
+    if not ("async" in firstTypeName.normalize or
+            "async" in secondTypeName.normalize):
+      return
+
+    if async:
+      if firstTypeName.normalize.startsWith("async"):
+        result = newIdentDefs(param[0], param[1][1])
+      elif secondTypeName.normalize.startsWith("async"):
+        result = newIdentDefs(param[0], param[1][2])
+    else:
+      if not firstTypeName.normalize.startsWith("async"):
+        result = newIdentDefs(param[0], param[1][1])
+      elif not secondTypeName.normalize.startsWith("async"):
+        result = newIdentDefs(param[0], param[1][2])
+
+proc stripReturnType(returnType: NimNode): NimNode =
+  # Strip out the 'Future' from 'Future[T]'.
+  result = returnType
+  if returnType.kind == nnkBracketExpr:
+    let fut = repr(returnType[0])
+    if fut != "Future":
+      error("Expected return type of 'Future' got '" & fut & "'")
+    result = returnType[1]
+
+proc splitProc(prc: NimNode): (NimNode, NimNode) =
+  ## Takes a procedure definition which takes a generic union of arguments,
+  ## for example: proc (socket: Socket | AsyncSocket).
+  ## It transforms them so that ``proc (socket: Socket)`` and
+  ## ``proc (socket: AsyncSocket)`` are returned.
+  result[0] = prc.copyNimTree()
+  result[0][3][0] = stripReturnType(result[0][3][0])
+  for i in 1 .. <result[0][3].len:
+    result[0][3][i] = splitParams(result[0][3][i], false)
+  result[0][6] = stripAwait(result[0][6])
+
+  result[1] = prc.copyNimTree()
+  for i in 1 .. <result[1][3].len:
+    result[1][3][i] = splitParams(result[1][3][i], true)
+
+macro multisync*(prc: untyped): untyped =
+  ## Macro which processes async procedures into both asynchronous and
+  ## synchronous procedures.
+  ##
+  ## The generated async procedures use the ``async`` macro, whereas the
+  ## generated synchronous procedures simply strip off the ``await`` calls.
+  hint("Processing " & prc[0].getName & " as a multisync proc.")
+
+  let (sync, asyncPrc) = splitProc(prc)
+  result = newStmtList()
+  result.add(asyncSingleProc(asyncPrc))
+  result.add(sync)