Merge branch 'devel' into araq-quirky-exceptions

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,37 +1,32 @@
 ---
 name: Bug report
-about: You found an unexpected behaviour? Use this template.
+about: Have you found an unexpected behavior? Use this template.
-
 ---
 
 <!-- Think about the title, twice. -->
 
-<!-- Summarize the Problem here, keep it simple. -->
+<!-- Summarize the problem here, keep it short and simple. -->
-<!-- e.g. `echo` outputs the wrong string. -->
+Function `echo` outputs the wrong string.
 
 
 ### Example
-<!-- This should be a source code block. 
+<!-- Paste your example in the code-block below. -->
 ```nim
 echo "Hello World!"
 ```
--->
 
 
 ### Current Output
-<!--
 ```
 Hola mundo!
 ```
--->
 
 
 ### Expected Output
-<!--
+<!-- What should be the correct output? -->
 ```
 Hello World!
 ```
--->
 
 
 ### Possible Solution
@@ -40,6 +35,11 @@ Hello World!
 
 ### Additional Information
 <!--- For Example:
-   A link to a project where the issue is relevant.
+* Your Nim version (output of `nim -v`).
-   A link to a related issue or discussion.
+* Was it working in the previous Nim releases?
+* A link to a related issue or discussion.
 -->
+```
+$ nim -v
+Nim Compiler Version 0.1.2
+```

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,9 +1,18 @@
 ---
 name: Feature request
-about: You want to suggest a new feature? Use this template.
+about: Do you want to suggest a new feature? Use this template.
 
 ---
 
+<!---
+Notice there is now a separate repository for the detailed RFCs and proposals:
+https://github.com/nim-lang/RFCs
+
+If you have a simple feature request, you can post it here using this template,
+but bear in mind that adding new features to the language is currently a low priority.
+-->
+
+
 ### Summary
 <!--- Short summary of your proposed feature -->
 
@@ -18,6 +27,6 @@ about: You want to suggest a new feature? Use this template.
 
 ### Additional Information
 <!--- For Example:
-   A link to a project where the issue is relevant.
+* A link to a project where the issue is relevant.
-   A link to a related issue or discussion.
+* A link to a related issue or discussion.
-   -->
+-->

.gitignore

@@ -32,7 +32,7 @@ doc/*.html
 doc/*.pdf
 doc/*.idx
 /web/upload
-build/*
+/build/*
 bin/*
 
 # iOS specific wildcards.
@@ -52,6 +52,7 @@ xcuserdata/
 /run.json
 # for `nim doc foo.nim`
 /*.html
+lib/**/*.html
 #/testresults.html #covered by /*.html
 
 /testresults.json
@@ -71,3 +72,7 @@ test.txt
 tweeter.db
 tweeter_test.db
 megatest.nim
+
+/outputExpected.txt
+/outputGotten.txt
+

.travis.yml

@@ -14,8 +14,9 @@ matrix:
     - os: osx
       env: NIM_COMPILE_TO_CPP=true
 
-  allow_failures:
+# To allow failures for a failing configuration, use something like:
-    - env: NIM_COMPILE_TO_CPP=true
+#  allow_failures:
+#    - env: NIM_COMPILE_TO_CPP=true
 #    - os: osx
 
 addons:
@@ -40,26 +41,14 @@ before_script:
   - sh build.sh
   - cd ..
   - export PATH=$(pwd)/bin${PATH:+:$PATH}
+  - echo PATH:${PATH}
+  - set +e   # prevents breaking after_failure
+
 script:
+  - echo "travis_fold:start:nim_c_koch"
   - nim c koch
-  - env NIM_COMPILE_TO_CPP=false ./koch boot
+  - echo "travis_fold:end:nim_c_koch"
-  - ./koch boot -d:release
+  - ./koch runCI
-  - ./koch nimble
-  - nim e tests/test_nimscript.nims
-  #- nimble install zip -y
-  #- nimble install opengl
-  #- nimble install sdl1
-  #- nimble install jester@#head -y
-  #- nimble install niminst
-  - nim c -d:nimCoroutines testament/tester
-  - testament/tester --pedantic all -d:nimCoroutines
-  - nim c -o:bin/nimpretty nimpretty/nimpretty.nim
-  - nim c -r nimpretty/tester.nim
-  - ./koch docs --git.commit:devel
-  - ./koch csource
-  - ./koch nimsuggest
-  - nim c -r nimsuggest/tester
-  - nim c -r nimdoc/tester
 
 before_deploy:
   # Make https://nim-lang.github.io/Nim work the same as https://nim-lang.github.io/Nim/overview.html
@@ -73,3 +62,6 @@ deploy:                         # https://nim-lang.github.io/Nim
   keep-history: false
   on:
     branch: devel
+
+# Extract failed tests
+after_failure: nim c -r tools/ci_testresults.nim

appveyor.yml

@@ -44,22 +44,6 @@ install:
 
 build_script:
   - bin\nim c koch
-  - koch boot -d:release
+  - koch runCI
-  - koch nimble
-  - nim e tests/test_nimscript.nims
-  - nim c -o:bin/nimpretty.exe nimpretty/nimpretty.nim
-  - nim c -r nimpretty/tester.nim
-#  - nimble install zip -y
-#  - nimble install opengl -y
-#  - nimble install sdl1 -y
-#  - nimble install jester@#head -y
-  - nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester
-  - nim c -d:nimCoroutines testament/tester
-
-test_script:
-  - testament\tester --pedantic all -d:nimCoroutines
-  - nim c -r nimdoc\tester
-#  - koch csource
-#  - koch zip
 
 deploy: off

build_all.sh

@@ -26,6 +26,9 @@ build_nim_csources(){
 
 [ -f $nim_csources ] || echo_run build_nim_csources
 
-echo_run bin/nim c koch # Note: if fails, may need to `cd csources && git pull`
+# Note: if fails, may need to `cd csources && git pull`
+# see D20190115T162028
+echo_run bin/nim c --skipUserCfg --skipParentCfg koch
+
 echo_run ./koch boot -d:release
 echo_run ./koch tools # Compile Nimble and other tools.

changelog.md

@@ -1,4 +1,5 @@
-## v0.20.0 - XX/XX/2018
+## v0.20.0 - XX/XX/2019
+
 
 ### Changes affecting backwards compatibility
 
@@ -23,13 +24,21 @@
 - The undocumented ``#? strongSpaces`` parsing mode has been removed.
 - The `not` operator is now always a unary operator, this means that code like
   ``assert not isFalse(3)`` compiles.
+- `getImpl` on a `var` or `let` symbol will now return the full `IdentDefs`
+  tree from the symbol declaration instead of just the initializer portion.
 
 
 #### Breaking changes in the standard library
 
 - `osproc.execProcess` now also takes a `workingDir` parameter.
 
-- `options.UnpackError` is no longer a ref type and inherits from `System.Defect` instead of `System.ValueError`.
+- `options.UnpackError` is no longer a ref type and inherits from `system.Defect` instead of `system.ValueError`.
+
+- `system.ValueError` now inherits from `system.CatchableError` instead of `system.Defect`.
+
+- The procs `parseutils.parseBiggsetInt`, `parseutils.parseInt`, `parseutils.parseBiggestUInt` and `parseutils.parseUInt` now raise a `ValueError` when the parsed integer is outside of the valid range. Previously they sometimes raised a `OverflowError` and sometimes returned `0`.
+
+- `streams.StreamObject` now restricts its fields to only raise `system.Defect`, `system.IOError` and `system.OSError`. This change only affects custom stream implementations.
 
 - nre's `RegexMatch.{captureBounds,captures}[]`  no longer return `Option` or
   `nil`/`""`, respectivly. Use the newly added `n in p.captures` method to
@@ -52,6 +61,16 @@
 
 - splitFile now correctly handles edge cases, see #10047
 
+- `isNil` is no longer false for undefined in the JavaScript backend: now it's true for both nil and undefined. Use `isNull` or `isUndefined` if you need exact equallity: `isNil` is consistent with `===`, `isNull` and `isUndefined` with `==`.
+
+- several deprecated modules were removed: `ssl`, `matchers`, `httpserver`,
+  `unsigned`, `actors`, `parseurl`
+
+- two poorly documented and not used modules (`subexes`, `scgi`) were moved to
+  graveyard (they are available as Nimble packages)
+
+
+
 #### Breaking changes in the compiler
 
 - The compiler now implements the "generic symbol prepass" for `when` statements
@@ -92,13 +111,20 @@ proc enumToString*(enums: openArray[enum]): string =
   is instantiation of generic proc symbol.
 
 - Added the parameter ``isSorted`` for the ``sequtils.deduplicate`` proc.
+
 - There is a new stdlib module `std/diff` to compute the famous "diff"
   of two texts by line.
 
 - Added `os.relativePath`.
+
 - Added `parseopt.remainingArgs`.
+
 - Added `os.getCurrentCompilerExe` (implmented as `getAppFilename` at CT),
-  can be used to retrive the currently executing compiler.
+  can be used to retrieve the currently executing compiler.
+
+- Added `xmltree.toXmlAttributes`.
+
+- Added `Rusage`, `getrusage`, `wait4` to posix interface.
 
 
 ### Library changes
@@ -123,16 +149,20 @@ proc enumToString*(enums: openArray[enum]): string =
 - `os.joinPath` and `os.normalizePath` handle edge cases like ``"a/b/../../.."``
   differently.
 
+- `securehash` is moved to `lib/deprecated`
+
 
 ### Language additions
 
-- Vm suport for float32<->int32 and float64<->int64 casts was added.
+- Vm support for float32<->int32 and float64<->int64 casts was added.
 - There is a new pragma block `noSideEffect` that works like
   the `gcsafe` pragma block.
 - added os.getCurrentProcessId()
 - User defined pragmas are now allowed in the pragma blocks
-- Pragma blocks are now longer eliminated from the typed AST tree to preserve
+- Pragma blocks are no longer eliminated from the typed AST tree to preserve
   pragmas for further analysis by macros
+- Custom pragmas are now supported for `var` and `let` symbols.
+
 
 ### Language changes
 
@@ -141,10 +171,10 @@ proc enumToString*(enums: openArray[enum]): string =
   it's more recognizable and allows tools like github to recognize it as Nim,
   see [#9647](https://github.com/nim-lang/Nim/issues/9647).
   The previous extension will continue to work.
-- Pragma syntax is now consistent. Previous syntax where type pragmas did not 
+- Pragma syntax is now consistent. Previous syntax where type pragmas did not
   follow the type name is now deprecated. Also pragma before generic parameter
   list is deprecated to be consistent with how pragmas are used with a proc. See
-  [#8514](https://github.com/nim-lang/Nim/issues/8514) and 
+  [#8514](https://github.com/nim-lang/Nim/issues/8514) and
   [#1872](https://github.com/nim-lang/Nim/issues/1872) for further details.
 
 
@@ -152,8 +182,12 @@ proc enumToString*(enums: openArray[enum]): string =
 - `jsondoc` now include a `moduleDescription` field with the module
   description. `jsondoc0` shows comments as it's own objects as shown in the
   documentation.
+- `nimpretty`: --backup now defaults to `off` instead of `on` and the flag was
+  un-documented; use `git` instead of relying on backup files.
+
 
 ### Compiler changes
 - The deprecated `fmod` proc is now unavailable on the VM'.
 
+
 ### Bugfixes

compiler/asciitables.nim

@@ -0,0 +1,83 @@
+#[
+move to std/asciitables.nim once stable, or to a nimble paackage
+once compiler can depend on nimble
+]#
+
+type Cell* = object
+  text*: string
+  width*, row*, col*, ncols*, nrows*: int
+
+iterator parseTableCells*(s: string, delim = '\t'): Cell =
+  ## iterates over all cells in a `delim`-delimited `s`, after a 1st
+  ## pass that computes number of rows, columns, and width of each column.
+  var widths: seq[int]
+  var cell: Cell
+  template update() =
+    if widths.len<=cell.col:
+      widths.setLen cell.col+1
+      widths[cell.col] = cell.width
+    else:
+      widths[cell.col] = max(widths[cell.col], cell.width)
+    cell.width = 0
+
+  for a in s:
+    case a
+    of '\n':
+      update()
+      cell.col = 0
+      cell.row.inc
+    elif a == delim:
+      update()
+      cell.col.inc
+    else:
+      # todo: consider multi-width chars when porting to non-ascii implementation
+      cell.width.inc
+  if s.len > 0 and s[^1] != '\n':
+    update()
+
+  cell.ncols = widths.len
+  cell.nrows = cell.row + 1
+  cell.row = 0
+  cell.col = 0
+  cell.width = 0
+
+  template update2() =
+    cell.width = widths[cell.col]
+    yield cell
+    cell.text = ""
+    cell.width = 0
+    cell.col.inc
+
+  template finishRow() =
+    for col in cell.col..<cell.ncols:
+      cell.col = col
+      update2()
+    cell.col = 0
+
+  for a in s:
+    case a
+    of '\n':
+      finishRow()
+      cell.row.inc
+    elif a == delim:
+      update2()
+    else:
+      cell.width.inc
+      cell.text.add a
+
+  if s.len > 0 and s[^1] != '\n':
+    finishRow()
+
+proc alignTable*(s: string, delim = '\t', fill = ' ', sep = " "): string =
+  ## formats a `delim`-delimited `s` representing a table; each cell is aligned
+  ## to a width that's computed for each column; consecutive columns are
+  ## delimted by `sep`, and alignment space is filled using `fill`.
+  ## More customized formatting can be done by calling `parseTableCells` directly.
+  for cell in parseTableCells(s, delim):
+    result.add cell.text
+    for i in cell.text.len..<cell.width:
+      result.add fill
+    if cell.col < cell.ncols-1:
+      result.add sep
+    if cell.col == cell.ncols-1 and cell.row < cell.nrows - 1:
+      result.add '\n'

compiler/ast.nim

@@ -655,7 +655,7 @@ type
     mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym,
     mNHint, mNWarning, mNError,
     mInstantiationInfo, mGetTypeInfo,
-    mNimvm, mIntDefine, mStrDefine, mRunnableExamples,
+    mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples,
     mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf,
     mSymIsInstantiationOf
 
@@ -1087,6 +1087,13 @@ proc newSym*(symKind: TSymKind, name: PIdent, owner: PSym,
   when debugIds:
     registerId(result)
 
+proc astdef*(s: PSym): PNode =
+  # get only the definition (initializer) portion of the ast
+  if s.ast != nil and s.ast.kind == nkIdentDefs:
+    s.ast[2]
+  else:
+    s.ast
+
 proc isMetaType*(t: PType): bool =
   return t.kind in tyMetaTypes or
          (t.kind == tyStatic and t.n == nil) or
@@ -1345,7 +1352,6 @@ proc copySym*(s: PSym): PSym =
   result = newSym(s.kind, s.name, s.owner, s.info, s.options)
   #result.ast = nil            # BUGFIX; was: s.ast which made problems
   result.typ = s.typ
-  result.id = getID()
   when debugIds: registerId(result)
   result.flags = s.flags
   result.magic = s.magic

compiler/ccgexprs.nim

@@ -460,7 +460,7 @@ proc binaryStmtAddr(p: BProc, e: PNode, d: var TLoc, frmt: string) =
   if d.k != locNone: internalError(p.config, e.info, "binaryStmtAddr")
   initLocExpr(p, e.sons[1], a)
   initLocExpr(p, e.sons[2], b)
-  lineCg(p, cpsStmts, frmt, addrLoc(p.config, a), rdLoc(b))
+  lineCg(p, cpsStmts, frmt, byRefLoc(p, a), rdLoc(b))
 
 proc unaryStmt(p: BProc, e: PNode, d: var TLoc, frmt: string) =
   var a: TLoc
@@ -1028,7 +1028,7 @@ proc gcUsage(conf: ConfigRef; n: PNode) =
 
 proc strLoc(p: BProc; d: TLoc): Rope =
   if p.config.selectedGc == gcDestructors:
-    result = addrLoc(p.config, d)
+    result = byRefLoc(p, d)
   else:
     result = rdLoc(d)
 
@@ -1110,7 +1110,7 @@ proc genStrAppend(p: BProc, e: PNode, d: var TLoc) =
                         strLoc(p, dest), rdLoc(a)))
   if p.config.selectedGC == gcDestructors:
     linefmt(p, cpsStmts, "#prepareAdd($1, $2$3);$n",
-            addrLoc(p.config, dest), lens, rope(L))
+            byRefLoc(p, dest), lens, rope(L))
   else:
     initLoc(call, locCall, e, OnHeap)
     call.r = ropecg(p.module, "#resizeString($1, $2$3)", [rdLoc(dest), lens, rope(L)])
@@ -1123,7 +1123,7 @@ proc genSeqElemAppend(p: BProc, e: PNode, d: var TLoc) =
   #    seq = (typeof seq) incrSeq(&seq->Sup, sizeof(x));
   #    seq->data[seq->len-1] = x;
   let seqAppendPattern = if not p.module.compileToCpp:
-                           "($2) #incrSeqV3(&($1)->Sup, $3)"
+                           "($2) #incrSeqV3((TGenericSeq*)($1), $3)"
                          else:
                            "($2) #incrSeqV3($1, $3)"
   var a, b, dest, tmpL, call: TLoc
@@ -1501,7 +1501,7 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) =
                 addrLoc(p.config, a), genTypeInfo(p.module, t, e.info)]), a.storage)
   of tyOpenArray, tyVarargs:
     var b: TLoc
-    case a.t.kind
+    case skipTypes(a.t, abstractVarRange).kind
     of tyOpenArray, tyVarargs:
       putIntoDest(p, b, e, "$1, $1Len_0" % [rdLoc(a)], a.storage)
     of tyString, tySequence:

compiler/ccgstmts.nim

@@ -124,13 +124,14 @@ proc endBlock(p: BProc, blockEnd: Rope) =
 
 proc endBlock(p: BProc) =
   let topBlock = p.blocks.len - 1
-  var blockEnd = if p.blocks[topBlock].label != nil:
-      ropecg(p.module, "} $1: ;$n", p.blocks[topBlock].label)
-    else:
-      ~"}$n"
   let frameLen = p.blocks[topBlock].frameLen
+  var blockEnd: Rope
   if frameLen > 0:
     blockEnd.addf("FR_.len-=$1;$n", [frameLen.rope])
+  if p.blocks[topBlock].label != nil:
+    blockEnd.addf("} $1: ;$n", [p.blocks[topBlock].label])
+  else:
+    blockEnd.addf("}$n", [])
   endBlock(p, blockEnd)
 
 proc genSimpleBlock(p: BProc, stmts: PNode) {.inline.} =

compiler/ccgtypes.nim

@@ -165,7 +165,6 @@ proc mapType(conf: ConfigRef; typ: PType): TCTypeKind =
     of tySet:
       if mapSetType(conf, base) == ctArray: result = ctPtrToArray
       else: result = ctPtr
-    # XXX for some reason this breaks the pegs module
     else: result = ctPtr
   of tyPointer: result = ctPtr
   of tySequence: result = ctNimSeq
@@ -1041,6 +1040,8 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope;
       else: internalError(m.config, n.info, "genObjectFields(nkRecCase)")
   of nkSym:
     var field = n.sym
+    # Do not produce code for void types
+    if isEmptyType(field.typ): return
     if field.bitsize == 0:
       if field.loc.r == nil: fillObjectFields(m, typ)
       if field.loc.t == nil:

compiler/cgen.nim

@@ -16,7 +16,7 @@ import
   condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases,
   lowerings, tables, sets, ndi, lineinfos, pathutils, transf
 
-import system/helpers2
+import system/indexerrors
 
 when not defined(leanCompiler):
   import semparallel
@@ -264,6 +264,12 @@ proc addrLoc(conf: ConfigRef; a: TLoc): Rope =
   if lfIndirect notin a.flags and mapType(conf, a.t) != ctArray:
     result = "(&" & result & ")"
 
+proc byRefLoc(p: BProc; a: TLoc): Rope =
+  result = a.r
+  if lfIndirect notin a.flags and mapType(p.config, a.t) != ctArray and not
+      p.module.compileToCpp:
+    result = "(&" & result & ")"
+
 proc rdCharLoc(a: TLoc): Rope =
   # read a location that may need a char-cast:
   result = rdLoc(a)
@@ -321,7 +327,7 @@ proc resetLoc(p: BProc, loc: var TLoc) =
   else:
     if optNilCheck in p.options:
       linefmt(p, cpsStmts, "#chckNil((void*)$1);$n", addrLoc(p.config, loc))
-    if loc.storage != OnStack:
+    if loc.storage != OnStack and containsGcRef:
       linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
               addrLoc(p.config, loc), genTypeInfo(p.module, loc.t, loc.lode.info))
       # XXX: generated reset procs should not touch the m_type
@@ -1056,9 +1062,22 @@ proc genVarPrototype(m: BModule, n: PNode) =
       if sfVolatile in sym.flags: add(m.s[cfsVars], " volatile")
       addf(m.s[cfsVars], " $1;$n", [sym.loc.r])
 
+const
+  frameDefines = """
+$1  define nimfr_(proc, file) \
+    TFrame FR_; \
+    FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = 0; #nimFrame(&FR_);
+
+$1  define nimfrs_(proc, file, slots, length) \
+    struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; VarSlot s[slots];} FR_; \
+    FR_.procname = proc; FR_.filename = file; FR_.line = 0; FR_.len = length; #nimFrame((TFrame*)&FR_);
+
+$1  define nimln_(n, file) \
+    FR_.line = n; FR_.filename = file;
+"""
+
 proc addIntTypes(result: var Rope; conf: ConfigRef) {.inline.} =
-  addf(result, "#define NIM_NEW_MANGLING_RULES\L" &
+  addf(result, "#define NIM_INTBITS $1\L", [
-               "#define NIM_INTBITS $1\L", [
     platform.CPU[conf.target.targetCPU].intSize.rope])
   if conf.cppCustomNamespace.len > 0:
     result.add("#define USE_NIM_NAMESPACE ")
@@ -1103,9 +1122,8 @@ proc genMainProc(m: BModule) =
       "}$N$N" &
       "void PreMain(void) {$N" &
       "\tvoid (*volatile inner)(void);$N" &
-      "$1" &
       "\tinner = PreMainInner;$N" &
-      "$4$5" &
+      "$1" &
       "\t(*inner)();$N" &
       "}$N$N"
 
@@ -1222,12 +1240,7 @@ proc genMainProc(m: BModule) =
     else: ropecg(m, "\t#initStackBottomWith((void *)&inner);$N")
   inc(m.labels)
   appcg(m, m.s[cfsProcs], PreMainBody, [
-    m.g.mainDatInit, m.g.breakpoints, m.g.otherModsInit,
+    m.g.mainDatInit, m.g.breakpoints, m.g.otherModsInit])
-     if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
-       ropecg(m, "\t#initThreadVarsEmulation();$N")
-     else:
-       "".rope,
-     initStackBottomCall])
 
   appcg(m, m.s[cfsProcs], nimMain,
         [m.g.mainModInit, initStackBottomCall, rope(m.labels)])
@@ -1256,21 +1269,32 @@ proc getInitName(m: PSym): Rope =
 
 proc getDatInitName(m: PSym): Rope = getSomeInitName(m, "DatInit000")
 
+
 proc registerModuleToMain(g: BModuleList; m: BModule) =
+  if m.s[cfsDatInitProc].len > 0:
+    let datInit = m.module.getDatInitName
+    addf(g.mainModProcs, "N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit])
+    addf(g.mainDatInit, "\t$1();$N", [datInit])
+
+  # Initialization of TLS and GC should be done in between
+  # systemDatInit and systemInit calls if any
+  if sfSystemModule in m.module.flags:
+    if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
+      add(g.mainDatInit, ropecg(m, "\t#initThreadVarsEmulation();$N"))
+    if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone:
+      add(g.mainDatInit, ropecg(m, "\t#initStackBottomWith((void *)&inner);$N"))
+
   if m.s[cfsInitProc].len > 0:
     let init = m.module.getInitName
     addf(g.mainModProcs, "N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init])
     let initCall = "\t$1();$N" % [init]
     if sfMainModule in m.module.flags:
       add(g.mainModInit, initCall)
+    elif sfSystemModule in m.module.flags:
+      add(g.mainDatInit, initCall) # systemInit must called right after systemDatInit if any
     else:
       add(g.otherModsInit, initCall)
 
-  if m.s[cfsDatInitProc].len > 0:
-    let datInit = m.module.getDatInitName
-    addf(g.mainModProcs, "N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit])
-    addf(g.mainDatInit, "\t$1();$N", [datInit])
-
 proc genDatInitCode(m: BModule) =
   ## this function is called in cgenWriteModules after all modules are closed,
   ## it means raising dependency on the symbols is too late as it will not propogate
@@ -1297,6 +1321,7 @@ proc genInitCode(m: BModule) =
   ## this function is called in cgenWriteModules after all modules are closed,
   ## it means raising dependency on the symbols is too late as it will not propogate
   ## into other modules, only simple rope manipulations are allowed
+  appcg(m, m.s[cfsForwardTypes], frameDefines, [rope("#")])
 
   var moduleInitRequired = false
   let initname = getInitName(m.module)
@@ -1308,9 +1333,9 @@ proc genInitCode(m: BModule) =
     appcg(m, m.s[cfsTypeInit1], "static #TNimType $1[$2];$n",
           [m.nimTypesName, rope(m.nimTypes)])
 
-  # Give this small function its own scope
+  if m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0:
-  addf(prc, "{$N", [])
+    # Give this small function its own scope
-  block:
+    addf(prc, "{$N", [])
     # Keep a bogus frame in case the code needs one
     add(prc, ~"\tTFrame FR_; FR_.len = 0;$N")
 
@@ -1331,8 +1356,11 @@ proc genInitCode(m: BModule) =
       add(prc, genSectionStart(cpsStmts, m.config))
       add(prc, m.preInitProc.s(cpsStmts))
       add(prc, genSectionEnd(cpsStmts, m.config))
-  addf(prc, "}$N", [])
+    addf(prc, "}$N", [])
 
+  # add new scope for following code, because old vcc compiler need variable
+  # be defined at the top of the block
+  addf(prc, "{$N", [])
   if m.initProc.gcFrameId > 0:
     moduleInitRequired = true
     add(prc, initGCFrame(m.initProc))
@@ -1369,6 +1397,7 @@ proc genInitCode(m: BModule) =
   if m.initProc.gcFrameId > 0:
     moduleInitRequired = true
     add(prc, deinitGCFrame(m.initProc))
+  addf(prc, "}$N", [])
 
   addf(prc, "}$N$N", [])
 

compiler/commands.nim

@@ -142,7 +142,7 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC
 proc processOnOffSwitch(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass,
                         info: TLineInfo) =
   case arg.normalize
-  of "on": conf.options = conf.options + op
+  of "","on": conf.options = conf.options + op
   of "off": conf.options = conf.options - op
   else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
 
@@ -158,7 +158,7 @@ proc processOnOffSwitchOrList(conf: ConfigRef; op: TOptions, arg: string, pass:
 proc processOnOffSwitchG(conf: ConfigRef; op: TGlobalOptions, arg: string, pass: TCmdLinePass,
                          info: TLineInfo) =
   case arg.normalize
-  of "on": conf.globalOptions = conf.globalOptions + op
+  of "", "on": conf.globalOptions = conf.globalOptions + op
   of "off": conf.globalOptions = conf.globalOptions - op
   else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
 
@@ -224,7 +224,7 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo
     case arg.normalize
     of "boehm":        result = conf.selectedGC == gcBoehm
     of "refc":         result = conf.selectedGC == gcRefc
-    of "v2":           result = conf.selectedGC == gcV2
+    of "v2":           result = false
     of "markandsweep": result = conf.selectedGC == gcMarkAndSweep
     of "generational": result = false
     of "destructors":  result = conf.selectedGC == gcDestructors
@@ -291,6 +291,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "patterns": result = contains(conf.options, optPatterns)
   of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace)
   of "nilseqs": result = contains(conf.options, optNilSeqs)
+  of "oldast": result = contains(conf.options, optOldAst)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc processPath(conf: ConfigRef; path: string, info: TLineInfo,
@@ -413,26 +414,19 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if pass in {passCmd2, passPP}:
       addExternalFileToLink(conf, AbsoluteFile arg)
   of "debuginfo":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optCDebug}, arg, pass, info)
-    incl(conf.globalOptions, optCDebug)
   of "embedsrc":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optEmbedOrigSrc}, arg, pass, info)
-    incl(conf.globalOptions, optEmbedOrigSrc)
   of "compileonly", "c":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info)
-    incl(conf.globalOptions, optCompileOnly)
   of "nolinking":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optNoLinking}, arg, pass, info)
-    incl(conf.globalOptions, optNoLinking)
   of "nomain":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optNoMain}, arg, pass, info)
-    incl(conf.globalOptions, optNoMain)
   of "forcebuild", "f":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info)
-    incl(conf.globalOptions, optForceFullMake)
   of "project":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info)
-    incl conf.globalOptions, optWholeProject
   of "gc":
     expectArg(conf, switch, arg, pass, info)
     case arg.normalize
@@ -442,7 +436,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "refc":
       conf.selectedGC = gcRefc
     of "v2":
-      conf.selectedGC = gcV2
+      message(conf, info, warnDeprecated, "--gc:v2 is deprecated; using default gc")
     of "markandsweep":
       conf.selectedGC = gcMarkAndSweep
       defineSymbol(conf.symbols, "gcmarkandsweep")
@@ -498,7 +492,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     else: undefSymbol(conf.symbols, "hotcodereloading")
   of "oldnewlines":
     case arg.normalize
-    of "on":
+    of "","on":
       conf.oldNewlines = true
       defineSymbol(conf.symbols, "nimOldNewlines")
     of "off":
@@ -508,6 +502,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       localError(conf, info, errOnOrOffExpectedButXFound % arg)
   of "laxstrings": processOnOffSwitch(conf, {optLaxStrings}, arg, pass, info)
   of "nilseqs": processOnOffSwitch(conf, {optNilSeqs}, arg, pass, info)
+  of "oldast": processOnOffSwitch(conf, {optOldAst}, arg, pass, info)
   of "checks", "x": processOnOffSwitch(conf, ChecksOptions, arg, pass, info)
   of "floatchecks":
     processOnOffSwitch(conf, {optNaNCheck, optInfCheck}, arg, pass, info)
@@ -597,11 +592,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if pass in {passCmd2, passPP}:
       conf.implicitIncludes.add findModule(conf, arg, toFullPath(conf, info)).string
   of "listcmd":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optListCmd}, arg, pass, info)
-    incl(conf.globalOptions, optListCmd)
   of "genmapping":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optGenMapping}, arg, pass, info)
-    incl(conf.globalOptions, optGenMapping)
   of "os":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd1, passPP}:
@@ -617,8 +610,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       elif cpu != conf.target.hostCPU:
         setTarget(conf.target, conf.target.targetOS, cpu)
   of "run", "r":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optRun}, arg, pass, info)
-    incl(conf.globalOptions, optRun)
   of "errormax":
     expectArg(conf, switch, arg, pass, info)
     # Note: `nim check` (etc) can overwrite this.
@@ -666,21 +658,16 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "v2": conf.symbolFiles = v2Sf
     else: localError(conf, info, "invalid option for --incremental: " & arg)
   of "skipcfg":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optSkipSystemConfigFile}, arg, pass, info)
-    incl(conf.globalOptions, optSkipSystemConfigFile)
   of "skipprojcfg":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optSkipProjConfigFile}, arg, pass, info)
-    incl(conf.globalOptions, optSkipProjConfigFile)
   of "skipusercfg":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optSkipUserConfigFile}, arg, pass, info)
-    incl(conf.globalOptions, optSkipUserConfigFile)
   of "skipparentcfg":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optSkipParentConfigFiles}, arg, pass, info)
-    incl(conf.globalOptions, optSkipParentConfigFiles)
   of "genscript", "gendeps":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optGenScript}, arg, pass, info)
-    incl(conf.globalOptions, optGenScript)
+    processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info)
-    incl(conf.globalOptions, optCompileOnly)
   of "colors": processOnOffSwitchG(conf, {optUseColors}, arg, pass, info)
   of "lib":
     expectArg(conf, switch, arg, pass, info)
@@ -714,16 +701,13 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectNoArg(conf, switch, arg, pass, info)
     conf.ideCmd = ideUse
   of "stdout":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optStdout}, arg, pass, info)
-    incl(conf.globalOptions, optStdout)
   of "listfullpaths":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optListFullPaths}, arg, pass, info)
-    incl conf.globalOptions, optListFullPaths
   of "dynliboverride":
     dynlibOverride(conf, switch, arg, pass, info)
   of "dynliboverrideall":
-    expectNoArg(conf, switch, arg, pass, info)
+    processOnOffSwitchG(conf, {optDynlibOverrideAll}, arg, pass, info)
-    incl conf.globalOptions, optDynlibOverrideAll
   of "cs":
     # only supported for compatibility. Does nothing.
     expectArg(conf, switch, arg, pass, info)
@@ -762,6 +746,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     else:
       conf.cppCustomNamespace = "Nim"
     defineSymbol(conf.symbols, "cppCompileToNamespace", conf.cppCustomNamespace)
+  of "docinternal":
+    processOnOffSwitchG(conf, {optDocInternal}, arg, pass, info)
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
     else: invalidCmdLineOption(conf, pass, switch, info)

compiler/condsyms.nim

@@ -15,26 +15,21 @@ import
 from options import Feature
 from lineinfos import HintsToStr, WarningsToStr
 
-const
-  catNone = "false"
-
 proc defineSymbol*(symbols: StringTableRef; symbol: string, value: string = "true") =
   symbols[symbol] = value
 
 proc undefSymbol*(symbols: StringTableRef; symbol: string) =
-  symbols[symbol] = catNone
+  symbols.del(symbol)
 
 #proc lookupSymbol*(symbols: StringTableRef; symbol: string): string =
 #  result = if isDefined(symbol): gSymbols[symbol] else: nil
 
 iterator definedSymbolNames*(symbols: StringTableRef): string =
   for key, val in pairs(symbols):
-    if val != catNone: yield key
+    yield key
 
 proc countDefinedSymbols*(symbols: StringTableRef): int =
-  result = 0
+  symbols.len
-  for key, val in pairs(symbols):
-    if val != catNone: inc(result)
 
 proc initDefines*(symbols: StringTableRef) =
   # for bootstrapping purposes and old code:

compiler/destroyer.nim

@@ -132,61 +132,36 @@ type
     emptyNode: PNode
     otherRead: PNode
 
-
+proc isLastRead(s: PSym; c: var Con; pc, comesFrom: int): int =
-proc isHarmlessVar*(s: PSym; c: Con): bool =
+  var pc = pc
-  # 's' is harmless if it used only once and its
+  while pc < c.g.len:
-  # definition/usage are not split by any labels:
+    case c.g[pc].kind
-  #
-  # let s = foo()
-  # while true:
-  #   a[i] = s
-  #
-  # produces:
-  #
-  # def s
-  # L1:
-  #   use s
-  # goto L1
-  #
-  # let s = foo()
-  # if cond:
-  #   a[i] = s
-  # else:
-  #   a[j] = s
-  #
-  # produces:
-  #
-  # def s
-  # fork L2
-  # use s
-  # goto L3
-  # L2:
-  # use s
-  # L3
-  #
-  # So this analysis is for now overly conservative, but correct.
-  var defsite = -1
-  var usages = 0
-  for i in 0..<c.g.len:
-    case c.g[i].kind
     of def:
-      if c.g[i].sym == s:
+      if c.g[pc].sym == s:
-        if defsite < 0: defsite = i
+        # the path lead to a redefinition of 's' --> abandon it.
-        else: return false
+        return high(int) 
+      inc pc
     of use:
-      if c.g[i].sym == s:
+      if c.g[pc].sym == s:
-        if defsite < 0: return false
+        c.otherRead = c.g[pc].n
-        for j in defsite .. i:
+        return -1
-          # not within the same basic block?
+      inc pc
-          if j in c.jumpTargets: return false
+    of goto:
-        # if we want to die after the first 'use':
+      pc = pc + c.g[pc].dest
-        if usages > 1: return false
+    of fork:
-        inc usages
+      # every branch must lead to the last read of the location:
-    #of useWithinCall:
+      var variantA = isLastRead(s, c, pc+1, pc)
-    #  if c.g[i].sym == s: return false
+      if variantA < 0: return -1
-    of goto, fork:
+      let variantB = isLastRead(s, c, pc + c.g[pc].dest, pc)
-      discard "we do not perform an abstract interpretation yet"
+      if variantB < 0: return -1
-  result = usages <= 1
+      elif variantA == high(int): 
+        variantA = variantB
+      pc = variantA
+    of InstrKind.join:
+      let dest = pc + c.g[pc].dest
+      if dest == comesFrom: return pc + 1
+      inc pc
+  return pc
 
 proc isLastRead(n: PNode; c: var Con): bool =
   # first we need to search for the instruction that belongs to 'n':
@@ -195,59 +170,52 @@ proc isLastRead(n: PNode; c: var Con): bool =
   var instr = -1
   for i in 0..<c.g.len:
     if c.g[i].n == n:
-      if instr < 0: instr = i
+      if instr < 0:
-      else:
+        instr = i
-        # eh, we found two positions that belong to 'n'?
+        break
-        # better return 'false' then:
+
-        return false
   if instr < 0: return false
   # we go through all paths beginning from 'instr+1' and need to
   # ensure that we don't find another 'use X' instruction.
   if instr+1 >= c.g.len: return true
-  let s = n.sym
+  when true:
-  var pcs: seq[int] = @[instr+1]
+    result = isLastRead(n.sym, c, instr+1, -1) >= 0
-  var takenGotos: IntSet
+  else:
-  var takenForks = initIntSet()
+    let s = n.sym
-  while pcs.len > 0:
+    var pcs: seq[int] = @[instr+1]
-    var pc = pcs.pop
+    var takenGotos: IntSet
+    var takenForks = initIntSet()
+    while pcs.len > 0:
+      var pc = pcs.pop
 
-    takenGotos = initIntSet()
+      takenGotos = initIntSet()
-    while pc < c.g.len:
+      while pc < c.g.len:
-      case c.g[pc].kind
+        case c.g[pc].kind
-      of def:
+        of def:
-        if c.g[pc].sym == s:
+          if c.g[pc].sym == s:
-          # the path lead to a redefinition of 's' --> abandon it.
+            # the path lead to a redefinition of 's' --> abandon it.
-          when false:
+            break
-            # Too complex thinking ahead: In reality it is enough to find
-            # the 'def x' here on the current path to make the 'use x' valid.
-            # but for this the definition needs to dominate the usage:
-            var dominates = true
-            for j in pc+1 .. instr:
-              # not within the same basic block?
-              if c.g[j].kind in {goto, fork} and (j + c.g[j].dest) in (pc+1 .. instr):
-                #if j in c.jumpTargets:
-                dominates = false
-            if dominates: break
-          break
-        inc pc
-      of use:
-        if c.g[pc].sym == s:
-          c.otherRead = c.g[pc].n
-          return false
-        inc pc
-      of goto:
-        # we must leave endless loops eventually:
-        if not takenGotos.containsOrIncl(pc):
-          pc = pc + c.g[pc].dest
-        else:
           inc pc
-      of fork:
+        of use:
-        # we follow the next instruction but push the dest onto our "work" stack:
+          if c.g[pc].sym == s:
-        if not takenForks.containsOrIncl(pc):
+            c.otherRead = c.g[pc].n
-          pcs.add pc + c.g[pc].dest
+            return false
-        inc pc
+          inc pc
-  #echo c.graph.config $ n.info, " last read here!"
+        of goto:
-  return true
+          # we must leave endless loops eventually:
+          if not takenGotos.containsOrIncl(pc):
+            pc = pc + c.g[pc].dest
+          else:
+            inc pc
+        of fork:
+          # we follow the next instruction but push the dest onto our "work" stack:
+          if not takenForks.containsOrIncl(pc):
+            pcs.add pc + c.g[pc].dest
+          inc pc
+        of InstrKind.join:
+          inc pc
+    #echo c.graph.config $ n.info, " last read here!"
+    return true
 
 template interestingSym(s: PSym): bool =
   s.owner == c.owner and s.kind in InterestingSyms and hasDestructor(s.typ)
@@ -276,7 +244,10 @@ proc patchHead(n: PNode) =
 
 proc patchHead(s: PSym) =
   if sfFromGeneric in s.flags:
-    patchHead(s.ast[bodyPos])
+    # do not patch the builtin type bound operators for seqs:
+    let dest = s.typ.sons[1].skipTypes(abstractVar)
+    if dest.kind != tySequence:
+      patchHead(s.ast[bodyPos])
 
 proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) =
   var m = "'" & opname & "' is not available for type <" & typeToString(t) & ">"
@@ -299,7 +270,8 @@ template genOp(opr, opname, ri) =
     globalError(c.graph.config, dest.info, "internal error: '" & opname &
       "' operator not found for type " & typeToString(t))
   elif op.ast[genericParamsPos].kind != nkEmpty:
-    globalError(c.graph.config, dest.info, "internal error: '" & opname & "' operator is generic")
+    globalError(c.graph.config, dest.info, "internal error: '" & opname &
+      "' operator is generic")
   patchHead op
   if sfError in op.flags: checkForErrorPragma(c, t, ri, opname)
   let addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ))
@@ -307,6 +279,12 @@ template genOp(opr, opname, ri) =
   result = newTree(nkCall, newSymNode(op), addrExp)
 
 proc genSink(c: Con; t: PType; dest, ri: PNode): PNode =
+  when false:
+    if t.kind != tyString:
+      echo "this one ", c.graph.config$dest.info, " for ", typeToString(t, preferDesc)
+      debug t.sink.typ.sons[2]
+      echo t.sink.id, " owner ", t.id
+      quit 1
   let t = t.skipTypes({tyGenericInst, tyAlias, tySink})
   genOp(if t.sink != nil: t.sink else: t.assignment, "=sink", ri)
 

compiler/dfa.nim

@@ -34,12 +34,12 @@ import ast, astalgo, types, intsets, tables, msgs, options, lineinfos
 
 type
   InstrKind* = enum
-    goto, fork, def, use
+    goto, fork, join, def, use
   Instr* = object
     n*: PNode
     case kind*: InstrKind
     of def, use: sym*: PSym
-    of goto, fork: dest*: int
+    of goto, fork, join: dest*: int
 
   ControlFlowGraph* = seq[Instr]
 
@@ -56,6 +56,7 @@ type
     inCall, inTryStmt: int
     blocks: seq[TBlock]
     tryStmtFixups: seq[TPosition]
+    forks: seq[TPosition]
     owner: PSym
 
 proc debugInfo(info: TLineInfo): string =
@@ -67,18 +68,18 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
   var jumpTargets = initIntSet()
   let last = if last < 0: c.len-1 else: min(last, c.len-1)
   for i in start..last:
-    if c[i].kind in {goto, fork}:
+    if c[i].kind in {goto, fork, join}:
       jumpTargets.incl(i+c[i].dest)
   var i = start
   while i <= last:
     if i in jumpTargets: result.add("L" & $i & ":\n")
     result.add "\t"
-    result.add $c[i].kind
+    result.add ($i & " " & $c[i].kind)
     result.add "\t"
     case c[i].kind
     of def, use:
       result.add c[i].sym.name.s
-    of goto, fork:
+    of goto, fork, join:
       result.add "L"
       result.add c[i].dest+i
     result.add("\t#")
@@ -86,7 +87,7 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
     result.add("\n")
     inc i
   if i in jumpTargets: result.add("L" & $i & ": End\n")
-
+  # consider calling `asciitables.alignTable`
 
 proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
   ## echos the ControlFlowGraph for debugging purposes.
@@ -98,11 +99,166 @@ proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
 proc forkI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
   c.code.add Instr(n: n, kind: fork, dest: 0)
+  c.forks.add result
 
 proc gotoI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
   c.code.add Instr(n: n, kind: goto, dest: 0)
 
+#[
+
+Design of join
+==============
+
+block:
+  if cond: break
+  def(x)
+
+use(x)
+
+Generates:
+
+L0: fork L1
+  join L0  # patched.
+  goto Louter
+L1:
+  def x
+  join L0
+Louter:
+  use x
+
+
+block outer:
+  while a:
+    while b:
+      if foo:
+        if bar:
+          break outer # --> we need to 'join' every pushed 'fork' here
+
+
+This works and then our abstract interpretation needs to deal with 'fork'
+differently. It really causes a split in execution. Two threads are
+"spawned" and both need to reach the 'join L' instruction. Afterwards
+the abstract interpretations are joined and execution resumes single
+threaded.
+
+
+Abstract Interpretation
+-----------------------
+
+proc interpret(pc, state, comesFrom): state =
+  result = state
+  # we need an explicit 'create' instruction (an explicit heap), in order
+  # to deal with 'var x = create(); var y = x; var z = y; destroy(z)'
+  while true:
+    case pc
+    of fork:
+      let a = interpret(pc+1, result, pc)
+      let b = interpret(forkTarget, result, pc)
+      result = a ++ b # ++ is a union operation
+      inc pc
+    of join:
+      if joinTarget == comesFrom: return result
+      else: inc pc
+    of use X:
+      if not result.contains(x):
+        error "variable not initialized " & x
+      inc pc
+    of def X:
+      if not result.contains(x):
+        result.incl X
+      else:
+        error "overwrite of variable causes memory leak " & x
+      inc pc
+    of destroy X:
+      result.excl X
+
+This is correct but still can lead to false positives:
+
+proc p(cond: bool) =
+  if cond:
+    new(x)
+  otherThings()
+  if cond:
+    destroy x
+
+Is not a leak. We should find a way to model *data* flow, not just
+control flow. One solution is to rewrite the 'if' without a fork
+instruction. The unstructured aspect can now be easily dealt with
+the 'goto' and 'join' instructions.
+
+proc p(cond: bool) =
+  L0: fork Lend
+    new(x)
+    # do not 'join' here!
+
+  Lend:
+    otherThings()
+    join L0  # SKIP THIS FOR new(x) SOMEHOW
+  destroy x
+  join L0 # but here.
+
+
+
+But if we follow 'goto Louter' we will never come to the join point.
+We restore the bindings after popping pc from the stack then there
+"no" problem?!
+
+
+while cond:
+  prelude()
+  if not condB: break
+  postlude()
+
+--->
+var setFlag = true
+while cond and not setFlag:
+  prelude()
+  if not condB:
+    setFlag = true   # BUT: Dependency
+  if not setFlag:    # HERE
+    postlude()
+
+--->
+var setFlag = true
+while cond and not setFlag:
+  prelude()
+  if not condB:
+    postlude()
+    setFlag = true
+
+
+-------------------------------------------------
+
+while cond:
+  prelude()
+  if more:
+    if not condB: break
+    stuffHere()
+  postlude()
+
+-->
+var setFlag = true
+while cond and not setFlag:
+  prelude()
+  if more:
+    if not condB:
+      setFlag = false
+    else:
+      stuffHere()
+      postlude()
+  else:
+    postlude()
+
+This is getting complicated. Instead we keep the whole 'join' idea but
+duplicate the 'join' instructions on breaks and return exits!
+
+]#
+
+proc joinI(c: var Con; fromFork: TPosition; n: PNode) =
+  let dist = fromFork.int - c.code.len
+  c.code.add Instr(n: n, kind: join, dest: dist)
+
 proc genLabel(c: Con): TPosition =
   result = TPosition(c.code.len)
 
@@ -135,30 +291,97 @@ proc isTrue(n: PNode): bool =
 
 proc gen(c: var Con; n: PNode) # {.noSideEffect.}
 
-proc genWhile(c: var Con; n: PNode) =
+when true:
-  # L1:
+  proc genWhile(c: var Con; n: PNode) =
-  #   cond, tmp
+    # We unroll every loop 3 times. We emulate 0, 1, 2 iterations
-  #   fork tmp, L2
+    # through the loop. We need to prove this is correct for our
-  #   body
+    # purposes. But Herb Sutter claims it is. (Proof by authority.)
-  #   jmp L1
+    #[
-  # L2:
+    while cond:
-  let L1 = c.genLabel
+      body
-  withBlock(nil):
+
+    Becomes:
+
+    if cond:
+      body
+      if cond:
+        body
+        if cond:
+          body
+
+    We still need to ensure 'break' resolves properly, so an AST to AST
+    translation is impossible.
+
+    So the code to generate is:
+
+      cond
+      fork L4  # F1
+      body
+      cond
+      fork L5  # F2
+      body
+      cond
+      fork L6  # F3
+      body
+    L6:
+      join F3
+    L5:
+      join F2
+    L4:
+      join F1
+    ]#
     if isTrue(n.sons[0]):
-      c.gen(n.sons[1])
+      # 'while true' is an idiom in Nim and so we produce
-      c.jmpBack(n, L1)
+      # better code for it:
+      for i in 0..2:
+        withBlock(nil):
+          c.gen(n.sons[1])
     else:
-      c.gen(n.sons[0])
+      let oldForksLen = c.forks.len
-      let L2 = c.forkI(n)
+      var endings: array[3, TPosition]
-      c.gen(n.sons[1])
+      for i in 0..2:
-      c.jmpBack(n, L1)
+        withBlock(nil):
-      c.patch(L2)
+          c.gen(n.sons[0])
+          endings[i] = c.forkI(n)
+          c.gen(n.sons[1])
+      for i in countdown(endings.high, 0):
+        let endPos = endings[i]
+        c.patch(endPos)
+        c.joinI(c.forks.pop(), n)
+      doAssert(c.forks.len == oldForksLen)
+
+else:
+
+  proc genWhile(c: var Con; n: PNode) =
+    # L1:
+    #   cond, tmp
+    #   fork tmp, L2
+    #   body
+    #   jmp L1
+    # L2:
+    let oldForksLen = c.forks.len
+    let L1 = c.genLabel
+    withBlock(nil):
+      if isTrue(n.sons[0]):
+        c.gen(n.sons[1])
+        c.jmpBack(n, L1)
+      else:
+        c.gen(n.sons[0])
+        let L2 = c.forkI(n)
+        c.gen(n.sons[1])
+        c.jmpBack(n, L1)
+        c.patch(L2)
+    setLen(c.forks, oldForksLen)
 
 proc genBlock(c: var Con; n: PNode) =
   withBlock(n.sons[0].sym):
     c.gen(n.sons[1])
 
+proc genJoins(c: var Con; n: PNode) =
+  for i in countdown(c.forks.high, 0): joinI(c, c.forks[i], n)
+
 proc genBreak(c: var Con; n: PNode) =
+  genJoins(c, n)
   let L1 = c.gotoI(n)
   if n.sons[0].kind == nkSym:
     #echo cast[int](n.sons[0].sym)
@@ -170,28 +393,76 @@ proc genBreak(c: var Con; n: PNode) =
   else:
     c.blocks[c.blocks.high].fixups.add L1
 
+template forkT(n, body) =
+  let oldLen = c.forks.len
+  let L1 = c.forkI(n)
+  body
+  c.patch(L1)
+  c.joinI(L1, n)
+  setLen(c.forks, oldLen)
+
 proc genIf(c: var Con, n: PNode) =
+  #[
+
+  if cond:
+    A
+  elif condB:
+    B
+  elif condC:
+    C
+  else:
+    D
+
+  cond
+  fork L1
+  A
+  goto Lend
+  L1:
+    condB
+    fork L2
+    B
+    goto Lend2
+  L2:
+    condC
+    fork L3
+    C
+    goto Lend3
+  L3:
+    D
+    goto Lend3 # not eliminated to simplify the join generation
+  Lend3:
+    join F3
+  Lend2:
+    join F2
+  Lend:
+    join F1
+
+  ]#
+  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
   for i in countup(0, len(n) - 1):
     var it = n.sons[i]
     c.gen(it.sons[0])
     if it.len == 2:
-      let elsePos = c.forkI(it.sons[1])
+      let elsePos = forkI(c, it[1])
       c.gen(it.sons[1])
-      if i < sonsLen(n)-1:
+      endings.add(c.gotoI(it.sons[1]))
-        endings.add(c.gotoI(it.sons[1]))
       c.patch(elsePos)
-  for endPos in endings: c.patch(endPos)
+  for i in countdown(endings.high, 0):
+    let endPos = endings[i]
+    c.patch(endPos)
+    c.joinI(c.forks.pop(), n)
+  doAssert(c.forks.len == oldLen)
 
 proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, a
   #   fork L1
   #   asgn dest, b
   # L1:
+  #   join F1
   c.gen(n.sons[1])
-  let L1 = c.forkI(n)
+  forkT(n):
-  c.gen(n.sons[2])
+    c.gen(n.sons[2])
-  c.patch(L1)
 
 proc genCase(c: var Con; n: PNode) =
   #  if (!expr1) goto L1;
@@ -204,72 +475,94 @@ proc genCase(c: var Con; n: PNode) =
   #  L2:
   #    elsePart
   #  Lend:
-  when false:
+  let isExhaustive = skipTypes(n.sons[0].typ,
-    # XXX Exhaustiveness is not yet mapped to the control flow graph as
+    abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString}
-    # it seems to offer no benefits for the 'last read of' question.
-    let isExhaustive = skipTypes(n.sons[0].typ,
-      abstractVarRange-{tyTypeDesc}).kind in {tyFloat..tyFloat128, tyString} or
-      lastSon(n).kind == nkElse
 
   var endings: seq[TPosition] = @[]
+  let oldLen = c.forks.len
   c.gen(n.sons[0])
   for i in 1 ..< n.len:
     let it = n.sons[i]
     if it.len == 1:
       c.gen(it.sons[0])
+    elif i == n.len-1 and isExhaustive:
+      # treat the last branch as 'else' if this is an exhaustive case statement.
+      c.gen(it.lastSon)
     else:
       let elsePos = c.forkI(it.lastSon)
       c.gen(it.lastSon)
-      if i < sonsLen(n)-1:
+      endings.add(c.gotoI(it.lastSon))
-        endings.add(c.gotoI(it.lastSon))
       c.patch(elsePos)
-  for endPos in endings: c.patch(endPos)
+  for i in countdown(endings.high, 0):
+    let endPos = endings[i]
+    c.patch(endPos)
+    c.joinI(c.forks.pop(), n)
+  doAssert(c.forks.len == oldLen)
 
 proc genTry(c: var Con; n: PNode) =
+  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
   inc c.inTryStmt
-  var newFixups: seq[TPosition]
+  let oldFixups = c.tryStmtFixups.len
-  swap(newFixups, c.tryStmtFixups)
 
-  let elsePos = c.forkI(n)
+  #let elsePos = c.forkI(n)
   c.gen(n.sons[0])
   dec c.inTryStmt
-  for f in newFixups:
+  for i in oldFixups..c.tryStmtFixups.high:
+    let f = c.tryStmtFixups[i]
     c.patch(f)
-  swap(newFixups, c.tryStmtFixups)
+    # we also need to produce join instructions
+    # for the 'fork' that might preceed the goto instruction
+    if f.int-1 >= 0 and c.code[f.int-1].kind == fork:
+      c.joinI(TPosition(f.int-1), n)
 
-  c.patch(elsePos)
+  setLen(c.tryStmtFixups, oldFixups)
+
+  #c.patch(elsePos)
   for i in 1 ..< n.len:
     let it = n.sons[i]
     if it.kind != nkFinally:
       var blen = len(it)
       let endExcept = c.forkI(it)
       c.gen(it.lastSon)
-      if i < sonsLen(n)-1:
+      endings.add(c.gotoI(it))
-        endings.add(c.gotoI(it))
       c.patch(endExcept)
-  for endPos in endings: c.patch(endPos)
+  for i in countdown(endings.high, 0):
+    let endPos = endings[i]
+    c.patch(endPos)
+    c.joinI(c.forks.pop(), n)
+
+  # join the 'elsePos' forkI instruction:
+  #c.joinI(c.forks.pop(), n)
+
   let fin = lastSon(n)
   if fin.kind == nkFinally:
     c.gen(fin.sons[0])
+  doAssert(c.forks.len == oldLen)
+
+template genNoReturn(c: var Con; n: PNode) =
+  # leave the graph
+  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 proc genRaise(c: var Con; n: PNode) =
+  genJoins(c, n)
   gen(c, n.sons[0])
   if c.inTryStmt > 0:
     c.tryStmtFixups.add c.gotoI(n)
   else:
-    c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
+    genNoReturn(c, n)
 
 proc genImplicitReturn(c: var Con) =
   if c.owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter} and resultPos < c.owner.ast.len:
     gen(c, c.owner.ast.sons[resultPos])
 
 proc genReturn(c: var Con; n: PNode) =
+  genJoins(c, n)
   if n.sons[0].kind != nkEmpty:
     gen(c, n.sons[0])
   else:
     genImplicitReturn(c)
-  c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
+  genNoReturn(c, n)
 
 const
   InterestingSyms = {skVar, skResult, skLet, skParam}
@@ -287,6 +580,14 @@ proc genDef(c: var Con; n: PNode) =
   if n.kind == nkSym and n.sym.kind in InterestingSyms:
     c.code.add Instr(n: n, kind: def, sym: n.sym)
 
+proc canRaise(fn: PNode): bool =
+  const magicsThatCanRaise = {
+    mNone, mSlurp, mStaticExec, mParseExprToAst, mParseStmtToAst}
+  if fn.kind == nkSym and fn.sym.magic notin magicsThatCanRaise:
+    result = false
+  else:
+    result = true
+
 proc genCall(c: var Con; n: PNode) =
   gen(c, n[0])
   var t = n[0].typ
@@ -297,8 +598,16 @@ proc genCall(c: var Con; n: PNode) =
     if t != nil and i < t.len and t.sons[i].kind == tyVar:
       genDef(c, n[i])
   # every call can potentially raise:
-  if c.inTryStmt > 0:
+  if c.inTryStmt > 0 and canRaise(n[0]):
-    c.tryStmtFixups.add c.forkI(n)
+    # we generate the instruction sequence:
+    # fork L1
+    # goto exceptionHandler (except or finally)
+    # L1:
+    # join F1
+    let endGoto = c.forkI(n)
+    c.tryStmtFixups.add c.gotoI(n)
+    c.patch(endGoto)
+    c.joinI(c.forks.pop(), n)
   dec c.inCall
 
 proc genMagic(c: var Con; n: PNode; m: TMagic) =
@@ -307,9 +616,6 @@ proc genMagic(c: var Con; n: PNode; m: TMagic) =
   of mNew, mNewFinalize:
     genDef(c, n[1])
     for i in 2..<n.len: gen(c, n[i])
-  of mExit:
-    genCall(c, n)
-    c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
   else:
     genCall(c, n)
 
@@ -334,6 +640,8 @@ proc gen(c: var Con; n: PNode) =
         genMagic(c, n, s.magic)
       else:
         genCall(c, n)
+      if sfNoReturn in n.sons[0].sym.flags:
+        genNoReturn(c, n)
     else:
       genCall(c, n)
   of nkCharLit..nkNilLit: discard
@@ -368,114 +676,48 @@ proc gen(c: var Con; n: PNode) =
     doAssert false, "dfa construction pass requires the elimination of 'defer'"
   else: discard
 
-proc dfa(code: seq[Instr]; conf: ConfigRef) =
-  var u = newSeq[IntSet](code.len) # usages
-  var d = newSeq[IntSet](code.len) # defs
-  var c = newSeq[IntSet](code.len) # consumed
-  var backrefs = initTable[int, int]()
-  for i in 0..<code.len:
-    u[i] = initIntSet()
-    d[i] = initIntSet()
-    c[i] = initIntSet()
-    case code[i].kind
-    of use: u[i].incl(code[i].sym.id)
-    of def: d[i].incl(code[i].sym.id)
-    of fork, goto:
-      let d = i+code[i].dest
-      backrefs.add(d, i)
-
-  var w = @[0]
-  var maxIters = 50
-  var someChange = true
-  var takenGotos = initIntSet()
-  var consuming = -1
-  while w.len > 0 and maxIters > 0: # and someChange:
-    dec maxIters
-    var pc = w.pop() # w[^1]
-    var prevPc = -1
-    # this simulates a single linear control flow execution:
-    while pc < code.len:
-      if prevPc >= 0:
-        someChange = false
-        # merge step and test for changes (we compute the fixpoints here):
-        # 'u' needs to be the union of prevPc, pc
-        # 'd' needs to be the intersection of 'pc'
-        for id in u[prevPc]:
-          if not u[pc].containsOrIncl(id):
-            someChange = true
-        # in (a; b) if ``a`` sets ``v`` so does ``b``. The intersection
-        # is only interesting on merge points:
-        for id in d[prevPc]:
-          if not d[pc].containsOrIncl(id):
-            someChange = true
-        # if this is a merge point, we take the intersection of the 'd' sets:
-        if backrefs.hasKey(pc):
-          var intersect = initIntSet()
-          assign(intersect, d[pc])
-          var first = true
-          for prevPc in backrefs.allValues(pc):
-            for def in d[pc]:
-              if def notin d[prevPc]:
-                excl(intersect, def)
-                someChange = true
-                when defined(debugDfa):
-                  echo "Excluding ", pc, " prev ", prevPc
-          assign d[pc], intersect
-      if consuming >= 0:
-        if not c[pc].containsOrIncl(consuming):
-          someChange = true
-        consuming = -1
-
-      # our interpretation ![I!]:
-      prevPc = pc
-      case code[pc].kind
-      of goto:
-        # we must leave endless loops eventually:
-        if not takenGotos.containsOrIncl(pc) or someChange:
-          pc = pc + code[pc].dest
-        else:
-          inc pc
-      of fork:
-        # we follow the next instruction but push the dest onto our "work" stack:
-        #if someChange:
-        w.add pc + code[pc].dest
-        inc pc
-      of use:
-        #if not d[prevPc].missingOrExcl():
-        # someChange = true
-        consuming = code[pc].sym.id
-        inc pc
-      of def:
-        if not d[pc].containsOrIncl(code[pc].sym.id):
-          someChange = true
-        inc pc
-
-  when defined(useDfa) and defined(debugDfa):
-    for i in 0..<code.len:
-      echo "PC ", i, ": defs: ", d[i], "; uses ", u[i], "; consumes ", c[i]
-
-  # now check the condition we're interested in:
-  for i in 0..<code.len:
-    case code[i].kind
-    of use:
-      let s = code[i].sym
-      if s.id notin d[i]:
-        localError(conf, code[i].n.info, "usage of uninitialized variable: " & s.name.s)
-      if s.id in c[i]:
-        localError(conf, code[i].n.info, "usage of an already consumed variable: " & s.name.s)
-
-    else: discard
-
-proc dataflowAnalysis*(s: PSym; body: PNode; conf: ConfigRef) =
-  var c = Con(code: @[], blocks: @[])
-  gen(c, body)
-  genImplicitReturn(c)
-  when defined(useDfa) and defined(debugDfa): echoCfg(c.code)
-  dfa(c.code, conf)
-
 proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
   ## constructs a control flow graph for ``body``.
   var c = Con(code: @[], blocks: @[], owner: s)
   gen(c, body)
   genImplicitReturn(c)
   shallowCopy(result, c.code)
+
+proc interpret(code: ControlFlowGraph; pc: int, state: seq[PSym], comesFrom: int; threadId: int): (seq[PSym], int) =
+  var res = state
+  var pc = pc
+  while pc < code.len:
+    #echo threadId, " ", code[pc].kind
+    case code[pc].kind
+    of goto:
+      pc = pc + code[pc].dest
+    of fork:
+      let target = pc + code[pc].dest
+      let (branchA, pcA) = interpret(code, pc+1, res, pc, threadId+1)
+      let (branchB, _) = interpret(code, target, res, pc, threadId+2)
+      # we add vars if they are in both branches:
+      for v in branchB:
+        if v in branchA:
+          if v notin res:
+            res.add v
+      pc = pcA+1
+    of join:
+      let target = pc + code[pc].dest
+      if comesFrom == target: return (res, pc)
+      inc pc
+    of use:
+      let v = code[pc].sym
+      if v notin res and v.kind != skParam:
+        echo "attempt to read uninitialized variable ", v.name.s
+      inc pc
+    of def:
+      let v = code[pc].sym
+      if v notin res:
+        res.add v
+      inc pc
+  return (res, pc)
+
+proc dataflowAnalysis*(s: PSym; body: PNode) =
+  let c = constructCfg(s, body)
+  #echoCfg c
+  discard interpret(c, 0, @[], -1, 1)

compiler/docgen.nim

@@ -40,6 +40,7 @@ type
                     # already. See bug #3655
     destFile*: AbsoluteFile
     thisDir*: AbsoluteDir
+    examples: string
 
   PDoc* = ref TDocumentor ## Alias to type less.
 
@@ -118,7 +119,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   result.conf = conf
   result.cache = cache
   initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex),
-                   conf.configVars, filename.string, {roSupportRawDirective},
+                   conf.configVars, filename.string, {roSupportRawDirective, roSupportMarkdown},
                    docgenFindFile, compilerMsgHandler)
 
   if conf.configVars.hasKey("doc.googleAnalytics"):
@@ -143,28 +144,30 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   initStrTable result.types
   result.onTestSnippet =
     proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) =
-    var d = TDocumentor(gen)
+      var d = TDocumentor(gen)
-    var outp: AbsoluteFile
+      var outp: AbsoluteFile
-    if filename.len == 0:
+      if filename.len == 0:
-      inc(d.id)
+        inc(d.id)
-      let nameOnly = splitFile(d.filename).name
+        let nameOnly = splitFile(d.filename).name
-      let subdir = getNimcacheDir(conf) / RelativeDir(nameOnly)
+        outp = getNimcacheDir(conf) / RelativeDir(nameOnly) /
-      createDir(subdir)
+               RelativeFile(nameOnly & "_snippet_" & $d.id & ".nim")
-      outp = subdir / RelativeFile(nameOnly & "_snippet_" & $d.id & ".nim")
+      elif isAbsolute(filename):
-    elif isAbsolute(filename):
+        outp = AbsoluteFile(filename)
-      outp = AbsoluteFile filename
+      else:
-    else:
+        # Nim's convention: every path is relative to the file it was written in:
-      # Nim's convention: every path is relative to the file it was written in:
+        let nameOnly = splitFile(d.filename).name
-      outp = splitFile(d.filename).dir.AbsoluteDir / RelativeFile(filename)
+        outp = AbsoluteDir(nameOnly) / RelativeFile(filename)
-    # Include the current file if we're parsing a nim file
+      # Make sure the destination directory exists
-    let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename]
+      createDir(outp.splitFile.dir)
-    writeFile(outp, importStmt & content)
+      # Include the current file if we're parsing a nim file
-    let c = if cmd.startsWith("nim "): os.getAppFilename() & cmd.substr(3)
+      let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename]
-            else: cmd
+      writeFile(outp, importStmt & content)
-    let c2 = c % quoteShell(outp)
+      let c = if cmd.startsWith("nim "): os.getAppFilename() & cmd.substr(3)
-    rawMessage(conf, hintExecuting, c2)
+              else: cmd
-    if execShellCmd(c2) != status:
+      let c2 = c % quoteShell(outp)
-      rawMessage(conf, errGenerated, "executing of external program failed: " & c2)
+      rawMessage(conf, hintExecuting, c2)
+      if execShellCmd(c2) != status:
+        rawMessage(conf, errGenerated, "executing of external program failed: " & c2)
   result.emitted = initIntSet()
   result.destFile = getOutFile2(conf, relativeTo(filename, conf.projectPath),
                                 outExt, RelativeDir"htmldocs", false)
@@ -243,7 +246,7 @@ proc genComment(d: PDoc, n: PNode): string =
   result = ""
   var dummyHasToc: bool
   if n.comment.len > 0:
-    renderRstToOut(d[], parseRst(n.comment, toFilename(d.conf, n.info),
+    renderRstToOut(d[], parseRst(n.comment, toFullPath(d.conf, n.info),
                                toLinenumber(n.info), toColumn(n.info),
                                dummyHasToc, d.options, d.conf), result)
 
@@ -299,13 +302,17 @@ proc externalDep(d: PDoc; module: PSym): string =
   else:
     result = extractFilename toFullPath(d.conf, FileIndex module.position)
 
-proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRenderFlags = {}) =
+proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRenderFlags = {};
+                           procLink: Rope) =
   var r: TSrcGen
   var literal = ""
   initTokRender(r, n, renderFlags)
   var kind = tkEof
+  var tokenPos = 0
+  var procTokenPos = 0
   while true:
     getNextTok(r, kind, literal)
+    inc tokenPos
     case kind
     of tkEof:
       break
@@ -313,6 +320,8 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe
       dispA(d.conf, result, "<span class=\"Comment\">$1</span>", "\\spanComment{$1}",
             [rope(esc(d.target, literal))])
     of tokKeywordLow..tokKeywordHigh:
+      if kind in {tkProc, tkMethod, tkIterator, tkMacro, tkTemplate, tkFunc, tkConverter}:
+        procTokenPos = tokenPos
       dispA(d.conf, result, "<span class=\"Keyword\">$1</span>", "\\spanKeyword{$1}",
             [rope(literal)])
     of tkOpr:
@@ -332,7 +341,11 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe
             "\\spanFloatNumber{$1}", [rope(esc(d.target, literal))])
     of tkSymbol:
       let s = getTokSym(r)
-      if s != nil and s.kind == skType and sfExported in s.flags and
+      # -2 because of the whitespace in between:
+      if procTokenPos == tokenPos-2 and procLink != nil:
+        dispA(d.conf, result, "<a href=\"#$2\"><span class=\"Identifier\">$1</span></a>",
+              "\\spanIdentifier{$1}", [rope(esc(d.target, literal)), procLink])
+      elif s != nil and s.kind == skType and sfExported in s.flags and
           s.owner != nil and belongsToPackage(d.conf, s.owner) and
           d.target == outHtml:
         let external = externalDep(d, s.owner)
@@ -378,6 +391,14 @@ proc testExample(d: PDoc; ex: PNode) =
       "_examples" & $d.exampleCounter & ".nim"))
   #let nimcache = outp.changeFileExt"" & "_nimcache"
   renderModule(ex, d.filename, outp.string, conf = d.conf)
+  d.examples.add "import r\"" & outp.string & "\"\n"
+
+proc runAllExamples(d: PDoc) =
+  if d.examples.len == 0: return
+  let outputDir = d.conf.getNimcacheDir / RelativeDir"runnableExamples"
+  let outp = outputDir / RelativeFile(extractFilename(d.filename.changeFileExt"" &
+      "_examples.nim"))
+  writeFile(outp, d.examples)
   let backend = if isDefined(d.conf, "js"): "js"
                 elif isDefined(d.conf, "cpp"): "cpp"
                 elif isDefined(d.conf, "objc"): "objc"
@@ -400,11 +421,9 @@ proc extractImports(n: PNode; result: PNode) =
   for i in 0..<n.safeLen: extractImports(n[i], result)
 
 proc prepareExamples(d: PDoc; n: PNode) =
-
   var docComment = newTree(nkCommentStmt)
   let loc = d.conf.toFileLineCol(n.info)
   docComment.comment = "autogenerated by docgen from " & loc
-
   var runnableExamples = newTree(nkStmtList,
       docComment,
       newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
@@ -438,7 +457,7 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope) =
       for b in body:
         if i > 0: dest.add "\n"
         inc i
-        nodeToHighlightedHtml(d, b, dest, {})
+        nodeToHighlightedHtml(d, b, dest, {}, nil)
       dest.add(d.config.getOrDefault"doc.listing_end" % id)
   else: discard
   for i in 0 ..< n.safeLen:
@@ -457,7 +476,10 @@ proc isVisible(d: PDoc; n: PNode): bool =
     # we cannot generate code for forwarded symbols here as we have no
     # exception tracking information here. Instead we copy over the comment
     # from the proc header.
-    result = {sfExported, sfFromGeneric, sfForward}*n.sym.flags == {sfExported}
+    if optDocInternal in d.conf.globalOptions:
+      result = {sfFromGeneric, sfForward}*n.sym.flags == {}
+    else:
+      result = {sfExported, sfFromGeneric, sfForward}*n.sym.flags == {sfExported}
     if result and containsOrIncl(d.emitted, n.sym.id):
       result = false
   elif n.kind == nkPragmaExpr:
@@ -538,16 +560,19 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string =
   ## If you modify the output of this proc, please update the anchor generation
   ## section of ``doc/docgen.txt``.
   result = baseName
-  case k:
+  case k
-  of skProc, skFunc: result.add(defaultParamSeparator)
+  of skProc, skFunc: discard
-  of skMacro: result.add(".m" & defaultParamSeparator)
+  of skMacro: result.add(".m")
-  of skMethod: result.add(".e" & defaultParamSeparator)
+  of skMethod: result.add(".e")
-  of skIterator: result.add(".i" & defaultParamSeparator)
+  of skIterator: result.add(".i")
-  of skTemplate: result.add(".t" & defaultParamSeparator)
+  of skTemplate: result.add(".t")
-  of skConverter: result.add(".c" & defaultParamSeparator)
+  of skConverter: result.add(".c")
   else: discard
   if len(n) > paramsPos and n[paramsPos].kind == nkFormalParams:
-    result.add(renderParamTypes(n[paramsPos]))
+    let params = renderParamTypes(n[paramsPos])
+    if params.len > 0:
+      result.add(defaultParamSeparator)
+      result.add(params)
 
 proc isCallable(n: PNode): bool =
   ## Returns true if `n` contains a callable node.
@@ -605,9 +630,6 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
       break
     plainName.add(literal)
 
-  nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments,
-    renderDocComments, renderSyms})
-
   inc(d.id)
   let
     plainNameRope = rope(xmltree.escape(plainName.strip))
@@ -620,6 +642,9 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
     symbolOrIdRope = symbolOrId.rope
     symbolOrIdEncRope = encodeUrl(symbolOrId).rope
 
+  nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments,
+    renderDocComments, renderSyms}, symbolOrIdEncRope)
+
   var seeSrcRope: Rope = nil
   let docItemSeeSrc = getConfigVar(d.conf, "doc.item.seesrc")
   if docItemSeeSrc.len > 0:
@@ -631,7 +656,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
         path = path[cwd.len+1 .. ^1].replace('\\', '/')
     let gitUrl = getConfigVar(d.conf, "git.url")
     if gitUrl.len > 0:
-      let commit = getConfigVar(d.conf, "git.commit", "master")
+      let defaultBranch = if NimPatch mod 2 == 1: "devel" else: "master"
+      let commit = getConfigVar(d.conf, "git.commit", defaultBranch)
       let develBranch = getConfigVar(d.conf, "git.devel", "devel")
       dispA(d.conf, seeSrcRope, "$1", "", [ropeFormatNamedVars(d.conf, docItemSeeSrc,
           ["path", "line", "url", "commit", "devel"], [rope path.string,
@@ -935,6 +961,7 @@ proc generateIndex*(d: PDoc) =
     writeIndexFile(d[], dest.string)
 
 proc writeOutput*(d: PDoc, useWarning = false) =
+  runAllExamples(d)
   var content = genOutFile(d)
   if optStdout in d.conf.globalOptions:
     writeRope(stdout, content)
@@ -947,6 +974,7 @@ proc writeOutput*(d: PDoc, useWarning = false) =
         outfile.string)
 
 proc writeOutputJson*(d: PDoc, useWarning = false) =
+  runAllExamples(d)
   var modDesc: string
   for desc in d.modDesc:
     modDesc &= desc
@@ -982,7 +1010,7 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef;
 
   d.isPureRst = true
   var rst = parseRst(readFile(filen.string), filen.string, 0, 1, d.hasToc,
-                     {roSupportRawDirective}, conf)
+                     {roSupportRawDirective, roSupportMarkdown}, conf)
   var modDesc = newStringOfCap(30_000)
   renderRstToOut(d[], rst, modDesc)
   d.modDesc = rope(modDesc)

compiler/extccomp.nim

@@ -63,8 +63,8 @@ compiler gcc:
   result = (
     name: "gcc",
     objExt: "o",
-    optSpeed: " -O3 -ffast-math ",
+    optSpeed: " -O3 ",
-    optSize: " -Os -ffast-math ",
+    optSize: " -Os ",
     compilerExe: "gcc",
     cppCompiler: "g++",
     compileTmpl: "-c $options $include -o $objfile $file",
@@ -88,8 +88,8 @@ compiler nintendoSwitchGCC:
   result = (
     name: "switch_gcc",
     objExt: "o",
-    optSpeed: " -O3 -ffast-math ",
+    optSpeed: " -O3 ",
-    optSize: " -Os -ffast-math ",
+    optSize: " -Os ",
     compilerExe: "aarch64-none-elf-gcc",
     cppCompiler: "aarch64-none-elf-g++",
     compileTmpl: "-w -MMD -MP -MF $dfile -c $options $include -o $objfile $file",

compiler/gorgeimpl.nim

@@ -24,11 +24,11 @@ proc readOutput(p: Process): (string, int) =
 
 proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (string, int) =
   let workingDir = parentDir(toFullPath(conf, info))
-  if cache.len > 0:# and optForceFullMake notin gGlobalOptions:
+  if cache.len > 0:
     let h = secureHash(cmd & "\t" & input & "\t" & cache)
     let filename = toGeneratedFile(conf, AbsoluteFile("gorge_" & $h), "txt").string
     var f: File
-    if open(f, filename):
+    if optForceFullMake notin conf.globalOptions and open(f, filename):
       result = (f.readAll, 0)
       f.close
       return

compiler/guards.nim

@@ -257,9 +257,9 @@ proc canon*(n: PNode; o: Operators): PNode =
     for i in 0 ..< n.len:
       result.sons[i] = canon(n.sons[i], o)
   elif n.kind == nkSym and n.sym.kind == skLet and
-      n.sym.ast.getMagic in (someEq + someAdd + someMul + someMin +
+      n.sym.astdef.getMagic in (someEq + someAdd + someMul + someMin +
       someMax + someHigh + {mUnaryLt} + someSub + someLen + someDiv):
-    result = n.sym.ast.copyTree
+    result = n.sym.astdef.copyTree
   else:
     result = n
   case result.getMagic
@@ -395,8 +395,8 @@ proc usefulFact(n: PNode; o: Operators): PNode =
     #   if a:
     #     ...
     # We make can easily replace 'a' by '2 < x' here:
-    if n.sym.ast != nil:
+    if n.sym.astdef != nil:
-      result = usefulFact(n.sym.ast, o)
+      result = usefulFact(n.sym.astdef, o)
   elif n.kind == nkStmtListExpr:
     result = usefulFact(n.lastSon, o)
 

compiler/importer.nim

@@ -162,9 +162,9 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
         localError(c.config, n.info, "A module cannot import itself")
     if sfDeprecated in result.flags:
       if result.constraint != nil:
-        message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s)
+        message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated")
       else:
-        message(c.config, n.info, warnDeprecated, result.name.s)
+        message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated")
     suggestSym(c.config, n.info, result, c.graph.usageSym, false)
     importStmtResult.add newSymNode(result, n.info)
     #newStrNode(toFullPath(c.config, f), n.info)

compiler/jsgen.nim

@@ -925,7 +925,7 @@ const
 
 proc needsNoCopy(p: PProc; y: PNode): bool =
   return y.kind in nodeKindsNeedNoCopy or
-        ((mapType(y.typ) != etyBaseIndex or y.sym.kind == skParam) and
+        ((mapType(y.typ) != etyBaseIndex or (y.kind == nkSym and y.sym.kind == skParam)) and
           (skipTypes(y.typ, abstractInst).kind in
             {tyRef, tyPtr, tyLent, tyVar, tyCString, tyProc} + IntegralTypes))
 
@@ -950,7 +950,7 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
       lineF(p, "$1 = nimCopy(null, $2, $3);$n",
                [a.rdLoc, b.res, genTypeInfo(p, y.typ)])
   of etyObject:
-    if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
+    if x.typ.kind == tyVar or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
       lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
@@ -1502,6 +1502,8 @@ proc createRecordVarAux(p: PProc, rec: PNode, excludedFieldIDs: IntSet, output:
     for i in countup(1, sonsLen(rec) - 1):
       createRecordVarAux(p, lastSon(rec.sons[i]), excludedFieldIDs, output)
   of nkSym:
+    # Do not produce code for void types
+    if isEmptyType(rec.sym.typ): return
     if rec.sym.id notin excludedFieldIDs:
       if output.len > 0: output.add(", ")
       output.addf("$#: ", [mangleName(p.module, rec.sym)])

compiler/lambdalifting.nim

@@ -320,17 +320,30 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
     rawAddSon(result, obj)
     c.ownerToType[owner.id] = result
 
+proc getEnvTypeForOwnerUp(c: var DetectionPass; owner: PSym;
+                          info: TLineInfo): PType =
+  var r = c.getEnvTypeForOwner(owner, info)
+  result = newType(tyPtr, owner)
+  rawAddSon(result, r.base)
+
 proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
   let refObj = c.getEnvTypeForOwner(dest, info) # getHiddenParam(dest).typ
   let obj = refObj.lastSon
-  let fieldType = c.getEnvTypeForOwner(dep, info) #getHiddenParam(dep).typ
+  # The assumption here is that gcDestructors means we cannot deal
+  # with cycles properly, so it's better to produce a weak ref (=ptr) here.
+  # This seems to be generally correct but since it's a bit risky it's only
+  # enabled for gcDestructors.
+  let fieldType = if c.graph.config.selectedGc == gcDestructors:
+                    c.getEnvTypeForOwnerUp(dep, info) #getHiddenParam(dep).typ
+                  else:
+                    c.getEnvTypeForOwner(dep, info)
   if refObj == fieldType:
     localError(c.graph.config, dep.info, "internal error: invalid up reference computed")
 
   let upIdent = getIdent(c.graph.cache, upName)
   let upField = lookupInRecord(obj.n, upIdent)
   if upField != nil:
-    if upField.typ != fieldType:
+    if upField.typ.base != fieldType.base:
       localError(c.graph.config, dep.info, "internal error: up references do not agree")
   else:
     let result = newSym(skField, upIdent, obj.owner, obj.owner.info)
@@ -452,7 +465,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
           w = up
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit,
      nkTemplateDef, nkTypeSection, nkProcDef, nkMethodDef,
-     nkConverterDef, nkMacroDef, nkFuncDef:
+     nkConverterDef, nkMacroDef, nkFuncDef, nkCommentStmt:
     discard
   of nkLambdaKinds, nkIteratorDef:
     if n.typ != nil:
@@ -555,7 +568,7 @@ proc rawClosureCreation(owner: PSym;
   let upField = lookupInRecord(env.typ.lastSon.n, getIdent(d.graph.cache, upName))
   if upField != nil:
     let up = getUpViaParam(d.graph, owner)
-    if up != nil and upField.typ == up.typ:
+    if up != nil and upField.typ.base == up.typ.base:
       result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info),
                  up, env.info))
     #elif oldenv != nil and oldenv.typ == upField.typ:
@@ -586,7 +599,7 @@ proc closureCreationForIter(iter: PNode;
   let upField = lookupInRecord(v.typ.lastSon.n, getIdent(d.graph.cache, upName))
   if upField != nil:
     let u = setupEnvVar(owner, d, c)
-    if u.typ == upField.typ:
+    if u.typ.base == upField.typ.base:
       result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info),
                  u, iter.info))
     else:

compiler/lexer.nim

@@ -17,7 +17,7 @@
 
 import
   hashes, options, msgs, strutils, platform, idents, nimlexbase, llstream,
-  wordrecg, lineinfos, pathutils
+  wordrecg, lineinfos, pathutils, parseutils
 
 const
   MaxLineLength* = 80         # lines longer than this lead to a warning
@@ -307,20 +307,6 @@ template tokenEndPrevious(tok, pos) =
   when defined(nimpretty):
     tok.offsetB = L.offsetBase + pos
 
-{.push overflowChecks: off.}
-# We need to parse the largest uint literal without overflow checks
-proc unsafeParseUInt(s: string, b: var BiggestInt, start = 0): int =
-  var i = start
-  if i < s.len and s[i] in {'0'..'9'}:
-    b = 0
-    while i < s.len and s[i] in {'0'..'9'}:
-      b = b * 10 + (ord(s[i]) - ord('0'))
-      inc(i)
-      while i < s.len and s[i] == '_': inc(i) # underscores are allowed and ignored
-    result = i - start
-{.pop.} # overflowChecks
-
-
 template eatChar(L: var TLexer, t: var TToken, replacementChar: char) =
   add(t.literal, replacementChar)
   inc(L.bufpos)
@@ -586,33 +572,43 @@ proc getNumber(L: var TLexer, result: var TToken) =
       of floatTypes:
         result.fNumber = parseFloat(result.literal)
       of tkUint64Lit:
-        xi = 0
+        var iNumber: uint64
-        let len = unsafeParseUInt(result.literal, xi)
+        var len: int
-        if len != result.literal.len or len == 0:
+        try:
-          raise newException(ValueError, "invalid integer: " & $xi)
+          len = parseBiggestUInt(result.literal, iNumber)
-        result.iNumber = xi
+        except ValueError:
+          raise newException(OverflowError, "number out of range: " & $result.literal)
+        if len != result.literal.len:
+          raise newException(ValueError, "invalid integer: " & $result.literal)
+        result.iNumber = cast[int64](iNumber)
       else:
-        result.iNumber = parseBiggestInt(result.literal)
+        var iNumber: int64
+        var len: int
+        try:
+          len = parseBiggestInt(result.literal, iNumber)
+        except ValueError:
+          raise newException(OverflowError, "number out of range: " & $result.literal)
+        if len != result.literal.len:
+          raise newException(ValueError, "invalid integer: " & $result.literal)
+        result.iNumber = iNumber
 
-      # Explicit bounds checks
+      # Explicit bounds checks. Only T.high needs to be considered
+      # since result.iNumber can't be negative.
       let outOfRange =
         case result.tokType
-        of tkInt8Lit: (result.iNumber < int8.low or result.iNumber > int8.high)
+        of tkInt8Lit: result.iNumber > int8.high
-        of tkUInt8Lit: (result.iNumber < BiggestInt(uint8.low) or
+        of tkUInt8Lit: result.iNumber > BiggestInt(uint8.high)
-                        result.iNumber > BiggestInt(uint8.high))
+        of tkInt16Lit: result.iNumber > int16.high
-        of tkInt16Lit: (result.iNumber < int16.low or result.iNumber > int16.high)
+        of tkUInt16Lit: result.iNumber > BiggestInt(uint16.high)
-        of tkUInt16Lit: (result.iNumber < BiggestInt(uint16.low) or
+        of tkInt32Lit: result.iNumber > int32.high
-                        result.iNumber > BiggestInt(uint16.high))
+        of tkUInt32Lit: result.iNumber > BiggestInt(uint32.high)
-        of tkInt32Lit: (result.iNumber < int32.low or result.iNumber > int32.high)
-        of tkUInt32Lit: (result.iNumber < BiggestInt(uint32.low) or
-                        result.iNumber > BiggestInt(uint32.high))
         else: false
 
       if outOfRange: lexMessageLitNum(L, "number out of range: '$1'", startpos)
 
     # Promote int literal to int64? Not always necessary, but more consistent
     if result.tokType == tkIntLit:
-      if (result.iNumber < low(int32)) or (result.iNumber > high(int32)):
+      if result.iNumber > high(int32):
         result.tokType = tkInt64Lit
 
   except ValueError:
@@ -622,7 +618,12 @@ proc getNumber(L: var TLexer, result: var TToken) =
   tokenEnd(result, postPos-1)
   L.bufpos = postPos
 
-proc handleHexChar(L: var TLexer, xi: var int) =
+proc handleHexChar(L: var TLexer, xi: var int; position: range[0..4]) =
+  template invalid() =
+    lexMessage(L, errGenerated,
+      "expected a hex digit, but found: " & L.buf[L.bufpos] &
+        "; maybe prepend with 0")
+
   case L.buf[L.bufpos]
   of '0'..'9':
     xi = (xi shl 4) or (ord(L.buf[L.bufpos]) - ord('0'))
@@ -633,10 +634,12 @@ proc handleHexChar(L: var TLexer, xi: var int) =
   of 'A'..'F':
     xi = (xi shl 4) or (ord(L.buf[L.bufpos]) - ord('A') + 10)
     inc(L.bufpos)
+  of '"', '\'':
+    if position <= 1: invalid()
+    # do not progress the bufpos here.
+    if position == 0: inc(L.bufpos)
   else:
-    lexMessage(L, errGenerated,
+    invalid()
-      "expected a hex digit, but found: " & L.buf[L.bufpos] &
-        " ; maybe prepend with 0")
     # Need to progress for `nim check`
     inc(L.bufpos)
 
@@ -731,8 +734,8 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
   of 'x', 'X':
     inc(L.bufpos)
     var xi = 0
-    handleHexChar(L, xi)
+    handleHexChar(L, xi, 1)
-    handleHexChar(L, xi)
+    handleHexChar(L, xi, 2)
     add(tok.literal, chr(xi))
   of 'u', 'U':
     if tok.tokType == tkCharLit:
@@ -743,7 +746,7 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
       inc(L.bufpos)
       var start = L.bufpos
       while L.buf[L.bufpos] != '}':
-        handleHexChar(L, xi)
+        handleHexChar(L, xi, 0)
       if start == L.bufpos:
         lexMessage(L, errGenerated,
           "Unicode codepoint cannot be empty")
@@ -753,10 +756,10 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
         lexMessage(L, errGenerated,
           "Unicode codepoint must be lower than 0x10FFFF, but was: " & hex)
     else:
-      handleHexChar(L, xi)
+      handleHexChar(L, xi, 1)
-      handleHexChar(L, xi)
+      handleHexChar(L, xi, 2)
-      handleHexChar(L, xi)
+      handleHexChar(L, xi, 3)
-      handleHexChar(L, xi)
+      handleHexChar(L, xi, 4)
     addUnicodeCodePoint(tok.literal, xi)
   of '0'..'9':
     if matchTwoChars(L, '0', {'0'..'9'}):

compiler/lineinfos.nim

@@ -65,7 +65,7 @@ const
     warnOctalEscape: "octal escape sequences do not exist; leading zero is ignored",
     warnXIsNeverRead: "'$1' is never read",
     warnXmightNotBeenInit: "'$1' might not have been initialized",
-    warnDeprecated: "$1 is deprecated",
+    warnDeprecated: "$1",
     warnConfigDeprecated: "config file '$1' is deprecated",
     warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)",
     warnUnknownMagic: "unknown magic '$1' might crash the compiler",
@@ -171,7 +171,7 @@ proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,
     hintSource, hintGlobalVar, hintGCStats}
   result[0] = result[1] - {hintSuccessX, hintSuccess, hintConf,
-    hintProcessing, hintPattern, hintExecuting, hintLinking}
+    hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC}
 
 const
   NotesVerbosity* = computeNotesVerbosity()

compiler/lists.nim

@@ -1,28 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2012 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-# This module is deprecated, don't use it.
-# TODO Remove this
-
-import os
-
-static:
-  echo "WARNING: imported deprecated module compiler/lists.nim, use seq ore lists from the standard library"
-
-proc appendStr*(list: var seq[string]; data: string) {.deprecated.} =
-  # just use system.add
-  list.add(data)
-
-proc includeStr(list: var seq[string]; data: string): bool {.deprecated.} =
-  if list.contains(data):
-    result = true
-  else:
-    result = false
-    list.add data
-

compiler/lookups.nim

@@ -89,7 +89,7 @@ proc skipAlias*(s: PSym; n: PNode; conf: ConfigRef): PSym =
       prettybase.replaceDeprecated(conf, n.info, s, result)
     else:
       message(conf, n.info, warnDeprecated, "use " & result.name.s & " instead; " &
-              s.name.s)
+              s.name.s & " is deprecated")
 
 proc localSearchInScope*(c: PContext, s: PIdent): PSym =
   result = strTableGet(c.currentScope.symbols, s)
@@ -150,7 +150,7 @@ type
 
 proc getSymRepr*(conf: ConfigRef; s: PSym): string =
   case s.kind
-  of skProc, skFunc, skMethod, skConverter, skIterator:
+  of routineKinds, skType:
     result = getProcHeader(conf, s)
   else:
     result = s.name.s
@@ -190,7 +190,7 @@ proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) =
     wrongRedefinition(c, info, sym.name.s, conflict.info)
 
 proc addDecl*(c: PContext, sym: PSym) =
-  let conflict = c.currentScope.addUniqueSym(sym)
+  let conflict = strTableInclReportConflict(c.currentScope.symbols, sym, true)
   if conflict != nil:
     wrongRedefinition(c, sym.info, sym.name.s, conflict.info)
 

compiler/main.nim

@@ -92,6 +92,7 @@ proc commandJsonScript(graph: ModuleGraph) =
 
 when not defined(leanCompiler):
   proc commandCompileToJS(graph: ModuleGraph) =
+    let conf = graph.config
     #incl(gGlobalOptions, optSafeCode)
     setTarget(graph.config.target, osJS, cpuJS)
     #initDefines()
@@ -100,6 +101,8 @@ when not defined(leanCompiler):
     semanticPasses(graph)
     registerPass(graph, JSgenPass)
     compileProject(graph)
+    if optGenScript in graph.config.globalOptions:
+      writeDepsFile(graph, toGeneratedFile(conf, conf.projectFull, ""))
 
 proc interactivePasses(graph: ModuleGraph) =
   initDefines(graph.config.symbols)

compiler/modulepaths.nim

@@ -114,7 +114,6 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
     try:
       result =
         pathSubs(conf, n.strVal, toFullPath(conf, n.info).splitFile().dir)
-          .replace(" ")
     except ValueError:
       localError(conf, n.info, "invalid path: " & n.strVal)
       result = n.strVal
@@ -147,7 +146,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
     # hacky way to implement 'x / y /../ z':
     result = renderTree(n, {renderNoComments}).replace(" ")
   of nkDotExpr:
-    localError(conf, n.info, warnDeprecated, "using '.' instead of '/' in import paths")
+    localError(conf, n.info, warnDeprecated, "using '.' instead of '/' in import paths is deprecated")
     result = renderTree(n, {renderNoComments}).replace(".", "/")
   of nkImportAs:
     result = getModuleName(conf, n.sons[0])

compiler/msgs.nim

@@ -135,6 +135,10 @@ const
   WarningColor = fgYellow
   HintTitle    = "Hint: "
   HintColor    = fgGreen
+  # NOTE: currently line info line numbers start with 1,
+  # but column numbers start with 0, however most editors expect
+  # first column to be 1, so we need to +1 here
+  ColOffset*   = 1
 
 proc getInfoContextLen*(conf: ConfigRef): int = return conf.m.msgContext.len
 proc setInfoContextLen*(conf: ConfigRef; L: int) = setLen(conf.m.msgContext, L)
@@ -155,7 +159,10 @@ template toFilename*(conf: ConfigRef; fileIdx: FileIndex): string =
   if fileIdx.int32 < 0 or conf == nil:
     "???"
   else:
-    conf.m.fileInfos[fileIdx.int32].projPath.string
+    if optListFullPaths in conf.globalOptions:
+      conf.m.fileInfos[fileIdx.int32].fullPath.string
+    else:
+      conf.m.fileInfos[fileIdx.int32].projPath.string
 
 proc toFullPath*(conf: ConfigRef; fileIdx: FileIndex): string =
   if fileIdx.int32 < 0 or conf == nil: result = "???"
@@ -164,6 +171,7 @@ proc toFullPath*(conf: ConfigRef; fileIdx: FileIndex): string =
 proc setDirtyFile*(conf: ConfigRef; fileIdx: FileIndex; filename: AbsoluteFile) =
   assert fileIdx.int32 >= 0
   conf.m.fileInfos[fileIdx.int32].dirtyFile = filename
+  setLen conf.m.fileInfos[fileIdx.int32].lines, 0
 
 proc setHash*(conf: ConfigRef; fileIdx: FileIndex; hash: string) =
   assert fileIdx.int32 >= 0
@@ -187,16 +195,19 @@ template toFilename*(conf: ConfigRef; info: TLineInfo): string =
 template toFullPath*(conf: ConfigRef; info: TLineInfo): string =
   toFullPath(conf, info.fileIndex)
 
+template toFullPathConsiderDirty*(conf: ConfigRef; info: TLineInfo): string =
+  string toFullPathConsiderDirty(conf, info.fileIndex)
+
 proc toMsgFilename*(conf: ConfigRef; info: TLineInfo): string =
   if info.fileIndex.int32 < 0:
     result = "???"
     return
   let absPath = conf.m.fileInfos[info.fileIndex.int32].fullPath.string
-  let relPath = conf.m.fileInfos[info.fileIndex.int32].projPath.string
   if optListFullPaths in conf.globalOptions:
     result = absPath
   else:
-    result = if absPath.len < relPath.len: absPath else: relPath
+    let relPath = conf.m.fileInfos[info.fileIndex.int32].projPath.string
+    result = if relPath.count("..") > 2: absPath else: relPath
 
 proc toLinenumber*(info: TLineInfo): int {.inline.} =
   result = int info.line
@@ -208,7 +219,9 @@ proc toFileLine*(conf: ConfigRef; info: TLineInfo): string {.inline.} =
   result = toFilename(conf, info) & ":" & $info.line
 
 proc toFileLineCol*(conf: ConfigRef; info: TLineInfo): string {.inline.} =
-  result = toFilename(conf, info) & "(" & $info.line & ", " & $info.col & ")"
+  # consider calling `helpers.lineInfoToString` instead
+  result = toFilename(conf, info) & "(" & $info.line & ", " &
+    $(info.col + ColOffset) & ")"
 
 proc `$`*(conf: ConfigRef; info: TLineInfo): string = toFileLineCol(conf, info)
 
@@ -359,7 +372,7 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) =
         styledMsgWriteln(styleBright,
                          PosFormat % [toMsgFilename(conf, context.info),
                                       coordToStr(context.info.line.int),
-                                      coordToStr(context.info.col+1)],
+                                      coordToStr(context.info.col+ColOffset)],
                          resetStyle,
                          message)
     info = context.info
@@ -423,9 +436,9 @@ proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
 proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
   if i.fileIndex.int32 < 0: return ""
 
-  if not optPreserveOrigSource(conf) and conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
+  if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
     try:
-      for line in lines(toFullPath(conf, i)):
+      for line in lines(toFullPathConsiderDirty(conf, i)):
         addSourceLine conf, i.fileIndex, line.string
     except IOError:
       discard
@@ -446,7 +459,7 @@ proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): s
               of hintMin..hintMax: HintTitle
               else: ErrorTitle
   result = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int),
-                        coordToStr(info.col+1)] &
+                        coordToStr(info.col+ColOffset)] &
            title &
            getMessageStr(msg, arg)
 
@@ -483,11 +496,8 @@ proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
     color = HintColor
     if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)]
     inc(conf.hintCounter)
-  # NOTE: currently line info line numbers start with 1,
-  # but column numbers start with 0, however most editors expect
-  # first column to be 1, so we need to +1 here
   let x = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int),
-                       coordToStr(info.col+1)]
+                       coordToStr(info.col+ColOffset)]
   let s = getMessageStr(msg, arg)
 
   if not ignoreMsg:

compiler/options.nim

@@ -40,7 +40,8 @@ type                          # please make sure we have under 32 options
     optMemTracker,
     optHotCodeReloading,
     optLaxStrings,
-    optNilSeqs
+    optNilSeqs,
+    optOldAst
 
   TOptions* = set[TOption]
   TGlobalOption* = enum       # **keep binary compatible**
@@ -76,8 +77,9 @@ type                          # please make sure we have under 32 options
     optExcessiveStackTrace    # fully qualified module filenames
     optShowAllMismatches      # show all overloading resolution candidates
     optWholeProject           # for 'doc2': output any dependency
+    optDocInternal            # generate documentation for non-exported symbols
     optMixedMode              # true if some module triggered C++ codegen
-    optListFullPaths
+    optListFullPaths          # use full paths in toMsgFilename, toFilename
     optNoNimblePath
     optDynlibOverrideAll
 
@@ -253,84 +255,6 @@ template depConfigFields*(fn) {.dirty.} =
   fn(globalOptions)
   fn(selectedGC)
 
-proc mergeConfigs*(dest, src: ConfigRef; mergeSymbols: bool) =
-  template merge[T: enum](a, b: T) =
-    a = b
-  template merge[T](a, b: set[T]) =
-    a = a + b
-  template merge(a, b: int) =
-    inc a, b
-  template merge[T](a, b: seq[T]) =
-    for bb in b: a.add b
-  template merge(a, b: string) =
-    a = b
-  template merge[T: AbsoluteFile|AbsoluteDir](a, b: T) =
-    if a.isEmpty and not b.isEmpty: a = b
-
-  template merge[T](a, b: HashSet[T]) =
-    for bb in b: a.incl b
-  template merge(a, b: StringTableRef) =
-    for k, v in b: a[k] = v
-  template merge[T: object](a, b: T) =
-    a = b
-
-  template m(field) =
-    merge(dest.field, src.field)
-
-  m target
-  m options
-  m globalOptions
-  m cmd
-  m selectedGC
-  dest.verbosity = src.verbosity
-  m numberOfProcessors
-  m evalExpr
-  m symbolFiles
-  m cppDefines
-  m headerFile
-  m features
-  m arguments
-  m ideCmd
-  m cCompiler
-  m enableNotes
-  m disableNotes
-  m foreignPackageNotes
-  m notes
-  m errorCounter
-  m hintCounter
-  m warnCounter
-  m errorMax
-  m configVars
-  if mergeSymbols:
-    m symbols
-  m projectName
-  m projectPath
-  m projectFull
-  m searchPaths
-  m lazyPaths
-  m outFile
-  m prefixDir
-  m libpath
-  m nimcacheDir
-  m dllOverrides
-  m moduleOverrides
-  m command
-  m commandArgs
-  m implicitImports
-  m implicitIncludes
-  m docSeeSrcUrl
-  m cIncludes
-  m cLibs
-  m cLinkedLibs
-  m externalToLink
-  m linkOptionsCmd
-  m compileOptionsCmd
-  m linkOptions
-  m compileOptions
-  m ccompilerpath
-  m toCompile
-  m cppCustomNamespace
-
 const oldExperimentalFeatures* = {implicitDeref, dotOperators, callOperator, parallel}
 
 const
@@ -436,7 +360,7 @@ proc cppDefine*(c: ConfigRef; define: string) =
 
 proc isDefined*(conf: ConfigRef; symbol: string): bool =
   if conf.symbols.hasKey(symbol):
-    result = conf.symbols[symbol] != "false"
+    result = true
   elif cmpIgnoreStyle(symbol, CPU[conf.target.targetCPU].name) == 0:
     result = true
   elif cmpIgnoreStyle(symbol, platform.OS[conf.target.targetOS].name) == 0:
@@ -560,16 +484,7 @@ proc setDefaultLibpath*(conf: ConfigRef) =
       conf.libpath = AbsoluteDir parentNimLibPath
 
 proc canonicalizePath*(conf: ConfigRef; path: AbsoluteFile): AbsoluteFile =
-  # on Windows, 'expandFilename' calls getFullPathName which doesn't do
+  result = AbsoluteFile path.string.expandFilename
-  # case corrections, so we have to use this convoluted way of retrieving
-  # the true filename (see tests/modules and Nimble uses 'import Uri' instead
-  # of 'import uri'):
-  when defined(windows):
-    result = AbsoluteFile path.string.expandFilename
-    for x in walkFiles(result.string):
-      return AbsoluteFile x
-  else:
-    result = AbsoluteFile path.string.expandFilename
 
 proc shortenDir*(conf: ConfigRef; dir: string): string {.
     deprecated: "use 'relativeTo' instead".} =

compiler/parampatterns.nim

@@ -115,20 +115,19 @@ proc compileConstraints(p: PNode, result: var TPatternCode; conf: ConfigRef) =
   else:
     patternError(p, conf)
 
-proc semNodeKindConstraints*(p: PNode; conf: ConfigRef): PNode =
+proc semNodeKindConstraints*(n: PNode; conf: ConfigRef; start: Natural): PNode =
   ## does semantic checking for a node kind pattern and compiles it into an
   ## efficient internal format.
-  assert p.kind == nkCurlyExpr
+  result = newNodeI(nkStrLit, n.info)
-  result = newNodeI(nkStrLit, p.info)
   result.strVal = newStringOfCap(10)
   result.strVal.add(chr(aqNone.ord))
-  if p.len >= 2:
+  if n.len >= 2:
-    for i in 1..<p.len:
+    for i in start..<n.len:
-      compileConstraints(p.sons[i], result.strVal, conf)
+      compileConstraints(n[i], result.strVal, conf)
     if result.strVal.len > MaxStackSize-1:
-      internalError(conf, p.info, "parameter pattern too complex")
+      internalError(conf, n.info, "parameter pattern too complex")
   else:
-    patternError(p, conf)
+    patternError(n, conf)
   result.strVal.add(ppEof)
 
 type

compiler/parser.nim

@@ -332,12 +332,15 @@ proc parseSymbol(p: var TParser, mode = smNormal): PNode =
           parMessage(p, errIdentifierExpected, p.tok)
         break
       of tkOpr, tkDot, tkDotDot, tkEquals, tkParLe..tkParDotRi:
+        let lineinfo = parLineinfo(p)
         var accm = ""
         while p.tok.tokType in {tkOpr, tkDot, tkDotDot, tkEquals,
                                 tkParLe..tkParDotRi}:
           accm.add(tokToStr(p.tok))
           getTok(p)
-        result.add(newIdentNodeP(p.lex.cache.getIdent(accm), p))
+        let node = newNodeI(nkIdent, lineinfo)
+        node.ident = p.lex.cache.getIdent(accm)
+        result.add(node)
       of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCharLit:
         result.add(newIdentNodeP(p.lex.cache.getIdent(tokToStr(p.tok)), p))
         getTok(p)
@@ -721,6 +724,14 @@ const
   tkTypeClasses = {tkRef, tkPtr, tkVar, tkStatic, tkType,
                    tkEnum, tkTuple, tkObject, tkProc}
 
+proc commandExpr(p: var TParser; r: PNode; mode: TPrimaryMode): PNode =
+  result = newNodeP(nkCommand, p)
+  addSon(result, r)
+  var isFirstParam = true
+  # progress NOT guaranteed
+  p.hasProgress = false
+  addSon result, commandParam(p, isFirstParam, mode)
+
 proc primarySuffix(p: var TParser, r: PNode,
                    baseIndent: int, mode: TPrimaryMode): PNode =
   #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks?
@@ -731,8 +742,6 @@ proc primarySuffix(p: var TParser, r: PNode,
   #|       | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax
   result = r
 
-  template somePar() =
-    if p.tok.strongSpaceA > 0: break
   # progress guaranteed
   while p.tok.indent < 0 or
        (p.tok.tokType == tkDot and p.tok.indent >= baseIndent):
@@ -746,6 +755,8 @@ proc primarySuffix(p: var TParser, r: PNode,
           result = newNodeP(nkCommand, p)
           result.addSon r
           result.addSon primary(p, pmNormal)
+        else:
+          result = commandExpr(p, result, mode)
         break
       result = namedParams(p, result, nkCall, tkParRi)
       if result.len > 1 and result.sons[1].kind == nkExprColonExpr:
@@ -756,39 +767,27 @@ proc primarySuffix(p: var TParser, r: PNode,
       result = parseGStrLit(p, result)
     of tkBracketLe:
       # progress guaranteed
-      somePar()
+      if p.tok.strongSpaceA > 0:
+        result = commandExpr(p, result, mode)
+        break
       result = namedParams(p, result, nkBracketExpr, tkBracketRi)
     of tkCurlyLe:
       # progress guaranteed
-      somePar()
+      if p.tok.strongSpaceA > 0:
+        result = commandExpr(p, result, mode)
+        break
       result = namedParams(p, result, nkCurlyExpr, tkCurlyRi)
     of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast,
        tkOpr, tkDotDot, tkTypeClasses - {tkRef, tkPtr}:
-        # XXX: In type sections we allow the free application of the
+      # XXX: In type sections we allow the free application of the
-        # command syntax, with the exception of expressions such as
+      # command syntax, with the exception of expressions such as
-        # `foo ref` or `foo ptr`. Unfortunately, these two are also
+      # `foo ref` or `foo ptr`. Unfortunately, these two are also
-        # used as infix operators for the memory regions feature and
+      # used as infix operators for the memory regions feature and
-        # the current parsing rules don't play well here.
+      # the current parsing rules don't play well here.
       if p.inPragma == 0 and (isUnary(p) or p.tok.tokType notin {tkOpr, tkDotDot}):
         # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet
         # solution, but pragmas.nim can't handle that
-        let a = result
+        result = commandExpr(p, result, mode)
-        result = newNodeP(nkCommand, p)
-        addSon(result, a)
-        var isFirstParam = true
-        when true:
-          # progress NOT guaranteed
-          p.hasProgress = false
-          addSon result, commandParam(p, isFirstParam, mode)
-          if not p.hasProgress: break
-        else:
-          while p.tok.tokType != tkEof:
-            let x = parseExpr(p)
-            addSon(result, x)
-            if p.tok.tokType != tkComma: break
-            getTok(p)
-            optInd(p, x)
-          result = postExprBlocks(p, result)
       break
     else:
       break
@@ -1916,7 +1915,7 @@ proc parseObject(p: var TParser): PNode =
   getTok(p)
   if p.tok.tokType == tkCurlyDotLe and p.validInd:
     # Deprecated since v0.20.0
-    parMessage(p, warnDeprecated, "type pragmas follow the type name; this form of writing pragmas")
+    parMessage(p, warnDeprecated, "type pragmas follow the type name; this form of writing pragmas is deprecated")
     addSon(result, parsePragma(p))
   else:
     addSon(result, p.emptyNode)
@@ -2008,7 +2007,7 @@ proc parseTypeDef(p: var TParser): PNode =
   if p.tok.tokType == tkBracketLe and p.validInd:
     if not noPragmaYet:
       # Deprecated since v0.20.0
-      parMessage(p, warnDeprecated, "pragma before generic parameter list")
+      parMessage(p, warnDeprecated, "pragma before generic parameter list is deprecated")
     genericParam = parseGenericParamList(p)
   else:
     genericParam = p.emptyNode

compiler/pragmas.nim

@@ -64,10 +64,10 @@ const
   varPragmas* = {wImportc, wExportc, wVolatile, wRegister, wThreadVar, wNodecl,
     wMagic, wHeader, wDeprecated, wCompilerProc, wCore, wDynlib, wExtern,
     wImportCpp, wImportObjC, wError, wNoInit, wCompileTime, wGlobal,
-    wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims, wUsed, wRaises}
+    wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims, wUsed}
   constPragmas* = {wImportc, wExportc, wHeader, wDeprecated, wMagic, wNodecl,
     wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject, wExportNims,
-    wIntDefine, wStrDefine, wUsed, wCompilerProc, wCore, wRaises}
+    wIntDefine, wStrDefine, wBoolDefine, wUsed, wCompilerProc, wCore}
   letPragmas* = varPragmas
   procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNosideeffect,
                       wThread, wRaises, wLocks, wTags, wGcSafe}
@@ -242,7 +242,7 @@ proc pragmaNoForward(c: PContext, n: PNode; flag=sfNoForward) =
   # deprecated as of 0.18.1
   message(c.config, n.info, warnDeprecated,
           "use {.experimental: \"codeReordering.\".} instead; " &
-          (if flag == sfNoForward: "{.noForward.}" else: "{.reorder.}"))
+          (if flag == sfNoForward: "{.noForward.}" else: "{.reorder.}") & " is deprecated")
 
 proc processCallConv(c: PContext, n: PNode) =
   if n.kind in nkPragmaCallKinds and n.len == 2 and n.sons[1].kind == nkIdent:
@@ -447,14 +447,14 @@ proc processPop(c: PContext, n: PNode) =
 proc processDefine(c: PContext, n: PNode) =
   if (n.kind in nkPragmaCallKinds and n.len == 2) and (n[1].kind == nkIdent):
     defineSymbol(c.config.symbols, n[1].ident.s)
-    message(c.config, n.info, warnDeprecated, "define")
+    message(c.config, n.info, warnDeprecated, "define is deprecated")
   else:
     invalidPragma(c, n)
 
 proc processUndef(c: PContext, n: PNode) =
   if (n.kind in nkPragmaCallKinds and n.len == 2) and (n[1].kind == nkIdent):
     undefSymbol(c.config.symbols, n[1].ident.s)
-    message(c.config, n.info, warnDeprecated, "undef")
+    message(c.config, n.info, warnDeprecated, "undef is deprecated")
   else:
     invalidPragma(c, n)
 
@@ -784,7 +784,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         if sym.kind in {skTemplate, skMacro}:
           incl(sym.flags, sfImmediate)
           incl(sym.flags, sfAllUntyped)
-          message(c.config, n.info, warnDeprecated, "use 'untyped' parameters instead; immediate")
+          message(c.config, n.info, warnDeprecated, "use 'untyped' parameters instead; immediate is deprecated")
         else: invalidPragma(c, it)
       of wDirty:
         if sym.kind == skTemplate: incl(sym.flags, sfDirty)
@@ -883,7 +883,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wExplain:
         sym.flags.incl sfExplain
       of wDeprecated:
-        if sym != nil and sym.kind in routineKinds + {skType}:
+        if sym != nil and sym.kind in routineKinds + {skType, skVar, skLet}:
           if it.kind in nkPragmaCallKinds: discard getStrLitNode(c, it)
           incl(sym.flags, sfDeprecated)
         elif sym != nil and sym.kind != skModule:
@@ -1056,14 +1056,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         noVal(c, it)
         if sym == nil: invalidPragma(c, it)
       of wLine: pragmaLine(c, it)
-      of wRaises, wTags:
+      of wRaises, wTags: pragmaRaisesOrTags(c, it)
-        if not sym.isNil and sym.kind in {skVar, skLet, skConst}:
-          if comesFromPush:
-            return
-          else:
-            invalidPragma(c, it)
-        else:
-          pragmaRaisesOrTags(c, it)
       of wLocks:
         if sym == nil: pragmaLockStmt(c, it)
         elif sym.typ == nil: invalidPragma(c, it)
@@ -1100,10 +1093,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wThis:
         if it.kind in nkPragmaCallKinds and it.len == 2:
           c.selfName = considerQuotedIdent(c, it[1])
-          message(c.config, n.info, warnDeprecated, "the '.this' pragma")
+          message(c.config, n.info, warnDeprecated, "the '.this' pragma is deprecated")
         elif it.kind == nkIdent or it.len == 1:
           c.selfName = getIdent(c.cache, "self")
-          message(c.config, n.info, warnDeprecated, "the '.this' pragma")
+          message(c.config, n.info, warnDeprecated, "the '.this' pragma is deprecated")
         else:
           localError(c.config, it.info, "'this' pragma is allowed to have zero or one arguments")
       of wNoRewrite:
@@ -1115,21 +1108,33 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         sym.magic = mIntDefine
       of wStrDefine:
         sym.magic = mStrDefine
+      of wBoolDefine:
+        sym.magic = mBoolDefine
       of wUsed:
         noVal(c, it)
         if sym == nil: invalidPragma(c, it)
         else: sym.flags.incl sfUsed
       of wLiftLocals: discard
       else: invalidPragma(c, it)
-    elif sym == nil or (sym != nil and sym.kind in {skVar, skLet, skParam,
+    elif comesFromPush and whichKeyword(ident) in {wTags, wRaises}:
-                      skField, skProc, skFunc, skConverter, skMethod, skType}):
+      discard "ignore the .push pragma; it doesn't apply"
-      n.sons[i] = semCustomPragma(c, it)
-    elif sym != nil:
-      illegalCustomPragma(c, it, sym)
     else:
-      invalidPragma(c, it)
+      if sym == nil or (sym != nil and sym.kind in {skVar, skLet, skParam,
+                        skField, skProc, skFunc, skConverter, skMethod, skType}):
+        n.sons[i] = semCustomPragma(c, it)
+      elif sym != nil:
+        illegalCustomPragma(c, it, sym)
+      else:
+        invalidPragma(c, it)
+
+proc overwriteLineInfo(n: PNode; info: TLineInfo) =
+  n.info = info
+  for i in 0..<safeLen(n):
+    overwriteLineInfo(n[i], info)
 
 proc mergePragmas(n, pragmas: PNode) =
+  var pragmas = copyTree(pragmas)
+  overwriteLineInfo pragmas, n.info
   if n[pragmasPos].kind == nkEmpty:
     n[pragmasPos] = pragmas
   else:

compiler/scriptconfig.nim

@@ -42,15 +42,18 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string;
       proc (a: VmArgs) =
         body
 
-  template cbos(name, body) {.dirty.} =
+  template cbexc(name, exc, body) {.dirty.} =
     result.registerCallback "stdlib.system." & astToStr(name),
       proc (a: VmArgs) =
         errorMsg = ""
         try:
           body
-        except OSError:
+        except exc:
           errorMsg = getCurrentExceptionMsg()
 
+  template cbos(name, body) {.dirty.} =
+    cbexc(name, OSError, body)
+
   # Idea: Treat link to file as a file, but ignore link to directory to prevent
   # endless recursions out of the box.
   cbos listFiles:
@@ -63,8 +66,10 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string;
     os.removeFile getString(a, 0)
   cbos createDir:
     os.createDir getString(a, 0)
-  cbos getOsError:
+
-    setResult(a, errorMsg)
+  result.registerCallback "stdlib.system.getError",
+    proc (a: VmArgs) = setResult(a, errorMsg)
+
   cbos setCurrentDir:
     os.setCurrentDir getString(a, 0)
   cbos getCurrentDir:
@@ -155,6 +160,12 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string;
     setResult(a, os.getAppFilename())
   cbconf cppDefine:
     options.cppDefine(conf, a.getString(0))
+  cbexc stdinReadLine, EOFError:
+    setResult(a, "")
+    setResult(a, stdin.readLine())
+  cbexc stdinReadAll, EOFError:
+    setResult(a, "")
+    setResult(a, stdin.readAll())
 
 proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile;
                    freshDefines=true; conf: ConfigRef) =

compiler/sem.nim

@@ -450,17 +450,18 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
                   flags: TExprFlags = {}): PNode =
   pushInfoContext(c.config, nOrig.info, sym.detailedInfo)
 
-  markUsed(c.config, n.info, sym, c.graph.usageSym)
+  let info = getCallLineInfo(n)
-  onUse(n.info, sym)
+  markUsed(c.config, info, sym, c.graph.usageSym)
+  onUse(info, sym)
   if sym == c.p.owner:
-    globalError(c.config, n.info, "recursive dependency: '$1'" % sym.name.s)
+    globalError(c.config, info, "recursive dependency: '$1'" % sym.name.s)
 
   let genericParams = if sfImmediate in sym.flags: 0
                       else: sym.ast[genericParamsPos].len
   let suppliedParams = max(n.safeLen - 1, 0)
 
   if suppliedParams < genericParams:
-    globalError(c.config, n.info, errMissingGenericParamsForTemplate % n.renderTree)
+    globalError(c.config, info, errMissingGenericParamsForTemplate % n.renderTree)
 
   #if c.evalContext == nil:
   #  c.evalContext = c.createEvalContext(emStatic)
@@ -556,8 +557,6 @@ proc isEmptyTree(n: PNode): bool =
   else: result = false
 
 proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =
-  if n.kind == nkDefer:
-    localError(c.config, n.info, "defer statement not supported at top level")
   if c.topStmts == 0 and not isImportSystemStmt(c.graph, n):
     if sfSystemModule notin c.module.flags and not isEmptyTree(n):
       c.importTable.addSym c.graph.systemModule # import the "System" identifier

compiler/semasgn.nim

@@ -14,7 +14,7 @@
 
 type
   TLiftCtx = object
-    c: PContext
+    graph: ModuleGraph
     info: TLineInfo # for construction
     kind: TTypeAttachedOp
     fn: PSym
@@ -22,7 +22,7 @@ type
     recurse: bool
 
 proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode)
-proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp;
+proc liftBody(g: ModuleGraph; typ: PType; kind: TTypeAttachedOp;
               info: TLineInfo): PSym {.discardable.}
 
 proc at(a, i: PNode, elemType: PType): PNode =
@@ -33,7 +33,7 @@ proc at(a, i: PNode, elemType: PType): PNode =
 
 proc liftBodyTup(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   for i in 0 ..< t.len:
-    let lit = lowerings.newIntLit(c.c.graph, x.info, i)
+    let lit = lowerings.newIntLit(c.graph, x.info, i)
     liftBodyAux(c, t.sons[i], body, x.at(lit, t.sons[i]), y.at(lit, t.sons[i]))
 
 proc dotField(x: PNode, f: PSym): PNode =
@@ -49,6 +49,14 @@ proc liftBodyObj(c: var TLiftCtx; n, body, x, y: PNode) =
     liftBodyAux(c, f.typ, body, x.dotField(f), y.dotField(f))
   of nkNilLit: discard
   of nkRecCase:
+    if c.kind in {attachedSink, attachedAsgn, attachedDeepCopy}:
+      ## the value needs to be destroyed before we assign the selector
+      ## or the value is lost
+      let prevKind = c.kind
+      c.kind = attachedDestructor
+      liftBodyObj(c, n, body, x, y)
+      c.kind = prevKind
+
     # copy the selector:
     liftBodyObj(c, n[0], body, x, y)
     # we need to generate a case statement:
@@ -66,26 +74,25 @@ proc liftBodyObj(c: var TLiftCtx; n, body, x, y: PNode) =
       liftBodyObj(c, n[i].lastSon, branch.sons[L-1], x, y)
       caseStmt.add(branch)
     body.add(caseStmt)
-    localError(c.c.config, c.info, "cannot lift assignment operator to 'case' object")
   of nkRecList:
     for t in items(n): liftBodyObj(c, t, body, x, y)
   else:
-    illFormedAstLocal(n, c.c.config)
+    illFormedAstLocal(n, c.graph.config)
 
-proc genAddr(c: PContext; x: PNode): PNode =
+proc genAddr(g: ModuleGraph; x: PNode): PNode =
   if x.kind == nkHiddenDeref:
-    checkSonsLen(x, 1, c.config)
+    checkSonsLen(x, 1, g.config)
     result = x.sons[0]
   else:
-    result = newNodeIT(nkHiddenAddr, x.info, makeVarType(c, x.typ))
+    result = newNodeIT(nkHiddenAddr, x.info, makeVarType(x.typ.owner, x.typ))
     addSon(result, x)
 
-proc newAsgnCall(c: PContext; op: PSym; x, y: PNode): PNode =
+proc newAsgnCall(g: ModuleGraph; op: PSym; x, y: PNode): PNode =
   #if sfError in op.flags:
   #  localError(c.config, x.info, "usage of '$1' is a user-defined error" % op.name.s)
   result = newNodeI(nkCall, x.info)
   result.add newSymNode(op)
-  result.add genAddr(c, x)
+  result.add genAddr(g, x)
   result.add y
 
 proc newAsgnStmt(le, ri: PNode): PNode =
@@ -98,10 +105,10 @@ proc newOpCall(op: PSym; x: PNode): PNode =
   result.add(newSymNode(op))
   result.add x
 
-proc destructorCall(c: PContext; op: PSym; x: PNode): PNode =
+proc destructorCall(g: ModuleGraph; op: PSym; x: PNode): PNode =
   result = newNodeIT(nkCall, x.info, op.typ.sons[0])
   result.add(newSymNode(op))
-  result.add genAddr(c, x)
+  result.add genAddr(g, x)
 
 proc newDeepCopyCall(op: PSym; x, y: PNode): PNode =
   result = newAsgnStmt(x, newOpCall(op, y))
@@ -120,13 +127,13 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
     else:
       op = field
       if op == nil:
-        op = liftBody(c.c, t, c.kind, c.info)
+        op = liftBody(c.graph, t, c.kind, c.info)
     if sfError in op.flags:
       incl c.fn.flags, sfError
     else:
-      markUsed(c.c.config, c.info, op, c.c.graph.usageSym)
+      markUsed(c.graph.config, c.info, op, c.graph.usageSym)
     onUse(c.info, op)
-    body.add newAsgnCall(c.c, op, x, y)
+    body.add newAsgnCall(c.graph, op, x, y)
     result = true
 
 proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
@@ -134,9 +141,9 @@ proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
   of attachedDestructor:
     let op = t.destructor
     if op != nil:
-      markUsed(c.c.config, c.info, op, c.c.graph.usageSym)
+      markUsed(c.graph.config, c.info, op, c.graph.usageSym)
       onUse(c.info, op)
-      body.add destructorCall(c.c, op, x)
+      body.add destructorCall(c.graph, op, x)
       result = true
   of attachedAsgn:
     result = considerAsgnOrSink(c, t, body, x, y, t.assignment)
@@ -145,7 +152,7 @@ proc considerOverloadedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
   of attachedDeepCopy:
     let op = t.deepCopy
     if op != nil:
-      markUsed(c.c.config, c.info, op, c.c.graph.usageSym)
+      markUsed(c.graph.config, c.info, op, c.graph.usageSym)
       onUse(c.info, op)
       body.add newDeepCopyCall(op, x, y)
       result = true
@@ -162,13 +169,13 @@ proc addVar(father, v, value: PNode) =
   addSon(father, vpart)
 
 proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
-  var temp = newSym(skTemp, getIdent(c.c.cache, lowerings.genPrefix), c.fn, c.info)
+  var temp = newSym(skTemp, getIdent(c.graph.cache, lowerings.genPrefix), c.fn, c.info)
-  temp.typ = getSysType(c.c.graph, body.info, tyInt)
+  temp.typ = getSysType(c.graph, body.info, tyInt)
   incl(temp.flags, sfFromGeneric)
 
   var v = newNodeI(nkVarSection, c.info)
   result = newSymNode(temp)
-  v.addVar(result, lowerings.newIntLit(c.c.graph, body.info, first))
+  v.addVar(result, lowerings.newIntLit(c.graph, body.info, first))
   body.add v
 
 proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode =
@@ -178,22 +185,22 @@ proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode =
 
 proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
   result = newNodeI(nkWhileStmt, c.info, 2)
-  let cmp = genBuiltin(c.c.graph, mLeI, "<=", i)
+  let cmp = genBuiltin(c.graph, mLeI, "<=", i)
-  cmp.add genHigh(c.c.graph, dest)
+  cmp.add genHigh(c.graph, dest)
-  cmp.typ = getSysType(c.c.graph, c.info, tyBool)
+  cmp.typ = getSysType(c.graph, c.info, tyBool)
   result.sons[0] = cmp
   result.sons[1] = newNodeI(nkStmtList, c.info)
 
 proc addIncStmt(c: var TLiftCtx; body, i: PNode) =
-  let incCall = genBuiltin(c.c.graph, mInc, "inc", i)
+  let incCall = genBuiltin(c.graph, mInc, "inc", i)
-  incCall.add lowerings.newIntLit(c.c.graph, c.info, 1)
+  incCall.add lowerings.newIntLit(c.graph, c.info, 1)
   body.add incCall
 
-proc newSeqCall(c: PContext; x, y: PNode): PNode =
+proc newSeqCall(g: ModuleGraph; x, y: PNode): PNode =
   # don't call genAddr(c, x) here:
-  result = genBuiltin(c.graph, mNewSeq, "newSeq", x)
+  result = genBuiltin(g, mNewSeq, "newSeq", x)
-  let lenCall = genBuiltin(c.graph, mLengthSeq, "len", y)
+  let lenCall = genBuiltin(g, mLengthSeq, "len", y)
-  lenCall.typ = getSysType(c.graph, x.info, tyInt)
+  lenCall.typ = getSysType(g, x.info, tyInt)
   result.add lenCall
 
 proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) =
@@ -204,7 +211,7 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     defaultOp(c, t, body, x, y)
   of tyArray:
     if tfHasAsgn in t.flags:
-      let i = declareCounter(c, body, firstOrd(c.c.config, t))
+      let i = declareCounter(c, body, firstOrd(c.graph.config, t))
       let whileLoop = genWhileLoop(c, i, x)
       let elemType = t.lastSon
       liftBodyAux(c, elemType, whileLoop.sons[1], x.at(i, elemType),
@@ -216,12 +223,12 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of tySequence:
     # note that tfHasAsgn is propagated so we need the check on
     # 'selectedGC' here to determine if we have the new runtime.
-    if c.c.config.selectedGC == gcDestructors:
+    if c.graph.config.selectedGC == gcDestructors:
       discard considerOverloadedOp(c, t, body, x, y)
     elif tfHasAsgn in t.flags:
       if c.kind != attachedDestructor:
-        body.add newSeqCall(c.c, x, y)
+        body.add newSeqCall(c.graph, x, y)
-      let i = declareCounter(c, body, firstOrd(c.c.config, t))
+      let i = declareCounter(c, body, firstOrd(c.graph.config, t))
       let whileLoop = genWhileLoop(c, i, x)
       let elemType = t.lastSon
       liftBodyAux(c, elemType, whileLoop.sons[1], x.at(i, elemType),
@@ -235,11 +242,12 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       discard considerOverloadedOp(c, t, body, x, y)
     else:
       defaultOp(c, t, body, x, y)
-  of tyObject, tyDistinct:
+  of tyObject:
     if not considerOverloadedOp(c, t, body, x, y):
-      if t.sons[0] != nil:
+      liftBodyObj(c, t.n, body, x, y)
-        liftBodyAux(c, t.sons[0].skipTypes(skipPtrs), body, x, y)
+  of tyDistinct:
-      if t.kind == tyObject: liftBodyObj(c, t.n, body, x, y)
+    if not considerOverloadedOp(c, t, body, x, y):
+      liftBodyAux(c, t.sons[0].skipTypes(skipPtrs), body, x, y)
   of tyTuple:
     liftBodyTup(c, t, body, x, y)
   of tyProc:
@@ -250,20 +258,20 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       # have to go through some indirection; we delegate this to the codegen:
       let call = newNodeI(nkCall, c.info, 2)
       call.typ = t
-      call.sons[0] = newSymNode(createMagic(c.c.graph, "deepCopy", mDeepCopy))
+      call.sons[0] = newSymNode(createMagic(c.graph, "deepCopy", mDeepCopy))
       call.sons[1] = y
       body.add newAsgnStmt(x, call)
   of tyVarargs, tyOpenArray:
-    localError(c.c.config, c.info, "cannot copy openArray")
+    localError(c.graph.config, c.info, "cannot copy openArray")
   of tyFromExpr, tyProxy, tyBuiltInTypeClass, tyUserTypeClass,
      tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, tyAnything,
      tyGenericParam, tyGenericBody, tyNil, tyExpr, tyStmt,
      tyTypeDesc, tyGenericInvocation, tyForward:
-    internalError(c.c.config, c.info, "assignment requested for type: " & typeToString(t))
+    internalError(c.graph.config, c.info, "assignment requested for type: " & typeToString(t))
   of tyOrdinal, tyRange, tyInferred,
      tyGenericInst, tyStatic, tyVar, tyLent, tyAlias, tySink:
     liftBodyAux(c, lastSon(t), body, x, y)
-  of tyOptAsRef: internalError(c.c.config, "liftBodyAux")
+  of tyOptAsRef: internalError(c.graph.config, "liftBodyAux")
 
 proc newProcType(info: TLineInfo; owner: PSym): PType =
   result = newType(tyProc, owner)
@@ -279,26 +287,59 @@ proc addParam(procType: PType; param: PSym) =
   addSon(procType.n, newSymNode(param))
   rawAddSon(procType, param.typ)
 
-proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp;
+proc liftBodyDistinctType(g: ModuleGraph; typ: PType; kind: TTypeAttachedOp; info: TLineInfo): PSym =
+  assert typ.kind == tyDistinct
+  let baseType = typ[0]
+  case kind
+    of attachedAsgn:
+      if baseType.assignment == nil:
+        discard liftBody(g, baseType, kind, info)
+      typ.assignment = baseType.assignment
+      result = typ.assignment
+    of attachedSink:
+      if baseType.sink == nil:
+        discard liftBody(g, baseType, kind, info)
+      typ.sink = baseType.sink
+      result = typ.sink
+    of attachedDeepCopy:
+      if baseType.deepCopy == nil:
+        discard liftBody(g, baseType, kind, info)
+      typ.deepCopy = baseType.deepCopy
+      result = typ.deepCopy
+    of attachedDestructor:
+      if baseType.destructor == nil:
+        discard liftBody(g, baseType, kind, info)
+      typ.destructor = baseType.destructor
+      result = typ.destructor
+
+proc liftBody(g: ModuleGraph; typ: PType; kind: TTypeAttachedOp;
               info: TLineInfo): PSym =
+  if typ.kind == tyDistinct:
+    return liftBodyDistinctType(g, typ, kind, info)
+  when false:
+    var typ = typ
+    if c.config.selectedGC == gcDestructors and typ.kind == tySequence:
+      # use the canonical type to access the =sink and =destroy etc.
+      typ = c.graph.sysTypes[tySequence]
+
   var a: TLiftCtx
   a.info = info
-  a.c = c
+  a.graph = g
   a.kind = kind
   let body = newNodeI(nkStmtList, info)
   let procname = case kind
-                 of attachedAsgn: getIdent(c.cache, "=")
+                 of attachedAsgn: getIdent(g.cache, "=")
-                 of attachedSink: getIdent(c.cache, "=sink")
+                 of attachedSink: getIdent(g.cache, "=sink")
-                 of attachedDeepCopy: getIdent(c.cache, "=deepcopy")
+                 of attachedDeepCopy: getIdent(g.cache, "=deepcopy")
-                 of attachedDestructor: getIdent(c.cache, "=destroy")
+                 of attachedDestructor: getIdent(g.cache, "=destroy")
 
   result = newSym(skProc, procname, typ.owner, info)
   a.fn = result
   a.asgnForType = typ
 
-  let dest = newSym(skParam, getIdent(c.cache, "dest"), result, info)
+  let dest = newSym(skParam, getIdent(g.cache, "dest"), result, info)
-  let src = newSym(skParam, getIdent(c.cache, "src"), result, info)
+  let src = newSym(skParam, getIdent(g.cache, "src"), result, info)
-  dest.typ = makeVarType(c, typ)
+  dest.typ = makeVarType(typ.owner, typ)
   src.typ = typ
 
   result.typ = newProcType(info, typ.owner)
@@ -309,7 +350,7 @@ proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp;
   liftBodyAux(a, typ, body, newSymNode(dest).newDeref, newSymNode(src))
   # recursion is handled explicitly, do not register the type based operation
   # before 'liftBodyAux':
-  if c.config.selectedGC == gcDestructors and
+  if g.config.selectedGC == gcDestructors and
       typ.kind in {tySequence, tyString} and body.len == 0:
     discard "do not cache it yet"
   else:
@@ -328,17 +369,17 @@ proc liftBody(c: PContext; typ: PType; kind: TTypeAttachedOp;
   incl result.flags, sfFromGeneric
 
 
-proc getAsgnOrLiftBody(c: PContext; typ: PType; info: TLineInfo): PSym =
+proc getAsgnOrLiftBody(g: ModuleGraph; typ: PType; info: TLineInfo): PSym =
   let t = typ.skipTypes({tyGenericInst, tyVar, tyLent, tyAlias, tySink})
   result = t.assignment
   if result.isNil:
-    result = liftBody(c, t, attachedAsgn, info)
+    result = liftBody(g, t, attachedAsgn, info)
 
-proc overloadedAsgn(c: PContext; dest, src: PNode): PNode =
+proc overloadedAsgn(g: ModuleGraph; dest, src: PNode): PNode =
-  let a = getAsgnOrLiftBody(c, dest.typ, dest.info)
+  let a = getAsgnOrLiftBody(g, dest.typ, dest.info)
-  result = newAsgnCall(c, a, dest, src)
+  result = newAsgnCall(g, a, dest, src)
 
-proc liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) =
+proc liftTypeBoundOps*(g: ModuleGraph; typ: PType; info: TLineInfo) =
   ## In the semantic pass this is called in strategic places
   ## to ensure we lift assignment, destructors and moves properly.
   ## The later 'destroyer' pass depends on it.
@@ -350,11 +391,11 @@ proc liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) =
   let typ = typ.skipTypes({tyGenericInst, tyAlias})
   # we generate the destructor first so that other operators can depend on it:
   if typ.destructor == nil:
-    liftBody(c, typ, attachedDestructor, info)
+    liftBody(g, typ, attachedDestructor, info)
   if typ.assignment == nil:
-    liftBody(c, typ, attachedAsgn, info)
+    liftBody(g, typ, attachedAsgn, info)
   if typ.sink == nil:
-    liftBody(c, typ, attachedSink, info)
+    liftBody(g, typ, attachedSink, info)
 
-#proc patchResolvedTypeBoundOp*(c: PContext; n: PNode): PNode =
+#proc patchResolvedTypeBoundOp*(g: ModuleGraph; n: PNode): PNode =
 #  if n.kind == nkCall and

compiler/semcall.nim

@@ -221,7 +221,7 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
       candidates.add(diag & "\n")
   if skipped > 0:
     candidates.add($skipped & " other mismatching symbols have been " &
-        " suppressed; compile with --showAllMismatches:on to see them\n")
+        "suppressed; compile with --showAllMismatches:on to see them\n")
   result = (prefer, candidates)
 
 const
@@ -288,7 +288,18 @@ proc getMsgDiagnostic(c: PContext, flags: TExprFlags, n, f: PNode): string =
 
   let ident = considerQuotedIdent(c, f, n).s
   if nfDotField in n.flags and nfExplicitCall notin n.flags:
-    result = errUndeclaredField % ident & result
+    let sym = n.sons[1].typ.sym
+    var typeHint = ""
+    if sym == nil:
+      # Perhaps we're in a `compiles(foo.bar)` expression, or
+      # in a concept, eg:
+      #   ExplainedConcept {.explain.} = concept x
+      #     x.foo is int
+      # We coudl use: `(c.config $ n.sons[1].info)` to get more context.
+      discard
+    else:
+      typeHint = " for type " & getProcHeader(c.config, sym)
+    result = errUndeclaredField % ident & typeHint & " " & result
   else:
     if result.len == 0: result = errUndeclaredRoutine % ident
     else: result = errBadRoutine % [ident, result]
@@ -452,16 +463,23 @@ proc updateDefaultParams(call: PNode) =
       if nfDefaultRefsParam in def.flags: call.flags.incl nfDefaultRefsParam
       call[i] = def
 
+proc getCallLineInfo(n: PNode): TLineInfo =
+  case n.kind
+  of nkAccQuoted, nkBracketExpr, nkCall, nkCommand: getCallLineInfo(n.sons[0])
+  of nkDotExpr: getCallLineInfo(n.sons[1])
+  else: n.info
+
 proc semResolvedCall(c: PContext, x: TCandidate,
                      n: PNode, flags: TExprFlags): PNode =
   assert x.state == csMatch
   var finalCallee = x.calleeSym
-  markUsed(c.config, n.sons[0].info, finalCallee, c.graph.usageSym)
+  let info = getCallLineInfo(n)
-  onUse(n.sons[0].info, finalCallee)
+  markUsed(c.config, info, finalCallee, c.graph.usageSym)
+  onUse(info, finalCallee)
   assert finalCallee.ast != nil
   if x.hasFauxMatch:
     result = x.call
-    result.sons[0] = newSymNode(finalCallee, result.sons[0].info)
+    result.sons[0] = newSymNode(finalCallee, getCallLineInfo(result.sons[0]))
     if containsGenericType(result.typ) or x.fauxMatch == tyUnknown:
       result.typ = newTypeS(x.fauxMatch, c)
     return
@@ -486,7 +504,7 @@ proc semResolvedCall(c: PContext, x: TCandidate,
 
   result = x.call
   instGenericConvertersSons(c, result, x)
-  result[0] = newSymNode(finalCallee, result[0].info)
+  result[0] = newSymNode(finalCallee, getCallLineInfo(result[0]))
   result.typ = finalCallee.typ.sons[0]
   updateDefaultParams(result)
 
@@ -541,7 +559,7 @@ proc semOverloadedCall(c: PContext, n, nOrig: PNode,
       notFoundError(c, n, errors)
 
 proc explicitGenericInstError(c: PContext; n: PNode): PNode =
-  localError(c.config, n.info, errCannotInstantiateX % renderTree(n))
+  localError(c.config, getCallLineInfo(n), errCannotInstantiateX % renderTree(n))
   result = n
 
 proc explicitGenericSym(c: PContext, n: PNode, s: PSym): PNode =
@@ -564,9 +582,10 @@ proc explicitGenericSym(c: PContext, n: PNode, s: PSym): PNode =
     if tm in {isNone, isConvertible}: return nil
   var newInst = generateInstance(c, s, m.bindings, n.info)
   newInst.typ.flags.excl tfUnresolved
-  markUsed(c.config, n.info, s, c.graph.usageSym)
+  let info = getCallLineInfo(n)
-  onUse(n.info, s)
+  markUsed(c.config, info, s, c.graph.usageSym)
-  result = newSymNode(newInst, n.info)
+  onUse(info, s)
+  result = newSymNode(newInst, info)
 
 proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
   assert n.kind == nkBracketExpr
@@ -583,7 +602,7 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
     # number of generic type parameters:
     if safeLen(s.ast.sons[genericParamsPos]) != n.len-1:
       let expected = safeLen(s.ast.sons[genericParamsPos])
-      localError(c.config, n.info, errGenerated, "cannot instantiate: '" & renderTree(n) &
+      localError(c.config, getCallLineInfo(n), errGenerated, "cannot instantiate: '" & renderTree(n) &
          "'; got " & $(n.len-1) & " type(s) but expected " & $expected)
       return n
     result = explicitGenericSym(c, n, s)
@@ -592,7 +611,7 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
     # choose the generic proc with the proper number of type parameters.
     # XXX I think this could be improved by reusing sigmatch.paramTypesMatch.
     # It's good enough for now.
-    result = newNodeI(a.kind, n.info)
+    result = newNodeI(a.kind, getCallLineInfo(n))
     for i in countup(0, len(a)-1):
       var candidate = a.sons[i].sym
       if candidate.kind in {skProc, skMethod, skConverter,

compiler/semdata.nim

@@ -286,6 +286,13 @@ proc makeVarType*(c: PContext, baseType: PType; kind = tyVar): PType =
     result = newTypeS(kind, c)
     addSonSkipIntLit(result, baseType)
 
+proc makeVarType*(owner: PSym, baseType: PType; kind = tyVar): PType =
+  if baseType.kind == kind:
+    result = baseType
+  else:
+    result = newType(kind, owner)
+    addSonSkipIntLit(result, baseType)
+
 proc makeTypeDesc*(c: PContext, typ: PType): PType =
   if typ.kind == tyTypeDesc:
     result = typ

compiler/semexprs.nim

@@ -15,7 +15,7 @@ const
   errXExpectsTypeOrValue = "'$1' expects a type or value"
   errVarForOutParamNeededX = "for a 'var' type a variable needs to be passed; but '$1' is immutable"
   errXStackEscape = "address of '$1' may not escape its stack frame"
-  errExprHasNoAddress = "expression has no address; maybe use 'unsafeAddr'"
+  errExprHasNoAddress = "expression has no address"
   errCannotInterpretNodeX = "cannot evaluate '$1'"
   errNamedExprExpected = "named expression expected"
   errNamedExprNotAllowed = "named expression not allowed here"
@@ -24,25 +24,29 @@ const
 
 proc semTemplateExpr(c: PContext, n: PNode, s: PSym,
                      flags: TExprFlags = {}): PNode =
-  markUsed(c.config, n.info, s, c.graph.usageSym)
+  let info = getCallLineInfo(n)
-  onUse(n.info, s)
+  markUsed(c.config, info, s, c.graph.usageSym)
+  onUse(info, s)
+  # Note: This is n.info on purpose. It prevents template from creating an info
+  # context when called from an another template
   pushInfoContext(c.config, n.info, s.detailedInfo)
   result = evalTemplate(n, s, getCurrOwner(c), c.config, efFromHlo in flags)
   if efNoSemCheck notin flags: result = semAfterMacroCall(c, n, result, s, flags)
   popInfoContext(c.config)
 
   # XXX: A more elaborate line info rewrite might be needed
-  result.info = n.info
+  result.info = info
 
 proc semFieldAccess(c: PContext, n: PNode, flags: TExprFlags = {}): PNode
 
+template rejectEmptyNode(n: PNode) =
+  # No matter what a nkEmpty node is not what we want here
+  if n.kind == nkEmpty: illFormedAst(n, c.config)
+
 proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
+  rejectEmptyNode(n)
   # same as 'semExprWithType' but doesn't check for proc vars
   result = semExpr(c, n, flags + {efOperand})
-  #if result.kind == nkEmpty and result.typ.isNil:
-    # do not produce another redundant error message:
-    #raiseRecoverableError("")
-  #  result = errorNode(c, n)
   if result.typ != nil:
     # XXX tyGenericInst here?
     if result.typ.kind == tyProc and tfUnresolved in result.typ.flags:
@@ -56,10 +60,10 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     result.typ = errorType(c)
 
 proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
+  rejectEmptyNode(n)
   result = semExpr(c, n, flags+{efWantValue})
-  if result.isNil or result.kind == nkEmpty:
+  if result.kind == nkEmpty:
     # do not produce another redundant error message:
-    #raiseRecoverableError("")
     result = errorNode(c, n)
   if result.typ == nil or result.typ == c.enforceVoidContext:
     localError(c.config, n.info, errExprXHasNoType %
@@ -69,7 +73,8 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     if result.typ.kind in {tyVar, tyLent}: result = newDeref(result)
 
 proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
-  result = semExpr(c, n, flags)
+  rejectEmptyNode(n)
+  result = semExpr(c, n, flags+{efWantValue})
   if result.kind == nkEmpty:
     # do not produce another redundant error message:
     result = errorNode(c, n)
@@ -108,6 +113,8 @@ const
 
 proc checkConvertible(c: PContext, castDest, src: PType): TConvStatus =
   result = convOK
+  # We're interested in the inner type and not in the static tag
+  var src = src.skipTypes({tyStatic})
   if sameType(castDest, src) and castDest.sym == src.sym:
     # don't annoy conversions that may be needed on another processor:
     if castDest.kind notin IntegralTypes+{tyRange}:
@@ -230,7 +237,7 @@ proc semConv(c: PContext, n: PNode): PNode =
   # special case to make MyObject(x = 3) produce a nicer error message:
   if n[1].kind == nkExprEqExpr and
       targetType.skipTypes(abstractPtrs).kind == tyObject:
-    localError(c.config, n.info, "object contruction uses ':', not '='")
+    localError(c.config, n.info, "object construction uses ':', not '='")
   var op = semExprWithType(c, n.sons[1])
   if targetType.isMetaType:
     let final = inferWithMetatype(c, targetType, op, true)
@@ -403,8 +410,8 @@ proc semIs(c: PContext, n: PNode, flags: TExprFlags): PNode =
       n[1] = makeTypeSymNode(c, lhsType, n[1].info)
       lhsType = n[1].typ
   else:
-    internalAssert c.config, lhsType.base.kind != tyNone
+    if lhsType.base.kind == tyNone or
-    if c.inGenericContext > 0 and lhsType.base.containsGenericType:
+        (c.inGenericContext > 0 and lhsType.base.containsGenericType):
       # BUGFIX: don't evaluate this too early: ``T is void``
       return
 
@@ -710,16 +717,20 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
       let a = getConstExpr(c.module, n.sons[i], c.graph)
       if a == nil: return n
       call.add(a)
+
     #echo "NOW evaluating at compile time: ", call.renderTree
-    if sfCompileTime in callee.flags:
+    if c.inStaticContext == 0 or sfNoSideEffect in callee.flags:
-      result = evalStaticExpr(c.module, c.graph, call, c.p.owner)
+      if sfCompileTime in callee.flags:
-      if result.isNil:
+        result = evalStaticExpr(c.module, c.graph, call, c.p.owner)
-        localError(c.config, n.info, errCannotInterpretNodeX % renderTree(call))
+        if result.isNil:
-      else: result = fixupTypeAfterEval(c, result, n)
+          localError(c.config, n.info, errCannotInterpretNodeX % renderTree(call))
+        else: result = fixupTypeAfterEval(c, result, n)
+      else:
+        result = evalConstExpr(c.module, c.graph, call)
+        if result.isNil: result = n
+        else: result = fixupTypeAfterEval(c, result, n)
     else:
-      result = evalConstExpr(c.module, c.graph, call)
+      result = n
-      if result.isNil: result = n
-      else: result = fixupTypeAfterEval(c, result, n)
     #if result != n:
     #  echo "SUCCESS evaluated at compile time: ", call.renderTree
 
@@ -798,7 +809,7 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode =
       result = magicsAfterOverloadResolution(c, result, flags)
     if result.typ != nil and
         not (result.typ.kind == tySequence and result.typ.sons[0].kind == tyEmpty):
-      liftTypeBoundOps(c, result.typ, n.info)
+      liftTypeBoundOps(c.graph, result.typ, n.info)
     #result = patchResolvedTypeBoundOp(c, result)
   if c.matchedConcept == nil:
     result = evalAtCompileTime(c, result)
@@ -913,7 +924,7 @@ proc semExprNoType(c: PContext, n: PNode): PNode =
   let isPush = hintExtendedContext in c.config.notes
   if isPush: pushInfoContext(c.config, n.info)
   result = semExpr(c, n, {efWantStmt})
-  discardCheck(c, result, {})
+  result = discardCheck(c, result, {})
   if isPush: popInfoContext(c.config)
 
 proc isTypeExpr(n: PNode): bool =
@@ -1039,7 +1050,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
   of skConst:
     markUsed(c.config, n.info, s, c.graph.usageSym)
     onUse(n.info, s)
-    case skipTypes(s.typ, abstractInst-{tyTypeDesc}).kind
+    let typ = skipTypes(s.typ, abstractInst-{tyTypeDesc})
+    case typ.kind
     of  tyNil, tyChar, tyInt..tyInt64, tyFloat..tyFloat128,
         tyTuple, tySet, tyUInt..tyUInt64:
       if s.magic == mNone: result = inlineConst(c, n, s)
@@ -1057,6 +1069,12 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
       # deal with two different ``[]``.
       if s.ast.len == 0: result = inlineConst(c, n, s)
       else: result = newSymNode(s, n.info)
+    of tyStatic:
+      if typ.n != nil:
+        result = typ.n
+        result.typ = typ.base
+      else:
+        result = newSymNode(s, n.info)
     else:
       result = newSymNode(s, n.info)
   of skMacro:
@@ -1071,8 +1089,9 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     if efNoEvaluateGeneric in flags and s.ast[genericParamsPos].len > 0 or
        (n.kind notin nkCallKinds and s.requiredParams > 0) or
        sfCustomPragma in sym.flags:
-      markUsed(c.config, n.info, s, c.graph.usageSym)
+      let info = getCallLineInfo(n)
-      onUse(n.info, s)
+      markUsed(c.config, info, s, c.graph.usageSym)
+      onUse(info, s)
       result = symChoice(c, n, s, scClosed)
     else:
       result = semTemplateExpr(c, n, s, flags)
@@ -1158,9 +1177,10 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     onUse(n.info, s)
     result = newSymNode(s, n.info)
   else:
-    markUsed(c.config, n.info, s, c.graph.usageSym)
+    let info = getCallLineInfo(n)
-    onUse(n.info, s)
+    markUsed(c.config, info, s, c.graph.usageSym)
-    result = newSymNode(s, n.info)
+    onUse(info, s)
+    result = newSymNode(s, info)
 
 proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
   ## returns nil if it's not a built-in field access
@@ -1273,7 +1293,11 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
       if ty.sons[0] == nil: break
       ty = skipTypes(ty.sons[0], skipPtrs)
     if f != nil:
-      if fieldVisible(c, f):
+      let visibilityCheckNeeded =
+        if n[1].kind == nkSym and n[1].sym == f:
+          false # field lookup was done already, likely by hygienic template or bindSym
+        else: true
+      if not visibilityCheckNeeded or fieldVisible(c, f):
         # is the access to a public field or in the same module or in a friend?
         markUsed(c.config, n.sons[1].info, f, c.graph.usageSym)
         onUse(n.sons[1].info, f)
@@ -1581,7 +1605,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
           typeMismatch(c.config, n.info, lhs.typ, rhsTyp)
 
     n.sons[1] = fitNode(c, le, rhs, goodLineInfo(n[1]))
-    liftTypeBoundOps(c, lhs.typ, lhs.info)
+    liftTypeBoundOps(c.graph, lhs.typ, lhs.info)
     #liftTypeBoundOps(c, n.sons[0].typ, n.sons[0].info)
 
     fixAbstractType(c, n)
@@ -1628,7 +1652,7 @@ proc semProcBody(c: PContext, n: PNode): PNode =
       a.sons[1] = result
       result = semAsgn(c, a)
   else:
-    discardCheck(c, result, {})
+    result = discardCheck(c, result, {})
 
   if c.p.owner.kind notin {skMacro, skTemplate} and
      c.p.resultSym != nil and c.p.resultSym.typ.isMetaType:
@@ -2324,7 +2348,6 @@ proc semExportExcept(c: PContext, n: PNode): PNode =
 
 proc semExport(c: PContext, n: PNode): PNode =
   result = newNodeI(nkExportStmt, n.info)
-
   for i in 0..<n.len:
     let a = n.sons[i]
     var o: TOverloadIter
@@ -2343,6 +2366,9 @@ proc semExport(c: PContext, n: PNode): PNode =
         it = nextIter(ti, s.tab)
     else:
       while s != nil:
+        if s.kind == skEnumField:
+          localError(c.config, a.info, errGenerated, "cannot export: " & renderTree(a) &
+            "; enum field cannot be exported individually")
         if s.kind in ExportableSymKinds+{skModule}:
           result.add(newSymNode(s, a.info))
           strTableAdd(c.module.tab, s)
@@ -2430,7 +2456,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
       result.kind = nkCall
       result = semExpr(c, result, flags)
   of nkBind:
-    message(c.config, n.info, warnDeprecated, "bind")
+    message(c.config, n.info, warnDeprecated, "bind is deprecated")
     result = semExpr(c, n.sons[0], flags)
   of nkTypeOfExpr, nkTupleTy, nkTupleClassTy, nkRefTy..nkEnumTy, nkStaticTy:
     if c.matchedConcept != nil and n.len == 1:
@@ -2620,6 +2646,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   of nkStaticStmt:
     result = semStaticStmt(c, n)
   of nkDefer:
+    if c.currentScope == c.topLevelScope:
+      localError(c.config, n.info, "defer statement not supported at top level")
     n.sons[0] = semExpr(c, n.sons[0])
     if not n.sons[0].typ.isEmptyType and not implicitlyDiscardable(n.sons[0]):
       localError(c.config, n.info, "'defer' takes a 'void' expression")

compiler/semfields.nim

@@ -19,6 +19,9 @@ type
     c: PContext
 
 proc instFieldLoopBody(c: TFieldInstCtx, n: PNode, forLoop: PNode): PNode =
+  if c.field != nil and isEmptyType(c.field.typ):
+    result = newNode(nkEmpty)
+    return
   case n.kind
   of nkEmpty..pred(nkIdent), succ(nkSym)..nkNilLit: result = n
   of nkIdent, nkSym:

compiler/semfold.nim

@@ -15,7 +15,7 @@ import
   nversion, platform, math, msgs, os, condsyms, idents, renderer, types,
   commands, magicsys, modulegraphs, strtabs, lineinfos
 
-import system/helpers2
+import system/indexerrors
 
 proc newIntNodeT*(intVal: BiggestInt, n: PNode; g: ModuleGraph): PNode =
   case skipTypes(n.typ, abstractVarRange).kind
@@ -569,10 +569,20 @@ proc getConstExpr(m: PSym, n: PNode; g: ModuleGraph): PNode =
           try:
             result = newIntNodeT(g.config.symbols[s.name.s].parseInt, n, g)
           except ValueError:
-            localError(g.config, n.info, "expression is not an integer literal")
+            localError(g.config, s.info,
+              "{.intdefine.} const was set to an invalid integer: '" &
+                g.config.symbols[s.name.s] & "'")
       of mStrDefine:
         if isDefined(g.config, s.name.s):
           result = newStrNodeT(g.config.symbols[s.name.s], n, g)
+      of mBoolDefine:
+        if isDefined(g.config, s.name.s):
+          try:
+            result = newIntNodeT(g.config.symbols[s.name.s].parseBool.int, n, g)
+          except ValueError:
+            localError(g.config, s.info,
+              "{.booldefine.} const was set to an invalid bool: '" &
+                g.config.symbols[s.name.s] & "'")
       else:
         result = copyTree(s.ast)
     of skProc, skFunc, skMethod:

compiler/semmagic.nim

@@ -16,7 +16,12 @@ proc semAddr(c: PContext; n: PNode; isUnsafeAddr=false): PNode =
   if x.kind == nkSym:
     x.sym.flags.incl(sfAddrTaken)
   if isAssignable(c, x, isUnsafeAddr) notin {arLValue, arLocalLValue}:
-    localError(c.config, n.info, errExprHasNoAddress)
+    # Do not suggest the use of unsafeAddr if this expression already is a
+    # unsafeAddr
+    if isUnsafeAddr:
+      localError(c.config, n.info, errExprHasNoAddress)
+    else:
+      localError(c.config, n.info, errExprHasNoAddress & "; maybe use 'unsafeAddr'")
   result.add x
   result.typ = makePtrType(c, x.typ)
 
@@ -410,4 +415,9 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
       result = n
     else:
       result = plugin(c, n)
+  of mNewFinalize:
+    # Make sure the finalizer procedure refers to a procedure
+    if n[^1].kind == nkSym and n[^1].sym.kind notin {skProc, skFunc}:
+      localError(c.config, n.info, "finalizer must be a direct reference to a procedure")
+    result = n
   else: result = n

compiler/sempass2.nim

@@ -232,7 +232,10 @@ proc listGcUnsafety(s: PSym; onlyWarning: bool; conf: ConfigRef) =
 proc useVar(a: PEffects, n: PNode) =
   let s = n.sym
   if isLocalVar(a, s):
-    if s.id notin a.init:
+    if sfNoInit in s.flags:
+      # If the variable is explicitly marked as .noinit. do not emit any error
+      a.init.add s.id
+    elif s.id notin a.init:
       if {tfNeedsInit, tfNotNil} * s.typ.flags != {}:
         message(a.config, n.info, warnProveInit, s.name.s)
       else:
@@ -353,6 +356,8 @@ proc trackTryStmt(tracked: PEffects, n: PNode) =
 
   var branches = 1
   var hasFinally = false
+
+  # Collect the exceptions caught by the except branches
   for i in 1 ..< n.len:
     let b = n.sons[i]
     let blen = sonsLen(b)
@@ -368,12 +373,18 @@ proc trackTryStmt(tracked: PEffects, n: PNode) =
           else:
             assert(b.sons[j].kind == nkType)
             catches(tracked, b.sons[j].typ)
+    else:
+      assert b.kind == nkFinally
+  # Add any other exception raised in the except bodies
+  for i in 1 ..< n.len:
+    let b = n.sons[i]
+    let blen = sonsLen(b)
+    if b.kind == nkExceptBranch:
       setLen(tracked.init, oldState)
       track(tracked, b.sons[blen-1])
       for i in oldState..<tracked.init.len:
         addToIntersection(inter, tracked.init[i])
     else:
-      assert b.kind == nkFinally
       setLen(tracked.init, oldState)
       track(tracked, b.sons[blen-1])
       hasFinally = true
@@ -710,11 +721,17 @@ proc track(tracked: PEffects, n: PNode) =
   of nkSym:
     useVar(tracked, n)
   of nkRaiseStmt:
-    n.sons[0].info = n.info
+    if n[0].kind != nkEmpty:
-    #throws(tracked.exc, n.sons[0])
+      n.sons[0].info = n.info
-    addEffect(tracked, n.sons[0], useLineInfo=false)
+      #throws(tracked.exc, n.sons[0])
-    for i in 0 ..< safeLen(n):
+      addEffect(tracked, n.sons[0], useLineInfo=false)
-      track(tracked, n.sons[i])
+      for i in 0 ..< safeLen(n):
+        track(tracked, n.sons[i])
+    else:
+      # A `raise` with no arguments means we're going to re-raise the exception
+      # being handled or, if outside of an `except` block, a `ReraiseError`.
+      # Here we add a `Exception` tag in order to cover both the cases.
+      addEffect(tracked, createRaise(tracked.graph, n))
   of nkCallKinds:
     if getConstExpr(tracked.owner_module, n, tracked.graph) != nil:
       return
@@ -1013,7 +1030,7 @@ proc trackProc*(g: ModuleGraph; s: PSym, body: PNode) =
       "declared lock level is $1, but real lock level is $2" %
         [$s.typ.lockLevel, $t.maxLockLevel])
   when defined(useDfa):
-    if s.kind == skFunc:
+    if s.name.s == "testp":
       dataflowAnalysis(s, body)
       when false: trackWrites(s, body)
 

compiler/semstmts.nim

@@ -130,13 +130,13 @@ proc fixNilType(c: PContext; n: PNode) =
     for it in n: fixNilType(c, it)
   n.typ = nil
 
-proc discardCheck(c: PContext, result: PNode, flags: TExprFlags) =
+proc discardCheck(c: PContext, expr: PNode, flags: TExprFlags): PNode =
+  result = expr
   if c.matchedConcept != nil or efInTypeof in flags: return
 
   if result.typ != nil and result.typ.kind notin {tyStmt, tyVoid}:
     if implicitlyDiscardable(result):
-      var n = newNodeI(nkDiscardStmt, result.info, 1)
+      result = newNode(nkDiscardStmt, result.info, @[result])
-      n[0] = result
     elif result.typ.kind != tyError and c.config.cmd != cmdInteractive:
       var n = result
       while n.kind in skipForDiscardable: n = n.lastSon
@@ -168,7 +168,8 @@ proc semIf(c: PContext, n: PNode; flags: TExprFlags): PNode =
     else: illFormedAst(it, c.config)
   if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or
       (not hasElse and efInTypeof notin flags):
-    for it in n: discardCheck(c, it.lastSon, flags)
+    for it in n:
+      it.sons[^1] = discardCheck(c, it.sons[^1], flags)
     result.kind = nkIfStmt
     # propagate any enforced VoidContext:
     if typ == c.enforceVoidContext: result.typ = c.enforceVoidContext
@@ -266,17 +267,20 @@ proc semTry(c: PContext, n: PNode; flags: TExprFlags): PNode =
 
   dec c.p.inTryStmt
   if isEmptyType(typ) or typ.kind in {tyNil, tyExpr}:
-    discardCheck(c, n.sons[0], flags)
+    n.sons[0] = discardCheck(c, n.sons[0], flags)
-    for i in 1..n.len-1: discardCheck(c, n.sons[i].lastSon, flags)
+    for i in 1..n.len-1:
+      n.sons[i].sons[^1] = discardCheck(c, n.sons[i].sons[^1], flags)
     if typ == c.enforceVoidContext:
       result.typ = c.enforceVoidContext
   else:
-    if n.lastSon.kind == nkFinally: discardCheck(c, n.lastSon.lastSon, flags)
+    if n.lastSon.kind == nkFinally:
+      n.sons[^1].sons[^1] = discardCheck(c, n.sons[^1].sons[^1], flags)
     n.sons[0] = fitNode(c, typ, n.sons[0], n.sons[0].info)
     for i in 1..last:
       var it = n.sons[i]
       let j = it.len-1
-      it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info)
+      if not endsInNoReturn(it.sons[j]):
+        it.sons[j] = fitNode(c, typ, it.sons[j], it.sons[j].info)
     result.typ = typ
 
 proc fitRemoveHiddenConv(c: PContext, typ: PType, n: PNode): PNode =
@@ -315,24 +319,24 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym =
     result = semIdentWithPragma(c, kind, n, {})
     if result.owner.kind == skModule:
       incl(result.flags, sfGlobal)
-  let info = case n.kind
+
-             of nkPostfix:
+  proc getLineInfo(n: PNode): TLineInfo =
-               n.sons[1].info
+    case n.kind
-             of nkPragmaExpr:
+    of nkPostfix:
-               if n.sons[0].kind == nkPostfix:
+      getLineInfo(n.sons[1])
-                 n.sons[0].sons[1].info
+    of nkAccQuoted, nkPragmaExpr:
-               else:
+      getLineInfo(n.sons[0])
-                 n.sons[0].info
+    else:
-             else:
+      n.info
-               n.info
+  let info = getLineInfo(n)
   suggestSym(c.config, info, result, c.graph.usageSym)
 
 proc checkNilable(c: PContext; v: PSym) =
   if {sfGlobal, sfImportC} * v.flags == {sfGlobal} and
       {tfNotNil, tfNeedsInit} * v.typ.flags != {}:
-    if v.ast.isNil:
+    if v.astdef.isNil:
       message(c.config, v.info, warnProveInit, v.name.s)
-    elif tfNotNil in v.typ.flags and tfNotNil notin v.ast.typ.flags:
+    elif tfNotNil in v.typ.flags and tfNotNil notin v.astdef.typ.flags:
       message(c.config, v.info, warnProveInit, v.name.s)
 
 include semasgn
@@ -429,7 +433,6 @@ proc setVarType(c: PContext; v: PSym, typ: PType) =
 proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
   var b: PNode
   result = copyNode(n)
-  var hasCompileTime = false
   for i in countup(0, sonsLen(n)-1):
     var a = n.sons[i]
     if c.config.cmd == cmdIdeTools: suggestStmt(c, a)
@@ -437,11 +440,11 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
     if a.kind notin {nkIdentDefs, nkVarTuple, nkConstDef}: illFormedAst(a, c.config)
     checkMinSonsLen(a, 3, c.config)
     var length = sonsLen(a)
-    var typ: PType
+
+    var typ: PType = nil
     if a.sons[length-2].kind != nkEmpty:
       typ = semTypeNode(c, a.sons[length-2], nil)
-    else:
+
-      typ = nil
     var def: PNode = c.graph.emptyNode
     if a.sons[length-1].kind != nkEmpty:
       def = semExprWithType(c, a.sons[length-1], {efAllowDestructor})
@@ -473,7 +476,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
     # this can only happen for errornous var statements:
     if typ == nil: continue
     typeAllowedCheck(c.config, a.info, typ, symkind, if c.matchedConcept != nil: {taConcept} else: {})
-    liftTypeBoundOps(c, typ, a.info)
+    liftTypeBoundOps(c.graph, typ, a.info)
     var tup = skipTypes(typ, {tyGenericInst, tyAlias, tySink})
     if a.kind == nkVarTuple:
       if tup.kind != tyTuple:
@@ -518,8 +521,6 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
               message(c.config, a.info, warnShadowIdent, v.name.s)
       if a.kind != nkVarTuple:
         if def.kind != nkEmpty:
-          # this is needed for the evaluation pass and for the guard checking:
-          v.ast = def
           if sfThread in v.flags: localError(c.config, def.info, errThreadvarCannotInit)
         setVarType(c, v, typ)
         b = newNodeI(nkIdentDefs, a.info)
@@ -531,6 +532,23 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
         addSon(b, a.sons[length-2])
         addSon(b, copyTree(def))
         addToVarSection(c, result, n, b)
+        if optOldAst in c.config.options:
+          if def.kind != nkEmpty:
+            v.ast = def
+        else:
+          # this is needed for the evaluation pass, guard checking
+          #  and custom pragmas:
+          var ast = newNodeI(nkIdentDefs, a.info)
+          if a[j].kind == nkPragmaExpr:
+            var p = newNodeI(nkPragmaExpr, a.info)
+            p.add newSymNode(v)
+            p.add a[j][1].copyTree
+            ast.add p
+          else:
+            ast.add newSymNode(v)
+          ast.add a.sons[length-2].copyTree
+          ast.add def
+          v.ast = ast
       else:
         if def.kind in {nkPar, nkTupleConstr}: v.ast = def[j]
         # bug #7663, for 'nim check' this can be a non-tuple:
@@ -538,16 +556,16 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
         else: v.typ = tup
         b.sons[j] = newSymNode(v)
       checkNilable(c, v)
-      if sfCompileTime in v.flags: hasCompileTime = true
+      if sfCompileTime in v.flags:
+        var x = newNodeI(result.kind, v.info)
+        addSon(x, result[i])
+        vm.setupCompileTimeVar(c.module, c.graph, x)
       if v.flags * {sfGlobal, sfThread} == {sfGlobal}:
         message(c.config, v.info, hintGlobalVar)
-  if hasCompileTime:
-    vm.setupCompileTimeVar(c.module, c.graph, result)
-    # handled by the VM codegen:
-    #c.graph.recordStmt(c.graph, c.module, result)
 
 proc semConst(c: PContext, n: PNode): PNode =
   result = copyNode(n)
+  inc c.inStaticContext
   for i in countup(0, sonsLen(n) - 1):
     var a = n.sons[i]
     if c.config.cmd == cmdIdeTools: suggestStmt(c, a)
@@ -564,9 +582,19 @@ proc semConst(c: PContext, n: PNode): PNode =
     if def == nil:
       localError(c.config, a.sons[length-1].info, errConstExprExpected)
       continue
+
+    if def.typ.kind == tyTypeDesc and c.p.owner.kind != skMacro:
+      # prevent the all too common 'const x = int' bug:
+      localError(c.config, def.info, "'typedesc' metatype is not valid here; typed '=' instead of ':'?")
+      def.typ = errorType(c)
+
     # check type compatibility between def.typ and typ:
     if typ != nil:
-      def = fitRemoveHiddenConv(c, typ, def)
+      if typ.isMetaType:
+        def = inferWithMetatype(c, typ, def)
+        typ = def.typ
+      else:
+        def = fitRemoveHiddenConv(c, typ, def)
     else:
       typ = def.typ
     if typ == nil:
@@ -607,6 +635,7 @@ proc semConst(c: PContext, n: PNode): PNode =
         v.ast = def[j]
         b.sons[j] = newSymNode(v)
     addSon(result,b)
+  dec c.inStaticContext
 
 include semfields
 
@@ -662,7 +691,7 @@ proc semForVars(c: PContext, n: PNode; flags: TExprFlags): PNode =
   openScope(c)
   n.sons[length-1] = semExprBranch(c, n.sons[length-1], flags)
   if efInTypeof notin flags:
-    discardCheck(c, n.sons[length-1], flags)
+    n.sons[^1] = discardCheck(c, n.sons[^1], flags)
   closeScope(c)
   dec(c.p.nestedLoopCounter)
 
@@ -849,7 +878,8 @@ proc semCase(c: PContext, n: PNode; flags: TExprFlags): PNode =
   closeScope(c)
   if isEmptyType(typ) or typ.kind in {tyNil, tyExpr} or
       (not hasElse and efInTypeof notin flags):
-    for i in 1..n.len-1: discardCheck(c, n.sons[i].lastSon, flags)
+    for i in 1..n.len-1:
+      n.sons[i].sons[^1] = discardCheck(c, n.sons[i].sons[^1], flags)
     # propagate any enforced VoidContext:
     if typ == c.enforceVoidContext:
       result.typ = c.enforceVoidContext
@@ -1099,6 +1129,13 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) =
         #debug s.typ
       s.ast = a
       popOwner(c)
+      # If the right hand side expression was a macro call we replace it with
+      # its evaluated result here so that we don't execute it once again in the
+      # final pass
+      if a[2].kind in nkCallKinds:
+        a[2] = newNodeIT(nkType, a[2].info, t)
+    if sfExportc in s.flags and s.typ.kind == tyAlias:
+      localError(c.config, name.info, "{.exportc.} not allowed for type aliases")
     let aa = a.sons[2]
     if aa.kind in {nkRefTy, nkPtrTy} and aa.len == 1 and
        aa.sons[0].kind == nkObjectTy:
@@ -1234,9 +1271,6 @@ proc semTypeSection(c: PContext, n: PNode): PNode =
 
 proc semParamList(c: PContext, n, genericParams: PNode, s: PSym) =
   s.typ = semProcTypeNode(c, n, genericParams, nil, s.kind)
-  if s.kind notin {skMacro, skTemplate}:
-    if s.typ.sons[0] != nil and s.typ.sons[0].kind == tyStmt:
-      localError(c.config, n.info, "invalid return type: 'stmt'")
 
 proc addParams(c: PContext, n: PNode, kind: TSymKind) =
   for i in countup(1, sonsLen(n)-1):
@@ -1459,7 +1493,8 @@ proc canonType(c: PContext, t: PType): PType =
     result = t
 
 proc semOverride(c: PContext, s: PSym, n: PNode) =
-  case s.name.s.normalize
+  let name = s.name.s.normalize
+  case name
   of "=destroy":
     let t = s.typ
     var noError = false
@@ -1478,6 +1513,9 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
           localError(c.config, n.info, errGenerated,
             "cannot bind another '" & s.name.s & "' to: " & typeToString(obj))
         noError = true
+        if obj.owner.getModule != s.getModule:
+          localError(c.config, n.info, errGenerated,
+            "type bound operation `=destroy` can be defined only in the same module with its type (" & obj.typeToString() & ")")
     if not noError and sfSystemModule notin s.owner.flags:
       localError(c.config, n.info, errGenerated,
         "signature for '" & s.name.s & "' must be proc[T: object](x: var T)")
@@ -1501,6 +1539,11 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
       else:
         localError(c.config, n.info, errGenerated,
                    "cannot bind 'deepCopy' to: " & typeToString(t))
+
+      if t.owner.getModule != s.getModule:
+          localError(c.config, n.info, errGenerated,
+            "type bound operation `" & name & "` can be defined only in the same module with its type (" & t.typeToString() & ")")
+
     else:
       localError(c.config, n.info, errGenerated,
                  "signature for 'deepCopy' must be proc[T: ptr|ref](x: T): T")
@@ -1525,12 +1568,17 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
       if obj.kind in {tyObject, tyDistinct, tySequence, tyString} and sameType(obj, objB):
         # attach these ops to the canonical tySequence
         obj = canonType(c, obj)
+        #echo "ATTACHING TO ", obj.id, " ", s.name.s, " ", cast[int](obj)
         let opr = if s.name.s == "=": addr(obj.assignment) else: addr(obj.sink)
         if opr[].isNil:
           opr[] = s
         else:
           localError(c.config, n.info, errGenerated,
                      "cannot bind another '" & s.name.s & "' to: " & typeToString(obj))
+        if obj.owner.getModule != s.getModule:
+            localError(c.config, n.info, errGenerated,
+              "type bound operation `" & name & "` can be defined only in the same module with its type (" & obj.typeToString() & ")")
+
         return
     if sfSystemModule notin s.owner.flags:
       localError(c.config, n.info, errGenerated,
@@ -1580,7 +1628,7 @@ proc semMethodPrototype(c: PContext; s: PSym; n: PNode) =
           foundObj = true
           x.methods.add((col,s))
     if not foundObj:
-      message(c.config, n.info, warnDeprecated, "generic method not attachable to object type")
+      message(c.config, n.info, warnDeprecated, "generic method not attachable to object type is deprecated")
   else:
     # why check for the body? bug #2400 has none. Checking for sfForward makes
     # no sense either.
@@ -2013,7 +2061,7 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
       n.typ = n.sons[i].typ
       if not isEmptyType(n.typ): n.kind = nkStmtListExpr
     elif i != last or voidContext:
-      discardCheck(c, n.sons[i], flags)
+      n.sons[i] = discardCheck(c, n.sons[i], flags)
     else:
       n.typ = n.sons[i].typ
       if not isEmptyType(n.typ): n.kind = nkStmtListExpr

compiler/semtempl.nim

@@ -58,25 +58,26 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule): PNode =
       inc(i)
       if i > 1: break
     a = nextOverloadIter(o, c, n)
+  let info = getCallLineInfo(n)
   if i <= 1 and r != scForceOpen:
     # XXX this makes more sense but breaks bootstrapping for now:
     # (s.kind notin routineKinds or s.magic != mNone):
     # for instance 'nextTry' is both in tables.nim and astalgo.nim ...
-    result = newSymNode(s, n.info)
+    result = newSymNode(s, info)
-    markUsed(c.config, n.info, s, c.graph.usageSym)
+    markUsed(c.config, info, s, c.graph.usageSym)
-    onUse(n.info, s)
+    onUse(info, s)
   else:
     # semantic checking requires a type; ``fitNode`` deals with it
     # appropriately
     let kind = if r == scClosed or n.kind == nkDotExpr: nkClosedSymChoice
                else: nkOpenSymChoice
-    result = newNodeIT(kind, n.info, newTypeS(tyNone, c))
+    result = newNodeIT(kind, info, newTypeS(tyNone, c))
     a = initOverloadIter(o, c, n)
     while a != nil:
       if a.kind != skModule:
         incl(a.flags, sfUsed)
-        addSon(result, newSymNode(a, n.info))
+        addSon(result, newSymNode(a, info))
-        onUse(n.info, a)
+        onUse(info, a)
       a = nextOverloadIter(o, c, n)
 
 proc semBindStmt(c: PContext, n: PNode, toBind: var IntSet): PNode =

compiler/semtypes.nim

@@ -141,7 +141,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
     if isPure and (let conflict = strTableInclReportConflict(symbols, e); conflict != nil):
       wrongRedefinition(c, e.info, e.name.s, conflict.info)
     inc(counter)
-  if not hasNull: incl(result.flags, tfNeedsInit)
+  if tfNotNil in e.typ.flags and not hasNull: incl(result.flags, tfNeedsInit)
 
 proc semSet(c: PContext, n: PNode, prev: PType): PType =
   result = newOrPrevType(tySet, prev, c)
@@ -210,7 +210,7 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType =
               tyError, tyObject}:
           message c.config, n[i].info, errGenerated, "region needs to be an object type"
         else:
-          message(c.config, n.info, warnDeprecated, "region for pointer types")
+          message(c.config, n.info, warnDeprecated, "region for pointer types is deprecated")
         addSonSkipIntLit(result, region)
     addSonSkipIntLit(result, t)
     if tfPartial in result.flags:
@@ -1015,8 +1015,8 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
     result = addImplicitGeneric(copyType(paramType, getCurrOwner(c), false))
 
   of tyGenericParam:
-    markUsed(c.config, info, paramType.sym, c.graph.usageSym)
+    markUsed(c.config, paramType.sym.info, paramType.sym, c.graph.usageSym)
-    onUse(info, paramType.sym)
+    onUse(paramType.sym.info, paramType.sym)
     if tfWildcard in paramType.flags:
       paramType.flags.excl tfWildcard
       paramType.sym.kind = skType
@@ -1028,7 +1028,12 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
 proc semParamType(c: PContext, n: PNode, constraint: var PNode): PType =
   if n.kind == nkCurlyExpr:
     result = semTypeNode(c, n.sons[0], nil)
-    constraint = semNodeKindConstraints(n, c.config)
+    constraint = semNodeKindConstraints(n, c.config, 1)
+  elif n.kind == nkCall and
+      n[0].kind in {nkIdent, nkSym, nkOpenSymChoice, nkClosedSymChoice} and
+      considerQuotedIdent(c, n[0]).s == "{}":
+    result = semTypeNode(c, n[1], nil)
+    constraint = semNodeKindConstraints(n, c.config, 2)
   else:
     result = semTypeNode(c, n, nil)
 
@@ -1153,12 +1158,15 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
     # turn explicit 'void' return type into 'nil' because the rest of the
     # compiler only checks for 'nil':
     if skipTypes(r, {tyGenericInst, tyAlias, tySink}).kind != tyVoid:
+      if kind notin {skMacro, skTemplate} and r.kind in {tyStmt, tyExpr}:
+        localError(c.config, n.sons[0].info, "return type '" & typeToString(r) &
+            "' is only valid for macros and templates")
       # 'auto' as a return type does not imply a generic:
-      if r.kind == tyAnything:
+      elif r.kind == tyAnything:
         # 'p(): auto' and 'p(): expr' are equivalent, but the rest of the
         # compiler is hardly aware of 'auto':
         r = newTypeS(tyExpr, c)
-      elif r.kind != tyExpr:
+      else:
         if r.sym == nil or sfAnon notin r.sym.flags:
           let lifted = liftParamType(c, kind, genericParams, r, "result",
                                      n.sons[0].info)
@@ -1310,7 +1318,19 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType =
   if tx != result and tx.kind == tyObject and tx.sons[0] != nil:
     semObjectTypeForInheritedGenericInst(c, n, tx)
 
-proc maybeAliasType(c: PContext; typeExpr, prev: PType): PType
+proc maybeAliasType(c: PContext; typeExpr, prev: PType): PType =
+  if typeExpr.kind in {tyObject, tyEnum, tyDistinct, tyForward} and prev != nil:
+    result = newTypeS(tyAlias, c)
+    result.rawAddSon typeExpr
+    result.sym = prev.sym
+    assignType(prev, result)
+
+proc fixupTypeOf(c: PContext, prev: PType, typExpr: PNode) =
+  if prev != nil:
+    let result = newTypeS(tyAlias, c)
+    result.rawAddSon typExpr.typ
+    result.sym = prev.sym
+    assignType(prev, result)
 
 proc semTypeExpr(c: PContext, n: PNode; prev: PType): PType =
   var n = semExprWithType(c, n, {efDetermineType})
@@ -1414,20 +1434,6 @@ proc semProcTypeWithScope(c: PContext, n: PNode,
     when useEffectSystem: setEffectsForProcType(c.graph, result, n.sons[1])
   closeScope(c)
 
-proc maybeAliasType(c: PContext; typeExpr, prev: PType): PType =
-  if typeExpr.kind in {tyObject, tyEnum, tyDistinct} and prev != nil:
-    result = newTypeS(tyAlias, c)
-    result.rawAddSon typeExpr
-    result.sym = prev.sym
-    assignType(prev, result)
-
-proc fixupTypeOf(c: PContext, prev: PType, typExpr: PNode) =
-  if prev != nil:
-    let result = newTypeS(tyAlias, c)
-    result.rawAddSon typExpr.typ
-    result.sym = prev.sym
-    assignType(prev, result)
-
 proc symFromExpectedTypeNode(c: PContext, n: PNode): PSym =
   if n.kind == nkType:
     result = symFromType(c, n.typ, n.info)
@@ -1468,7 +1474,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
 
   if c.config.cmd == cmdIdeTools: suggestExpr(c, n)
   case n.kind
-  of nkEmpty: discard
+  of nkEmpty: result = n.typ
   of nkTypeOfExpr:
     # for ``type(countup(1,3))``, see ``tests/ttoseq``.
     checkSonsLen(n, 1, c.config)
@@ -1571,11 +1577,16 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
         assert s != nil
         assert prev == nil
         result = copyType(s, s.owner, keepId=false)
-        # XXX figure out why this has children already...
+        # Remove the 'T' parameter from tySequence:
         result.sons.setLen 0
         result.n = nil
         result.flags = {tfHasAsgn}
         semContainerArg(c, n, "seq", result)
+        if result.len > 0:
+          var base = result[0]
+          if base.kind in {tyGenericInst, tyAlias, tySink}: base = lastSon(base)
+          if base.kind != tyGenericParam:
+            c.typesWithOps.add((result, result))
       else:
         result = semContainer(c, n, tySequence, "seq", prev)
         if c.config.selectedGc == gcDestructors:
@@ -1708,11 +1719,9 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     result = newOrPrevType(tyError, prev, c)
   n.typ = result
   dec c.inTypeContext
-  if c.inTypeContext == 0: instAllTypeBoundOp(c, n.info)
+  if c.inTypeContext == 0:
-
+    #if $n == "var seq[StackTraceEntry]":
-when false:
+    #  echo "begin ", n
-  proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
-    result = semTypeNodeInner(c, n, prev)
     instAllTypeBoundOp(c, n.info)
 
 proc setMagicType(conf: ConfigRef; m: PSym, kind: TTypeKind, size: int) =

compiler/semtypinst.nim

@@ -297,12 +297,6 @@ proc instCopyType*(cl: var TReplTypeVars, t: PType): PType =
       #result.destructor = nil
       result.sink = nil
 
-template typeBound(c, newty, oldty, field, info) =
-  let opr = newty.field
-  if opr != nil and sfFromGeneric notin opr.flags:
-    # '=' needs to be instantiated for generics when the type is constructed:
-    newty.field = c.instTypeBoundOp(c, opr, oldty, info, attachedAsgn, 1)
-
 proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   # tyGenericInvocation[A, tyGenericInvocation[A, B]]
   # is difficult to handle:
@@ -317,7 +311,10 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   else:
     result = searchInstTypes(t)
 
-  if result != nil and eqFlags*result.flags == eqFlags*t.flags: return
+  if result != nil and eqFlags*result.flags == eqFlags*t.flags:
+    when defined(reportCacheHits):
+      echo "Generic instantiation cached ", typeToString(result), " for ", typeToString(t)
+    return
   for i in countup(1, sonsLen(t) - 1):
     var x = t.sons[i]
     if x.kind in {tyGenericParam}:
@@ -332,7 +329,11 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   if header != t:
     # search again after first pass:
     result = searchInstTypes(header)
-    if result != nil and eqFlags*result.flags == eqFlags*t.flags: return
+    if result != nil and eqFlags*result.flags == eqFlags*t.flags:
+      when defined(reportCacheHits):
+        echo "Generic instantiation cached ", typeToString(result), " for ",
+          typeToString(t), " header ", typeToString(header)
+      return
   else:
     header = instCopyType(cl, t)
 
@@ -384,7 +385,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   rawAddSon(result, newbody)
   checkPartialConstructedType(cl.c.config, cl.info, newbody)
   let dc = newbody.deepCopy
-  if cl.allowMetaTypes == false:
+  if not cl.allowMetaTypes:
     if dc != nil and sfFromGeneric notin newbody.deepCopy.flags:
       # 'deepCopy' needs to be instantiated for
       # generics *when the type is constructed*:
@@ -402,6 +403,11 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
           discard
         else:
           newbody.lastSon.typeInst = result
+    # DESTROY: adding object|opt for opt[topttree.Tree]
+    # sigmatch: Formal opt[=destroy.T] real opt[topttree.Tree]
+    # adding myseq for myseq[system.int]
+    # sigmatch: Formal myseq[=destroy.T] real myseq[system.int]
+    #echo "DESTROY: adding ", typeToString(newbody), " for ", typeToString(result, preferDesc)
     cl.c.typesWithOps.add((newbody, result))
     let mm = skipTypes(bbody, abstractPtrs)
     if tfFromGeneric notin mm.flags:
@@ -432,7 +438,7 @@ proc eraseVoidParams*(t: PType) =
           inc pos
       setLen t.sons, pos
       setLen t.n.sons, pos
-      return
+      break
 
 proc skipIntLiteralParams*(t: PType) =
   for i in 0 ..< t.sonsLen:
@@ -490,7 +496,12 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
       result.kind = tyUserTypeClassInst
 
   of tyGenericBody:
-    localError(cl.c.config, cl.info, "cannot instantiate: '" & typeToString(t) & "'")
+    localError(
+      cl.c.config,
+      cl.info,
+      "cannot instantiate: '" &
+      typeToString(t, preferDesc) &
+      "'; Maybe generic arguments are missing?")
     result = errorType(cl.c)
     #result = replaceTypeVarsT(cl, lastSon(t))
 
@@ -555,6 +566,12 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
 
       for i in countup(0, sonsLen(result) - 1):
         if result.sons[i] != nil:
+          if result.sons[i].kind == tyGenericBody:
+            localError(cl.c.config, t.sym.info,
+              "cannot instantiate '" &
+              typeToString(result.sons[i], preferDesc) &
+              "' inside of type definition: '" &
+              t.owner.name.s & "'; Maybe generic arguments are missing?")
           var r = replaceTypeVarsT(cl, result.sons[i])
           if result.kind == tyObject:
             # carefully coded to not skip the precious tyGenericInst:
@@ -590,6 +607,13 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
         result.size = -1
         result.n = replaceObjBranches(cl, result.n)
 
+template typeBound(c, newty, oldty, field, info) =
+  let opr = newty.field
+  if opr != nil and sfFromGeneric notin opr.flags:
+    # '=' needs to be instantiated for generics when the type is constructed:
+    #echo "DESTROY: instantiating ", astToStr(field), " for ", typeToString(oldty)
+    newty.field = c.instTypeBoundOp(c, opr, oldty, info, attachedAsgn, 1)
+
 proc instAllTypeBoundOp*(c: PContext, info: TLineInfo) =
   var i = 0
   while i < c.typesWithOps.len:

compiler/sighashes.nim

@@ -196,18 +196,23 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) =
       else:
         c.hashSym(t.sym)
       if {sfAnon, sfGenSym} * t.sym.flags != {}:
-        # generated object names can be identical, so we need to
+        # Generated object names can be identical, so we need to
-        # disambiguate furthermore by hashing the field types and names:
+        # disambiguate furthermore by hashing the field types and names.
-        # mild hack to prevent endless recursions (makes nimforum compile again):
+        if t.n.len > 0:
-        let oldFlags = t.sym.flags
+          let oldFlags = t.sym.flags
-        t.sym.flags = t.sym.flags - {sfAnon, sfGenSym}
+          # Mild hack to prevent endless recursion.
-        let n = t.n
+          t.sym.flags = t.sym.flags - {sfAnon, sfGenSym}
-        for i in 0 ..< n.len:
+          for n in t.n:
-          assert n[i].kind == nkSym
+            assert(n.kind == nkSym)
-          let s = n[i].sym
+            let s = n.sym
-          c.hashSym s
+            c.hashSym s
-          c.hashType s.typ, flags
+            c.hashType s.typ, flags
-        t.sym.flags = oldFlags
+          t.sym.flags = oldFlags
+        else:
+          # The object has no fields: we _must_ add something here in order to
+          # make the hash different from the one we produce by hashing only the
+          # type name.
+          c &= ".empty"
     else:
       c &= t.id
     if t.len > 0 and t.sons[0] != nil:

compiler/sigmatch.nim

@@ -1316,7 +1316,7 @@ proc typeRelImpl(c: var TCandidate, f, aOrig: PType,
         if typeRel(c, f.sons[i], a.sons[i]) == isNone: return isNone
       result = typeRel(c, f.lastSon, a.lastSon, flags + {trNoCovariance})
       subtypeCheck()
-      if result <= isConvertible: result = isNone
+      if result <= isIntConv: result = isNone
       elif tfNotNil in f.flags and tfNotNil notin a.flags:
         result = isNilConversion
     elif a.kind == tyNil: result = f.allowsNil
@@ -2071,6 +2071,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType,
     # constructor in a call:
     if result == nil and f.kind == tyVarargs:
       if f.n != nil:
+        # Forward to the varargs converter
         result = localConvMatch(c, m, f, a, arg)
       else:
         r = typeRel(m, base(f), a)
@@ -2083,10 +2084,10 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType,
         # bug #4799, varargs accepting subtype relation object
         elif r == isSubtype:
           inc(m.subtypeMatches)
-          if f.kind == tyTypeDesc:
+          if base(f).kind == tyTypeDesc:
             result = arg
           else:
-            result = implicitConv(nkHiddenSubConv, f, arg, m, c)
+            result = implicitConv(nkHiddenSubConv, base(f), arg, m, c)
           m.baseTypeMatch = true
         else:
           result = userConvMatch(c, m, base(f), a, arg)
@@ -2267,8 +2268,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
 
       if n.sons[a].kind == nkHiddenStdConv:
         doAssert n.sons[a].sons[0].kind == nkEmpty and
-                 n.sons[a].sons[1].kind == nkArgList and
+                 n.sons[a].sons[1].kind in {nkBracket, nkArgList}
-                 n.sons[a].sons[1].len == 0
         # Steal the container and pass it along
         setSon(m.call, formal.position + 1, n.sons[a].sons[1])
       else:
@@ -2505,6 +2505,11 @@ proc instTypeBoundOp*(c: PContext; dc: PSym; t: PType; info: TLineInfo;
     if f.kind in {tyRef, tyPtr}: f = f.lastSon
   else:
     if f.kind == tyVar: f = f.lastSon
+  #if c.config.selectedGC == gcDestructors and f.kind == tySequence:
+  # use the canonical type to access the =sink and =destroy etc.
+  #  f = c.graph.sysTypes[tySequence]
+  #echo "YUP_---------Formal ", typeToString(f, preferDesc), " real ", typeToString(t, preferDesc), " ", f.id, " ", t.id
+
   if typeRel(m, f, t) == isNone:
     localError(c.config, info, "cannot instantiate: '" & dc.name.s & "'")
   else:

compiler/sizealignoffsetimpl.nim

@@ -154,12 +154,17 @@ proc computePackedObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode, initialOf
       if result == szIllegalRecursion:
         break
   of nkSym:
+    var size = szUnknownSize
     if n.sym.bitsize == 0:
       computeSizeAlign(conf, n.sym.typ)
-      n.sym.offset = initialOffset.int
+      size = n.sym.typ.size.int
-      result = n.sym.offset + n.sym.typ.size
+
-    else:
+    if initialOffset == szUnknownSize or size == szUnknownSize:
+      n.sym.offset = szUnknownSize
       result = szUnknownSize
+    else:
+      n.sym.offset = int(initialOffset)
+      result = initialOffset + n.sym.typ.size
   else:
     result = szUnknownSize
 

compiler/suggest.nim

@@ -32,8 +32,8 @@
 
 # included from sigmatch.nim
 
-import algorithm, prefixmatches, lineinfos, pathutils
+import algorithm, prefixmatches, lineinfos, pathutils, parseutils
-from wordrecg import wDeprecated, wError
+from wordrecg import wDeprecated, wError, wAddr, wYield, specialWords
 
 when defined(nimsuggest):
   import passes, tables # importer
@@ -81,6 +81,38 @@ proc cmpSuggestions(a, b: Suggest): int =
   # independent of hashing order:
   result = cmp(a.name[], b.name[])
 
+proc getTokenLenFromSource(conf: ConfigRef; ident: string; info: TLineInfo): int =
+  let
+    line = sourceLine(conf, info)
+    column = toColumn(info)
+
+  proc isOpeningBacktick(col: int): bool =
+    if col >= 0 and col < line.len:
+      if line[col] == '`':
+        not isOpeningBacktick(col - 1)
+      else:
+        isOpeningBacktick(col - 1)
+    else:
+      false
+
+  if column > line.len:
+    result = 0
+  elif column > 0 and line[column - 1] == '`' and isOpeningBacktick(column - 1):
+    result = skipUntil(line, '`', column)
+    if cmpIgnoreStyle(line[column..column + result - 1], ident) != 0:
+      result = 0
+  elif ident[0] in linter.Letters and ident[^1] != '=':
+    result = identLen(line, column)
+    if cmpIgnoreStyle(line[column..column + result - 1], ident) != 0:
+      result = 0
+  else:
+    result = skipWhile(line, OpChars + {'[', '(', '{', ']', ')', '}'}, column)
+    if ident[^1] == '=' and ident[0] in linter.Letters:
+      if line[column..column + result - 1] != "=":
+        result = 0
+    elif line[column..column + result - 1] != ident:
+      result = 0
+
 proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo;
                   quality: range[0..100]; prefix: PrefixMatch;
                   inTypeContext: bool; scope: int): Suggest =
@@ -88,7 +120,6 @@ proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info
   result.section = section
   result.quality = quality
   result.isGlobal = sfGlobal in s.flags
-  result.tokenLen = s.name.s.len
   result.prefix = prefix
   result.contextFits = inTypeContext == (s.kind in {skType, skGenericParam})
   result.scope = scope
@@ -109,7 +140,11 @@ proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info
         result.qualifiedPath.add(ow2.origModuleName)
       if ow != nil:
         result.qualifiedPath.add(ow.origModuleName)
-    result.qualifiedPath.add(s.name.s)
+    if s.name.s[0] in OpChars + {'[', '{', '('} or
+       s.name.id in ord(wAddr)..ord(wYield):
+      result.qualifiedPath.add('`' & s.name.s & '`')
+    else:
+      result.qualifiedPath.add(s.name.s)
 
     if s.typ != nil:
       result.forth = typeToString(s.typ)
@@ -122,6 +157,10 @@ proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info
   result.line = toLinenumber(infox)
   result.column = toColumn(infox)
   result.version = conf.suggestVersion
+  result.tokenLen = if section != ideHighlight:
+                      s.name.s.len
+                    else:
+                      getTokenLenFromSource(conf, s.name.s, infox)
 
 proc `$`*(suggest: Suggest): string =
   result = $suggest.section
@@ -456,23 +495,27 @@ proc suggestSym*(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym;
 proc extractPragma(s: PSym): PNode =
   if s.kind in routineKinds:
     result = s.ast[pragmasPos]
-  elif s.kind in {skType}:
+  elif s.kind in {skType, skVar, skLet}:
     # s.ast = nkTypedef / nkPragmaExpr / [nkSym, nkPragma]
     result = s.ast[0][1]
   doAssert result == nil or result.kind == nkPragma
 
 proc warnAboutDeprecated(conf: ConfigRef; info: TLineInfo; s: PSym) =
-  let pragmaNode = if s.kind == skEnumField: extractPragma(s.owner) else: extractPragma(s)
+  var pragmaNode: PNode
+  if optOldAst in conf.options and s.kind in {skVar, skLet}:
+    pragmaNode = nil
+  else:
+    pragmaNode = if s.kind == skEnumField: extractPragma(s.owner) else: extractPragma(s)
   let name =
-    if s.kind == skEnumField: "enum '" & s.owner.name.s & "' which contains field '" & s.name.s & "'"
+    if s.kind == skEnumField and sfDeprecated notin s.flags: "enum '" & s.owner.name.s & "' which contains field '" & s.name.s & "'"
     else: s.name.s
   if pragmaNode != nil:
     for it in pragmaNode:
       if whichPragma(it) == wDeprecated and it.safeLen == 2 and
           it[1].kind in {nkStrLit..nkTripleStrLit}:
-        message(conf, info, warnDeprecated, it[1].strVal & "; " & name)
+        message(conf, info, warnDeprecated, it[1].strVal & "; " & name & " is deprecated")
         return
-  message(conf, info, warnDeprecated, name)
+  message(conf, info, warnDeprecated, name & " is deprecated")
 
 proc userError(conf: ConfigRef; info: TLineInfo; s: PSym) =
   let pragmaNode = extractPragma(s)
@@ -490,7 +533,7 @@ proc markUsed(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym) =
   if s.kind == skEnumField and s.owner != nil:
     incl(s.owner.flags, sfUsed)
     if sfDeprecated in s.owner.flags:
-      incl(s.flags, sfDeprecated)
+      warnAboutDeprecated(conf, info, s)
   if {sfDeprecated, sfError} * s.flags != {}:
     if sfDeprecated in s.flags: warnAboutDeprecated(conf, info, s)
     if sfError in s.flags: userError(conf, info, s)

compiler/transf.nim

@@ -321,6 +321,38 @@ proc introduceNewLocalVars(c: PTransf, n: PNode): PTransNode =
     for i in countup(0, sonsLen(n)-1):
       result[i] = introduceNewLocalVars(c, n.sons[i])
 
+proc transformAsgn(c: PTransf, n: PNode): PTransNode =
+  let rhs = n[1]
+
+  if rhs.kind != nkTupleConstr:
+    return transformSons(c, n)
+
+  # Unpack the tuple assignment into N temporary variables and then pack them
+  # into a tuple: this allows us to get the correct results even when the rhs
+  # depends on the value of the lhs
+  let letSection = newTransNode(nkLetSection, n.info, rhs.len)
+  let newTupleConstr = newTransNode(nkTupleConstr, n.info, rhs.len)
+  for i, field in rhs:
+    let val = if field.kind == nkExprColonExpr: field[1] else: field
+    let def = newTransNode(nkIdentDefs, field.info, 3)
+    def[0] = PTransNode(newTemp(c, val.typ, field.info))
+    def[1] = PTransNode(newNodeI(nkEmpty, field.info))
+    def[2] = transform(c, val)
+    letSection[i] = def
+    # NOTE: We assume the constructor fields are in the correct order for the
+    # given tuple type
+    newTupleConstr[i] = def[0]
+
+  PNode(newTupleConstr).typ = rhs.typ
+
+  let asgnNode = newTransNode(nkAsgn, n.info, 2)
+  asgnNode[0] = transform(c, n[0])
+  asgnNode[1] = newTupleConstr
+
+  result = newTransNode(nkStmtList, n.info, 2)
+  result[0] = letSection
+  result[1] = asgnNode
+
 proc transformYield(c: PTransf, n: PNode): PTransNode =
   proc asgnTo(lhs: PNode, rhs: PTransNode): PTransNode =
     # Choose the right assignment instruction according to the given ``lhs``
@@ -914,6 +946,7 @@ proc transform(c: PTransf, n: PNode): PTransNode =
         let hoisted = hoistParamsUsedInDefault(c, call, hoistedParams, call[i])
         if hoisted != nil: call[i] = hoisted
       result = newTree(nkStmtListExpr, hoistedParams, call).PTransNode
+      PNode(result).typ = call.typ
   of nkAddr, nkHiddenAddr:
     result = transformAddrDeref(c, n, nkDerefExpr, nkHiddenDeref)
   of nkDerefExpr, nkHiddenDeref:
@@ -948,6 +981,8 @@ proc transform(c: PTransf, n: PNode): PTransNode =
       result = transformYield(c, n)
     else:
       result = transformSons(c, n)
+  #of nkAsgn:
+  #  result = transformAsgn(c, n)
   of nkIdentDefs, nkConstDef:
     result = PTransNode(n)
     result[0] = transform(c, n[0])

compiler/types.nim

@@ -107,21 +107,24 @@ proc isFloatLit*(t: PType): bool {.inline.} =
   result = t.kind == tyFloat and t.n != nil and t.n.kind == nkFloatLit
 
 proc getProcHeader*(conf: ConfigRef; sym: PSym; prefer: TPreferedDesc = preferName): string =
-  result = sym.owner.name.s & '.' & sym.name.s & '('
+  assert sym != nil
-  var n = sym.typ.n
+  result = sym.owner.name.s & '.' & sym.name.s
-  for i in countup(1, sonsLen(n) - 1):
+  if sym.kind in routineKinds:
-    let p = n.sons[i]
+    result.add '('
-    if p.kind == nkSym:
+    var n = sym.typ.n
-      add(result, p.sym.name.s)
+    for i in countup(1, sonsLen(n) - 1):
-      add(result, ": ")
+      let p = n.sons[i]
-      add(result, typeToString(p.sym.typ, prefer))
+      if p.kind == nkSym:
-      if i != sonsLen(n)-1: add(result, ", ")
+        add(result, p.sym.name.s)
-    else:
+        add(result, ": ")
-      result.add renderTree(p)
+        add(result, typeToString(p.sym.typ, prefer))
-  add(result, ')')
+        if i != sonsLen(n)-1: add(result, ", ")
-  if n.sons[0].typ != nil:
+      else:
-    result.add(": " & typeToString(n.sons[0].typ, prefer))
+        result.add renderTree(p)
-  result.add "[declared in "
+    add(result, ')')
+    if n.sons[0].typ != nil:
+      result.add(": " & typeToString(n.sons[0].typ, prefer))
+  result.add " [declared in "
   result.add(conf$sym.info)
   result.add "]"
 
@@ -429,7 +432,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
        sfAnon notin t.sym.flags:
     if t.kind == tyInt and isIntLit(t):
       result = t.sym.name.s & " literal(" & $t.n.intVal & ")"
-    elif t.kind == tyAlias:
+    elif t.kind == tyAlias and t.sons[0].kind != tyAlias:
       result = typeToString(t.sons[0])
     elif prefer in {preferName, preferTypeName} or t.sym.owner.isNil:
       result = t.sym.name.s
@@ -453,12 +456,18 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
         result = $t.n.intVal
       else:
         result = "int literal(" & $t.n.intVal & ")"
-  of tyGenericBody, tyGenericInst, tyGenericInvocation:
+  of tyGenericInst, tyGenericInvocation:
     result = typeToString(t.sons[0]) & '['
     for i in countup(1, sonsLen(t)-1-ord(t.kind != tyGenericInvocation)):
       if i > 1: add(result, ", ")
       add(result, typeToString(t.sons[i], preferGenericArg))
     add(result, ']')
+  of tyGenericBody:
+    result = typeToString(t.lastSon) & '['
+    for i in countup(0, sonsLen(t)-2):
+      if i > 0: add(result, ", ")
+      add(result, typeToString(t.sons[i], preferTypeName))
+    add(result, ']')
   of tyTypeDesc:
     if t.sons[0].kind == tyNone: result = "typedesc"
     else: result = "type " & typeToString(t.sons[0])
@@ -609,7 +618,6 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
     result = typeToStr[t.kind]
   result.addTypeFlags(t)
 
-
 proc firstOrd*(conf: ConfigRef; t: PType): BiggestInt =
   case t.kind
   of tyBool, tyChar, tySequence, tyOpenArray, tyString, tyVarargs, tyProxy:
@@ -756,7 +764,7 @@ type
 
   TTypeCmpFlags* = set[TTypeCmpFlag]
 
-  TSameTypeClosure = object {.pure.}
+  TSameTypeClosure = object
     cmp: TDistinctCompare
     recCheck: int
     flags: TTypeCmpFlags

compiler/unittest_light.nim

@@ -0,0 +1,38 @@
+# note: consider merging tests/assert/testhelper.nim here.
+
+proc mismatch*[T](lhs: T, rhs: T): string =
+  ## Simplified version of `unittest.require` that satisfies a common use case,
+  ## while avoiding pulling too many dependencies. On failure, diagnostic
+  ## information is provided that in particular makes it easy to spot
+  ## whitespace mismatches and where the mismatch is.
+  proc replaceInvisible(s: string): string =
+    for a in s:
+      case a
+      of '\n': result.add "\\n\n"
+      else: result.add a
+
+  proc quoted(s: string): string = result.addQuoted s
+
+  result.add "\n"
+  result.add "lhs:{" & replaceInvisible(
+      $lhs) & "}\nrhs:{" & replaceInvisible($rhs) & "}\n"
+  when compiles(lhs.len):
+    if lhs.len != rhs.len:
+      result.add "lhs.len: " & $lhs.len & " rhs.len: " & $rhs.len & "\n"
+    when compiles(lhs[0]):
+      var i = 0
+      while i < lhs.len and i < rhs.len:
+        if lhs[i] != rhs[i]: break
+        i.inc
+      result.add "first mismatch index: " & $i & "\n"
+      if i < lhs.len and i < rhs.len:
+        result.add "lhs[i]: {" & quoted($lhs[i]) & "}\nrhs[i]: {" & quoted(
+            $rhs[i]) & "}\n"
+      result.add "lhs[0..<i]:{" & replaceInvisible($lhs[
+          0..<i]) & "}"
+
+proc assertEquals*[T](lhs: T, rhs: T) =
+  when false: # can be useful for debugging to see all that's fed to this.
+    echo "----" & $lhs
+  if lhs!=rhs:
+    doAssert false, mismatch(lhs, rhs)

compiler/vm.nim

@@ -10,10 +10,6 @@
 ## This file implements the new evaluation engine for Nim code.
 ## An instruction is 1-3 int32s in memory, it is a register based VM.
 
-const
-  debugEchoCode = false
-  traceCode = debugEchoCode
-
 import ast except getstr
 
 import
@@ -26,6 +22,9 @@ from evaltempl import evalTemplate
 
 from modulegraphs import ModuleGraph, PPassContext
 
+const
+  traceCode = defined(nimVMDebug)
+
 when hasFFI:
   import evalffi
 
@@ -65,22 +64,27 @@ proc stackTraceAux(c: PCtx; x: PStackFrame; pc: int; recursionLimit=100) =
       return
     stackTraceAux(c, x.next, x.comesFrom, recursionLimit-1)
     var info = c.debug[pc]
-    # we now use the same format as in system/except.nim
+    # we now use a format similar to the one in lib/system/excpt.nim
-    var s = substr(toFilename(c.config, info), 0)
+    var s = ""
-    # this 'substr' prevents a strange corruption. XXX This needs to be
+    # todo: factor with quotedFilename
-    # investigated eventually but first attempts to fix it broke everything
+    if optExcessiveStackTrace in c.config.globalOptions:
-    # see the araq-wip-fixed-writebarrier branch.
+      s = toFullPath(c.config, info)
+    else:
+      s = toFilename(c.config, info)
     var line = toLinenumber(info)
+    var col = toColumn(info)
     if line > 0:
       add(s, '(')
       add(s, $line)
+      add(s, ", ")
+      add(s, $(col + ColOffset))
       add(s, ')')
     if x.prc != nil:
       for k in 1..max(1, 25-s.len): add(s, ' ')
       add(s, x.prc.name.s)
     msgWriteln(c.config, s)
 
-proc stackTraceB(c: PCtx, tos: PStackFrame, pc: int,
+proc stackTraceImpl(c: PCtx, tos: PStackFrame, pc: int,
                 msg: string, lineInfo: TLineInfo) =
   msgWriteln(c.config, "stack trace: (most recent call last)")
   stackTraceAux(c, tos, pc)
@@ -90,11 +94,11 @@ proc stackTraceB(c: PCtx, tos: PStackFrame, pc: int,
 
 template stackTrace(c: PCtx, tos: PStackFrame, pc: int,
                     msg: string, lineInfo: TLineInfo) =
-  stackTraceB(c, tos, pc, msg, lineInfo)
+  stackTraceImpl(c, tos, pc, msg, lineInfo)
   return
 
 template stackTrace(c: PCtx, tos: PStackFrame, pc: int, msg: string) =
-  stackTraceB(c, tos, pc, msg, c.debug[pc])
+  stackTraceImpl(c, tos, pc, msg, c.debug[pc])
   return
 
 proc bailOut(c: PCtx; tos: PStackFrame) =
@@ -255,64 +259,101 @@ proc pushSafePoint(f: PStackFrame; pc: int) =
   f.safePoints.add(pc)
 
 proc popSafePoint(f: PStackFrame) =
-  # XXX this needs a proper fix!
+  discard f.safePoints.pop()
-  if f.safePoints.len > 0:
-    discard f.safePoints.pop()
 
-proc cleanUpOnException(c: PCtx; tos: PStackFrame):
+type
-                                              tuple[pc: int, f: PStackFrame] =
+  ExceptionGoto = enum
-  let raisedType = c.currentExceptionA.typ.skipTypes(abstractPtrs)
+    ExceptionGotoHandler,
-  var f = tos
+    ExceptionGotoFinally,
-  while true:
+    ExceptionGotoUnhandled
-    while f.safePoints.len == 0:
-      f = f.next
-      if f.isNil: return (-1, nil)
-    var pc2 = f.safePoints[f.safePoints.high]
 
-    var nextExceptOrFinally = -1
+proc findExceptionHandler(c: PCtx, f: PStackFrame, exc: PNode):
-    if c.code[pc2].opcode == opcExcept:
+  tuple[why: ExceptionGoto, where: int] =
-      nextExceptOrFinally = pc2 + c.code[pc2].regBx - wordExcess
+  let raisedType = exc.typ.skipTypes(abstractPtrs)
-      inc pc2
-    while c.code[pc2].opcode == opcExcept:
-      let excIndex = c.code[pc2].regBx-wordExcess
-      let exceptType = if excIndex > 0: c.types[excIndex].skipTypes(
-                          abstractPtrs)
-                       else: nil
-      #echo typeToString(exceptType), " ", typeToString(raisedType)
-      if exceptType.isNil or inheritanceDiff(raisedType, exceptType) <= 0:
-        # mark exception as handled but keep it in B for
-        # the getCurrentException() builtin:
-        c.currentExceptionB = c.currentExceptionA
-        c.currentExceptionA = nil
-        # execute the corresponding handler:
-        while c.code[pc2].opcode == opcExcept: inc pc2
-        discard f.safePoints.pop
-        return (pc2, f)
-      inc pc2
-      if c.code[pc2].opcode != opcExcept and nextExceptOrFinally >= 0:
-        # we're at the end of the *except list*, but maybe there is another
-        # *except branch*?
-        pc2 = nextExceptOrFinally+1
-        if c.code[pc2].opcode == opcExcept:
-          nextExceptOrFinally = pc2 + c.code[pc2].regBx - wordExcess
 
-    if nextExceptOrFinally >= 0:
+  while f.safePoints.len > 0:
-      pc2 = nextExceptOrFinally
+    var pc = f.safePoints.pop()
-    if c.code[pc2].opcode == opcFinally:
+
-      # execute the corresponding handler, but don't quit walking the stack:
+    var matched = false
-      discard f.safePoints.pop
+    var pcEndExcept = pc
-      return (pc2+1, f)
+
-    # not the right one:
+    # Scan the chain of exceptions starting at pc.
-    discard f.safePoints.pop
+    # The structure is the following:
+    # pc - opcExcept, <end of this block>
+    #      - opcExcept, <pattern1>
+    #      - opcExcept, <pattern2>
+    #        ...
+    #      - opcExcept, <patternN>
+    #      - Exception handler body
+    #    - ... more opcExcept blocks may follow
+    #    - ... an optional opcFinally block may follow
+    #
+    # Note that the exception handler body already contains a jump to the
+    # finally block or, if that's not present, to the point where the execution
+    # should continue.
+    # Also note that opcFinally blocks are the last in the chain.
+    while c.code[pc].opcode == opcExcept:
+      # Where this Except block ends
+      pcEndExcept = pc + c.code[pc].regBx - wordExcess
+      inc pc
+
+      # A series of opcExcept follows for each exception type matched
+      while c.code[pc].opcode == opcExcept:
+        let excIndex = c.code[pc].regBx - wordExcess
+        let exceptType =
+          if excIndex > 0: c.types[excIndex].skipTypes(abstractPtrs)
+          else: nil
+
+        # echo typeToString(exceptType), " ", typeToString(raisedType)
+
+        # Determine if the exception type matches the pattern
+        if exceptType.isNil or inheritanceDiff(raisedType, exceptType) <= 0:
+          matched = true
+          break
+
+        inc pc
+
+      # Skip any further ``except`` pattern and find the first instruction of
+      # the handler body
+      while c.code[pc].opcode == opcExcept:
+        inc pc
+
+      if matched:
+        break
+
+      # If no handler in this chain is able to catch this exception we check if
+      # the "parent" chains are able to. If this chain ends with a `finally`
+      # block we must execute it before continuing.
+      pc = pcEndExcept
+
+    # Where the handler body starts
+    let pcBody = pc
+
+    if matched:
+      return (ExceptionGotoHandler, pcBody)
+    elif c.code[pc].opcode == opcFinally:
+      # The +1 here is here because we don't want to execute it since we've
+      # already pop'd this statepoint from the stack.
+      return (ExceptionGotoFinally, pc + 1)
+
+  return (ExceptionGotoUnhandled, 0)
 
 proc cleanUpOnReturn(c: PCtx; f: PStackFrame): int =
-  for s in f.safePoints:
+  # Walk up the chain of safepoints and return the PC of the first `finally`
-    var pc = s
+  # block we find or -1 if no such block is found.
+  # Note that the safepoint is removed once the function returns!
+  result = -1
+
+  # Traverse the stack starting from the end in order to execute the blocks in
+  # the inteded order
+  for i in 1 .. f.safePoints.len:
+    var pc = f.safePoints[^i]
+    # Skip the `except` blocks
     while c.code[pc].opcode == opcExcept:
-      pc = pc + c.code[pc].regBx - wordExcess
+      pc += c.code[pc].regBx - wordExcess
     if c.code[pc].opcode == opcFinally:
-      return pc
+      discard f.safePoints.pop
-  return -1
+      return pc + 1
 
 proc opConv(c: PCtx; dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): bool =
   if desttyp.kind == tyString:
@@ -395,6 +436,11 @@ proc opConv(c: PCtx; dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType):
         dest.floatVal = toBiggestFloat(src.intVal)
       else:
         dest.floatVal = src.floatVal
+    of tyObject:
+      if srctyp.skipTypes(abstractRange).kind != tyObject:
+        internalError(c.config, "invalid object-to-object conversion")
+      # A object-to-object conversion is essentially a no-op
+      moveConst(dest, src)
     else:
       asgnComplex(dest, src)
 
@@ -440,6 +486,9 @@ const
 proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
   var pc = start
   var tos = tos
+  # Used to keep track of where the execution is resumed.
+  var savedPC = -1
+  var savedFrame: PStackFrame
   var regs: seq[TFullReg] # alias to tos.slots for performance
   move(regs, tos.slots)
   #echo "NEW RUN ------------------------"
@@ -447,27 +496,31 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     #{.computedGoto.}
     let instr = c.code[pc]
     let ra = instr.regA
-    #if c.traceActive:
+
     when traceCode:
       echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra, " rb ", instr.regB, " rc ", instr.regC
-      # message(c.config, c.debug[pc], warnUser, "Trace")
 
     case instr.opcode
     of opcEof: return regs[ra]
     of opcRet:
-      # XXX perform any cleanup actions
+      let newPc = c.cleanUpOnReturn(tos)
-      pc = tos.comesFrom
+      # Perform any cleanup action before returning
-      tos = tos.next
+      if newPc < 0:
-      let retVal = regs[0]
+        pc = tos.comesFrom
-      if tos.isNil:
+        tos = tos.next
-        #echo "RET ", retVal.rendertree
+        let retVal = regs[0]
-        return retVal
+        if tos.isNil:
+          return retVal
 
-      move(regs, tos.slots)
+        move(regs, tos.slots)
-      assert c.code[pc].opcode in {opcIndCall, opcIndCallAsgn}
+        assert c.code[pc].opcode in {opcIndCall, opcIndCallAsgn}
-      if c.code[pc].opcode == opcIndCallAsgn:
+        if c.code[pc].opcode == opcIndCallAsgn:
-        regs[c.code[pc].regA] = retVal
+          regs[c.code[pc].regA] = retVal
-        #echo "RET2 ", retVal.rendertree, " ", c.code[pc].regA
+      else:
+        savedPC = pc
+        savedFrame = tos
+        # The -1 is needed because at the end of the loop we increment `pc`
+        pc = newPc - 1
     of opcYldYoid: assert false
     of opcYldVal: assert false
     of opcAsgnInt:
@@ -925,7 +978,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         stackTrace(c, tos, pc, errNilAccess)
     of opcGetImpl:
       decodeB(rkNode)
-      let a = regs[rb].node
+      var a = regs[rb].node
+      if a.kind == nkVarTy: a = a[0]
       if a.kind == nkSym:
         regs[ra].node = if a.sym.ast.isNil: newNode(nkNilLit)
                         else: copyTree(a.sym.ast)
@@ -1015,7 +1069,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         # it's a callback:
         c.callbacks[-prc.offset-2].value(
           VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[pointer](regs),
-                 currentException: c.currentExceptionB,
+                 currentException: c.currentExceptionA,
                  currentLineInfo: c.debug[pc]))
       elif sfImportc in prc.flags:
         if allowFFI notin c.features:
@@ -1108,44 +1162,55 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       tos.pushSafePoint(pc + rbx)
       assert c.code[pc+rbx].opcode in {opcExcept, opcFinally}
     of opcExcept:
-      # just skip it; it's followed by a jump;
+      # This opcode is never executed, it only holds informations for the
-      # we'll execute in the 'raise' handler
+      # exception handling routines.
-      let rbx = instr.regBx - wordExcess - 1 # -1 for the following 'inc pc'
+      doAssert(false)
-      inc pc, rbx
-      while c.code[pc+1].opcode == opcExcept:
-        let rbx = c.code[pc+1].regBx - wordExcess - 1
-        inc pc, rbx
-      #assert c.code[pc+1].opcode in {opcExcept, opcFinally}
-      if c.code[pc+1].opcode != opcFinally:
-        # in an except handler there is no active safe point for the 'try':
-        tos.popSafePoint()
     of opcFinally:
-      # just skip it; it's followed by the code we need to execute anyway
+      # Pop the last safepoint introduced by a opcTry. This opcode is only
+      # executed _iff_ no exception was raised in the body of the `try`
+      # statement hence the need to pop the safepoint here.
+      doAssert(savedPC < 0)
       tos.popSafePoint()
     of opcFinallyEnd:
-      if c.currentExceptionA != nil:
+      # The control flow may not resume at the next instruction since we may be
-        # we are in a cleanup run:
+      # raising an exception or performing a cleanup.
-        let (newPc, newTos) = cleanUpOnException(c, tos)
+      if not savedPC < 0:
-        if newPc-1 < 0:
+        pc = savedPC - 1
-          bailOut(c, tos)
+        savedPC = -1
-          return
+        if tos != savedFrame:
-        pc = newPc-1
+          tos = savedFrame
-        if tos != newTos:
-          tos = newTos
           move(regs, tos.slots)
     of opcRaise:
       let raised = regs[ra].node
       c.currentExceptionA = raised
       c.exceptionInstr = pc
-      let (newPc, newTos) = cleanUpOnException(c, tos)
+
-      # -1 because of the following 'inc'
+      var frame = tos
-      if newPc-1 < 0:
+      var jumpTo = findExceptionHandler(c, frame, raised)
+      while jumpTo.why == ExceptionGotoUnhandled and not frame.next.isNil:
+        frame = frame.next
+        jumpTo = findExceptionHandler(c, frame, raised)
+
+      case jumpTo.why:
+      of ExceptionGotoHandler:
+        # Jump to the handler, do nothing when the `finally` block ends.
+        savedPC = -1
+        pc = jumpTo.where - 1
+        if tos != frame:
+          tos = frame
+          move(regs, tos.slots)
+      of ExceptionGotoFinally:
+        # Jump to the `finally` block first then re-jump here to continue the
+        # traversal of the exception chain
+        savedPC = pc
+        savedFrame = tos
+        pc = jumpTo.where - 1
+        if tos != frame:
+          tos = frame
+          move(regs, tos.slots)
+      of ExceptionGotoUnhandled:
+        # Nobody handled this exception, error out.
         bailOut(c, tos)
-        return
-      pc = newPc-1
-      if tos != newTos:
-        tos = newTos
-        move(regs, tos.slots)
     of opcNew:
       ensureKind(rkNode)
       let typ = c.types[instr.regBx - wordExcess]
@@ -1245,8 +1310,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       let newLen = regs[rb].intVal.int
       if regs[ra].node.isNil: stackTrace(c, tos, pc, errNilAccess)
       else: c.setLenSeq(regs[ra].node, newLen, c.debug[pc])
-    of opcReset:
-      internalError(c.config, c.debug[pc], "too implement")
     of opcNarrowS:
       decodeB(rkInt)
       let min = -(1.BiggestInt shl (rb-1))
@@ -1256,6 +1319,11 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcNarrowU:
       decodeB(rkInt)
       regs[ra].intVal = regs[ra].intVal and ((1'i64 shl rb)-1)
+    of opcSignExtend:
+      # like opcNarrowS, but no out of range possible
+      decodeB(rkInt)
+      let imm = 64 - rb
+      regs[ra].intVal = ashr(regs[ra].intVal shl imm, imm)
     of opcIsNil:
       decodeB(rkInt)
       let node = regs[rb].node
@@ -1282,7 +1350,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         idx = int(regs[rb+rc-1].intVal)
         callback = c.callbacks[idx].value
         args = VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[pointer](regs),
-                currentException: c.currentExceptionB,
+                currentException: c.currentExceptionA,
                 currentLineInfo: c.debug[pc])
       callback(args)
       regs[ra].node.flags.incl nfIsRef

compiler/vmdef.nim

@@ -73,8 +73,9 @@ type
     opcContainsSet, opcRepr, opcSetLenStr, opcSetLenSeq,
     opcIsNil, opcOf, opcIs,
     opcSubStr, opcParseFloat, opcConv, opcCast,
-    opcQuit, opcReset,
+    opcQuit,
     opcNarrowS, opcNarrowU,
+    opcSignExtend,
 
     opcAddStrCh,
     opcAddStrStr,

compiler/vmgen.nim

@@ -33,6 +33,11 @@ import
 import platform
 from os import splitFile
 
+const
+  debugEchoCode* = defined(nimVMDebug)
+
+when debugEchoCode:
+  import asciitables
 when hasFFI:
   import evalffi
 
@@ -43,9 +48,10 @@ type
   TGenFlags = set[TGenFlag]
 
 proc debugInfo(c: PCtx; info: TLineInfo): string =
-  result = toFilename(c.config, info).splitFile.name & ":" & $info.line
+  result = toFileLineCol(c.config, info)
 
 proc codeListing(c: PCtx, result: var string, start=0; last = -1) =
+  ## for debugging purposes
   # first iteration: compute all necessary labels:
   var jumpTargets = initIntSet()
   let last = if last < 0: c.code.len-1 else: min(last, c.code.len-1)
@@ -54,7 +60,9 @@ proc codeListing(c: PCtx, result: var string, start=0; last = -1) =
     if x.opcode in relativeJumps:
       jumpTargets.incl(i+x.regBx-wordExcess)
 
-  # for debugging purposes
+  template toStr(opc: TOpcode): string = ($opc).substr(3)
+
+  result.add "code listing:\n"
   var i = start
   while i <= last:
     if i in jumpTargets: result.addf("L$1:\n", i)
@@ -62,34 +70,42 @@ proc codeListing(c: PCtx, result: var string, start=0; last = -1) =
 
     result.add($i)
     let opc = opcode(x)
-    if opc in {opcConv, opcCast}:
+    if opc in {opcIndCall, opcIndCallAsgn}:
+      result.addf("\t$#\tr$#, r$#, nargs:$#", opc.toStr, x.regA,
+                  x.regB, x.regC)
+    elif opc in {opcConv, opcCast}:
       let y = c.code[i+1]
       let z = c.code[i+2]
-      result.addf("\t$#\tr$#, r$#, $#, $#", ($opc).substr(3), x.regA, x.regB,
+      result.addf("\t$#\tr$#, r$#, $#, $#", opc.toStr, x.regA, x.regB,
         c.types[y.regBx-wordExcess].typeToString,
         c.types[z.regBx-wordExcess].typeToString)
       inc i, 2
     elif opc < firstABxInstr:
-      result.addf("\t$#\tr$#, r$#, r$#", ($opc).substr(3), x.regA,
+      result.addf("\t$#\tr$#, r$#, r$#", opc.toStr, x.regA,
                   x.regB, x.regC)
-    elif opc in relativeJumps:
+    elif opc in relativeJumps + {opcTry}:
-      result.addf("\t$#\tr$#, L$#", ($opc).substr(3), x.regA,
+      result.addf("\t$#\tr$#, L$#", opc.toStr, x.regA,
                   i+x.regBx-wordExcess)
+    elif opc in {opcExcept}:
+      let idx = x.regBx-wordExcess
+      result.addf("\t$#\t$#, $#", opc.toStr, x.regA, $idx)
     elif opc in {opcLdConst, opcAsgnConst}:
       let idx = x.regBx-wordExcess
-      result.addf("\t$#\tr$#, $# ($#)", ($opc).substr(3), x.regA,
+      result.addf("\t$#\tr$#, $# ($#)", opc.toStr, x.regA,
         c.constants[idx].renderTree, $idx)
     elif opc in {opcMarshalLoad, opcMarshalStore}:
       let y = c.code[i+1]
-      result.addf("\t$#\tr$#, r$#, $#", ($opc).substr(3), x.regA, x.regB,
+      result.addf("\t$#\tr$#, r$#, $#", opc.toStr, x.regA, x.regB,
         c.types[y.regBx-wordExcess].typeToString)
       inc i
     else:
-      result.addf("\t$#\tr$#, $#", ($opc).substr(3), x.regA, x.regBx-wordExcess)
+      result.addf("\t$#\tr$#, $#", opc.toStr, x.regA, x.regBx-wordExcess)
     result.add("\t#")
     result.add(debugInfo(c, c.debug[i]))
     result.add("\n")
     inc i
+  when debugEchoCode:
+    result = result.alignTable
 
 proc echoCode*(c: PCtx; start=0; last = -1) {.deprecated.} =
   var buf = ""
@@ -313,8 +329,14 @@ proc genWhile(c: PCtx; n: PNode) =
       c.patch(L2)
 
 proc genBlock(c: PCtx; n: PNode; dest: var TDest) =
+  let oldRegisterCount = c.prc.maxSlots
   withBlock(n.sons[0].sym):
     c.gen(n.sons[1], dest)
+
+  for i in oldRegisterCount ..< c.prc.maxSlots:
+    if c.prc.slots[i].kind in {slotFixedVar, slotFixedLet}:
+      c.prc.slots[i] = (inUse: false, kind: slotEmpty)
+
   c.clearDest(n, dest)
 
 proc genBreak(c: PCtx; n: PNode) =
@@ -461,10 +483,13 @@ proc genType(c: PCtx; typ: PType): int =
 proc genTry(c: PCtx; n: PNode; dest: var TDest) =
   if dest < 0 and not isEmptyType(n.typ): dest = getTemp(c, n.typ)
   var endings: seq[TPosition] = @[]
-  let elsePos = c.xjmp(n, opcTry, 0)
+  let ehPos = c.xjmp(n, opcTry, 0)
   c.gen(n.sons[0], dest)
   c.clearDest(n, dest)
-  c.patch(elsePos)
+  # Add a jump past the exception handling code
+  endings.add(c.xjmp(n, opcJmp, 0))
+  # This signals where the body ends and where the exception handling begins
+  c.patch(ehPos)
   for i in 1 ..< n.len:
     let it = n.sons[i]
     if it.kind != nkFinally:
@@ -480,14 +505,14 @@ proc genTry(c: PCtx; n: PNode; dest: var TDest) =
         c.gABx(it, opcExcept, 0, 0)
       c.gen(it.lastSon, dest)
       c.clearDest(n, dest)
-      if i < sonsLen(n)-1:
+      if i < sonsLen(n):
         endings.add(c.xjmp(it, opcJmp, 0))
       c.patch(endExcept)
-  for endPos in endings: c.patch(endPos)
   let fin = lastSon(n)
   # we always generate an 'opcFinally' as that pops the safepoint
-  # from the stack
+  # from the stack if no exception is raised in the body.
   c.gABx(fin, opcFinally, 0, 0)
+  for endPos in endings: c.patch(endPos)
   if fin.kind == nkFinally:
     c.gen(fin.sons[0])
     c.clearDest(n, dest)
@@ -967,11 +992,18 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     c.freeTemp(tmp)
     c.freeTemp(tmp2)
 
-  of mShlI: genBinaryABCnarrowU(c, n, dest, opcShlInt)
+  of mShlI:
-  of mAshrI: genBinaryABCnarrow(c, n, dest, opcAshrInt)
+    genBinaryABC(c, n, dest, opcShlInt)
-  of mBitandI: genBinaryABCnarrowU(c, n, dest, opcBitandInt)
+    # genNarrowU modified
-  of mBitorI: genBinaryABCnarrowU(c, n, dest, opcBitorInt)
+    let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
-  of mBitxorI: genBinaryABCnarrowU(c, n, dest, opcBitxorInt)
+    if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8):
+      c.gABC(n, opcNarrowU, dest, TRegister(t.size*8))
+    elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and t.size < 8):
+      c.gABC(n, opcSignExtend, dest, TRegister(t.size*8))
+  of mAshrI: genBinaryABC(c, n, dest, opcAshrInt)
+  of mBitandI: genBinaryABC(c, n, dest, opcBitandInt)
+  of mBitorI: genBinaryABC(c, n, dest, opcBitorInt)
+  of mBitxorI: genBinaryABC(c, n, dest, opcBitxorInt)
   of mAddU: genBinaryABCnarrowU(c, n, dest, opcAddu)
   of mSubU: genBinaryABCnarrowU(c, n, dest, opcSubu)
   of mMulU: genBinaryABCnarrowU(c, n, dest, opcMulu)
@@ -990,7 +1022,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mLtPtr, mLtU, mLtU64: genBinaryABC(c, n, dest, opcLtu)
   of mEqProc, mEqRef, mEqUntracedRef:
     genBinaryABC(c, n, dest, opcEqRef)
-  of mXor: genBinaryABCnarrowU(c, n, dest, opcXor)
+  of mXor: genBinaryABC(c, n, dest, opcXor)
   of mNot: genUnaryABC(c, n, dest, opcNot)
   of mUnaryMinusI, mUnaryMinusI64:
     genUnaryABC(c, n, dest, opcUnaryMinusInt)
@@ -999,7 +1031,10 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mUnaryPlusI, mUnaryPlusF64: gen(c, n.sons[1], dest)
   of mBitnotI:
     genUnaryABC(c, n, dest, opcBitnotInt)
-    genNarrowU(c, n, dest)
+    #genNarrowU modified, do not narrow signed types
+    let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
+    if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8):
+      c.gABC(n, opcNarrowU, dest, TRegister(t.size*8))
   of mToFloat, mToBiggestFloat, mToInt,
      mToBiggestInt, mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr,
      mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr:
@@ -1092,7 +1127,9 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mReset:
     unused(c, n, dest)
     var d = c.genx(n.sons[1])
-    c.gABC(n, opcReset, d)
+    c.gABx(n, opcLdNull, d, c.genType(n.sons[1].typ))
+    c.gABx(n, opcNodeToReg, d, d)
+    c.genAsgnPatch(n.sons[1], d)
   of mOf, mIs:
     if dest < 0: dest = c.getTemp(n.typ)
     var tmp = c.genx(n.sons[1])
@@ -2183,9 +2220,9 @@ proc genProc(c: PCtx; s: PSym): int =
     c.gABC(body, opcEof, eofInstr.regA)
     c.optimizeJumps(result)
     s.offset = c.prc.maxSlots
-    #if s.name.s == "calc":
+    # if s.name.s == "fun1":
-    #  echo renderTree(body)
+    #   echo renderTree(body)
-    #  c.echoCode(result)
+    #   c.echoCode(result)
     c.prc = oldPrc
   else:
     c.prc.maxSlots = s.offset

compiler/vmops.nim

@@ -62,6 +62,9 @@ template wrap2svoid(op, modop) {.dirty.} =
     op(getString(a, 0), getString(a, 1))
   modop op
 
+template ioop(op) {.dirty.} =
+  registerCallback(c, "stdlib.io." & astToStr(op), `op Wrapper`)
+
 proc getCurrentExceptionMsgWrapper(a: VmArgs) {.nimcall.} =
   setResult(a, if a.currentException.isNil: ""
                else: a.currentException.sons[3].skipColon.strVal)
@@ -113,8 +116,8 @@ proc registerAdditionalOps*(c: PCtx) =
     wrap2svoid(putEnv, osop)
     wrap1s(dirExists, osop)
     wrap1s(fileExists, osop)
-    wrap2svoid(writeFile, systemop)
+    wrap2svoid(writeFile, ioop)
-    wrap1s(readFile, systemop)
+    wrap1s(readFile, ioop)
     systemop getCurrentExceptionMsg
     registerCallback c, "stdlib.*.staticWalkDir", proc (a: VmArgs) {.nimcall.} =
       setResult(a, staticWalkDirImpl(getString(a, 0), getBool(a, 1)))

compiler/wordrecg.nim

@@ -35,7 +35,7 @@ type
     wColon, wColonColon, wEquals, wDot, wDotDot,
     wStar, wMinus,
     wMagic, wThread, wFinal, wProfiler, wMemTracker, wObjChecks,
-    wIntDefine, wStrDefine,
+    wIntDefine, wStrDefine, wBoolDefine
 
     wDestroy,
 
@@ -122,7 +122,8 @@ const
 
     ":", "::", "=", ".", "..",
     "*", "-",
-    "magic", "thread", "final", "profiler", "memtracker", "objchecks", "intdefine", "strdefine",
+    "magic", "thread", "final", "profiler", "memtracker", "objchecks",
+    "intdefine", "strdefine", "booldefine",
 
     "destroy",
 

config/nim.cfg

@@ -290,13 +290,16 @@ tcc.options.always = "-w"
 
 # Configuration for the Genode toolchain
 @if genode:
+  noCppExceptions # avoid std C++
+  tlsEmulation:on # no TLS segment register magic
   gcc.path = "/usr/local/genode-gcc/bin"
-  gcc.cpp.options.always = "-D__GENODE__ -fno-stack-protector"
   @if i386 or amd64:
     gcc.exe = "genode-x86-gcc"
     gcc.cpp.exe = "genode-x86-g++"
+    gcc.cpp.linkerexe = "genode-x86-ld"
   @elif arm:
     gcc.exe = "genode-arm-gcc"
     gcc.cpp.exe = "genode-arm-g++"
+    gcc.cpp.linkerexe = "genode-arm-ld"
   @end
 @end

doc/advopt.txt

@@ -18,9 +18,9 @@ Advanced commands:
 
 Advanced options:
   -o:FILE, --out:FILE       set the output filename
-  --stdout                  output to stdout
+  --stdout:on|off           output to stdout
   --colors:on|off           turn compiler messages coloring on|off
-  --listFullPaths           list full paths in messages
+  --listFullPaths:on|off    list full paths in messages
   -w:on|off|list, --warnings:on|off|list
                             turn all warnings on|off or list all available
   --warning[X]:on|off       turn specific warning X on|off
@@ -39,29 +39,28 @@ Advanced options:
   --nimcache:PATH           set the path used for generated files
   --header:FILE             the compiler should produce a .h file (FILE
                             is optional)
-  -c, --compileOnly         compile Nim files only; do not assemble or link
+  -c, --compileOnly:on|off  compile Nim files only; do not assemble or link
-  --noLinking               compile Nim and generated files but do not link
+  --noLinking:on|off        compile Nim and generated files but do not link
-  --noMain                  do not generate a main procedure
+  --noMain:on|off           do not generate a main procedure
-  --genScript               generate a compile script (in the 'nimcache'
+  --genScript:on|off        generate a compile script (in the 'nimcache'
                             subdirectory named 'compile_$$project$$scriptext'),
                             implies --compileOnly
-  --genDeps                 generate a '.deps' file containing the dependencies
+  --genDeps:on|off          generate a '.deps' file containing the dependencies
   --os:SYMBOL               set the target operating system (cross-compilation)
   --cpu:SYMBOL              set the target processor (cross-compilation)
-  --debuginfo               enables debug information
+  --debuginfo:on|off        enables debug information
   -t, --passC:OPTION        pass an option to the C compiler
   -l, --passL:OPTION        pass an option to the linker
   --cincludes:DIR           modify the C compiler header search path
   --clibdir:DIR             modify the linker library search path
   --clib:LIBNAME            link an additional C library
                             (you should omit platform-specific extensions)
-  --genMapping              generate a mapping file containing
-                            (Nim, mangled) identifier pairs
   --project                 document the whole project (doc2)
   --docSeeSrcUrl:url        activate 'see source' for doc and doc2 commands
                             (see doc.item.seesrc in config/nimdoc.cfg)
+  --docInternal             also generate documentation for non-exported symbols
   --lineDir:on|off          generation of #line directive on|off
-  --embedsrc                embeds the original source code as comments
+  --embedsrc:on|off         embeds the original source code as comments
                             in the generated output
   --threadanalysis:on|off   turn thread analysis on|off
   --tlsEmulation:on|off     turn thread local storage emulation on|off
@@ -78,11 +77,12 @@ Advanced options:
                             strings is allowed; only for backwards compatibility
   --nilseqs:on|off          allow 'nil' for strings/seqs for
                             backwards compatibility
-  --skipCfg                 do not read the general configuration file
+  --oldast:on|off           use old AST for backwards compatibility
-  --skipUserCfg             do not read the user's configuration file
+  --skipCfg:on|off          do not read the nim installation's configuration file
-  --skipParentCfg           do not read the parent dirs' configuration files
+  --skipUserCfg:on|off      do not read the user's configuration file
-  --skipProjCfg             do not read the project's configuration file
+  --skipParentCfg:on|off    do not read the parent dirs' configuration files
-  --gc:refc|v2|markAndSweep|boehm|go|none|regions
+  --skipProjCfg:on|off      do not read the project's configuration file
+  --gc:refc|markAndSweep|boehm|go|none|regions
                             select the GC to use; default is 'refc'
   --index:on|off            turn index file generation on|off
   --putenv:key=value        set an environment variable
@@ -98,7 +98,8 @@ Advanced options:
                             symbol matching is fuzzy so
                             that --dynlibOverride:lua matches
                             dynlib: "liblua.so.3"
-  --dynlibOverrideAll       makes the dynlib pragma have no effect
+  --dynlibOverrideAll
+                            disables the effects of the dynlib pragma
   --listCmd                 list the commands used to execute external programs
   --parallelBuild:0|1|...   perform a parallel build
                             value = number of processors (0 for auto-detect)

doc/basicopt.txt

@@ -15,7 +15,7 @@ Options:
                             (Optionally: Define the value for that symbol,
                             see: "compile time define pragmas")
   -u, --undef:SYMBOL        undefine a conditional symbol
-  -f, --forceBuild          force rebuilding of all modules
+  -f, --forceBuild:on|off   force rebuilding of all modules
   --stackTrace:on|off       turn stack tracing on|off
   --lineTrace:on|off        turn line tracing on|off
   --threads:on|off          turn support for multi-threading on|off

doc/contributing.rst

@@ -1,6 +1,10 @@
+============
 Contributing
 ============
 
+.. contents::
+
+
 Contributing happens via "Pull requests" (PR) on github. Every PR needs to be
 reviewed before it can be merged and the Continuous Integration should be green.
 
@@ -22,7 +26,7 @@ There are 3 types of tests:
 2. tests in ``when isMainModule:`` block, ran by ``nim c mymod.nim``
    ``nimble test`` also typially runs these in external nimble packages.
 
-3. testament tests, eg: tests/stdlib/tos.nim (only used for Nim repo).
+3. testament tests, eg: ``tests/stdlib/tos.nim`` (only used for Nim repo).
 
 Not all the tests follow the convention here, feel free to change the ones
 that don't. Always leave the code cleaner than you found it.
@@ -49,8 +53,8 @@ Sample test:
       seq2D[0][1] = true
       doAssert seq2D == @[@[true, true], @[true, false],
                           @[false, false], @[false, false]]
-      # doAssert with `not` can be done as follows:
+      # doAssert with `not` can now be done as follows:
-      doAssert: not 1 == 2
+      doAssert not (1 == 2)
 
 Newer tests tend to be run via ``testament`` rather than via ``when isMainModule:``,
 eg ``tests/stdlib/tos.nim``; this allows additional features such as custom
@@ -60,18 +64,19 @@ Compiler
 --------
 
 The tests for the compiler use a testing tool called ``testament``. They are all
-located in ``tests/`` (eg: ``tests/destructor/tdestructor3.nim``).
+located in ``tests/`` (e.g.: ``tests/destructor/tdestructor3.nim``).
 Each test has its own file. All test files are prefixed with ``t``. If you want
 to create a file for import into another test only, use the prefix ``m``.
 
 At the beginning of every test is the expected behavior of the test.
 Possible keys are:
 
-- output: The expected output, most likely via ``echo``
+- ``cmd``: A compilation command template e.g. ``nim $target --threads:on $options $file``
-- exitcode: Exit code of the test (via ``exit(number)``)
+- ``output``: The expected output (stdout + stderr), most likely via ``echo``
-- errormsg: The expected error message
+- ``exitcode``: Exit code of the test (via ``exit(number)``)
-- file: The file the errormsg was produced at
+- ``errormsg``: The expected compiler error message
-- line: The line the errormsg was produced at
+- ``file``: The file the errormsg was produced at
+- ``line``: The line the errormsg was produced at
 
 For a full spec, see here: ``testament/specs.nim``
 
@@ -113,8 +118,15 @@ list of these, see ``testament/categories.nim``, at the bottom.
 
 ::
 
-  ./koch tests c lib
+  ./koch tests c lib # compiles/runs stdlib modules, including `isMainModule` tests
+  ./koch tests c megatest # runs a set of tests that can be combined into 1
 
+To run a single test:
+
+::
+
+  ./koch test run <category>/<name>    # eg: tuples/ttuples_issues
+  ./koch test run tests/stdlib/tos.nim # can also provide relative path
 
 For reproducible tests (to reproduce an environment more similar to the one
 run by Continuous Integration on travis/appveyor), you may want to disable your
@@ -165,12 +177,15 @@ the old name and introduce a new name:
 .. code-block:: nim
 
   # for routines (proc/template/macro/iterator) and types:
-  proc oldProc() {.deprecated: "use `newImpl: string -> int` instead".} = ...
+  proc oldProc() {.deprecated: "use `newImpl: string -> int` instead".} = discard
 
-  # for (const/var/let) the msg is not yet supported:
+  # for (const/var/let/fields) the msg is not yet supported:
   const Foo {.deprecated.}  = 1
 
-  # for enum types ``deprecated`` is not yet supported.
+  # for enum types, you can deprecate the type or some elements
+  # (likewise with object types and their fields):
+  type Bar {.deprecated.} = enum bar0, bar1
+  type Barz  = enum baz0, baz1 {.deprecated.}, baz2
 
 
 See also `Deprecated <https://nim-lang.org/docs/manual.html#pragmas-deprecated-pragma>`_
@@ -181,21 +196,22 @@ Documentation
 =============
 
 When contributing new procs, be sure to add documentation, especially if
-the proc is public. Documentation begins on the line
+the proc is public. Even private procs benefit from documentation and can be
+viewed using ``nim doc --docInternal foo.nim``.
+Documentation begins on the line
 following the ``proc`` definition, and is prefixed by ``##`` on each line.
 
 Runnable code examples are also encouraged, to show typical behavior with a few
-test cases (typically 1 to 3 ``doAssert`` statements, depending on complexity).
+test cases (typically 1 to 3 ``assert`` statements, depending on complexity).
 These ``runnableExamples`` are automatically run by ``nim doc mymodule.nim``
 as well as ``testament`` and guarantee they stay in sync.
 
 .. code-block:: nim
   proc addBar*(a: string): string =
-    ## Adds "Bar" to ``a``.
+    ## Adds "Bar" to `a`.
     runnableExamples:
-      doAssert "baz".addBar == "bazBar"
+      assert "baz".addBar == "bazBar"
-
+    result = a & "Bar"
-     result = a & "Bar"
 
 See `parentDir <https://nim-lang.github.io/Nim/os.html#parentDir%2Cstring>`_
 example.
@@ -209,14 +225,13 @@ not guaranteed to stay in sync, so ``runnableExamples`` is usually preferred:
   proc someproc*(): string =
     ## Return "something"
     ##
-    ## .. code-block:: nim
+    ## .. code-block::
-    ##
     ##  echo someproc() # "something"
     result = "something" # single-hash comments do not produce documentation
 
 The ``.. code-block:: nim`` followed by a newline and an indentation instructs the
 ``nim doc`` command to produce syntax-highlighted example code with the
-documentation.
+documentation (``.. code-block::`` is sufficient from inside a nim module).
 
 When forward declaration is used, the documentation should be included with the
 first appearance of the proc.
@@ -236,14 +251,14 @@ the imperative (command) form. That is, between:
 .. code-block:: nim
 
   proc hello*(): string =
-    # Return "hello"
+    ## Return "hello"
     result = "hello"
 or
 
 .. code-block:: nim
 
   proc hello*(): string =
-    # says hello
+    ## says hello
     result = "hello"
 
 the first is preferred.
@@ -282,12 +297,15 @@ rationale: https://forum.nim-lang.org/t/4089
   doAssert() # preferred
 
 .. _tests_use_doAssert:
-Use ``doAssert`` (or ``require``, etc), not ``assert`` in all tests.
+Use ``doAssert`` (or ``require``, etc), not ``assert`` in all tests so they'll
+be enabled even in release mode (except for tests in ``runnableExamples`` blocks
+which for which ``nim doc`` ignores ``-d:release``).
 
 .. code-block:: nim
 
-  runnableExamples: assert foo() # bad
+  when isMainModule:
-  runnableExamples: doAssert foo() # preferred
+    assert foo() # bad
+    doAssert foo() # preferred
 
 .. _delegate_printing:
 Delegate printing to caller: return ``string`` instead of calling ``echo``
@@ -316,7 +334,7 @@ https://github.com/nim-lang/Nim/pull/9335 and https://forum.nim-lang.org/t/4089
 
 .. code-block:: nim
 
-  echo foo() # adds a line in testament `discard` block.
+  echo foo() # adds a line for testament in `output:` block inside `discard`.
   doAssert foo() == [1, 2] # preferred, except when not possible to do so.
 
 The Git stuff
@@ -398,6 +416,26 @@ Code reviews
    saves time explaining the change or applying it; see also
    https://forum.nim-lang.org/t/4317
 
+2. When reviewing large diffs that may involve code moving around, github's interface
+   doesn't help much as it doesn't highlight moves. Instead you can use something
+   like this, see visual results `here <https://github.com/nim-lang/Nim/pull/10431#issuecomment-456968196>`_:
+
+   .. code-block:: sh
+
+      git fetch origin pull/10431/head && git checkout FETCH_HEAD
+      git show --color-moved-ws=allow-indentation-change --color-moved=blocks HEAD^
+
+3. In addition, you can view github-like diffs locally to identify what was changed
+   within a code block using `diff-highlight` or `diff-so-fancy`, eg:
+
+   .. code-block:: sh
+
+      # put this in ~/.gitconfig:
+      [core]
+        pager = "diff-so-fancy | less -R" # or: use: `diff-highlight`
+
+
+
 .. include:: docstyle.rst
 
 

doc/docgen.rst

@@ -283,7 +283,7 @@ symbols in the `system module <system.html>`_.
 * ``const NimVersion = "0.0.0"`` **=>**
   `#NimVersion <system.html#NimVersion>`_
 * ``proc getTotalMem(): int {.rtl, raises: [], tags: [].}`` **=>**
-  `#getTotalMem, <system.html#getTotalMem,>`_
+  `#getTotalMem, <system.html#getTotalMem>`_
 * ``proc len[T](x: seq[T]): int {.magic: "LengthSeq", noSideEffect.}`` **=>**
   `#len,seq[T] <system.html#len,seq[T]>`_
 * ``iterator pairs[T](a: seq[T]): tuple[key: int, val: T] {.inline.}`` **=>**

doc/docs.rst

@@ -6,6 +6,9 @@ The documentation consists of several documents:
 - | `Tutorial (part II) <tut2.html>`_
   | The Nim tutorial part two deals with the advanced language constructs.
 
+- | `Tutorial (part III) <tut3.html>`_
+  | The Nim tutorial part three about Nim's macro system.
+
 - | `Language Manual <manual.html>`_
   | The Nim manual is a draft that will evolve into a proper specification.
 

doc/docstyle.rst

@@ -4,23 +4,31 @@ Documentation Style
 General Guidelines
 ------------------
 
-* Authors should document anything that is exported.
+* Authors should document anything that is exported; documentation for private
+  procs can be useful too (visible via ``nim doc --docInternal foo.nim``).
 * Within documentation, a period (`.`) should follow each sentence (or sentence fragment) in a comment block. The documentation may be limited to one sentence fragment, but if multiple sentences are within the documentation, each sentence after the first should be complete and in present tense.
-* Documentation is parsed as ReStructuredText (RST).
+* Documentation is parsed as a custom ReStructuredText (RST) with partial markdown support.
-* Inline code should be surrounded by double tick marks ("``````"). If you would like a character to immediately follow inline code (e.g., "``int8``s are great!"), escape the following character with a backslash (``\``). The preceding is typed as ``` ``int8``\s are great!```.
+
+.. code-block:: nim
+
+  proc someproc*(s: string, foo: int) =
+    ## Use single backticks for inline code, eg: `s` or `someExpr(true)`.
+    ## Use a backlash to follow with alphanumeric char: `int8`\s are great.
+
 
 Module-level documentation
 --------------------------
 
 Documentation of a module is placed at the top of the module itself. Each line of documentation begins with double hashes (``##``).
+Sometimes ``##[ multiline docs containing code ]##`` is preferable, see ``lib/pure/times.nim``.
 Code samples are encouraged, and should follow the general RST syntax:
 
 .. code-block:: Nim
 
-  ## The ``universe`` module computes the answer to life, the universe, and everything.
+  ## The `universe` module computes the answer to life, the universe, and everything.
   ##
-  ## .. code-block:: Nim
+  ## .. code-block::
-  ##  echo computeAnswerString() # "42"
+  ##  doAssert computeAnswerString() == 42
 
 
 Within this top-level comment, you can indicate the authorship and copyright of the code, which will be featured in the produced documentation.
@@ -38,12 +46,12 @@ Leave a space between the last line of top-level documentation and the beginning
 Procs, Templates, Macros, Converters, and Iterators
 ---------------------------------------------------
 
-The documentation of a procedure should begin with a capital letter and should be in present tense. Variables referenced in the documentation should be surrounded by double tick marks (``````).
+The documentation of a procedure should begin with a capital letter and should be in present tense. Variables referenced in the documentation should be surrounded by single tick marks:
 
 .. code-block:: Nim
 
   proc example1*(x: int) =
-    ## Prints the value of ``x``.
+    ## Prints the value of `x`.
     echo x
 
 Whenever an example of usage would be helpful to the user, you should include one within the documentation in RST format as below.
@@ -51,11 +59,16 @@ Whenever an example of usage would be helpful to the user, you should include on
 .. code-block:: Nim
 
   proc addThree*(x, y, z: int8): int =
-    ## Adds three ``int8`` values, treating them as unsigned and
+    ## Adds three `int8` values, treating them as unsigned and
     ## truncating the result.
     ##
-    ## .. code-block:: nim
+    ## .. code-block::
-    ##  echo addThree(3, 125, 6) # -122
+    ##  # things that aren't suitable for a `runnableExamples` go in code-block:
+    ##  echo execCmdEx("git pull")
+    ##  drawOnScreen()
+    runnableExamples:
+      # `runnableExamples` is usually preferred to `code-block`, when possible.
+      doAssert addThree(3, 125, 6) == -122
     result = x +% y +% z
 
 The commands ``nim doc`` and ``nim doc2`` will then correctly syntax highlight the Nim code within the documentation.
@@ -94,7 +107,7 @@ Make sure to place the documentation beside or within the object.
 .. code-block:: Nim
 
   type
-    ## This documentation disappears because it annotates the ``type`` keyword
+    ## Bad: this documentation disappears because it annotates the ``type`` keyword
     ## above, not ``NamedQueue``.
     NamedQueue*[T] = object
       name*: string ## This becomes the main documentation for the object, which

doc/intern.rst

@@ -67,8 +67,8 @@ Coding Guidelines
 * Max line length is 80 characters.
 * Provide spaces around binary operators if that enhances readability.
 * Use a space after a colon, but not before it.
-* Start types with a capital ``T``, unless they are pointers/references which
+* [deprecated] Start types with a capital ``T``, unless they are
-  start with ``P``.
+  pointers/references which start with ``P``.
 
 See also the `API naming design <apis.html>`_ document.
 

doc/lib.rst

@@ -67,6 +67,8 @@ Core
 * `lenientops <lenientops.html>`_
   Provides binary operators for mixed integer/float expressions for convenience.
 
+* `bitops <bitops.html>`_
+  Provides a series of low level methods for bit manipulation.
 
 
 Collections and algorithms
@@ -74,26 +76,38 @@ Collections and algorithms
 
 * `algorithm <algorithm.html>`_
   Implements some common generic algorithms like sort or binary search.
+
 * `tables <tables.html>`_
   Nim hash table support. Contains tables, ordered tables and count tables.
+
 * `sets <sets.html>`_
   Nim hash and bit set support.
+
 * `lists <lists.html>`_
   Nim linked list support. Contains singly and doubly linked lists and
   circular lists ("rings").
+
 * `deques <deques.html>`_
   Implementation of a double-ended queue.
   The underlying implementation uses a ``seq``.
+
+* `heapqueue <heapqueue.html>`_
+  Implementation of a heap data structure that can be used as a priority queue.
+
 * `intsets <intsets.html>`_
   Efficient implementation of a set of ints as a sparse bit set.
+
 * `critbits <critbits.html>`_
   This module implements a *crit bit tree* which is an efficient
   container for a sorted set of strings, or for a sorted mapping of strings.
+
 * `sequtils <sequtils.html>`_
   This module implements operations for the built-in seq type
   which were inspired by functional programming languages.
+
 * `sharedtables <sharedtables.html>`_
   Nim shared hash table support. Contains shared tables.
+
 * `sharedlist <sharedlist.html>`_
   Nim shared linked list support. Contains shared singly linked list.
 
@@ -107,8 +121,8 @@ String handling
   substrings, replacing substrings.
 
 * `strformat <strformat.html>`_
-  Macro based standard string interpolation / formatting. Inpired by
+  Macro based standard string interpolation / formatting. Inspired by
-  Python's ```f``-strings.
+  Python's ``f``-strings.
 
 * `strmisc <strmisc.html>`_
   This module contains uncommon string handling operations that do not
@@ -129,6 +143,13 @@ String handling
 * `unicode <unicode.html>`_
   This module provides support to handle the Unicode UTF-8 encoding.
 
+* `unidecode <unidecode.html>`_
+  It provides a single proc that does Unicode to ASCII transliterations.
+  Based on Python's Unidecode module.
+
+* `punycode <punycode.html>`_
+  Implements a representation of Unicode with the limited ASCII character subset.
+
 * `encodings <encodings.html>`_
   Converts between different character encodings. On UNIX, this uses
   the ``iconv`` library, on Windows the Windows API.
@@ -141,12 +162,6 @@ String handling
   Ropes can represent very long strings efficiently; especially concatenation
   is done in O(1) instead of O(n).
 
-* `matchers <matchers.html>`_
-  This module contains various string matchers for email addresses, etc.
-
-* `subexes <subexes.html>`_
-  This module implements advanced string substitution operations.
-
 * `std/editdistance <editdistance.html>`_
   This module contains an algorithm to compute the edit distance between two
   Unicode strings.
@@ -200,10 +215,16 @@ Generic Operating System Services
   ``asyncdispatch``.
 
 * `distros <distros.html>`_
-  This module implements the basics for OS distribution ("distro") detection and the OS's native package manager.
+  This module implements the basics for OS distribution ("distro") detection
-  Its primary purpose is to produce output for Nimble packages, but it also contains the widely used **Distribution** enum
+  and the OS's native package manager.
+  Its primary purpose is to produce output for Nimble packages,
+  but it also contains the widely used **Distribution** enum
   that is useful for writing platform specific code.
 
+* `volatile <volatile.html>`_
+  This module contains code for generating volatile loads and stores,
+  which are useful in embedded and systems programming.
+
 
 Math libraries
 --------------
@@ -236,9 +257,6 @@ Internet Protocols and Support
 * `cgi <cgi.html>`_
   This module implements helpers for CGI applications.
 
-* `scgi <scgi.html>`_
-  This module implements helpers for SCGI applications.
-
 * `browsers <browsers.html>`_
   This module implements procs for opening URLs with the user's default
   browser.
@@ -275,8 +293,8 @@ Internet Protocols and Support
   module.
 
 * `net <net.html>`_
-  This module implements a high-level sockets API. It will replace the
+  This module implements a high-level sockets API. It replaces the
-  ``sockets`` module in the future.
+  ``sockets`` module.
 
 * `nativesockets <nativesockets.html>`_
   This module implements a low-level sockets API.
@@ -285,6 +303,7 @@ Internet Protocols and Support
   This module implements a selector API with backends specific to each OS.
   Currently epoll on Linux and select on other operating systems.
 
+
 Parsers
 -------
 
@@ -411,6 +430,7 @@ Miscellaneous
 * `segfaults <segfaults.html>`_
   Turns access violations or segfaults into a ``NilAccessError`` exception.
 
+
 Modules for JS backend
 ----------------------
 
@@ -439,7 +459,6 @@ Regular expressions
   expressions. The current implementation uses PCRE.
 
 
-
 Database support
 ----------------
 
@@ -456,20 +475,13 @@ Database support
   for other databases too.
 
 
-Other
------
-
-* `ssl <ssl.html>`_
-  This module provides an easy to use sockets-style
-  Nim interface to the OpenSSL library.
-
-
 Wrappers
 ========
 
 The generated HTML for some of these wrappers is so huge that it is
 not contained in the distribution. You can then find them on the website.
 
+
 Windows specific
 ----------------
 
@@ -525,5 +537,5 @@ Nimble is a package manager for the Nim programming language.
 For instructions on how to install Nimble packages see
 `its README <https://github.com/nim-lang/nimble#readme>`_.
 
-To see a list of Nimble's packages, check out `https://nimble.directory/ <https://nimble.directory/>`_
+To see a list of Nimble's packages, check out `<https://nimble.directory/>`_
 or the `packages repo <https://github.com/nim-lang/packages>`_ on GitHub.

doc/manual.rst

@@ -46,8 +46,8 @@ and ``a ^* b`` is short for ``(a (b a)*)?``. Example::
 
   arrayConstructor = '[' expr ^* ',' ']'
 
-Other parts of Nim - like scoping rules or runtime semantics are only
+Other parts of Nim, like scoping rules or runtime semantics, are
-described in the, more easily comprehensible, informal manner for now.
+described informally.
 
 
 
@@ -55,7 +55,7 @@ described in the, more easily comprehensible, informal manner for now.
 Definitions
 ===========
 
-A Nim program specifies a computation that acts on a memory consisting of
+Nim code specifies a computation that acts on a memory consisting of
 components called `locations`:idx:. A variable is basically a name for a
 location. Each variable and location is of a certain `type`:idx:. The
 variable's type is called `static type`:idx:, the location's type is called
@@ -71,21 +71,40 @@ identifier is declared unless overloading resolution rules suggest otherwise.
 An expression specifies a computation that produces a value or location.
 Expressions that produce locations are called `l-values`:idx:. An l-value
 can denote either a location or the value the location contains, depending on
-the context. Expressions whose values can be determined statically are called
+the context.
-`constant expressions`:idx:; they are never l-values.
 
-A `static error`:idx: is an error that the implementation detects before
+A Nim `program`:idx: consists of one or more text `source files`:idx: containing
-program execution. Unless explicitly classified, an error is a static error.
+Nim code. It is processed by a Nim `compiler`:idx: into an `executable`:idx:.
+The nature of this executable depends on the compiler implementation; it may,
+for example, be a native binary or JavaScript source code.
+
+In a typical Nim program, most of the code is compiled into the executable.
+However, some of the code may be executed at
+`compile time`:idx:. This can include constant expressions, macro definitions,
+and Nim procedures used by macro definitions. Most of the Nim language is
+supported at compile time, but there are some restrictions -- see `Restrictions
+on Compile-Time Execution <#restrictions-on-compileminustime-execution>`_ for
+details. We use the term `runtime`:idx: to cover both compile-time execution
+and code execution in the executable.
+
+The compiler parses Nim source code into an internal data structure called the
+`abstract syntax tree`:idx: (`AST`:idx:). Then, before executing the code or
+compiling it into the executable, it transforms the AST through `semantic
+analysis`:idx:. This adds semantic information such as expression types,
+identifier meanings, and in some cases expression values. An error detected
+during semantic analysis is called a `static error`:idx:. Errors described in
+this manual are static errors when not otherwise specified.
 
 A `checked runtime error`:idx: is an error that the implementation detects
-and reports at runtime. The method for reporting such errors is via *raising
+and reports at runtime. The method for reporting such errors is via
-exceptions* or *dying with a fatal error*. However, the implementation
+*raising exceptions* or *dying with a fatal error*. However, the implementation
-provides a means to disable these runtime checks. See the section pragmas_
+provides a means to disable these `runtime checks`:idx:. See the section
-for details.
+pragmas_ for details.
 
-Whether a checked runtime error results in an exception or in a fatal error at
+Whether a checked runtime error results in an exception or in a fatal error is
-runtime is implementation specific. Thus the following program is always
+implementation specific. Thus the following program is invalid; even though the
-invalid:
+code purports to catch the `IndexError` from an out-of-bounds array access, the
+compiler may instead choose to allow the program to die with a fatal error.
 
 .. code-block:: nim
   var a: array[0..1, char]
@@ -98,8 +117,23 @@ invalid:
 An `unchecked runtime error`:idx: is an error that is not guaranteed to be
 detected, and can cause the subsequent behavior of the computation to
 be arbitrary. Unchecked runtime errors cannot occur if only `safe`:idx:
-language features are used.
+language features are used and if no runtime checks are disabled.
 
+A `constant expression`:idx: is an expression whose value can be computed during
+semantic analysis of the code in which it appears. It is never an l-value and
+never has side effects. Constant expressions are not limited to the capabilities
+of semantic analysis, such as constant folding; they can use all Nim language
+features that are supported for compile-time execution. Since constant
+expressions can be used as an input to semantic analysis (such as for defining
+array bounds), this flexibility requires the compiler to interleave semantic
+analysis and compile-time code execution.
+
+It is mostly accurate to picture semantic analysis proceeding top to bottom and
+left to right in the source code, with compile-time code execution interleaved
+when necessary to compute values that are required for subsequent semantic
+analysis. We will see much later in this document that macro invocation not only
+requires this interleaving, but also creates a situation where semantic analyis
+does not entirely proceed top to bottom and left to right.
 
 
 Lexical Analysis
@@ -251,6 +285,10 @@ Another advantage is that it frees the programmer from remembering
 the exact spelling of an identifier. The exception with respect to the first
 letter allows common code like ``var foo: Foo`` to be parsed unambiguously.
 
+Note that this rule also applies to keywords, meaning that ``notin`` is
+the same as ``notIn`` and ``not_in`` (all-lowercase version (``notin``, ``isnot``)
+is the preferred way of writing keywords).
+
 Historically, Nim was a fully `style-insensitive`:idx: language. This meant that
 it was not case-sensitive and underscores were ignored and there was not even a
 distinction between ``foo`` and ``Foo``.
@@ -480,7 +518,6 @@ The type suffixes are:
   ``'d``             float64
   ``'f32``           float32
   ``'f64``           float64
-  ``'f128``          float128
 =================    =========================
 
 Floating point literals may also be in binary, octal or hexadecimal
@@ -674,10 +711,87 @@ Rationale: Consistency with overloaded assignment or assignment-like operations,
 ``a = b`` can be read as ``performSomeCopy(a, b)``.
 
 
+Constants and Constant Expressions
+==================================
+
+A `constant`:idx: is a symbol that is bound to the value of a constant
+expression. Constant expressions are restricted to depend only on the following
+categories of values and operations, because these are either built into the
+language or declared and evaluated before semantic analysis of the constant
+expression:
+
+* literals
+* built-in operators
+* previously declared constants and compile-time variables
+* previously declared macros and templates
+* previously declared procedures that have no side effects beyond
+  possibly modifying compile-time variables
+
+A constant expression can contain code blocks that may internally use all Nim
+features supported at compile time (as detailed in the next section below).
+Within such a code block, it is possible to declare variables and then later
+read and update them, or declare variables and pass them to procedures that
+modify them. However, the code in such a block must still adhere to the
+retrictions listed above for referencing values and operations outside the
+block.
+
+The ability to access and modify compile-time variables adds flexibility to
+constant expressions that may be surprising to those coming from other
+statically typed languages. For example, the following code echoes the beginning
+of the Fibonacci series **at compile time**. (This is a demonstration of
+flexibility in defining constants, not a recommended style for solving this
+problem!)
+
+.. code-block:: nim
+    :test: "nim c $1"
+  import strformat
+
+  var fib_n {.compileTime.}: int
+  var fib_prev {.compileTime.}: int
+  var fib_prev_prev {.compileTime.}: int
+
+  proc next_fib(): int =
+    result = if fib_n < 2:
+      fib_n
+    else:
+      fib_prev_prev + fib_prev
+    inc(fib_n)
+    fib_prev_prev = fib_prev
+    fib_prev = result
+
+  const f0 = next_fib()
+  const f1 = next_fib()
+
+  const display_fib = block:
+    const f2 = next_fib()
+    var result = fmt"Fibonacci sequence: {f0}, {f1}, {f2}"
+    for i in 3..12:
+      add(result, fmt", {next_fib()}")
+    result
+
+  static:
+    echo display_fib
+
+
+Restrictions on Compile-Time Execution
+======================================
+
+Nim code that will be executed at compile time cannot use the following
+language features:
+
+* methods
+* closure iterators
+* the ``cast`` operator
+* reference (pointer) types
+* the FFI
+
+Some or all of these restrictions are likely to be lifted over time.
+
+
 Types
 =====
 
-All expressions have a type which is known at compile time. Nim
+All expressions have a type which is known during semantic analysis. Nim
 is statically typed. One can declare new types, which is in essence defining
 an identifier that can be used to denote this custom type.
 
@@ -798,8 +912,8 @@ For further details, see `Convertible relation
 Subrange types
 --------------
 A subrange type is a range of values from an ordinal or floating point type (the base
-type). To define a subrange type, one must specify it's limiting values: the
+type). To define a subrange type, one must specify its limiting values -- the
-lowest and highest value of the type:
+lowest and highest value of the type. For example:
 
 .. code-block:: nim
   type
@@ -811,8 +925,8 @@ lowest and highest value of the type:
 to 5. ``PositiveFloat`` defines a subrange of all positive floating point values.
 NaN does not belong to any subrange of floating point types.
 Assigning any other value to a variable of type ``Subrange`` is a
-checked runtime error (or static error if it can be statically
+checked runtime error (or static error if it can be determined during
-determined). Assignments from the base type to one of its subrange types
+semantic analysis). Assignments from the base type to one of its subrange types
 (and vice versa) are allowed.
 
 A subrange type has the same size as its base type (``int`` in the
@@ -855,7 +969,7 @@ The IEEE standard defines five types of floating-point exceptions:
 * Inexact: operation produces a result that cannot be represented with infinite
   precision, for example, 2.0 / 3.0, log(1.1) and 0.1 in input.
 
-The IEEE exceptions are either ignored at runtime or mapped to the
+The IEEE exceptions are either ignored during execution or mapped to the
 Nim exceptions: `FloatInvalidOpError`:idx:, `FloatDivByZeroError`:idx:,
 `FloatOverflowError`:idx:, `FloatUnderflowError`:idx:,
 and `FloatInexactError`:idx:.
@@ -882,8 +996,9 @@ The only operations that are affected by the ``floatChecks`` pragma are
 the ``+``, ``-``, ``*``, ``/`` operators for floating point types.
 
 An implementation should always use the maximum precision available to evaluate
-floating pointer values at compile time; this means expressions like
+floating pointer values during semantic analysis; this means expressions like
-``0.09'f32 + 0.01'f32 == 0.09'f64 + 0.01'f64`` are true.
+``0.09'f32 + 0.01'f32 == 0.09'f64 + 0.01'f64`` that are evaluating during
+constant folding are true.
 
 
 Boolean type
@@ -1110,9 +1225,9 @@ tuples, objects and sets belong to the structured types.
 
 Array and sequence types
 ------------------------
-Arrays are a homogeneous type, meaning that each element in the array
+Arrays are a homogeneous type, meaning that each element in the array has the
-has the same type. Arrays always have a fixed length which is specified at
+same type. Arrays always have a fixed length specified as a constant expression
-compile time (except for open arrays). They can be indexed by any ordinal type.
+(except for open arrays). They can be indexed by any ordinal type.
 A parameter ``A`` may be an *open array*, in which case it is indexed by
 integers from 0 to ``len(A)-1``. An array expression may be constructed by the
 array constructor ``[]``. The element type of this array expression is
@@ -1154,7 +1269,7 @@ operator, and remove (and get) the last element of a sequence with the
 
 The notation ``x[i]`` can be used to access the i-th element of ``x``.
 
-Arrays are always bounds checked (at compile-time or at runtime). These
+Arrays are always bounds checked (statically or at runtime). These
 checks can be disabled via pragmas or invoking the compiler with the
 ``--boundChecks:off`` command line switch.
 
@@ -1317,10 +1432,10 @@ can also be defined with indentation instead of ``[]``:
       name: string   # a person consists of a name
       age: natural   # and an age
 
-Objects provide many features that tuples do not. Object provide inheritance
+Objects provide many features that tuples do not. Object provide inheritance and
-and information hiding. Objects have access to their type at runtime, so that
+information hiding. Objects have access to their type during at runtime, so that
-the ``of`` operator can be used to determine the object's type. The ``of`` operator
+the ``of`` operator can be used to determine the object's type. The ``of``
-is similar to the ``instanceof`` operator in Java.
+operator is similar to the ``instanceof`` operator in Java.
 
 .. code-block:: nim
   type
@@ -1415,8 +1530,8 @@ In the example the ``kind`` field is called the `discriminator`:idx:\: For
 safety its address cannot be taken and assignments to it are restricted: The
 new value must not lead to a change of the active object branch. For an object
 branch switch ``system.reset`` has to be used. Also, when the fields of a
-particular branch are specified during object construction, the correct value
+particular branch are specified during object construction, the corresponding
-for the discriminator must be supplied at compile-time.
+discriminator value must be specified as a constant expression.
 
 Package level objects
 ---------------------
@@ -1430,8 +1545,8 @@ contexts (``var/ref/ptr IncompleteObject``) in general since the compiler does
 not yet know the size of the object. To complete an incomplete object
 the ``package`` pragma has to be used. ``package`` implies ``byref``.
 
-As long as a type ``T`` is incomplete ``sizeof(T)`` or "runtime type
+As long as a type ``T`` is incomplete, neither ``sizeof(T)`` nor runtime
-information" for ``T`` is not available.
+type information for ``T`` is available.
 
 
 Example:
@@ -1587,7 +1702,7 @@ details like this when mixing garbage collected data with unmanaged memory.
 Not nil annotation
 ------------------
 
-All types for that ``nil`` is a valid value can be annotated to
+All types for which ``nil`` is a valid value can be annotated to
 exclude ``nil`` as a valid value with the ``not nil`` annotation:
 
 .. code-block:: nim
@@ -1664,12 +1779,12 @@ Nim supports these `calling conventions`:idx:\:
     and another one for the pointer to implicitly passed environment.
 
 `stdcall`:idx:
-    This the stdcall convention as specified by Microsoft. The generated C
+    This is the stdcall convention as specified by Microsoft. The generated C
     procedure is declared with the ``__stdcall`` keyword.
 
 `cdecl`:idx:
     The cdecl convention means that a procedure shall use the same convention
-    as the C compiler. Under windows the generated C procedure is declared with
+    as the C compiler. Under Windows the generated C procedure is declared with
     the ``__cdecl`` keyword.
 
 `safecall`:idx:
@@ -1860,7 +1975,7 @@ that don't. Distinct types provide a means to introduce a new string type
     username: string
 
   db.query("SELECT FROM users WHERE name = '$1'" % username)
-  # Error at compile time: `query` expects an SQL string!
+  # Static error: `query` expects an SQL string!
 
 
 It is an essential property of abstract types that they **do not** imply a
@@ -2247,7 +2362,8 @@ Overloading resolution
 ======================
 
 In a call ``p(args)`` the routine ``p`` that matches best is selected. If
-multiple routines match equally well, the ambiguity is reported at compiletime.
+multiple routines match equally well, the ambiguity is reported during
+semantic analysis.
 
 Every arg in args needs to match. There are multiple different categories how an
 argument can match. Let ``f`` be the formal parameter's type and ``a`` the type
@@ -2628,53 +2744,36 @@ identifier ``_`` can be used to ignore some parts of the tuple:
 Const section
 -------------
 
-`Constants`:idx: are symbols which are bound to a value. The constant's value
+A const section declares constants whose values are constant expressions:
-cannot change. The compiler must be able to evaluate the expression in a
-constant declaration at compile time.
 
-Nim contains a sophisticated compile-time evaluator, so procedures which
+.. code-block::
-have no side-effect can be used in constant expressions too:
-
-.. code-block:: nim
   import strutils
   const
+    roundPi = 3.1415
     constEval = contains("abc", 'b') # computed at compile time!
 
+Once declared, a constant's symbol can be used as a constant expression.
 
-The rules for compile-time computability are:
+See `Constants and Constant Expressions <#constants-and-constant-expressions>`_
-
+for details.
-1. Literals are compile-time computable.
-2. Type conversions are compile-time computable.
-3. Procedure calls of the form ``p(X)`` are compile-time computable if
-   ``p`` is a proc without side-effects (see the `noSideEffect pragma
-   <#pragmas-nosideeffect-pragma>`_ for details) and if ``X`` is a
-   (possibly empty) list of compile-time computable arguments.
-
-
-Constants cannot be of type ``ptr``, ``ref`` or ``var``, nor can
-they contain such a type.
-
 
 Static statement/expression
 ---------------------------
 
-A static statement/expression can be used to enforce compile
+A static statement/expression explicitly requires compile-time execution.
-time evaluation explicitly. Enforced compile time evaluation can even evaluate
+Even some code that has side effects is permitted in a static block:
-code that has side effects:
 
 .. code-block::
 
   static:
     echo "echo at compile time"
 
-It's a static error if the compiler cannot perform the evaluation at compile
+There are limitations on what Nim code can be executed at compile time;
+see `Restrictions on Compile-Time Execution
+<#restrictions-on-compileminustime-execution>`_ for details.
+It's a static error if the compiler cannot execute the block at compile
 time.
 
-The current implementation poses some restrictions for compile time
-evaluation: Code which contains ``cast`` or makes use of the foreign function
-interface cannot be evaluated at compile time. Later versions of Nim will
-support the FFI at compile time.
-
 
 If statement
 ------------
@@ -2748,9 +2847,9 @@ empty ``discard`` statement should be used.
 For non ordinal types it is not possible to list every possible value and so
 these always require an ``else`` part.
 
-As case statements perform compile-time exhaustiveness checks, the value in
+Because case statements are checked for exhaustiveness during semantic analysis,
-every ``of`` branch must be known at compile time. This fact is also exploited
+the value in every ``of`` branch must be a constant expression.
-to generate more performant code.
+This restriction also allows the compiler to generate more performant code.
 
 As a special semantic extension, an expression in an ``of`` branch of a case
 statement may evaluate to a set or array constructor; the set or array is then
@@ -2808,17 +2907,18 @@ When nimvm statement
 --------------------
 
 ``nimvm`` is a special symbol, that may be used as expression of ``when nimvm``
-statement to differentiate execution path between runtime and compile time.
+statement to differentiate execution path between compile time and the
+executable.
 
 Example:
 
 .. code-block:: nim
   proc someProcThatMayRunInCompileTime(): bool =
     when nimvm:
-      # This code runs in compile time
+      # This branch is taken at compile time.
       result = true
     else:
-      # This code runs in runtime
+      # This branch is taken in the executable.
       result = false
   const ctValue = someProcThatMayRunInCompileTime()
   let rtValue = someProcThatMayRunInCompileTime()
@@ -3558,7 +3658,7 @@ returned value is an l-value and can be modified by the caller:
   writeAccessToG() = 6
   assert g == 6
 
-It is a compile time error if the implicitly introduced pointer could be
+It is a static error if the implicitly introduced pointer could be
 used to access a location beyond its lifetime:
 
 .. code-block:: nim
@@ -3674,7 +3774,7 @@ Invocation of a multi-method cannot be ambiguous: collide 2 is preferred over
 collide 1 because the resolution works from left to right.
 In the example ``Unit, Thing`` is preferred over ``Thing, Unit``.
 
-**Note**: Compile time evaluation is not (yet) supported for methods.
+**Note**: Compile-time execution is not (yet) supported for methods.
 
 
 Inhibit dynamic method resolution via procCall
@@ -3770,7 +3870,14 @@ First class iterators
 There are 2 kinds of iterators in Nim: *inline* and *closure* iterators.
 An `inline iterator`:idx: is an iterator that's always inlined by the compiler
 leading to zero overhead for the abstraction, but may result in a heavy
-increase in code size. Inline iterators are second class citizens;
+increase in code size.
+
+Caution: the body of a for loop over an inline iterator is inlined into
+each ``yield`` statement appearing in the iterator code,
+so ideally the code should be refactored to contain a single yield when possible
+to avoid code bloat.
+
+Inline iterators are second class citizens;
 They can be passed as parameters only to other inlining code facilities like
 templates, macros and other inline iterators.
 
@@ -3795,7 +3902,7 @@ In contrast to that, a `closure iterator`:idx: can be passed around more freely:
 Closure iterators have other restrictions than inline iterators:
 
 1. ``yield`` in a closure iterator can not occur in a ``try`` statement.
-2. For now, a closure iterator cannot be evaluated at compile time.
+2. For now, a closure iterator cannot be executed at compile time.
 3. ``return`` is allowed in a closure iterator (but rarely useful) and ends
    iteration.
 4. Neither inline nor closure iterators can be recursive.
@@ -4114,8 +4221,8 @@ The exception tree is defined in the `system <system.html>`_ module.
 Every exception inherits from ``system.Exception``. Exceptions that indicate
 programming bugs inherit from ``system.Defect`` (which is a subtype of ``Exception``)
 and are stricly speaking not catchable as they can also be mapped to an operation
-that terminates the whole process. Exceptions that indicate any other runtime error
+that terminates the whole process. Exceptions that indicate any other runtime
-that can be caught inherit from ``system.CatchableError``
+error that can be caught inherit from ``system.CatchableError``
 (which is a subtype of ``Exception``).
 
 
@@ -4352,8 +4459,9 @@ a `type variable`:idx:.
 Is operator
 -----------
 
-The ``is`` operator checks for type equivalence at compile time. It is
+The ``is`` operator is evaluated during semantic analysis to check for type
-therefore very useful for type specialization within generic code:
+equivalence. It is therefore very useful for type specialization within generic
+code:
 
 .. code-block:: nim
   type
@@ -5349,14 +5457,22 @@ chance to convert it into a sequence.
 Macros
 ======
 
-A macro is a special function that is executed at compile-time.
+A macro is a special function that is executed at compile time.
 Normally the input for a macro is an abstract syntax
 tree (AST) of the code that is passed to it. The macro can then do
-transformations on it and return the transformed AST. The
+transformations on it and return the transformed AST. This can be used to
-transformed AST is then passed to the compiler as if the macro
+add custom language features and implement `domain specific languages`:idx:.
-invocation would have been replaced by its result in the source
+
-code. This can be used to implement `domain specific
+Macro invocation is a case where semantic analyis does **not** entirely proceed
-languages`:idx:.
+top to bottom and left to right. Instead, semantic analysis happens at least
+twice:
+
+* Semantic analysis recognizes and resolves the macro invocation.
+* The compiler executes the macro body (which may invoke other procs).
+* It replaces the AST of the macro invocation with the AST returned by the macro.
+* It repeats semantic analysis of that region of the code.
+* If the AST returned by the macro contains other macro invocations,
+  this process iterates.
 
 While macros enable advanced compile-time code transformations, they
 cannot change Nim's syntax. However, this is no real restriction because
@@ -5640,7 +5756,7 @@ static[T]
 
 **Note**: static[T] is still in development.
 
-As their name suggests, static parameters must be known at compile-time:
+As their name suggests, static parameters must be constant expressions:
 
 .. code-block:: nim
 
@@ -5652,7 +5768,7 @@ As their name suggests, static parameters must be known at compile-time:
                           # regex, stored in a global variable
 
   precompiledRegex(paramStr(1)) # Error, command-line options
-                                # are not known at compile-time
+                                # are not constant expressions
 
 
 For the purposes of code generation, all static params are treated as
@@ -5666,7 +5782,7 @@ Static params can also appear in the signatures of generic types:
   type
     Matrix[M,N: static int; T: Number] = array[0..(M*N - 1), T]
       # Note how `Number` is just a type constraint here, while
-      # `static int` requires us to supply a compile-time int value
+      # `static int` requires us to supply an int value
 
     AffineTransform2D[T] = Matrix[3, 3, T]
     AffineTransform3D[T] = Matrix[4, 4, T]
@@ -5674,14 +5790,13 @@ Static params can also appear in the signatures of generic types:
   var m1: AffineTransform3D[float]  # OK
   var m2: AffineTransform2D[string] # Error, `string` is not a `Number`
 
-Please note that ``static T`` is just a syntactic convenience for the
+Please note that ``static T`` is just a syntactic convenience for the underlying
-underlying generic type ``static[T]``. The type param can be omitted
+generic type ``static[T]``. The type param can be omitted to obtain the type
-to obtain the type class of all values known at compile-time. A more
+class of all constant expressions. A more specific type class can be created by
-specific type class can be created by instantiating ``static`` with
+instantiating ``static`` with another type class.
-another type class.
 
-You can force the evaluation of a certain expression at compile-time by
+You can force an expression to be evaluated at compile time as a constant
-coercing it to a corresponding ``static`` type:
+expression by coercing it to a corresponding ``static`` type:
 
 .. code-block:: nim
   import math
@@ -6538,6 +6653,13 @@ modules don't need to import a module's dependencies:
 When the exported symbol is another module, all of its definitions will
 be forwarded. You can use an ``except`` list to exclude some of the symbols.
 
+Notice that when exporting, you need to specify only the module name:
+
+.. code-block:: nim
+  import foo/bar/baz
+  export baz
+
+
 
 Scope rules
 -----------
@@ -6767,11 +6889,11 @@ pragma block can be used:
 
 compileTime pragma
 ------------------
-The ``compileTime`` pragma is used to mark a proc or variable to be used at
+The ``compileTime`` pragma is used to mark a proc or variable to be used only
-compile time only. No code will be generated for it. Compile time procs are
+during compile-time execution. No code will be generated for it. Compile-time
-useful as helpers for macros. Since version 0.12.0 of the language, a proc
+procs are useful as helpers for macros. Since version 0.12.0 of the language, a
-that uses ``system.NimNode`` within its parameter types is implicitly declared
+proc that uses ``system.NimNode`` within its parameter types is implicitly
-``compileTime``:
+declared ``compileTime``:
 
 .. code-block:: nim
   proc astHelper(n: NimNode): NimNode =
@@ -6858,8 +6980,8 @@ structure:
 
 pure pragma
 -----------
-An object type can be marked with the ``pure`` pragma so that its type
+An object type can be marked with the ``pure`` pragma so that its type field
-field which is used for runtime type identification is omitted. This used to be
+which is used for runtime type identification is omitted. This used to be
 necessary for binary compatibility with other compiled languages.
 
 An enum type can be marked as ``pure``. Then access of its fields always
@@ -6885,7 +7007,7 @@ though.
 
 The ``error`` pragma can also be used to
 annotate a symbol (like an iterator or proc). The *usage* of the symbol then
-triggers a compile-time error. This is especially useful to rule out that some
+triggers a static error. This is especially useful to rule out that some
 operation is valid due to overloading and type conversions:
 
 .. code-block:: nim
@@ -6923,7 +7045,7 @@ statement as seen in stack backtraces:
   template myassert*(cond: untyped, msg = "") =
     if not cond:
       # change run-time line information of the 'raise' statement:
-      {.line: InstantiationInfo().}:
+      {.line: instantiationInfo().}:
         raise newException(EAssertionFailed, msg)
 
 If the ``line`` pragma is used with a parameter, the parameter needs be a
@@ -7007,7 +7129,7 @@ extension the pragma is simply ignored.
 unroll pragma
 -------------
 The ``unroll`` pragma can be used to tell the compiler that it should unroll
-a `for`:idx: or `while`:idx: loop for runtime efficiency:
+a `for`:idx: or `while`:idx: loop for execution efficiency:
 
 .. code-block:: nim
   proc searchChar(s: string, c: char): int =
@@ -7322,7 +7444,8 @@ compiler like you would using the commandline switch ``--passC``:
   {.passC: "-Wall -Werror".}
 
 Note that you can use ``gorge`` from the `system module <system.html>`_ to
-embed parameters from an external command at compile time:
+embed parameters from an external command that will be executed
+during semantic analysis:
 
 .. code-block:: Nim
   {.passC: gorge("pkg-config --cflags sdl").}
@@ -7336,7 +7459,8 @@ like you would using the commandline switch ``--passL``:
   {.passL: "-lSDLmain -lSDL".}
 
 Note that you can use ``gorge`` from the `system module <system.html>`_ to
-embed parameters from an external command at compile time:
+embed parameters from an external command that will be executed
+during semantic analysis:
 
 .. code-block:: Nim
   {.passL: gorge("pkg-config --libs sdl").}
@@ -7741,6 +7865,7 @@ pragma             description
 =================  ============================================
 `intdefine`:idx:   Reads in a build-time define as an integer
 `strdefine`:idx:   Reads in a build-time define as a string
+`booldefine`:idx:  Reads in a build-time define as a bool
 =================  ============================================
 
 .. code-block:: nim
@@ -7748,13 +7873,14 @@ pragma             description
    echo FooBar
 
 ::
-   nim c -d:FooBar=42 foobar.c
+   nim c -d:FooBar=42 foobar.nim
 
 In the above example, providing the -d flag causes the symbol
 ``FooBar`` to be overwritten at compile time, printing out 42. If the
 ``-d:FooBar=42`` were to be omitted, the default value of 5 would be
-used.
+used. To see if a value was provided, `defined(FooBar)` can be used.
 
+The syntax `-d:flag` is actually just a shortcut for `-d:flag=true`.
 
 Custom annotations
 ------------------
@@ -7947,7 +8073,7 @@ interoperability with C. Combining packed pragma with inheritance is not
 defined, and it should not be used with GC'ed memory (ref's).
 
 **Future directions**: Using GC'ed memory in packed pragma will result in
-compile-time error. Usage with inheritance should be defined and documented.
+a static error. Usage with inheritance should be defined and documented.
 
 
 Dynlib pragma for import
@@ -8228,11 +8354,11 @@ Example:
   echo formatFloat(pi(5000))
 
 
-The parallel statement is the preferred mechanism to introduce parallelism
+The parallel statement is the preferred mechanism to introduce parallelism in a
-in a Nim program. A subset of the Nim language is valid within a
+Nim program. A subset of the Nim language is valid within a ``parallel``
-``parallel`` section. This subset is checked to be free of data races at
+section. This subset is checked during semantic analysis to be free of data
-compile time. A sophisticated `disjoint checker`:idx: ensures that no data
+races. A sophisticated `disjoint checker`:idx: ensures that no data races are
-races are possible even though shared memory is extensively supported!
+possible even though shared memory is extensively supported!
 
 The subset is in fact the full language with the following
 restrictions / changes:
@@ -8264,8 +8390,8 @@ pragmas:
 1) A `guard`:idx: annotation is introduced to prevent data races.
 2) Every access of a guarded memory location needs to happen in an
    appropriate `locks`:idx: statement.
-3) Locks and routines can be annotated with `lock levels`:idx: to prevent
+3) Locks and routines can be annotated with `lock levels`:idx: to allow
-   deadlocks at compile time.
+   potential deadlocks to be detected during semantic analysis.
 
 
 Guards and the locks section
@@ -8385,9 +8511,10 @@ This means the following compiles (for now) even though it really should not:
 Lock levels
 -----------
 
-Lock levels are used to enforce a global locking order in order to prevent
+Lock levels are used to enforce a global locking order in order to detect
-deadlocks at compile-time. A lock level is an constant integer in the range
+potential deadlocks during semantic analysis. A lock level is an constant
-0..1_000. Lock level 0 means that no lock is acquired at all.
+integer in the range 0..1_000. Lock level 0 means that no lock is acquired at
+all.
 
 If a section of code holds a lock of level ``M`` than it can also acquire any
 lock of level ``N < M``. Another lock of level ``M`` cannot be acquired. Locks

doc/nimc.rst

@@ -347,18 +347,22 @@ complete list.
 Define                   Effect
 ======================   =========================================================
 ``release``              Turns off runtime checks and turns on the optimizer.
+                         More aggressive optimizations are possible, eg:
+                         ``--passC:-ffast-math`` (but see issue #10305)
+                         ``--stacktrace:off``
 ``useWinAnsi``           Modules like ``os`` and ``osproc`` use the Ansi versions
                          of the Windows API. The default build uses the Unicode
                          version.
 ``useFork``              Makes ``osproc`` use ``fork`` instead of ``posix_spawn``.
 ``useNimRtl``            Compile and link against ``nimrtl.dll``.
 ``useMalloc``            Makes Nim use C's `malloc`:idx: instead of Nim's
-                         own memory manager, ableit prefixing each allocation with
+                         own memory manager, albeit prefixing each allocation with
                          its size to support clearing memory on reallocation.
                          This only works with ``gc:none``.
 ``useRealtimeGC``        Enables support of Nim's GC for *soft* realtime
                          systems. See the documentation of the `gc <gc.html>`_
                          for further information.
+``logGC``                Enable GC logging to stdout.
 ``nodejs``               The JS target is actually ``node.js``.
 ``ssl``                  Enables OpenSSL support for the sockets module.
 ``memProfiler``          Enables memory profiling for the native GC.

doc/nims.rst

@@ -26,9 +26,28 @@ previous settings):
    ``$project.nim``. This file can be skipped with the same
    ``--skipProjCfg`` command line option.
 
-The VM cannot deal with ``importc`` because the FFI is not
+Limitations
-available. So the stdlib modules using ``importc`` cannot be used with
+=================================
-Nim's VM. However, at least the following modules are available:
+
+NimScript is subject to some limitations caused by the implementation of the VM
+(virtual machine):
+
+* Nim's FFI (foreign function interface) is not available in NimScript. This
+  means that any stdlib module which relies on ``importc`` can not be used in
+  the VM.
+
+* ``ptr`` operations are are hard to emulate with the symbolic representation
+  the VM uses. They are available and tested extensively but there are bugs left.
+
+* ``var T`` function arguments rely on ``ptr`` operations internally and might
+  also be problematic in some cases.
+
+* More than one level of `ref` is generally not supported (for example, the type 
+  `ref ref int`).
+
+* multimethods are not available.
+
+Given the above restrictions, at least the following modules are available:
 
 * `macros <macros.html>`_
 * `os <os.html>`_
@@ -98,17 +117,6 @@ Task          Description
 =========     ===================================================
 
 
-If the task runs an external command via ``exec`` it should afterwards call
-``setCommand "nop"`` to tell the Nim compiler that nothing else needs to be
-done:
-
-.. code-block:: nim
-
-  task tests, "test regular expressions":
-    exec "nim c -r tests"
-    setCommand "nop"
-
-
 Look at the module `distros <distros.html>`_ for some support of the
 OS's native package managers.
 

doc/packaging.rst

@@ -0,0 +1,69 @@
+=============
+Packaging Nim
+=============
+
+
+Supported architectures
+-----------------------
+
+Nim runs on a wide variety of platforms. Support on amd64 and i386 is tested regularly, while less popular platforms are tested by the community.
+
+- amd64
+- arm64 (aka aarch64)
+- armel
+- armhf
+- i386
+- m68k
+- mips64el
+- mipsel
+- powerpc
+- ppc64
+- ppc64el (aka ppc64le)
+- riscv64
+
+The following platforms are seldomly tested:
+
+- alpha
+- hppa
+- ia64
+- mips
+- s390x
+- sparc64
+
+Packaging for Linux
+-------------------
+
+See https://github.com/nim-lang/Nim/labels/Installation for installation-related bugs.
+
+Build Nim from the released tarball at https://nim-lang.org/install_unix.html
+It is different from the GitHub sources as it contains Nimble, C sources & other tools.
+
+The Debian package ships bash and ksh completion and manpages that can be reused.
+
+Hints on the build process:
+
+::
+
+   # build from C sources and then using koch
+   ./build.sh --os $os_type --cpu $cpu_arch
+   ./bin/nim c koch
+   ./koch boot -d:release
+
+   # optionally generate docs into doc/html
+   ./koch docs
+
+   ./koch tools -d:release
+
+   # extract files to be really installed
+   ./install.sh <tempdir>
+
+   # also include the tools
+   for fn in nimble nimsuggest nimgrep; do cp ./bin/$fn <tempdir>/nim/bin/; done
+
+What to install:
+
+- The expected stdlib location is /usr/lib/nim
+- Global configuration files under /etc/nim
+- Optionally: manpages, documentation, shell completion
+- When installing documentation, .idx files are not required
+- The "compiler" directory contains compiler sources and should not be part of the compiler binary package

doc/regexprs.txt

@@ -80,13 +80,13 @@ meta character     meaning
 ``|``              start of alternative branch
 ``(``              start subpattern
 ``)``              end subpattern
-``?``              extends the meaning of ``(``
-                   also 0 or 1 quantifier
-                   also quantifier minimizer
-``*``              0 or more quantifier
-``+``              1 or more quantifier
-                   also "possessive quantifier"
 ``{``              start min/max quantifier
+``?``              extends the meaning of ``(``
+                   | also 0 or 1 quantifier (equal to ``{0,1}``)
+                   | also quantifier minimizer
+``*``              0 or more quantifier (equal to ``{0,}``)
+``+``              1 or more quantifier (equal to ``{1,}``)
+                   | also "possessive quantifier"
 ==============     ============================================================
 
 

doc/tut3.rst

@@ -19,16 +19,16 @@ a Nim syntax tree into a different tree.
 
 Examples of things that can be implemented in macros:
 
- * An assert macro that prints both sides of a comparison operator, if
+* An assert macro that prints both sides of a comparison operator, if
-the assertion fails. ``myAssert(a == b)`` is converted to
+  the assertion fails. ``myAssert(a == b)`` is converted to
-``if a != b: quit($a " != " $b)``
+  ``if a != b: quit($a " != " $b)``
 
- * A debug macro that prints the value and the name of the symbol.
+* A debug macro that prints the value and the name of the symbol.
-``myDebugEcho(a)`` is converted to ``echo "a: ", a``
+  ``myDebugEcho(a)`` is converted to ``echo "a: ", a``
 
- * Symbolic differentiation of an expression.
+* Symbolic differentiation of an expression.
-``diff(a*pow(x,3) + b*pow(x,2) + c*x + d, x)``  is converted to
+  ``diff(a*pow(x,3) + b*pow(x,2) + c*x + d, x)`` is converted to
-``3*a*pow(x,2) + 2*a*x + c``
+  ``3*a*pow(x,2) + 2*a*x + c``
 
 
 Macro Arguments

koch.nim

@@ -46,6 +46,8 @@ Possible Commands:
   boot [options]           bootstraps with given command line options
   distrohelper [bindir]    helper for distro packagers
   tools                    builds Nim related tools
+  toolsNoNimble            builds Nim related tools (except nimble)
+                           doesn't require network connectivity
   nimble                   builds the Nimble tool
 Boot options:
   -d:release               produce a release version of the compiler
@@ -56,6 +58,7 @@ Boot options:
                            for bootstrapping
 
 Commands for core developers:
+  runCI                    runs continuous integration (CI), eg from travis
   docs [options]           generates the full documentation
   csource -d:release       builds the C sources for installation
   pdf                      builds the PDF documentation
@@ -71,13 +74,22 @@ Web options:
                            build the official docs, use UA-48159761-1
 """
 
+let kochExe* = when isMainModule: os.getAppFilename() # always correct when koch is main program, even if `koch` exe renamed eg: `nim c -o:koch_debug koch.nim`
+               else: getAppDir() / "koch".exe # works for winrelease
+
+proc kochExec*(cmd: string) =
+  exec kochExe.quoteShell & " " & cmd
+
+proc kochExecFold*(desc, cmd: string) =
+  execFold(desc, kochExe.quoteShell & " " & cmd)
+
 template withDir(dir, body) =
   let old = getCurrentDir()
   try:
     setCurrentDir(dir)
     body
   finally:
-    setCurrentdir(old)
+    setCurrentDir(old)
 
 let origDir = getCurrentDir()
 setCurrentDir(getAppDir())
@@ -116,14 +128,16 @@ proc bundleNimbleSrc(latest: bool) =
       exec("git checkout -f stable")
       exec("git pull")
 
+proc bundleC2nim() =
+  if not dirExists("dist/c2nim/.git"):
+    exec("git clone https://github.com/nim-lang/c2nim.git dist/c2nim")
+  nimCompile("dist/c2nim/c2nim", options = "--noNimblePath --path:.")
+
 proc bundleNimbleExe(latest: bool) =
   bundleNimbleSrc(latest)
   # installer.ini expects it under $nim/bin
   nimCompile("dist/nimble/src/nimble.nim", options = "-d:release --nilseqs:on")
 
-proc buildNimfind() =
-  nimCompile("tools/nimfind.nim", options = "-d:release")
-
 proc buildNimble(latest: bool) =
   # old installations created nim/nimblepkg/*.nim files. We remove these
   # here so that it cannot cause problems (nimble bug #306):
@@ -154,7 +168,7 @@ proc bundleNimsuggest() =
   nimCompile("nimsuggest/nimsuggest.nim", options = "-d:release")
 
 proc buildVccTool() =
-  nimCompile("tools/vccexe/vccexe.nim")
+  nimCompileFold("Compile Vcc", "tools/vccexe/vccexe.nim")
 
 proc bundleWinTools() =
   # TODO: consider building under `bin` instead of `.`
@@ -163,6 +177,7 @@ proc bundleWinTools() =
   buildVccTool()
   nimCompile("tools/nimgrab.nim", options = "-d:ssl")
   nimCompile("tools/nimgrep.nim")
+  bundleC2nim()
   when false:
     # not yet a tool worth including
     nimCompile(r"tools\downloader.nim", options = r"--cc:vcc --app:gui -d:ssl --noNimblePath --path:..\ui")
@@ -197,13 +212,12 @@ proc buildTool(toolname, args: string) =
   nimexec("cc $# $#" % [args, toolname])
   copyFile(dest="bin" / splitFile(toolname).name.exe, source=toolname.exe)
 
-proc buildTools(latest: bool) =
+proc buildTools() =
   bundleNimsuggest()
-  nimCompile("tools/nimgrep.nim", options = "-d:release")
+  nimCompileFold("Compile nimgrep", "tools/nimgrep.nim", options = "-d:release")
   when defined(windows): buildVccTool()
-  nimCompile("nimpretty/nimpretty.nim", options = "-d:release")
+  nimCompileFold("Compile nimpretty", "nimpretty/nimpretty.nim", options = "-d:release")
-  buildNimble(latest)
+  nimCompileFold("Compile nimfind", "tools/nimfind.nim", options = "-d:release")
-  buildNimfind()
 
 proc nsis(latest: bool; args: string) =
   bundleNimbleExe(latest)
@@ -272,16 +286,30 @@ proc boot(args: string) =
   var output = "compiler" / "nim".exe
   var finalDest = "bin" / "nim".exe
   # default to use the 'c' command:
-  let defaultCommand = if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": "cpp" else: "c"
+  let useCpp = getEnv("NIM_COMPILE_TO_CPP", "false") == "true"
-  let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: ""
   let smartNimcache = (if "release" in args: "nimcache/r_" else: "nimcache/d_") &
                       hostOs & "_" & hostCpu
 
-  copyExe(findStartNim(), 0.thVersion)
+  let nimStart = findStartNim()
+  copyExe(nimStart, 0.thVersion)
   for i in 0..2:
+    let defaultCommand = if useCpp: "cpp" else: "c"
+    let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: ""
     echo "iteration: ", i+1
-    exec i.thVersion & " $# $# --nimcache:$# compiler" / "nim.nim" % [bootOptions, args,
+    var extraOption = ""
-        smartNimcache]
+    if i == 0:
+      extraOption.add " --skipUserCfg --skipParentCfg"
+        # Note(D20190115T162028:here): the configs are skipped for bootstrap
+        # (1st iteration) to prevent newer flags from breaking bootstrap phase.
+        # fixes #10030.
+      let ret = execCmdEx(nimStart & " --version")
+      doAssert ret.exitCode == 0
+      let version = ret.output.splitLines[0]
+      if version.startsWith "Nim Compiler Version 0.19.0":
+        extraOption.add " -d:nimBoostrapCsources0_19_0"
+        # remove this when csources get updated
+    exec i.thVersion & " $# $# $# --nimcache:$# compiler" / "nim.nim" %
+      [bootOptions, extraOption, args, smartNimcache]
     if sameFileContent(output, i.thVersion):
       copyExe(output, finalDest)
       echo "executables are equal: SUCCESS!"
@@ -351,9 +379,10 @@ proc winReleaseArch(arch: string) =
   withMingw r"..\mingw" & arch & r"\bin":
     # Rebuilding koch is necessary because it uses its pointer size to
     # determine which mingw link to put in the NSIS installer.
-    nimexec "c --cpu:$# koch" % cpu
+    inFold "winrelease koch":
-    exec "koch boot -d:release --cpu:$#" % cpu
+      nimexec "c --cpu:$# koch" % cpu
-    exec "koch --latest zip -d:release"
+    kochExecFold("winrelease boot", "boot -d:release --cpu:$#" % cpu)
+    kochExecFold("winrelease zip", "--latest zip -d:release")
     overwriteFile r"build\nim-$#.zip" % VersionAsString,
              r"web\upload\download\nim-$#_x$#.zip" % [VersionAsString, arch]
 
@@ -362,13 +391,16 @@ proc winRelease*() =
   # anymore!
   # Build -docs file:
   when true:
-    buildDocs(gaCode)
+    inFold "winrelease buildDocs":
+      buildDocs(gaCode)
     withDir "web/upload/" & VersionAsString:
-      exec "7z a -tzip docs-$#.zip *.html" % VersionAsString
+      inFold "winrelease zipdocs":
+        exec "7z a -tzip docs-$#.zip *.html" % VersionAsString
     overwriteFile "web/upload/$1/docs-$1.zip" % VersionAsString,
                   "web/upload/download/docs-$1.zip" % VersionAsString
   when true:
-    csource("-d:release")
+    inFold "winrelease csource":
+      csource("-d:release")
   when sizeof(pointer) == 4:
     winReleaseArch "32"
   when sizeof(pointer) == 8:
@@ -428,6 +460,47 @@ proc xtemp(cmd: string) =
   finally:
     copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe)
 
+proc runCI(cmd: string) =
+  doAssert cmd.len == 0, cmd # avoid silently ignoring
+  echo "runCI:", cmd
+  # note(@araq): Do not replace these commands with direct calls (eg boot())
+  # as that would weaken our testing efforts.
+  when defined(posix): # appveyor (on windows) didn't run this
+    kochExecFold("Boot", "boot")
+  kochExecFold("Boot in release mode", "boot -d:release")
+
+  ## build nimble early on to enable remainder to depend on it if needed
+  kochExecFold("Build Nimble", "nimble")
+
+  when false:
+    for pkg in "zip opengl sdl1 jester@#head niminst".split:
+      exec "nimble install -y" & pkg
+
+  buildTools() # altenatively, kochExec "tools --toolsNoNimble"
+
+  ## run tests
+  execFold("Test nimscript", "nim e tests/test_nimscript.nims")
+  when defined(windows):
+    # note: will be over-written below
+    execFold("Compile tester", "nim c -d:nimCoroutines --os:genode -d:posix --compileOnly testament/tester")
+
+  # main bottleneck here
+  execFold("Run tester", "nim c -r -d:nimCoroutines testament/tester --pedantic all -d:nimCoroutines")
+
+  execFold("Run nimdoc tests", "nim c -r nimdoc/tester")
+  execFold("Run nimpretty tests", "nim c -r nimpretty/tester.nim")
+  when defined(posix):
+    execFold("Run nimsuggest tests", "nim c -r nimsuggest/tester")
+
+  ## remaining actions
+  when defined(posix):
+    kochExecFold("Docs", "docs --git.commit:devel")
+    kochExecFold("C sources", "csource")
+  elif defined(windows):
+    when false:
+      kochExec "csource"
+      kochExec "zip"
+
 proc pushCsources() =
   if not dirExists("../csources/.git"):
     quit "[Error] no csources git repository found"
@@ -475,8 +548,7 @@ proc testUnixInstall(cmdLineRest: string) =
       execCleanPath("./koch --latest tools")
       # check the tests work:
       putEnv("NIM_EXE_NOT_IN_PATH", "NOT_IN_PATH")
-      execCleanPath("./koch tests", destDir / "bin")
+      execCleanPath("./koch tests cat megatest", destDir / "bin")
-      #execCleanPath("./koch tests cat newconfig", destDir / "bin")
     else:
       echo "Version check: failure"
   finally:
@@ -512,6 +584,10 @@ when isMainModule:
   var op = initOptParser()
   var latest = false
   var stable = false
+  template isLatest(): bool =
+    if stable: false
+    else:
+      existsDir(".git") or latest
   while true:
     op.next()
     case op.kind
@@ -537,19 +613,21 @@ when isMainModule:
       of "distrohelper": geninstall()
       of "install": install(op.cmdLineRest)
       of "testinstall": testUnixInstall(op.cmdLineRest)
+      of "runci": runCI(op.cmdLineRest)
       of "test", "tests": tests(op.cmdLineRest)
       of "temp": temp(op.cmdLineRest)
       of "xtemp": xtemp(op.cmdLineRest)
       of "wintools": bundleWinTools()
-      of "nimble":
+      of "nimble": buildNimble(isLatest())
-        if stable: buildNimble(false)
-        else: buildNimble(existsDir(".git") or latest)
       of "nimsuggest": bundleNimsuggest()
+      of "toolsnonimble":
+        buildTools()
       of "tools":
-        if stable: buildTools(false)
+        buildTools()
-        else: buildTools(existsDir(".git") or latest)
+        buildNimble(isLatest())
       of "pushcsource", "pushcsources": pushCsources()
       of "valgrind": valgrind(op.cmdLineRest)
+      of "c2nim": bundleC2nim()
       else: showHelp()
       break
     of cmdEnd: break

lib/core/allocators.nim

@@ -18,6 +18,7 @@ type
     realloc*: proc (a: Allocator; p: pointer; oldSize, newSize: int): pointer {.nimcall.}
     deallocAll*: proc (a: Allocator) {.nimcall.}
     flags*: set[AllocatorFlag]
+    name*: cstring
     allocCount: int
     deallocCount: int
 
@@ -40,6 +41,7 @@ proc getLocalAllocator*(): Allocator =
       result = system.realloc(p, newSize)
     result.deallocAll = nil
     result.flags = {ThreadLocal}
+    result.name = "nim_local"
     localAllocator = result
 
 proc setLocalAllocator*(a: Allocator) =

lib/core/macros.nim

@@ -151,7 +151,7 @@ proc `==`*(a, b: NimNode): bool {.magic: "EqNimrodNode", noSideEffect.}
 
 proc `==`*(a, b: NimSym): bool {.magic: "EqNimrodNode", noSideEffect, deprecated.}
   ## compares two Nim symbols
-  ## **Deprecated since version 0.18.1**; Use ```==`(NimNode,NimNode)`` instead.
+  ## **Deprecated since version 0.18.1**; Use ``==(NimNode, NimNode)`` instead.
 
 
 proc sameType*(a, b: NimNode): bool {.magic: "SameNodeType", noSideEffect.} =
@@ -377,7 +377,9 @@ proc copyNimNode*(n: NimNode): NimNode {.magic: "NCopyNimNode", noSideEffect.}
 proc copyNimTree*(n: NimNode): NimNode {.magic: "NCopyNimTree", noSideEffect.}
 
 proc error*(msg: string, n: NimNode = nil) {.magic: "NError", benign.}
-  ## writes an error message at compile time
+  ## writes an error message at compile time. The optional ``n: NimNode``
+  ## parameter is used as the source for file and line number information in
+  ## the compilation error message.
 
 proc warning*(msg: string, n: NimNode = nil) {.magic: "NWarning", benign.}
   ## writes a warning message at compile time
@@ -1038,8 +1040,8 @@ proc newProc*(name = newEmptyNode(); params: openArray[NimNode] = [newEmptyNode(
     name,
     newEmptyNode(),
     newEmptyNode(),
-    newNimNode(nnkFormalParams).add(params), ##params
+    newNimNode(nnkFormalParams).add(params),
-    newEmptyNode(),  ## pragmas
+    newEmptyNode(),  # pragmas
     newEmptyNode(),
     body)
 
@@ -1402,12 +1404,19 @@ proc customPragmaNode(n: NimNode): NimNode =
     let impl = n.getImpl()
     if impl.kind in RoutineNodes:
       return impl.pragma
+    elif impl.kind == nnkIdentDefs and impl[0].kind == nnkPragmaExpr:
+      return impl[0][1]
     else:
-      return typ.getImpl()[0][1]
+      let timpl = typ.getImpl()
+      if timpl.len>0 and timpl[0].len>1:
+        return timpl[0][1]
+      else:
+        return timpl
 
   if n.kind in {nnkDotExpr, nnkCheckedFieldExpr}:
     let name = $(if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1])
-    var typDef = getImpl(getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0]))
+    let typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0])
+    var typDef = getImpl(if typInst.kind == nnkVarTy: typInst[0] else: typInst)
     while typDef != nil:
       typDef.expectKind(nnkTypeDef)
       let typ = typDef[2]
@@ -1492,9 +1501,18 @@ macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped =
   let pragmaNode = customPragmaNode(n)
   for p in pragmaNode:
     if p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp:
-      return p[1]
+      if p.len == 2:
-
+        result = p[1]
-  error(n.repr & " doesn't have a pragma named " & cp.repr()) # returning an empty node results in most cases in a cryptic error,
+      else:
+        let def = p[0].getImpl[3]
+        result = newTree(nnkPar)
+        for i in 1..<p.len:
+          let key = def[i][0]
+          let val = p[i]
+          result.add newTree(nnkExprColonExpr, key, val)
+      break
+  if result.kind == nnkEmpty:
+    error(n.repr & " doesn't have a pragma named " & cp.repr()) # returning an empty node results in most cases in a cryptic error,
 
 
 when not defined(booting):

lib/core/seqs.nim

@@ -15,7 +15,7 @@ proc supportsCopyMem(t: typedesc): bool {.magic: "TypeTrait".}
 
 ## Default seq implementation used by Nim's core.
 type
-  NimSeqPayload {.core.}[T] = object
+  NimSeqPayload[T] = object
     cap: int
     region: Allocator
     data: UncheckedArray[T]
@@ -40,6 +40,7 @@ proc `=destroy`[T](s: var seq[T]) =
   var x = cast[ptr NimSeqV2[T]](addr s)
   var p = x.p
   if p != nil:
+    mixin `=destroy`
     when not supportsCopyMem(T):
       for i in 0..<x.len: `=destroy`(p.data[i])
     p.region.dealloc(p.region, p, payloadSize(p.cap))
@@ -47,11 +48,12 @@ proc `=destroy`[T](s: var seq[T]) =
     x.len = 0
 
 proc `=`[T](x: var seq[T]; y: seq[T]) =
+  mixin `=destroy`
   var a = cast[ptr NimSeqV2[T]](addr x)
   var b = cast[ptr NimSeqV2[T]](unsafeAddr y)
 
   if a.p == b.p: return
-  `=destroy`(a)
+  `=destroy`(x)
   a.len = b.len
   if b.p != nil:
     a.p = cast[type(a.p)](alloc(payloadSize(a.len)))
@@ -63,10 +65,11 @@ proc `=`[T](x: var seq[T]; y: seq[T]) =
         a.p.data[i] = b.p.data[i]
 
 proc `=sink`[T](x: var seq[T]; y: seq[T]) =
+  mixin `=destroy`
   var a = cast[ptr NimSeqV2[T]](addr x)
   var b = cast[ptr NimSeqV2[T]](unsafeAddr y)
   if a.p != nil and a.p != b.p:
-    `=destroy`(a)
+    `=destroy`(x)
   a.len = b.len
   a.p = b.p
 
@@ -109,6 +112,7 @@ proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize: int): pointer {.
       result = q
 
 proc shrink*[T](x: var seq[T]; newLen: Natural) =
+  mixin `=destroy`
   sysAssert newLen <= x.len, "invalid newLen parameter for 'shrink'"
   when not supportsCopyMem(T):
     for i in countdown(x.len - 1, newLen - 1):

lib/core/strs.nim

@@ -51,15 +51,12 @@ proc `=destroy`(s: var string) =
   a.len = 0
   a.p = nil
 
-template lose(a) =
-  frees(a)
-
 proc `=sink`(x: var string, y: string) =
   var a = cast[ptr NimStringV2](addr x)
   var b = cast[ptr NimStringV2](unsafeAddr y)
   # we hope this is optimized away for not yet alive objects:
   if unlikely(a.p == b.p): return
-  lose(a)
+  frees(a)
   a.len = b.len
   a.p = b.p
 
@@ -67,13 +64,13 @@ proc `=`(x: var string, y: string) =
   var a = cast[ptr NimStringV2](addr x)
   var b = cast[ptr NimStringV2](unsafeAddr y)
   if unlikely(a.p == b.p): return
-  lose(a)
+  frees(a)
   a.len = b.len
   if isLiteral(b):
     # we can shallow copy literals:
     a.p = b.p
   else:
-    let region = if a.p.region != nil: a.p.region else: getLocalAllocator()
+    let region = if a.p != nil and a.p.region != nil: a.p.region else: getLocalAllocator()
     # we have to allocate the 'cap' here, consider
     # 'let y = newStringOfCap(); var x = y'
     # on the other hand... These get turned into moves now.
@@ -136,6 +133,7 @@ proc appendString(dest: var NimStringV2; src: NimStringV2) {.compilerproc, inlin
   if src.len > 0:
     # also copy the \0 terminator:
     copyMem(unsafeAddr dest.p.data[dest.len], unsafeAddr src.p.data[0], src.len+1)
+    inc dest.len, src.len
 
 proc appendChar(dest: var NimStringV2; c: char) {.compilerproc, inline.} =
   dest.p.data[dest.len] = c
@@ -166,7 +164,6 @@ proc mnewString(len: int): NimStringV2 {.compilerProc.} =
 proc setLengthStrV2(s: var NimStringV2, newLen: int) {.compilerRtl.} =
   if newLen > s.len:
     prepareAdd(s, newLen - s.len)
-  else:
+  s.len = newLen
-    s.len = newLen
+  # this also only works because the destructor
-    # this also only works because the destructor
+  # looks at s.p and not s.len
-    # looks at s.p and not s.len

lib/deprecated/core/unsigned.nim

@@ -1,18 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## **Warning:** Since version 0.11.4 this module is deprecated.
-##
-## This module implemented basic arithmetic operators for unsigned integers.
-## These operators are now available in the ``system`` module directly.
-
-{.deprecated.}
-
-export `shr`, `shl`, `and`, `or`, `xor`, `==`, `+`, `-`, `*`, `div`, `mod`,
-  `<=`, `<`

lib/deprecated/pure/actors.nim

@@ -1,239 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## `Actor`:idx: support for Nim. An actor is implemented as a thread with
-## a channel as its inbox. This module requires the ``--threads:on``
-## command line switch.
-##
-## Example:
-##
-## .. code-block:: nim
-##
-##      var
-##        a: ActorPool[int, void]
-##      createActorPool(a)
-##      for i in 0 ..< 300:
-##        a.spawn(i, proc (x: int) {.thread.} = echo x)
-##      a.join()
-##
-## **Note**: This whole module is deprecated. Use `threadpool` and ``spawn``
-## instead.
-
-{.deprecated.}
-
-from os import sleep
-
-type
-  Task*[In, Out] = object{.pure, final.} ## a task
-    when Out isnot void:
-      receiver*: ptr Channel[Out] ## the receiver channel of the response
-    action*: proc (x: In): Out {.thread.} ## action to execute;
-                                            ## sometimes useful
-    shutDown*: bool ## set to tell an actor to shut-down
-    data*: In ## the data to process
-
-  Actor[In, Out] = object{.pure, final.}
-    i: Channel[Task[In, Out]]
-    t: Thread[ptr Actor[In, Out]]
-
-  PActor*[In, Out] = ptr Actor[In, Out] ## an actor
-
-proc spawn*[In, Out](action: proc(
-    self: PActor[In, Out]){.thread.}): PActor[In, Out] =
-  ## creates an actor; that is a thread with an inbox. The caller MUST call
-  ## ``join`` because that also frees the actor's associated resources.
-  result = cast[PActor[In, Out]](allocShared0(sizeof(result[])))
-  open(result.i)
-  createThread(result.t, action, result)
-
-proc inbox*[In, Out](self: PActor[In, Out]): ptr Channel[In] =
-  ## gets a pointer to the associated inbox of the actor `self`.
-  result = addr(self.i)
-
-proc running*[In, Out](a: PActor[In, Out]): bool =
-  ## returns true if the actor `a` is running.
-  result = running(a.t)
-
-proc ready*[In, Out](a: PActor[In, Out]): bool =
-  ## returns true if the actor `a` is ready to process new messages.
-  result = ready(a.i)
-
-proc join*[In, Out](a: PActor[In, Out]) =
-  ## joins an actor.
-  joinThread(a.t)
-  close(a.i)
-  deallocShared(a)
-
-proc recv*[In, Out](a: PActor[In, Out]): Task[In, Out] =
-  ## receives a task from `a`'s inbox.
-  result = recv(a.i)
-
-proc send*[In, Out, X, Y](receiver: PActor[In, Out], msg: In,
-                            sender: PActor[X, Y]) =
-  ## sends a message to `a`'s inbox.
-  var t: Task[In, Out]
-  t.receiver = addr(sender.i)
-  shallowCopy(t.data, msg)
-  send(receiver.i, t)
-
-proc send*[In, Out](receiver: PActor[In, Out], msg: In,
-                      sender: ptr Channel[Out] = nil) =
-  ## sends a message to `receiver`'s inbox.
-  var t: Task[In, Out]
-  t.receiver = sender
-  shallowCopy(t.data, msg)
-  send(receiver.i, t)
-
-proc sendShutdown*[In, Out](receiver: PActor[In, Out]) =
-  ## send a shutdown message to `receiver`.
-  var t: Task[In, Out]
-  t.shutdown = true
-  send(receiver.i, t)
-
-proc reply*[In, Out](t: Task[In, Out], m: Out) =
-  ## sends a message to io's output message box.
-  when Out is void:
-    {.error: "you cannot reply to a void outbox".}
-  assert t.receiver != nil
-  send(t.receiver[], m)
-
-
-# ----------------- actor pools ----------------------------------------------
-
-type
-  ActorPool*[In, Out] = object{.pure, final.}  ## an actor pool
-    actors: seq[PActor[In, Out]]
-    when Out isnot void:
-      outputs: Channel[Out]
-
-proc `^`*[T](f: ptr Channel[T]): T =
-  ## alias for 'recv'.
-  result = recv(f[])
-
-proc poolWorker[In, Out](self: PActor[In, Out]) {.thread.} =
-  while true:
-    var m = self.recv
-    if m.shutDown: break
-    when Out is void:
-      m.action(m.data)
-    else:
-      send(m.receiver[], m.action(m.data))
-      #self.reply()
-
-proc createActorPool*[In, Out](a: var ActorPool[In, Out], poolSize = 4) =
-  ## creates an actor pool.
-  newSeq(a.actors, poolSize)
-  when Out isnot void:
-    open(a.outputs)
-  for i in 0 ..< a.actors.len:
-    a.actors[i] = spawn(poolWorker[In, Out])
-
-proc sync*[In, Out](a: var ActorPool[In, Out], polling=50) =
-  ## waits for every actor of `a` to finish with its work. Currently this is
-  ## implemented as polling every `polling` ms and has a slight chance
-  ## of failing since we check for every actor to be in `ready` state and not
-  ## for messages still in ether. This will change in a later
-  ## version, however.
-  var allReadyCount = 0
-  while true:
-    var wait = false
-    for i in 0..high(a.actors):
-      if not a.actors[i].i.ready:
-        wait = true
-        allReadyCount = 0
-        break
-    if not wait:
-      # it's possible that some actor sent a message to some other actor but
-      # both appeared to be non-working as the message takes some time to
-      # arrive. We assume that this won't take longer than `polling` and
-      # simply attempt a second time and declare victory then. ;-)
-      inc allReadyCount
-      if allReadyCount > 1: break
-    sleep(polling)
-
-proc terminate*[In, Out](a: var ActorPool[In, Out]) =
-  ## terminates each actor in the actor pool `a` and frees the
-  ## resources attached to `a`.
-  var t: Task[In, Out]
-  t.shutdown = true
-  for i in 0..<a.actors.len: send(a.actors[i].i, t)
-  for i in 0..<a.actors.len: join(a.actors[i])
-  when Out isnot void:
-    close(a.outputs)
-  a.actors = @[]
-
-proc join*[In, Out](a: var ActorPool[In, Out]) =
-  ## short-cut for `sync` and then `terminate`.
-  sync(a)
-  terminate(a)
-
-template setupTask =
-  t.action = action
-  shallowCopy(t.data, input)
-
-template schedule =
-  # extremely simple scheduler: We always try the first thread first, so that
-  # it remains 'hot' ;-). Round-robin hurts for keeping threads hot.
-  for i in 0..high(p.actors):
-    if p.actors[i].i.ready:
-      p.actors[i].i.send(t)
-      return
-  # no thread ready :-( --> send message to the thread which has the least
-  # messages pending:
-  var minIdx = -1
-  var minVal = high(int)
-  for i in 0..high(p.actors):
-    var curr = p.actors[i].i.peek
-    if curr == 0:
-      # ok, is ready now:
-      p.actors[i].i.send(t)
-      return
-    if curr < minVal and curr >= 0:
-      minVal = curr
-      minIdx = i
-  if minIdx >= 0:
-    p.actors[minIdx].i.send(t)
-  else:
-    raise newException(DeadThreadError, "cannot send message; thread died")
-
-proc spawn*[In, Out](p: var ActorPool[In, Out], input: In,
-                       action: proc (input: In): Out {.thread.}
-                       ): ptr Channel[Out] =
-  ## uses the actor pool to run ``action(input)`` concurrently.
-  ## `spawn` is guaranteed to not block.
-  var t: Task[In, Out]
-  setupTask()
-  result = addr(p.outputs)
-  t.receiver = result
-  schedule()
-
-proc spawn*[In](p: var ActorPool[In, void], input: In,
-                 action: proc (input: In) {.thread.}) =
-  ## uses the actor pool to run ``action(input)`` concurrently.
-  ## `spawn` is guaranteed to not block.
-  var t: Task[In, void]
-  setupTask()
-  schedule()
-
-when not defined(testing) and isMainModule:
-  var
-    a: ActorPool[int, void]
-  createActorPool(a)
-  for i in 0 ..< 300:
-    a.spawn(i, proc (x: int) {.thread.} = echo x)
-
-  when false:
-    proc treeDepth(n: PNode): int {.thread.} =
-      var x = a.spawn(treeDepth, n.le)
-      var y = a.spawn(treeDepth, n.ri)
-      result = max(^x, ^y) + 1
-
-  a.join()
-
-

lib/deprecated/pure/actors.nim.cfg

@@ -1,3 +0,0 @@
-# to shut up the tester:
---threads:on
-

lib/deprecated/pure/asyncio.nim

@@ -1,711 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2012 Andreas Rumpf, Dominik Picheta
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-include "system/inclrtl"
-
-import sockets, os
-
-##
-## **Warning:** This module is deprecated since version 0.10.2.
-## Use the brand new `asyncdispatch <asyncdispatch.html>`_ module together
-## with the `asyncnet <asyncnet.html>`_ module.
-
-## This module implements an asynchronous event loop together with asynchronous
-## sockets which use this event loop.
-## It is akin to Python's asyncore module. Many modules that use sockets
-## have an implementation for this module, those modules should all have a
-## ``register`` function which you should use to add the desired objects to a
-## dispatcher which you created so
-## that you can receive the events associated with that module's object.
-##
-## Once everything is registered in a dispatcher, you need to call the ``poll``
-## function in a while loop.
-##
-## **Note:** Most modules have tasks which need to be ran regularly, this is
-## why you should not call ``poll`` with a infinite timeout, or even a
-## very long one. In most cases the default timeout is fine.
-##
-## **Note:** This module currently only supports select(), this is limited by
-## FD_SETSIZE, which is usually 1024. So you may only be able to use 1024
-## sockets at a time.
-##
-## Most (if not all) modules that use asyncio provide a userArg which is passed
-## on with the events. The type that you set userArg to must be inheriting from
-## ``RootObj``!
-##
-## **Note:** If you want to provide async ability to your module please do not
-## use the ``Delegate`` object, instead use ``AsyncSocket``. It is possible
-## that in the future this type's fields will not be exported therefore breaking
-## your code.
-##
-## **Warning:** The API of this module is unstable, and therefore is subject
-## to change.
-##
-## Asynchronous sockets
-## ====================
-##
-## For most purposes you do not need to worry about the ``Delegate`` type. The
-## ``AsyncSocket`` is what you are after. It's a reference to
-## the ``AsyncSocketObj`` object. This object defines events which you should
-## overwrite by your own procedures.
-##
-## For server sockets the only event you need to worry about is the ``handleAccept``
-## event, in your handleAccept proc you should call ``accept`` on the server
-## socket which will give you the client which is connecting. You should then
-## set any events that you want to use on that client and add it to your dispatcher
-## using the ``register`` procedure.
-##
-## An example ``handleAccept`` follows:
-##
-## .. code-block:: nim
-##
-##    var disp = newDispatcher()
-##    ...
-##    proc handleAccept(s: AsyncSocket) =
-##      echo("Accepted client.")
-##      var client: AsyncSocket
-##      new(client)
-##      s.accept(client)
-##      client.handleRead = ...
-##      disp.register(client)
-##    ...
-##
-## For client sockets you should only be interested in the ``handleRead`` and
-## ``handleConnect`` events. The former gets called whenever the socket has
-## received messages and can be read from and the latter gets called whenever
-## the socket has established a connection to a server socket; from that point
-## it can be safely written to.
-##
-## Getting a blocking client from an AsyncSocket
-## =============================================
-##
-## If you need a asynchronous server socket but you wish to process the clients
-## synchronously then you can use the ``getSocket`` converter to get
-## a ``Socket`` from the ``AsyncSocket`` object, this can then be combined
-## with ``accept`` like so:
-##
-## .. code-block:: nim
-##
-##    proc handleAccept(s: AsyncSocket) =
-##      var client: Socket
-##      getSocket(s).accept(client)
-
-{.deprecated.}
-
-when defined(windows):
-  from winlean import TimeVal, SocketHandle, FD_SET, FD_ZERO, TFdSet,
-    FD_ISSET, select
-else:
-  from posix import TimeVal, Time, Suseconds, SocketHandle, FD_SET, FD_ZERO,
-    TFdSet, FD_ISSET, select
-
-type
-  DelegateObj* = object
-    fd*: SocketHandle
-    deleVal*: RootRef
-
-    handleRead*: proc (h: RootRef) {.nimcall, gcsafe.}
-    handleWrite*: proc (h: RootRef) {.nimcall, gcsafe.}
-    handleError*: proc (h: RootRef) {.nimcall, gcsafe.}
-    hasDataBuffered*: proc (h: RootRef): bool {.nimcall, gcsafe.}
-
-    open*: bool
-    task*: proc (h: RootRef) {.nimcall, gcsafe.}
-    mode*: FileMode
-
-  Delegate* = ref DelegateObj
-
-  Dispatcher* = ref DispatcherObj
-  DispatcherObj = object
-    delegates: seq[Delegate]
-
-  AsyncSocket* = ref AsyncSocketObj
-  AsyncSocketObj* = object of RootObj
-    socket: Socket
-    info: SocketStatus
-
-    handleRead*: proc (s: AsyncSocket) {.closure, gcsafe.}
-    handleWrite: proc (s: AsyncSocket) {.closure, gcsafe.}
-    handleConnect*: proc (s:  AsyncSocket) {.closure, gcsafe.}
-
-    handleAccept*: proc (s:  AsyncSocket) {.closure, gcsafe.}
-
-    handleTask*: proc (s: AsyncSocket) {.closure, gcsafe.}
-
-    lineBuffer: TaintedString ## Temporary storage for ``readLine``
-    sendBuffer: string ## Temporary storage for ``send``
-    sslNeedAccept: bool
-    proto: Protocol
-    deleg: Delegate
-
-  SocketStatus* = enum
-    SockIdle, SockConnecting, SockConnected, SockListening, SockClosed,
-    SockUDPBound
-
-
-proc newDelegate*(): Delegate =
-  ## Creates a new delegate.
-  new(result)
-  result.handleRead = (proc (h: RootRef) = discard)
-  result.handleWrite = (proc (h: RootRef) = discard)
-  result.handleError = (proc (h: RootRef) = discard)
-  result.hasDataBuffered = (proc (h: RootRef): bool = return false)
-  result.task = (proc (h: RootRef) = discard)
-  result.mode = fmRead
-
-proc newAsyncSocket(): AsyncSocket =
-  new(result)
-  result.info = SockIdle
-
-  result.handleRead = (proc (s: AsyncSocket) = discard)
-  result.handleWrite = nil
-  result.handleConnect = (proc (s: AsyncSocket) = discard)
-  result.handleAccept = (proc (s: AsyncSocket) = discard)
-  result.handleTask = (proc (s: AsyncSocket) = discard)
-
-  result.lineBuffer = "".TaintedString
-  result.sendBuffer = ""
-
-proc asyncSocket*(domain: Domain = AF_INET, typ: SockType = SOCK_STREAM,
-                  protocol: Protocol = IPPROTO_TCP,
-                  buffered = true): AsyncSocket =
-  ## Initialises an AsyncSocket object. If a socket cannot be initialised
-  ## OSError is raised.
-  result = newAsyncSocket()
-  result.socket = socket(domain, typ, protocol, buffered)
-  result.proto = protocol
-  if result.socket == invalidSocket: raiseOSError(osLastError())
-  result.socket.setBlocking(false)
-
-proc toAsyncSocket*(sock: Socket, state: SocketStatus = SockConnected): AsyncSocket =
-  ## Wraps an already initialized ``Socket`` into a AsyncSocket.
-  ## This is useful if you want to use an already connected Socket as an
-  ## asynchronous AsyncSocket in asyncio's event loop.
-  ##
-  ## ``state`` may be overriden, i.e. if ``sock`` is not connected it should be
-  ## adjusted properly. By default it will be assumed that the socket is
-  ## connected. Please note this is only applicable to TCP client sockets, if
-  ## ``sock`` is a different type of socket ``state`` needs to be adjusted!!!
-  ##
-  ## ================  ================================================================
-  ## Value             Meaning
-  ## ================  ================================================================
-  ##  SockIdle          Socket has only just been initialised, not connected or closed.
-  ##  SockConnected     Socket is connected to a server.
-  ##  SockConnecting    Socket is in the process of connecting to a server.
-  ##  SockListening     Socket is a server socket and is listening for connections.
-  ##  SockClosed        Socket has been closed.
-  ##  SockUDPBound      Socket is a UDP socket which is listening for data.
-  ## ================  ================================================================
-  ##
-  ## **Warning**: If ``state`` is set incorrectly the resulting ``AsyncSocket``
-  ## object may not work properly.
-  ##
-  ## **Note**: This will set ``sock`` to be non-blocking.
-  result = newAsyncSocket()
-  result.socket = sock
-  result.proto = if state == SockUDPBound: IPPROTO_UDP else: IPPROTO_TCP
-  result.socket.setBlocking(false)
-  result.info = state
-
-proc asyncSockHandleRead(h: RootRef) =
-  when defined(ssl):
-    if AsyncSocket(h).socket.isSSL and not
-         AsyncSocket(h).socket.gotHandshake:
-      return
-
-  if AsyncSocket(h).info != SockListening:
-    if AsyncSocket(h).info != SockConnecting:
-      AsyncSocket(h).handleRead(AsyncSocket(h))
-  else:
-    AsyncSocket(h).handleAccept(AsyncSocket(h))
-
-proc close*(sock: AsyncSocket) {.gcsafe.}
-proc asyncSockHandleWrite(h: RootRef) =
-  when defined(ssl):
-    if AsyncSocket(h).socket.isSSL and not
-         AsyncSocket(h).socket.gotHandshake:
-      return
-
-  if AsyncSocket(h).info == SockConnecting:
-    AsyncSocket(h).handleConnect(AsyncSocket(h))
-    AsyncSocket(h).info = SockConnected
-    # Stop receiving write events if there is no handleWrite event.
-    if AsyncSocket(h).handleWrite == nil:
-      AsyncSocket(h).deleg.mode = fmRead
-    else:
-      AsyncSocket(h).deleg.mode = fmReadWrite
-  else:
-    if AsyncSocket(h).sendBuffer != "":
-      let sock = AsyncSocket(h)
-      try:
-        let bytesSent = sock.socket.sendAsync(sock.sendBuffer)
-        if bytesSent == 0:
-          # Apparently the socket cannot be written to. Even though select
-          # just told us that it can be... This used to be an assert. Just
-          # do nothing instead.
-          discard
-        elif bytesSent != sock.sendBuffer.len:
-          sock.sendBuffer = sock.sendBuffer[bytesSent .. ^1]
-        elif bytesSent == sock.sendBuffer.len:
-          sock.sendBuffer = ""
-
-        if AsyncSocket(h).handleWrite != nil:
-          AsyncSocket(h).handleWrite(AsyncSocket(h))
-      except OSError:
-        # Most likely the socket closed before the full buffer could be sent to it.
-        sock.close() # TODO: Provide a handleError for users?
-    else:
-      if AsyncSocket(h).handleWrite != nil:
-        AsyncSocket(h).handleWrite(AsyncSocket(h))
-      else:
-        AsyncSocket(h).deleg.mode = fmRead
-
-when defined(ssl):
-  proc asyncSockDoHandshake(h: RootRef) {.gcsafe.} =
-    if AsyncSocket(h).socket.isSSL and not
-         AsyncSocket(h).socket.gotHandshake:
-      if AsyncSocket(h).sslNeedAccept:
-        var d = ""
-        let ret = AsyncSocket(h).socket.acceptAddrSSL(AsyncSocket(h).socket, d)
-        assert ret != AcceptNoClient
-        if ret == AcceptSuccess:
-          AsyncSocket(h).info = SockConnected
-      else:
-        # handshake will set socket's ``sslNoHandshake`` field.
-        discard AsyncSocket(h).socket.handshake()
-
-
-proc asyncSockTask(h: RootRef) =
-  when defined(ssl):
-    h.asyncSockDoHandshake()
-
-  AsyncSocket(h).handleTask(AsyncSocket(h))
-
-proc toDelegate(sock: AsyncSocket): Delegate =
-  result = newDelegate()
-  result.deleVal = sock
-  result.fd = getFD(sock.socket)
-  # We need this to get write events, just to know when the socket connects.
-  result.mode = fmReadWrite
-  result.handleRead = asyncSockHandleRead
-  result.handleWrite = asyncSockHandleWrite
-  result.task = asyncSockTask
-  # TODO: Errors?
-  #result.handleError = (proc (h: PObject) = assert(false))
-
-  result.hasDataBuffered =
-    proc (h: RootRef): bool {.nimcall.} =
-      return AsyncSocket(h).socket.hasDataBuffered()
-
-  sock.deleg = result
-  if sock.info notin {SockIdle, SockClosed}:
-    sock.deleg.open = true
-  else:
-    sock.deleg.open = false
-
-proc connect*(sock: AsyncSocket, name: string, port = Port(0),
-                   af: Domain = AF_INET) =
-  ## Begins connecting ``sock`` to ``name``:``port``.
-  sock.socket.connectAsync(name, port, af)
-  sock.info = SockConnecting
-  if sock.deleg != nil:
-    sock.deleg.open = true
-
-proc close*(sock: AsyncSocket) =
-  ## Closes ``sock``. Terminates any current connections.
-  sock.socket.close()
-  sock.info = SockClosed
-  if sock.deleg != nil:
-    sock.deleg.open = false
-
-proc bindAddr*(sock: AsyncSocket, port = Port(0), address = "") =
-  ## Equivalent to ``sockets.bindAddr``.
-  sock.socket.bindAddr(port, address)
-  if sock.proto == IPPROTO_UDP:
-    sock.info = SockUDPBound
-    if sock.deleg != nil:
-      sock.deleg.open = true
-
-proc listen*(sock: AsyncSocket) =
-  ## Equivalent to ``sockets.listen``.
-  sock.socket.listen()
-  sock.info = SockListening
-  if sock.deleg != nil:
-    sock.deleg.open = true
-
-proc acceptAddr*(server: AsyncSocket, client: var AsyncSocket,
-                 address: var string) =
-  ## Equivalent to ``sockets.acceptAddr``. This procedure should be called in
-  ## a ``handleAccept`` event handler **only** once.
-  ##
-  ## **Note**: ``client`` needs to be initialised.
-  assert(client != nil)
-  client = newAsyncSocket()
-  var c: Socket
-  new(c)
-  when defined(ssl):
-    if server.socket.isSSL:
-      var ret = server.socket.acceptAddrSSL(c, address)
-      # The following shouldn't happen because when this function is called
-      # it is guaranteed that there is a client waiting.
-      # (This should be called in handleAccept)
-      assert(ret != AcceptNoClient)
-      if ret == AcceptNoHandshake:
-        client.sslNeedAccept = true
-      else:
-        client.sslNeedAccept = false
-        client.info = SockConnected
-    else:
-      server.socket.acceptAddr(c, address)
-      client.sslNeedAccept = false
-      client.info = SockConnected
-  else:
-    server.socket.acceptAddr(c, address)
-    client.sslNeedAccept = false
-    client.info = SockConnected
-
-  if c == invalidSocket: raiseSocketError(server.socket)
-  c.setBlocking(false) # TODO: Needs to be tested.
-
-  # deleg.open is set in ``toDelegate``.
-
-  client.socket = c
-  client.lineBuffer = "".TaintedString
-  client.sendBuffer = ""
-  client.info = SockConnected
-
-proc accept*(server: AsyncSocket, client: var AsyncSocket) =
-  ## Equivalent to ``sockets.accept``.
-  var dummyAddr = ""
-  server.acceptAddr(client, dummyAddr)
-
-proc acceptAddr*(server: AsyncSocket): tuple[sock: AsyncSocket,
-                                              address: string] {.deprecated.} =
-  ## Equivalent to ``sockets.acceptAddr``.
-  ##
-  ## **Deprecated since version 0.9.0:** Please use the function above.
-  var client = newAsyncSocket()
-  var address: string = ""
-  acceptAddr(server, client, address)
-  return (client, address)
-
-proc accept*(server: AsyncSocket): AsyncSocket {.deprecated.} =
-  ## Equivalent to ``sockets.accept``.
-  ##
-  ## **Deprecated since version 0.9.0:** Please use the function above.
-  new(result)
-  var address = ""
-  server.acceptAddr(result, address)
-
-proc newDispatcher*(): Dispatcher =
-  new(result)
-  result.delegates = @[]
-
-proc register*(d: Dispatcher, deleg: Delegate) =
-  ## Registers delegate ``deleg`` with dispatcher ``d``.
-  d.delegates.add(deleg)
-
-proc register*(d: Dispatcher, sock: AsyncSocket): Delegate {.discardable.} =
-  ## Registers async socket ``sock`` with dispatcher ``d``.
-  result = sock.toDelegate()
-  d.register(result)
-
-proc unregister*(d: Dispatcher, deleg: Delegate) =
-  ## Unregisters deleg ``deleg`` from dispatcher ``d``.
-  for i in 0..len(d.delegates)-1:
-    if d.delegates[i] == deleg:
-      d.delegates.del(i)
-      return
-  raise newException(IndexError, "Could not find delegate.")
-
-proc isWriteable*(s: AsyncSocket): bool =
-  ## Determines whether socket ``s`` is ready to be written to.
-  var writeSock = @[s.socket]
-  return selectWrite(writeSock, 1) != 0 and s.socket notin writeSock
-
-converter getSocket*(s: AsyncSocket): Socket =
-  return s.socket
-
-proc isConnected*(s: AsyncSocket): bool =
-  ## Determines whether ``s`` is connected.
-  return s.info == SockConnected
-proc isListening*(s: AsyncSocket): bool =
-  ## Determines whether ``s`` is listening for incoming connections.
-  return s.info == SockListening
-proc isConnecting*(s: AsyncSocket): bool =
-  ## Determines whether ``s`` is connecting.
-  return s.info == SockConnecting
-proc isClosed*(s: AsyncSocket): bool =
-  ## Determines whether ``s`` has been closed.
-  return s.info == SockClosed
-proc isSendDataBuffered*(s: AsyncSocket): bool =
-  ## Determines whether ``s`` has data waiting to be sent, i.e. whether this
-  ## socket's sendBuffer contains data.
-  return s.sendBuffer.len != 0
-
-proc setHandleWrite*(s: AsyncSocket,
-    handleWrite: proc (s: AsyncSocket) {.closure, gcsafe.}) =
-  ## Setter for the ``handleWrite`` event.
-  ##
-  ## To remove this event you should use the ``delHandleWrite`` function.
-  ## It is advised to use that function instead of just setting the event to
-  ## ``proc (s: AsyncSocket) = nil`` as that would mean that that function
-  ## would be called constantly.
-  s.deleg.mode = fmReadWrite
-  s.handleWrite = handleWrite
-
-proc delHandleWrite*(s: AsyncSocket) =
-  ## Removes the ``handleWrite`` event handler on ``s``.
-  s.handleWrite = nil
-
-{.push warning[deprecated]: off.}
-proc recvLine*(s: AsyncSocket, line: var TaintedString): bool {.deprecated.} =
-  ## Behaves similar to ``sockets.recvLine``, however it handles non-blocking
-  ## sockets properly. This function guarantees that ``line`` is a full line,
-  ## if this function can only retrieve some data; it will save this data and
-  ## add it to the result when a full line is retrieved.
-  ##
-  ## Unlike ``sockets.recvLine`` this function will raise an OSError or SslError
-  ## exception if an error occurs.
-  ##
-  ## **Deprecated since version 0.9.2**: This function has been deprecated in
-  ## favour of readLine.
-  setLen(line.string, 0)
-  var dataReceived = "".TaintedString
-  var ret = s.socket.recvLineAsync(dataReceived)
-  case ret
-  of RecvFullLine:
-    if s.lineBuffer.len > 0:
-      string(line).add(s.lineBuffer.string)
-      setLen(s.lineBuffer.string, 0)
-    string(line).add(dataReceived.string)
-    if string(line) == "":
-      line = "\c\L".TaintedString
-    result = true
-  of RecvPartialLine:
-    string(s.lineBuffer).add(dataReceived.string)
-    result = false
-  of RecvDisconnected:
-    result = true
-  of RecvFail:
-    s.raiseSocketError(async = true)
-    result = false
-{.pop.}
-
-proc readLine*(s: AsyncSocket, line: var TaintedString): bool =
-  ## Behaves similar to ``sockets.readLine``, however it handles non-blocking
-  ## sockets properly. This function guarantees that ``line`` is a full line,
-  ## if this function can only retrieve some data; it will save this data and
-  ## add it to the result when a full line is retrieved, when this happens
-  ## False will be returned. True will only be returned if a full line has been
-  ## retrieved or the socket has been disconnected in which case ``line`` will
-  ## be set to "".
-  ##
-  ## This function will raise an OSError exception when a socket error occurs.
-  setLen(line.string, 0)
-  var dataReceived = "".TaintedString
-  var ret = s.socket.readLineAsync(dataReceived)
-  case ret
-  of ReadFullLine:
-    if s.lineBuffer.len > 0:
-      string(line).add(s.lineBuffer.string)
-      setLen(s.lineBuffer.string, 0)
-    string(line).add(dataReceived.string)
-    if string(line) == "":
-      line = "\c\L".TaintedString
-    result = true
-  of ReadPartialLine:
-    string(s.lineBuffer).add(dataReceived.string)
-    result = false
-  of ReadNone:
-    result = false
-  of ReadDisconnected:
-    result = true
-
-proc send*(sock: AsyncSocket, data: string) =
-  ## Sends ``data`` to socket ``sock``. This is basically a nicer implementation
-  ## of ``sockets.sendAsync``.
-  ##
-  ## If ``data`` cannot be sent immediately it will be buffered and sent
-  ## when ``sock`` becomes writeable (during the ``handleWrite`` event).
-  ## It's possible that only a part of ``data`` will be sent immediately, while
-  ## the rest of it will be buffered and sent later.
-  if sock.sendBuffer.len != 0:
-    sock.sendBuffer.add(data)
-    return
-  let bytesSent = sock.socket.sendAsync(data)
-  assert bytesSent >= 0
-  if bytesSent == 0:
-    sock.sendBuffer.add(data)
-    sock.deleg.mode = fmReadWrite
-  elif bytesSent != data.len:
-    sock.sendBuffer.add(data[bytesSent .. ^1])
-    sock.deleg.mode = fmReadWrite
-
-proc timeValFromMilliseconds(timeout = 500): Timeval =
-  if timeout != -1:
-    var seconds = timeout div 1000
-    when defined(posix):
-      result.tv_sec = seconds.Time
-      result.tv_usec = ((timeout - seconds * 1000) * 1000).Suseconds
-    else:
-      result.tv_sec = seconds.int32
-      result.tv_usec = ((timeout - seconds * 1000) * 1000).int32
-
-proc createFdSet(fd: var TFdSet, s: seq[Delegate], m: var int) =
-  FD_ZERO(fd)
-  for i in items(s):
-    m = max(m, int(i.fd))
-    FD_SET(i.fd, fd)
-
-proc pruneSocketSet(s: var seq[Delegate], fd: var TFdSet) =
-  var i = 0
-  var L = s.len
-  while i < L:
-    if FD_ISSET(s[i].fd, fd) != 0'i32:
-      s[i] = s[L-1]
-      dec(L)
-    else:
-      inc(i)
-  setLen(s, L)
-
-proc select(readfds, writefds, exceptfds: var seq[Delegate],
-             timeout = 500): int =
-  var tv {.noInit.}: Timeval = timeValFromMilliseconds(timeout)
-
-  var rd, wr, ex: TFdSet
-  var m = 0
-  createFdSet(rd, readfds, m)
-  createFdSet(wr, writefds, m)
-  createFdSet(ex, exceptfds, m)
-
-  if timeout != -1:
-    result = int(select(cint(m+1), addr(rd), addr(wr), addr(ex), addr(tv)))
-  else:
-    result = int(select(cint(m+1), addr(rd), addr(wr), addr(ex), nil))
-
-  pruneSocketSet(readfds, (rd))
-  pruneSocketSet(writefds, (wr))
-  pruneSocketSet(exceptfds, (ex))
-
-proc poll*(d: Dispatcher, timeout: int = 500): bool =
-  ## This function checks for events on all the delegates in the `PDispatcher`.
-  ## It then proceeds to call the correct event handler.
-  ##
-  ## This function returns ``True`` if there are file descriptors that are still
-  ## open, otherwise ``False``. File descriptors that have been
-  ## closed are immediately removed from the dispatcher automatically.
-  ##
-  ## **Note:** Each delegate has a task associated with it. This gets called
-  ## after each select() call, if you set timeout to ``-1`` the tasks will
-  ## only be executed after one or more file descriptors becomes readable or
-  ## writeable.
-  result = true
-  var readDg, writeDg, errorDg: seq[Delegate] = @[]
-  var len = d.delegates.len
-  var dc = 0
-
-  while dc < len:
-    let deleg = d.delegates[dc]
-    if (deleg.mode != fmWrite or deleg.mode != fmAppend) and deleg.open:
-      readDg.add(deleg)
-    if (deleg.mode != fmRead) and deleg.open:
-      writeDg.add(deleg)
-    if deleg.open:
-      errorDg.add(deleg)
-      inc dc
-    else:
-      # File/socket has been closed. Remove it from dispatcher.
-      d.delegates[dc] = d.delegates[len-1]
-      dec len
-
-  d.delegates.setLen(len)
-
-  var hasDataBufferedCount = 0
-  for d in d.delegates:
-    if d.hasDataBuffered(d.deleVal):
-      hasDataBufferedCount.inc()
-      d.handleRead(d.deleVal)
-  if hasDataBufferedCount > 0: return true
-
-  if readDg.len() == 0 and writeDg.len() == 0:
-    ## TODO: Perhaps this shouldn't return if errorDg has something?
-    return false
-
-  if select(readDg, writeDg, errorDg, timeout) != 0:
-    for i in 0..len(d.delegates)-1:
-      if i > len(d.delegates)-1: break # One delegate might've been removed.
-      let deleg = d.delegates[i]
-      if not deleg.open: continue # This delegate might've been closed.
-      if (deleg.mode != fmWrite or deleg.mode != fmAppend) and
-          deleg notin readDg:
-        deleg.handleRead(deleg.deleVal)
-      if (deleg.mode != fmRead) and deleg notin writeDg:
-        deleg.handleWrite(deleg.deleVal)
-      if deleg notin errorDg:
-        deleg.handleError(deleg.deleVal)
-
-  # Execute tasks
-  for i in items(d.delegates):
-    i.task(i.deleVal)
-
-proc len*(disp: Dispatcher): int =
-  ## Retrieves the amount of delegates in ``disp``.
-  return disp.delegates.len
-
-when not defined(testing) and isMainModule:
-
-  proc testConnect(s: AsyncSocket, no: int) =
-    echo("Connected! " & $no)
-
-  proc testRead(s: AsyncSocket, no: int) =
-    echo("Reading! " & $no)
-    var data = ""
-    if not s.readLine(data): return
-    if data == "":
-      echo("Closing connection. " & $no)
-      s.close()
-    echo(data)
-    echo("Finished reading! " & $no)
-
-  proc testAccept(s: AsyncSocket, disp: Dispatcher, no: int) =
-    echo("Accepting client! " & $no)
-    var client: AsyncSocket
-    new(client)
-    var address = ""
-    s.acceptAddr(client, address)
-    echo("Accepted ", address)
-    client.handleRead =
-      proc (s: AsyncSocket) =
-        testRead(s, 2)
-    disp.register(client)
-
-  proc main =
-    var d = newDispatcher()
-
-    var s = asyncSocket()
-    s.connect("amber.tenthbit.net", Port(6667))
-    s.handleConnect =
-      proc (s: AsyncSocket) =
-        testConnect(s, 1)
-    s.handleRead =
-      proc (s: AsyncSocket) =
-        testRead(s, 1)
-    d.register(s)
-
-    var server = asyncSocket()
-    server.handleAccept =
-      proc (s: AsyncSocket) =
-        testAccept(s, d, 78)
-    server.bindAddr(Port(5555))
-    server.listen()
-    d.register(server)
-
-    while d.poll(-1): discard
-  main()

lib/deprecated/pure/ftpclient.nim

@@ -1,669 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2015 Dominik Picheta
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-include "system/inclrtl"
-
-import sockets, strutils, parseutils, times, os, asyncio
-
-from asyncnet import nil
-from nativesockets import nil
-from asyncdispatch import Future
-## **Note**: This module is deprecated since version 0.11.3.
-## You should use the async version of this module
-## `asyncftpclient <asyncftpclient.html>`_.
-##
-## ----
-##
-## This module **partially** implements an FTP client as specified
-## by `RFC 959 <http://tools.ietf.org/html/rfc959>`_.
-##
-## This module provides both a synchronous and asynchronous implementation.
-## The asynchronous implementation requires you to use the ``asyncFTPClient``
-## function. You are then required to register the ``AsyncFTPClient`` with a
-## asyncio dispatcher using the ``register`` function. Take a look at the
-## asyncio module documentation for more information.
-##
-## **Note**: The asynchronous implementation is only asynchronous for long
-## file transfers, calls to functions which use the command socket will block.
-##
-## Here is some example usage of this module:
-##
-## .. code-block:: Nim
-##    var ftp = ftpClient("example.org", user = "user", pass = "pass")
-##    ftp.connect()
-##    ftp.retrFile("file.ext", "file.ext")
-##
-## **Warning:** The API of this module is unstable, and therefore is subject
-## to change.
-
-{.deprecated.}
-
-type
-  FtpBase*[SockType] = ref FtpBaseObj[SockType]
-  FtpBaseObj*[SockType] = object
-    csock*: SockType
-    dsock*: SockType
-    when SockType is asyncio.AsyncSocket:
-      handleEvent*: proc (ftp: AsyncFTPClient, ev: FTPEvent){.closure,gcsafe.}
-      disp: Dispatcher
-      asyncDSockID: Delegate
-    user*, pass*: string
-    address*: string
-    when SockType is asyncnet.AsyncSocket:
-      port*: nativesockets.Port
-    else:
-      port*: Port
-
-    jobInProgress*: bool
-    job*: FTPJob[SockType]
-
-    dsockConnected*: bool
-
-  FTPJobType* = enum
-    JRetrText, JRetr, JStore
-
-  FtpJob[T] = ref FtpJobObj[T]
-  FTPJobObj[T] = object
-    prc: proc (ftp: FTPBase[T], async: bool): bool {.nimcall, gcsafe.}
-    case typ*: FTPJobType
-    of JRetrText:
-      lines: string
-    of JRetr, JStore:
-      file: File
-      filename: string
-      total: BiggestInt # In bytes.
-      progress: BiggestInt # In bytes.
-      oneSecond: BiggestInt # Bytes transferred in one second.
-      lastProgressReport: float # Time
-      toStore: string # Data left to upload (Only used with async)
-
-  FtpClientObj* = FtpBaseObj[Socket]
-  FtpClient* = ref FtpClientObj
-
-  AsyncFtpClient* = ref AsyncFtpClientObj ## Async alternative to TFTPClient.
-  AsyncFtpClientObj* = FtpBaseObj[asyncio.AsyncSocket]
-
-  FTPEventType* = enum
-    EvTransferProgress, EvLines, EvRetr, EvStore
-
-  FTPEvent* = object ## Event
-    filename*: string
-    case typ*: FTPEventType
-    of EvLines:
-      lines*: string ## Lines that have been transferred.
-    of EvRetr, EvStore: ## Retr/Store operation finished.
-      nil
-    of EvTransferProgress:
-      bytesTotal*: BiggestInt     ## Bytes total.
-      bytesFinished*: BiggestInt  ## Bytes transferred.
-      speed*: BiggestInt          ## Speed in bytes/s
-      currentJob*: FTPJobType     ## The current job being performed.
-
-  ReplyError* = object of IOError
-  FTPError* = object of IOError
-
-
-const multiLineLimit = 10000
-
-proc ftpClient*(address: string, port = Port(21),
-                user, pass = ""): FtpClient =
-  ## Create a ``FtpClient`` object.
-  new(result)
-  result.user = user
-  result.pass = pass
-  result.address = address
-  result.port = port
-
-  result.dsockConnected = false
-  result.csock = socket()
-  if result.csock == invalidSocket: raiseOSError(osLastError())
-
-template blockingOperation(sock: Socket, body: untyped) =
-  body
-
-template blockingOperation(sock: asyncio.AsyncSocket, body: untyped) =
-  sock.setBlocking(true)
-  body
-  sock.setBlocking(false)
-
-proc expectReply[T](ftp: FtpBase[T]): TaintedString =
-  result = TaintedString""
-  blockingOperation(ftp.csock):
-    when T is Socket:
-      ftp.csock.readLine(result)
-    else:
-      discard ftp.csock.readLine(result)
-    var count = 0
-    while result[3] == '-':
-      ## Multi-line reply.
-      var line = TaintedString""
-      when T is Socket:
-        ftp.csock.readLine(line)
-      else:
-        discard ftp.csock.readLine(line)
-      result.add("\n" & line)
-      count.inc()
-      if count >= multiLineLimit:
-        raise newException(ReplyError, "Reached maximum multi-line reply count.")
-
-proc send*[T](ftp: FtpBase[T], m: string): TaintedString =
-  ## Send a message to the server, and wait for a primary reply.
-  ## ``\c\L`` is added for you.
-  ##
-  ## **Note:** The server may return multiple lines of coded replies.
-  blockingOperation(ftp.csock):
-    ftp.csock.send(m & "\c\L")
-  return ftp.expectReply()
-
-proc assertReply(received: TaintedString, expected: string) =
-  if not received.string.startsWith(expected):
-    raise newException(ReplyError,
-                       "Expected reply '$1' got: $2" % [
-                       expected, received.string])
-
-proc assertReply(received: TaintedString, expected: varargs[string]) =
-  for i in items(expected):
-    if received.string.startsWith(i): return
-  raise newException(ReplyError,
-                     "Expected reply '$1' got: $2" %
-                     [expected.join("' or '"), received.string])
-
-proc createJob[T](ftp: FtpBase[T],
-               prc: proc (ftp: FtpBase[T], async: bool): bool {.
-                          nimcall,gcsafe.},
-               cmd: FTPJobType) =
-  if ftp.jobInProgress:
-    raise newException(FTPError, "Unable to do two jobs at once.")
-  ftp.jobInProgress = true
-  new(ftp.job)
-  ftp.job.prc = prc
-  ftp.job.typ = cmd
-  case cmd
-  of JRetrText:
-    ftp.job.lines = ""
-  of JRetr, JStore:
-    ftp.job.toStore = ""
-
-proc deleteJob[T](ftp: FtpBase[T]) =
-  assert ftp.jobInProgress
-  ftp.jobInProgress = false
-  case ftp.job.typ
-  of JRetrText:
-    ftp.job.lines = ""
-  of JRetr, JStore:
-    ftp.job.file.close()
-  ftp.dsock.close()
-
-proc handleTask(s: AsyncSocket, ftp: AsyncFTPClient) =
-  if ftp.jobInProgress:
-    if ftp.job.typ in {JRetr, JStore}:
-      if epochTime() - ftp.job.lastProgressReport >= 1.0:
-        var r: FTPEvent
-        ftp.job.lastProgressReport = epochTime()
-        r.typ = EvTransferProgress
-        r.bytesTotal = ftp.job.total
-        r.bytesFinished = ftp.job.progress
-        r.speed = ftp.job.oneSecond
-        r.filename = ftp.job.filename
-        r.currentJob = ftp.job.typ
-        ftp.job.oneSecond = 0
-        ftp.handleEvent(ftp, r)
-
-proc handleWrite(s: AsyncSocket, ftp: AsyncFTPClient) =
-  if ftp.jobInProgress:
-    if ftp.job.typ == JStore:
-      assert (not ftp.job.prc(ftp, true))
-
-proc handleConnect(s: AsyncSocket, ftp: AsyncFTPClient) =
-  ftp.dsockConnected = true
-  assert(ftp.jobInProgress)
-  if ftp.job.typ == JStore:
-    s.setHandleWrite(proc (s: AsyncSocket) = handleWrite(s, ftp))
-  else:
-    s.delHandleWrite()
-
-proc handleRead(s: AsyncSocket, ftp: AsyncFTPClient) =
-  assert ftp.jobInProgress
-  assert ftp.job.typ != JStore
-  # This can never return true, because it shouldn't check for code
-  # 226 from csock.
-  assert(not ftp.job.prc(ftp, true))
-
-proc pasv[T](ftp: FtpBase[T]) =
-  ## Negotiate a data connection.
-  when T is Socket:
-    ftp.dsock = socket()
-    if ftp.dsock == invalidSocket: raiseOSError(osLastError())
-  elif T is AsyncSocket:
-    ftp.dsock = asyncSocket()
-    ftp.dsock.handleRead =
-      proc (s: AsyncSocket) =
-        handleRead(s, ftp)
-    ftp.dsock.handleConnect =
-      proc (s: AsyncSocket) =
-        handleConnect(s, ftp)
-    ftp.dsock.handleTask =
-      proc (s: AsyncSocket) =
-        handleTask(s, ftp)
-    ftp.disp.register(ftp.dsock)
-  else:
-    {.fatal: "Incorrect socket instantiation".}
-
-  var pasvMsg = ftp.send("PASV").string.strip.TaintedString
-  assertReply(pasvMsg, "227")
-  var betweenParens = captureBetween(pasvMsg.string, '(', ')')
-  var nums = betweenParens.split(',')
-  var ip = nums[0.. ^3]
-  var port = nums[^2.. ^1]
-  var properPort = port[0].parseInt()*256+port[1].parseInt()
-  ftp.dsock.connect(ip.join("."), Port(properPort.toU16))
-  when T is AsyncSocket:
-    ftp.dsockConnected = false
-  else:
-    ftp.dsockConnected = true
-
-proc normalizePathSep(path: string): string =
-  return replace(path, '\\', '/')
-
-proc connect*[T](ftp: FtpBase[T]) =
-  ## Connect to the FTP server specified by ``ftp``.
-  when T is AsyncSocket:
-    blockingOperation(ftp.csock):
-      ftp.csock.connect(ftp.address, ftp.port)
-  elif T is Socket:
-    ftp.csock.connect(ftp.address, ftp.port)
-  else:
-    {.fatal: "Incorrect socket instantiation".}
-
-  var reply = ftp.expectReply()
-  if reply.startsWith("120"):
-    # 120 Service ready in nnn minutes.
-    # We wait until we receive 220.
-    reply = ftp.expectReply()
-
-  # Handle 220 messages from the server
-  assertReply ftp.expectReply(), "220"
-
-  if ftp.user != "":
-    assertReply(ftp.send("USER " & ftp.user), "230", "331")
-
-  if ftp.pass != "":
-    assertReply ftp.send("PASS " & ftp.pass), "230"
-
-proc pwd*[T](ftp: FtpBase[T]): string =
-  ## Returns the current working directory.
-  var wd = ftp.send("PWD")
-  assertReply wd, "257"
-  return wd.string.captureBetween('"') # "
-
-proc cd*[T](ftp: FtpBase[T], dir: string) =
-  ## Changes the current directory on the remote FTP server to ``dir``.
-  assertReply ftp.send("CWD " & dir.normalizePathSep), "250"
-
-proc cdup*[T](ftp: FtpBase[T]) =
-  ## Changes the current directory to the parent of the current directory.
-  assertReply ftp.send("CDUP"), "200"
-
-proc getLines[T](ftp: FtpBase[T], async: bool = false): bool =
-  ## Downloads text data in ASCII mode
-  ## Returns true if the download is complete.
-  ## It doesn't if `async` is true, because it doesn't check for 226 then.
-  if ftp.dsockConnected:
-    var r = TaintedString""
-    when T is AsyncSocket:
-      if ftp.asyncDSock.readLine(r):
-        if r.string == "":
-          ftp.dsockConnected = false
-        else:
-          ftp.job.lines.add(r.string & "\n")
-    elif T is Socket:
-      assert(not async)
-      ftp.dsock.readLine(r)
-      if r.string == "":
-        ftp.dsockConnected = false
-      else:
-        ftp.job.lines.add(r.string & "\n")
-    else:
-      {.fatal: "Incorrect socket instantiation".}
-
-  if not async:
-    var readSocks: seq[Socket] = @[ftp.csock]
-    # This is only needed here. Asyncio gets this socket...
-    blockingOperation(ftp.csock):
-      if readSocks.select(1) != 0 and ftp.csock in readSocks:
-        assertReply ftp.expectReply(), "226"
-        return true
-
-proc listDirs*[T](ftp: FtpBase[T], dir: string = "",
-               async = false): seq[string] =
-  ## Returns a list of filenames in the given directory. If ``dir`` is "",
-  ## the current directory is used. If ``async`` is true, this
-  ## function will return immediately and it will be your job to
-  ## use asyncio's ``poll`` to progress this operation.
-
-  ftp.createJob(getLines[T], JRetrText)
-  ftp.pasv()
-
-  assertReply ftp.send("NLST " & dir.normalizePathSep), ["125", "150"]
-
-  if not async:
-    while not ftp.job.prc(ftp, false): discard
-    result = splitLines(ftp.job.lines)
-    ftp.deleteJob()
-  else: return @[]
-
-proc fileExists*(ftp: FtpClient, file: string): bool {.deprecated.} =
-  ## **Deprecated since version 0.9.0:** Please use ``existsFile``.
-  ##
-  ## Determines whether ``file`` exists.
-  ##
-  ## Warning: This function may block. Especially on directories with many
-  ## files, because a full list of file names must be retrieved.
-  var files = ftp.listDirs()
-  for f in items(files):
-    if f.normalizePathSep == file.normalizePathSep: return true
-
-proc existsFile*(ftp: FtpClient, file: string): bool =
-  ## Determines whether ``file`` exists.
-  ##
-  ## Warning: This function may block. Especially on directories with many
-  ## files, because a full list of file names must be retrieved.
-  var files = ftp.listDirs()
-  for f in items(files):
-    if f.normalizePathSep == file.normalizePathSep: return true
-
-proc createDir*[T](ftp: FtpBase[T], dir: string, recursive: bool = false) =
-  ## Creates a directory ``dir``. If ``recursive`` is true, the topmost
-  ## subdirectory of ``dir`` will be created first, following the secondmost...
-  ## etc. this allows you to give a full path as the ``dir`` without worrying
-  ## about subdirectories not existing.
-  if not recursive:
-    assertReply ftp.send("MKD " & dir.normalizePathSep), "257"
-  else:
-    var reply = TaintedString""
-    var previousDirs = ""
-    for p in split(dir, {os.DirSep, os.AltSep}):
-      if p != "":
-        previousDirs.add(p)
-        reply = ftp.send("MKD " & previousDirs)
-        previousDirs.add('/')
-    assertReply reply, "257"
-
-proc chmod*[T](ftp: FtpBase[T], path: string,
-            permissions: set[FilePermission]) =
-  ## Changes permission of ``path`` to ``permissions``.
-  var userOctal = 0
-  var groupOctal = 0
-  var otherOctal = 0
-  for i in items(permissions):
-    case i
-    of fpUserExec: userOctal.inc(1)
-    of fpUserWrite: userOctal.inc(2)
-    of fpUserRead: userOctal.inc(4)
-    of fpGroupExec: groupOctal.inc(1)
-    of fpGroupWrite: groupOctal.inc(2)
-    of fpGroupRead: groupOctal.inc(4)
-    of fpOthersExec: otherOctal.inc(1)
-    of fpOthersWrite: otherOctal.inc(2)
-    of fpOthersRead: otherOctal.inc(4)
-
-  var perm = $userOctal & $groupOctal & $otherOctal
-  assertReply ftp.send("SITE CHMOD " & perm &
-                       " " & path.normalizePathSep), "200"
-
-proc list*[T](ftp: FtpBase[T], dir: string = "", async = false): string =
-  ## Lists all files in ``dir``. If ``dir`` is ``""``, uses the current
-  ## working directory. If ``async`` is true, this function will return
-  ## immediately and it will be your job to call asyncio's
-  ## ``poll`` to progress this operation.
-  ftp.createJob(getLines[T], JRetrText)
-  ftp.pasv()
-
-  assertReply(ftp.send("LIST" & " " & dir.normalizePathSep), ["125", "150"])
-
-  if not async:
-    while not ftp.job.prc(ftp, false): discard
-    result = ftp.job.lines
-    ftp.deleteJob()
-  else:
-    return ""
-
-proc retrText*[T](ftp: FtpBase[T], file: string, async = false): string =
-  ## Retrieves ``file``. File must be ASCII text.
-  ## If ``async`` is true, this function will return immediately and
-  ## it will be your job to call asyncio's ``poll`` to progress this operation.
-  ftp.createJob(getLines[T], JRetrText)
-  ftp.pasv()
-  assertReply ftp.send("RETR " & file.normalizePathSep), ["125", "150"]
-
-  if not async:
-    while not ftp.job.prc(ftp, false): discard
-    result = ftp.job.lines
-    ftp.deleteJob()
-  else:
-    return ""
-
-proc getFile[T](ftp: FtpBase[T], async = false): bool =
-  if ftp.dsockConnected:
-    var r = "".TaintedString
-    var bytesRead = 0
-    var returned = false
-    if async:
-      when T is Socket:
-        raise newException(FTPError, "FTPClient must be async.")
-      else:
-        bytesRead = ftp.dsock.recvAsync(r, BufferSize)
-        returned = bytesRead != -1
-    else:
-      bytesRead = ftp.dsock.recv(r, BufferSize)
-      returned = true
-    let r2 = r.string
-    if r2 != "":
-      ftp.job.progress.inc(r2.len)
-      ftp.job.oneSecond.inc(r2.len)
-      ftp.job.file.write(r2)
-    elif returned and r2 == "":
-      ftp.dsockConnected = false
-
-  when T is Socket:
-    if not async:
-      var readSocks: seq[Socket] = @[ftp.csock]
-      blockingOperation(ftp.csock):
-        if readSocks.select(1) != 0 and ftp.csock in readSocks:
-          assertReply ftp.expectReply(), "226"
-          return true
-
-proc retrFile*[T](ftp: FtpBase[T], file, dest: string, async = false) =
-  ## Downloads ``file`` and saves it to ``dest``. Usage of this function
-  ## asynchronously is recommended to view the progress of the download.
-  ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function
-  ## when the download is finished, and the ``filename`` field will be equal
-  ## to ``file``.
-  ftp.createJob(getFile[T], JRetr)
-  ftp.job.file = open(dest, mode = fmWrite)
-  ftp.pasv()
-  var reply = ftp.send("RETR " & file.normalizePathSep)
-  assertReply reply, ["125", "150"]
-  if {'(', ')'} notin reply.string:
-    raise newException(ReplyError, "Reply has no file size.")
-  var fileSize: BiggestInt
-  if reply.string.captureBetween('(', ')').parseBiggestInt(fileSize) == 0:
-    raise newException(ReplyError, "Reply has no file size.")
-
-  ftp.job.total = fileSize
-  ftp.job.lastProgressReport = epochTime()
-  ftp.job.filename = file.normalizePathSep
-
-  if not async:
-    while not ftp.job.prc(ftp, false): discard
-    ftp.deleteJob()
-
-proc doUpload[T](ftp: FtpBase[T], async = false): bool =
-  if ftp.dsockConnected:
-    if ftp.job.toStore.len() > 0:
-      assert(async)
-      let bytesSent = ftp.dsock.sendAsync(ftp.job.toStore)
-      if bytesSent == ftp.job.toStore.len:
-        ftp.job.toStore = ""
-      elif bytesSent != ftp.job.toStore.len and bytesSent != 0:
-        ftp.job.toStore = ftp.job.toStore[bytesSent .. ^1]
-      ftp.job.progress.inc(bytesSent)
-      ftp.job.oneSecond.inc(bytesSent)
-    else:
-      var s = newStringOfCap(4000)
-      var len = ftp.job.file.readBuffer(addr(s[0]), 4000)
-      setLen(s, len)
-      if len == 0:
-        # File finished uploading.
-        ftp.dsock.close()
-        ftp.dsockConnected = false
-
-        if not async:
-          assertReply ftp.expectReply(), "226"
-          return true
-        return false
-
-      if not async:
-        ftp.dsock.send(s)
-      else:
-        let bytesSent = ftp.dsock.sendAsync(s)
-        if bytesSent == 0:
-          ftp.job.toStore.add(s)
-        elif bytesSent != s.len:
-          ftp.job.toStore.add(s[bytesSent .. ^1])
-        len = bytesSent
-
-      ftp.job.progress.inc(len)
-      ftp.job.oneSecond.inc(len)
-
-proc store*[T](ftp: FtpBase[T], file, dest: string, async = false) =
-  ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this
-  ## function asynchronously is recommended to view the progress of
-  ## the download.
-  ## The ``EvStore`` event is passed to the specified ``handleEvent`` function
-  ## when the upload is finished, and the ``filename`` field will be
-  ## equal to ``file``.
-  ftp.createJob(doUpload[T], JStore)
-  ftp.job.file = open(file)
-  ftp.job.total = ftp.job.file.getFileSize()
-  ftp.job.lastProgressReport = epochTime()
-  ftp.job.filename = file
-  ftp.pasv()
-
-  assertReply ftp.send("STOR " & dest.normalizePathSep), ["125", "150"]
-
-  if not async:
-    while not ftp.job.prc(ftp, false): discard
-    ftp.deleteJob()
-
-proc close*[T](ftp: FtpBase[T]) =
-  ## Terminates the connection to the server.
-  assertReply ftp.send("QUIT"), "221"
-  if ftp.jobInProgress: ftp.deleteJob()
-  ftp.csock.close()
-  ftp.dsock.close()
-
-proc csockHandleRead(s: AsyncSocket, ftp: AsyncFTPClient) =
-  if ftp.jobInProgress:
-    assertReply ftp.expectReply(), "226" # Make sure the transfer completed.
-    var r: FTPEvent
-    case ftp.job.typ
-    of JRetrText:
-      r.typ = EvLines
-      r.lines = ftp.job.lines
-    of JRetr:
-      r.typ = EvRetr
-      r.filename = ftp.job.filename
-      if ftp.job.progress != ftp.job.total:
-        raise newException(FTPError, "Didn't download full file.")
-    of JStore:
-      r.typ = EvStore
-      r.filename = ftp.job.filename
-      if ftp.job.progress != ftp.job.total:
-        raise newException(FTPError, "Didn't upload full file.")
-    ftp.deleteJob()
-
-    ftp.handleEvent(ftp, r)
-
-proc asyncFTPClient*(address: string, port = Port(21),
-                     user, pass = "",
-    handleEvent: proc (ftp: AsyncFTPClient, ev: FTPEvent) {.closure,gcsafe.} =
-      (proc (ftp: AsyncFTPClient, ev: FTPEvent) = discard)): AsyncFTPClient =
-  ## Create a ``AsyncFTPClient`` object.
-  ##
-  ## Use this if you want to use asyncio's dispatcher.
-  var dres: AsyncFtpClient
-  new(dres)
-  dres.user = user
-  dres.pass = pass
-  dres.address = address
-  dres.port = port
-  dres.dsockConnected = false
-  dres.handleEvent = handleEvent
-  dres.csock = asyncSocket()
-  dres.csock.handleRead =
-    proc (s: AsyncSocket) =
-      csockHandleRead(s, dres)
-  result = dres
-
-proc register*(d: Dispatcher, ftp: AsyncFTPClient): Delegate {.discardable.} =
-  ## Registers ``ftp`` with dispatcher ``d``.
-  ftp.disp = d
-  return ftp.disp.register(ftp.csock)
-
-when not defined(testing) and isMainModule:
-  proc main =
-    var d = newDispatcher()
-    let hev =
-      proc (ftp: AsyncFTPClient, event: FTPEvent) =
-        case event.typ
-        of EvStore:
-          echo("Upload finished!")
-          ftp.retrFile("payload.jpg", "payload2.jpg", async = true)
-        of EvTransferProgress:
-          var time: int64 = -1
-          if event.speed != 0:
-            time = (event.bytesTotal - event.bytesFinished) div event.speed
-          echo(event.currentJob)
-          echo(event.speed div 1000, " kb/s. - ",
-               event.bytesFinished, "/", event.bytesTotal,
-               " - ", time, " seconds")
-          echo(d.len)
-        of EvRetr:
-          echo("Download finished!")
-          ftp.close()
-          echo d.len
-        else: assert(false)
-    var ftp = asyncFTPClient("example.com", user = "foo", pass = "bar", handleEvent = hev)
-
-    d.register(ftp)
-    d.len.echo()
-    ftp.connect()
-    echo "connected"
-    ftp.store("payload.jpg", "payload.jpg", async = true)
-    d.len.echo()
-    echo "uploading..."
-    while true:
-      if not d.poll(): break
-  main()
-
-when not defined(testing) and isMainModule:
-  var ftp = ftpClient("example.com", user = "foo", pass = "bar")
-  ftp.connect()
-  echo ftp.pwd()
-  echo ftp.list()
-  echo("uploading")
-  ftp.store("payload.jpg", "payload.jpg", async = false)
-
-  echo("Upload complete")
-  ftp.retrFile("payload.jpg", "payload2.jpg", async = false)
-
-  echo("Download complete")
-  sleep(5000)
-  ftp.close()
-  sleep(200)

lib/deprecated/pure/parseopt2.nim

@@ -55,15 +55,6 @@ proc initOptParser*(cmdline: seq[string]): OptParser {.rtl.} =
 
   result.cmd = @cmdline
 
-proc initOptParser*(cmdline: string): OptParser {.rtl, deprecated.} =
-  ## Initalizes option parses with cmdline. Splits cmdline in on spaces
-  ## and calls initOptParser(openarray[string])
-  ## Do not use.
-  if cmdline == "": # backward compatibility
-    return initOptParser(@[])
-  else:
-    return initOptParser(cmdline.split)
-
 when not defined(createNimRtl):
   proc initOptParser*(): OptParser =
     ## Initializes option parser from current command line arguments.
@@ -112,10 +103,10 @@ proc next(p: var OptParser) =
     p.key = token
     p.val = ""
 
-proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo2$1", deprecated.} =
+proc cmdLineRest*(p: OptParser): TaintedString {.rtl, extern: "npo2$1".} =
-  ## Returns part of command line string that has not been parsed yet.
+  ## Returns the part of command line string that has not been parsed yet,
-  ## Do not use - does not correctly handle whitespace.
+  ## properly quoted.
-  return p.cmd[p.pos..p.cmd.len-1].join(" ")
+  return p.cmd[p.pos..p.cmd.len-1].quoteShellCommand
 
 type
   GetoptResult* = tuple[kind: CmdLineKind, key, val: TaintedString]

lib/deprecated/pure/parseurl.nim

@@ -1,112 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2015 Dominik Picheta
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## **Warnings:** This module is deprecated since version 0.10.2.
-## Use the `uri <uri.html>`_ module instead.
-##
-## Parses & constructs URLs.
-
-{.deprecated.}
-
-import strutils
-
-type
-  Url* = tuple[      ## represents a *Uniform Resource Locator* (URL)
-                     ## any optional component is "" if it does not exist
-    scheme, username, password,
-    hostname, port, path, query, anchor: string]
-
-proc parseUrl*(url: string): Url {.deprecated.} =
-  var i = 0
-
-  var scheme, username, password: string = ""
-  var hostname, port, path, query, anchor: string = ""
-
-  var temp = ""
-
-  if url[i] != '/': # url isn't a relative path
-    while true:
-      # Scheme
-      if url[i] == ':':
-        if url[i+1] == '/' and url[i+2] == '/':
-          scheme = temp
-          temp.setLen(0)
-          inc(i, 3) # Skip the //
-      # Authority(username, password)
-      if url[i] == '@':
-        username = temp
-        let colon = username.find(':')
-        if colon >= 0:
-          password = username.substr(colon+1)
-          username = username.substr(0, colon-1)
-        temp.setLen(0)
-        inc(i) #Skip the @
-      # hostname(subdomain, domain, port)
-      if url[i] == '/' or url[i] == '\0':
-        hostname = temp
-        let colon = hostname.find(':')
-        if colon >= 0:
-          port = hostname.substr(colon+1)
-          hostname = hostname.substr(0, colon-1)
-
-        temp.setLen(0)
-        break
-
-      temp.add(url[i])
-      inc(i)
-
-  if url[i] == '/': inc(i) # Skip the '/'
-  # Path
-  while true:
-    if url[i] == '?':
-      path = temp
-      temp.setLen(0)
-    if url[i] == '#':
-      if temp[0] == '?':
-        query = temp
-      else:
-        path = temp
-      temp.setLen(0)
-
-    if url[i] == '\0':
-      if temp[0] == '?':
-        query = temp
-      elif temp[0] == '#':
-        anchor = temp
-      else:
-        path = temp
-      break
-
-    temp.add(url[i])
-    inc(i)
-
-  return (scheme, username, password, hostname, port, path, query, anchor)
-
-proc `$`*(u: Url): string {.deprecated.} =
-  ## turns the URL `u` into its string representation.
-  result = ""
-  if u.scheme.len > 0:
-    result.add(u.scheme)
-    result.add("://")
-  if u.username.len > 0:
-    result.add(u.username)
-    if u.password.len > 0:
-      result.add(":")
-      result.add(u.password)
-    result.add("@")
-  result.add(u.hostname)
-  if u.port.len > 0:
-    result.add(":")
-    result.add(u.port)
-  if u.path.len > 0:
-    result.add("/")
-    result.add(u.path)
-  result.add(u.query)
-  result.add(u.anchor)
-

lib/deprecated/pure/rawsockets.nim

@@ -1,14 +0,0 @@
-import nativesockets
-export nativesockets
-
-{.warning: "rawsockets module is deprecated, use nativesockets instead".}
-
-template newRawSocket*(domain, sockType, protocol: cint): untyped =
-  {.warning: "newRawSocket is deprecated, use newNativeSocket instead".}
-  newNativeSocket(domain, sockType, protocol)
-
-template newRawSocket*(domain: Domain = AF_INET,
-                       sockType: SockType = SOCK_STREAM,
-                       protocol: Protocol = IPPROTO_TCP): untyped =
-  {.warning: "newRawSocket is deprecated, use newNativeSocket instead".}
-  newNativeSocket(domain, sockType, protocol)

lib/impure/db_odbc.nim

@@ -128,10 +128,10 @@ proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {.
               cast[PSQLCHAR](sqlState.addr),
               cast[PSQLCHAR](nativeErr.addr),
               cast[PSQLCHAR](errMsg.addr),
-              511.TSqlSmallInt, retSz.addr.PSQLSMALLINT)
+              511.TSqlSmallInt, retSz.addr)
   except:
     discard
-  return (res.int, $sqlState, $nativeErr, $errMsg)
+  return (res.int, $(addr sqlState), $(addr nativeErr), $(addr errMsg))
 
 proc dbError*(db: var DbConn) {.
           tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} =
@@ -277,14 +277,9 @@ iterator fastRows*(db: var DbConn, query: SqlQuery,
   ## Rows are retrieved from the server at each iteration.
   var
     rowRes: Row
-    sz: TSqlSmallInt = 0
+    sz: TSqlInteger = 0
-    cCnt: TSqlSmallInt = 0.TSqlSmallInt
+    cCnt: TSqlSmallInt = 0
-    res: TSqlSmallInt = 0.TSqlSmallInt
+    res: TSqlSmallInt = 0
-    tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled.
-    # tempcCnt,A field to store the number of temporary variables, for unknown reasons,
-    # after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0,
-    # so the values of the temporary variable to store the cCnt.
-    # After every cycle and specified to cCnt. To ensure the traversal of all fields.
   res = db.prepareFetch(query, args)
   if res == SQL_NO_DATA:
     discard
@@ -292,14 +287,13 @@ iterator fastRows*(db: var DbConn, query: SqlQuery,
     res = SQLNumResultCols(db.stmt, cCnt)
     rowRes = newRow(cCnt)
     rowRes.setLen(max(cCnt,0))
-    tempcCnt = cCnt
     while res == SQL_SUCCESS:
       for colId in 1..cCnt:
         buf[0] = '\0'
         db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
-                                 cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr))
+                                 cast[cstring](buf.addr), 4095.TSqlSmallInt,
+                                 sz.addr))
         rowRes[colId-1] = $(addr buf)
-        cCnt = tempcCnt
       yield rowRes
       res = SQLFetch(db.stmt)
   properFreeResult(SQL_HANDLE_STMT, db.stmt)
@@ -312,14 +306,9 @@ iterator instantRows*(db: var DbConn, query: SqlQuery,
   ## on demand using []. Returned handle is valid only within the interator body.
   var
     rowRes: Row = @[]
-    sz: TSqlSmallInt = 0
+    sz: TSqlInteger = 0
-    cCnt: TSqlSmallInt = 0.TSqlSmallInt
+    cCnt: TSqlSmallInt = 0
-    res: TSqlSmallInt = 0.TSqlSmallInt
+    res: TSqlSmallInt = 0
-    tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled.
-    # tempcCnt,A field to store the number of temporary variables, for unknown reasons,
-    # after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0,
-    # so the values of the temporary variable to store the cCnt.
-    # After every cycle and specified to cCnt. To ensure the traversal of all fields.
   res = db.prepareFetch(query, args)
   if res == SQL_NO_DATA:
     discard
@@ -327,14 +316,13 @@ iterator instantRows*(db: var DbConn, query: SqlQuery,
     res = SQLNumResultCols(db.stmt, cCnt)
     rowRes = newRow(cCnt)
     rowRes.setLen(max(cCnt,0))
-    tempcCnt = cCnt
     while res == SQL_SUCCESS:
       for colId in 1..cCnt:
         buf[0] = '\0'
         db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
-                                 cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr))
+                                 cast[cstring](buf.addr), 4095.TSqlSmallInt,
+                                 sz.addr))
         rowRes[colId-1] = $(addr buf)
-        cCnt = tempcCnt
       yield (row: rowRes, len: cCnt.int)
       res = SQLFetch(db.stmt)
   properFreeResult(SQL_HANDLE_STMT, db.stmt)
@@ -355,14 +343,9 @@ proc getRow*(db: var DbConn, query: SqlQuery,
   ## will return a Row with empty strings for each column.
   var
     rowRes: Row
-    sz: TSqlSmallInt = 0.TSqlSmallInt
+    sz: TSqlInteger = 0
-    cCnt: TSqlSmallInt = 0.TSqlSmallInt
+    cCnt: TSqlSmallInt = 0
-    res: TSqlSmallInt = 0.TSqlSmallInt
+    res: TSqlSmallInt = 0
-    tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled.
-    ## tempcCnt,A field to store the number of temporary variables, for unknown reasons,
-    ## after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0,
-    ## so the values of the temporary variable to store the cCnt.
-    ## After every cycle and specified to cCnt. To ensure the traversal of all fields.
   res = db.prepareFetch(query, args)
   if res == SQL_NO_DATA:
     result = @[]
@@ -370,13 +353,12 @@ proc getRow*(db: var DbConn, query: SqlQuery,
     res = SQLNumResultCols(db.stmt, cCnt)
     rowRes = newRow(cCnt)
     rowRes.setLen(max(cCnt,0))
-    tempcCnt = cCnt
     for colId in 1..cCnt:
       buf[0] = '\0'
       db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
-                               cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr))
+                               cast[cstring](buf.addr), 4095.TSqlSmallInt,
+                               sz.addr))
       rowRes[colId-1] = $(addr buf)
-      cCnt = tempcCnt
     res = SQLFetch(db.stmt)
     result = rowRes
   properFreeResult(SQL_HANDLE_STMT, db.stmt)
@@ -389,14 +371,9 @@ proc getAllRows*(db: var DbConn, query: SqlQuery,
   var
     rows: seq[Row] = @[]
     rowRes: Row
-    sz: TSqlSmallInt = 0
+    sz: TSqlInteger = 0
-    cCnt: TSqlSmallInt = 0.TSqlSmallInt
+    cCnt: TSqlSmallInt = 0
-    res: TSqlSmallInt = 0.TSqlSmallInt
+    res: TSqlSmallInt = 0
-    tempcCnt: TSqlSmallInt # temporary cCnt,Fix the field values to be null when the release schema is compiled.
-    ## tempcCnt,A field to store the number of temporary variables, for unknown reasons,
-    ## after performing a sqlgetdata function and circulating variables cCnt value will be changed to 0,
-    ## so the values of the temporary variable to store the cCnt.
-    ## After every cycle and specified to cCnt. To ensure the traversal of all fields.
   res = db.prepareFetch(query, args)
   if res == SQL_NO_DATA:
     result = @[]
@@ -404,14 +381,13 @@ proc getAllRows*(db: var DbConn, query: SqlQuery,
     res = SQLNumResultCols(db.stmt, cCnt)
     rowRes = newRow(cCnt)
     rowRes.setLen(max(cCnt,0))
-    tempcCnt = cCnt
     while res == SQL_SUCCESS:
       for colId in 1..cCnt:
         buf[0] = '\0'
         db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
-                                 cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr))
+                                 cast[cstring](buf.addr), 4095.TSqlSmallInt,
+                                 sz.addr))
         rowRes[colId-1] = $(addr buf)
-        cCnt = tempcCnt
       rows.add(rowRes)
       res = SQLFetch(db.stmt)
     result = rows

lib/impure/db_sqlite.nim

@@ -166,10 +166,12 @@ iterator fastRows*(db: DbConn, query: SqlQuery,
   var stmt = setupQuery(db, query, args)
   var L = (column_count(stmt))
   var result = newRow(L)
-  while step(stmt) == SQLITE_ROW:
+  try:
-    setRow(stmt, result, L)
+    while step(stmt) == SQLITE_ROW:
-    yield result
+      setRow(stmt, result, L)
-  if finalize(stmt) != SQLITE_OK: dbError(db)
+      yield result
+  finally:
+    if finalize(stmt) != SQLITE_OK: dbError(db)
 
 iterator instantRows*(db: DbConn, query: SqlQuery,
                       args: varargs[string, `$`]): InstantRow
@@ -177,9 +179,11 @@ iterator instantRows*(db: DbConn, query: SqlQuery,
   ## same as fastRows but returns a handle that can be used to get column text
   ## on demand using []. Returned handle is valid only within the iterator body.
   var stmt = setupQuery(db, query, args)
-  while step(stmt) == SQLITE_ROW:
+  try:
-    yield stmt
+    while step(stmt) == SQLITE_ROW:
-  if finalize(stmt) != SQLITE_OK: dbError(db)
+      yield stmt
+  finally:
+    if finalize(stmt) != SQLITE_OK: dbError(db)
 
 proc toTypeKind(t: var DbType; x: int32) =
   case x
@@ -210,9 +214,11 @@ iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery,
   ## on demand using []. Returned handle is valid only within the iterator body.
   var stmt = setupQuery(db, query, args)
   setColumns(columns, stmt)
-  while step(stmt) == SQLITE_ROW:
+  try:
-    yield stmt
+    while step(stmt) == SQLITE_ROW:
-  if finalize(stmt) != SQLITE_OK: dbError(db)
+      yield stmt
+  finally:
+    if finalize(stmt) != SQLITE_OK: dbError(db)
 
 proc `[]`*(row: InstantRow, col: int32): string {.inline.} =
   ## returns text for given column of the row

lib/impure/nre.nim

@@ -540,7 +540,7 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt
       raise RegexInternalError(msg : "Unknown internal error: " & $execRet)
 
 proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] =
-  ## Like ```find(...)`` <#proc-find>`_, but anchored to the start of the
+  ## Like ` ``find(...)`` <#proc-find>`_, but anchored to the start of the
   ## string.
   ##
   runnableExamples:
@@ -550,11 +550,11 @@ proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[R
   return str.matchImpl(pattern, start, endpos, pcre.ANCHORED)
 
 iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): RegexMatch =
-  ## Works the same as ```find(...)`` <#proc-find>`_, but finds every
+  ## Works the same as ` ``find(...)`` <#proc-find>`_, but finds every
   ## non-overlapping match. ``"2222".find(re"22")`` is ``"22", "22"``, not
   ## ``"22", "22", "22"``.
   ##
-  ## Arguments are the same as ```find(...)`` <#proc-find>`_
+  ## Arguments are the same as ` ``find(...)`` <#proc-find>`_
   ##
   ## Variants:
   ##
@@ -633,7 +633,7 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string]
   ## Splits the string with the given regex. This works according to the
   ## rules that Perl and Javascript use.
   ##
-  ## ``start`` behaves the same as in ```find(...)`` <#proc-find>`_.
+  ## ``start`` behaves the same as in ` ``find(...)`` <#proc-find>`_.
   ##
   runnableExamples:
     # -  If the match is zero-width, then the string is still split:

lib/impure/rdstdin.nim

@@ -73,6 +73,15 @@ when defined(Windows):
          discard readConsoleInputW(hStdin, irInputRecord, 1, dwEventsRead)
          return result
 
+elif defined(genode):
+  proc readLineFromStdin*(prompt: string): TaintedString {.
+                          tags: [ReadIOEffect, WriteIOEffect].} =
+    stdin.readLine()
+
+  proc readLineFromStdin*(prompt: string, line: var TaintedString): bool {.
+                          tags: [ReadIOEffect, WriteIOEffect].} =
+    stdin.readLine(line)
+
 else:
   import linenoise, termios
 

lib/impure/re.nim

@@ -561,31 +561,6 @@ proc escapeRe*(s: string): string =
       result.add("\\x")
       result.add(toHex(ord(c), 2))
 
-const ## common regular expressions
-  reIdentifier* {.deprecated.} = r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b"
-    ## describes an identifier
-  reNatural* {.deprecated.} = r"\b\d+\b"
-    ## describes a natural number
-  reInteger* {.deprecated.} = r"\b[-+]?\d+\b"
-    ## describes an integer
-  reHex* {.deprecated.} = r"\b0[xX][0-9a-fA-F]+\b"
-    ## describes a hexadecimal number
-  reBinary* {.deprecated.} = r"\b0[bB][01]+\b"
-    ## describes a binary number (example: 0b11101)
-  reOctal* {.deprecated.} = r"\b0[oO][0-7]+\b"
-    ## describes an octal number (example: 0o777)
-  reFloat* {.deprecated.} = r"\b[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?\b"
-    ## describes a floating point number
-  reEmail* {.deprecated.} = r"\b[a-zA-Z0-9!#$%&'*+/=?^_`{|}~\-]+(?:\. &" &
-                            r"[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@" &
-                            r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+" &
-                            r"(?:[a-zA-Z]{2}|com|org|net|gov|mil|biz|" &
-                            r"info|mobi|name|aero|jobs|museum)\b"
-    ## describes a common email address
-  reURL* {.deprecated.} = r"\b(http(s)?|ftp|gopher|telnet|file|notes|ms-help)" &
-                          r":((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-\=\\\.\&]*\b"
-    ## describes an URL
-
 when isMainModule:
   doAssert match("(a b c)", rex"\( .* \)")
   doAssert match("WHiLe", re("while", {reIgnoreCase}))
@@ -595,7 +570,7 @@ when isMainModule:
   doAssert "ABC".match(rex"\d+ | \w+")
 
   {.push warnings:off.}
-  doAssert matchLen("key", re(reIdentifier)) == 3
+  doAssert matchLen("key", re"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b") == 3
   {.pop.}
 
   var pattern = re"[a-z0-9]+\s*=\s*[a-z0-9]+"

lib/impure/ssl.nim

@@ -1,99 +0,0 @@
-#
-#
-#            Nim's Runtime Library
-#        (c) Copyright 2012 Dominik Picheta
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module provides an easy to use sockets-style
-## nim interface to the OpenSSL library.
-##
-## **Warning:** This module is deprecated, use the SSL procedures defined in
-## the ``net`` module instead.
-
-{.deprecated.}
-
-import openssl, strutils, os
-
-type
-  SecureSocket* = object
-    ssl: SslPtr
-    bio: BIO
-
-proc connect*(sock: var SecureSocket, address: string,
-    port: int): int =
-  ## Connects to the specified `address` on the specified `port`.
-  ## Returns the result of the certificate validation.
-  SslLoadErrorStrings()
-  ERR_load_BIO_strings()
-
-  if SSL_library_init() != 1:
-    raiseOSError(osLastError())
-
-  var ctx = SSL_CTX_new(SSLv23_client_method())
-  if ctx == nil:
-    ERR_print_errors_fp(stderr)
-    raiseOSError(osLastError())
-
-  #if SSL_CTX_load_verify_locations(ctx,
-  #   "/tmp/openssl-0.9.8e/certs/vsign1.pem", NIL) == 0:
-  #  echo("Failed load verify locations")
-  #  ERR_print_errors_fp(stderr)
-
-  sock.bio = BIO_new_ssl_connect(ctx)
-  if BIO_get_ssl(sock.bio, addr(sock.ssl)) == 0:
-    raiseOSError(osLastError())
-
-  if BIO_set_conn_hostname(sock.bio, address & ":" & $port) != 1:
-    raiseOSError(osLastError())
-
-  if BIO_do_connect(sock.bio) <= 0:
-    ERR_print_errors_fp(stderr)
-    raiseOSError(osLastError())
-
-  result = SSL_get_verify_result(sock.ssl)
-
-proc recvLine*(sock: SecureSocket, line: var TaintedString): bool =
-  ## Acts in a similar fashion to the `recvLine` in the sockets module.
-  ## Returns false when no data is available to be read.
-  ## `Line` must be initialized and not nil!
-  setLen(line.string, 0)
-  while true:
-    var c: array[0..0, char]
-    var n = BIO_read(sock.bio, addr c, c.len.cint)
-    if n <= 0: return false
-    if c[0] == '\r':
-      n = BIO_read(sock.bio, addr c, c.len.cint)
-      if n > 0 and c[0] == '\L':
-        return true
-      elif n <= 0:
-        return false
-    elif c[0] == '\L': return true
-    add(line.string, c[0])
-
-
-proc send*(sock: SecureSocket, data: string) =
-  ## Writes `data` to the socket.
-  if BIO_write(sock.bio, data, data.len.cint) <= 0:
-    raiseOSError(osLastError())
-
-proc close*(sock: SecureSocket) =
-  ## Closes the socket
-  if BIO_free(sock.bio) <= 0:
-    ERR_print_errors_fp(stderr)
-    raiseOSError(osLastError())
-
-when not defined(testing) and isMainModule:
-  var s: SecureSocket
-  echo connect(s, "smtp.gmail.com", 465)
-
-  #var buffer: array[0..255, char]
-  #echo BIO_read(bio, buffer, buffer.len)
-  var buffer: string = ""
-
-  echo s.recvLine(buffer)
-  echo buffer
-  echo buffer.len
-

lib/js/jscore.nim

@@ -73,7 +73,7 @@ proc parse*(d: DateLib, s: cstring): int {.importcpp.}
 proc newDate*(): DateTime {.
   importcpp: "new Date()".}
 
-proc newDate*(date: int|string): DateTime {.
+proc newDate*(date: int|int64|string): DateTime {.
   importcpp: "new Date(#)".}
 
 proc newDate*(year, month, day, hours, minutes,
@@ -90,6 +90,16 @@ proc getSeconds*(d: DateTime): int {.importcpp.}
 proc getYear*(d: DateTime): int {.importcpp.}
 proc getTime*(d: DateTime): int {.importcpp.}
 proc toString*(d: DateTime): cstring {.importcpp.}
+proc getUTCDate*(d: DateTime): int {.importcpp.}
+proc getUTCFullYear*(d: DateTime): int {.importcpp.}
+proc getUTCHours*(d: DateTime): int {.importcpp.}
+proc getUTCMilliseconds*(d: DateTime): int {.importcpp.}
+proc getUTCMinutes*(d: DateTime): int {.importcpp.}
+proc getUTCMonth*(d: DateTime): int {.importcpp.}
+proc getUTCSeconds*(d: DateTime): int {.importcpp.}
+proc getUTCDay*(d: DateTime): int {.importcpp.}
+proc getTimezoneOffset*(d: DateTime): int {.importcpp.}
+proc setFullYear*(d: DateTime, year: int) {.importcpp.}
 
 #JSON library
 proc stringify*(l: JsonLib, s: JsRoot): cstring {.importcpp.}