Merge branch 'devel' into coroutines

compiler/ccgstmts.nim

@@ -102,7 +102,7 @@ proc assignLabel(b: var TBlock): Rope {.inline.} =
 proc blockBody(b: var TBlock): Rope =
   result = b.sections[cpsLocals]
   if b.frameLen > 0:
-    result.addf("F.len+=$1;$n", [b.frameLen.rope])
+    result.addf("FR.len+=$1;$n", [b.frameLen.rope])
   result.add(b.sections[cpsInit])
   result.add(b.sections[cpsStmts])
 
@@ -123,7 +123,7 @@ proc endBlock(p: BProc) =
       ~"}$n"
   let frameLen = p.blocks[topBlock].frameLen
   if frameLen > 0:
-    blockEnd.addf("F.len-=$1;$n", [frameLen.rope])
+    blockEnd.addf("FR.len-=$1;$n", [frameLen.rope])
   endBlock(p, blockEnd)
 
 proc genSimpleBlock(p: BProc, stmts: PNode) {.inline.} =

compiler/cgen.nim

@@ -375,7 +375,7 @@ proc localDebugInfo(p: BProc, s: PSym) =
   var a = "&" & s.loc.r
   if s.kind == skParam and ccgIntroducedPtr(s): a = s.loc.r
   lineF(p, cpsInit,
-       "F.s[$1].address = (void*)$3; F.s[$1].typ = $4; F.s[$1].name = $2;$n",
+       "FR.s[$1].address = (void*)$3; FR.s[$1].typ = $4; FR.s[$1].name = $2;$n",
        [p.maxFrameLen.rope, makeCString(normalize(s.name.s)), a,
         genTypeInfo(p.module, s.loc.t)])
   inc(p.maxFrameLen)
@@ -597,7 +597,7 @@ proc cgsym(m: BModule, name: string): Rope =
     of skProc, skMethod, skConverter, skIterators: genProc(m, sym)
     of skVar, skResult, skLet: genVarPrototype(m, sym)
     of skType: discard getTypeDesc(m, sym.typ)
-    else: internalError("cgsym: " & name)
+    else: internalError("cgsym: " & name & ": " & $sym.kind)
   else:
     # we used to exclude the system module from this check, but for DLL
     # generation support this sloppyness leads to hard to detect bugs, so
@@ -623,7 +623,7 @@ proc retIsNotVoid(s: PSym): bool =
 proc initFrame(p: BProc, procname, filename: Rope): Rope =
   discard cgsym(p.module, "nimFrame")
   if p.maxFrameLen > 0:
-    discard cgsym(p.module, "TVarSlot")
+    discard cgsym(p.module, "VarSlot")
     result = rfmt(nil, "\tnimfrs($1, $2, $3, $4)$N",
                   procname, filename, p.maxFrameLen.rope,
                   p.blocks[0].frameLen.rope)
@@ -1017,7 +1017,7 @@ proc genInitCode(m: BModule) =
       var procname = makeCString(m.module.name.s)
       add(prc, initFrame(m.initProc, procname, m.module.info.quotedFilename))
     else:
-      add(prc, ~"\tTFrame F; F.len = 0;$N")
+      add(prc, ~"\tTFrame F; FR.len = 0;$N")
 
   add(prc, genSectionStart(cpsInit))
   add(prc, m.preInitProc.s(cpsInit))
@@ -1329,4 +1329,3 @@ proc cgenWriteModules* =
   if generatedHeader != nil: writeHeader(generatedHeader)
 
 const cgenPass* = makePass(myOpen, myOpenCached, myProcess, myClose)
-

compiler/commands.nim

@@ -645,7 +645,7 @@ proc processSwitch*(pass: TCmdLinePass; p: OptParser) =
 proc processArgument*(pass: TCmdLinePass; p: OptParser;
                       argsCount: var int): bool =
   if argsCount == 0:
-    options.command = p.key
+    if pass != passCmd2: options.command = p.key
   else:
     if pass == passCmd1: options.commandArgs.add p.key
     if argsCount == 1:

compiler/condsyms.nim

@@ -90,3 +90,4 @@ proc initDefines*() =
   defineSymbol("nimnode")
   defineSymbol("nimnomagic64")
   defineSymbol("nimvarargstyped")
+  defineSymbol("nimtypedescfixed")

compiler/evaltempl.nim

@@ -25,16 +25,21 @@ proc copyNode(ctx: TemplCtx, a, b: PNode): PNode =
   if ctx.instLines: result.info = b.info
 
 proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
+  template handleParam(param) =
+    let x = param
+    if x.kind == nkArgList:
+      for y in items(x): result.add(y)
+    else:
+      result.add copyTree(x)
+  
   case templ.kind
   of nkSym:
     var s = templ.sym
     if s.owner.id == c.owner.id:
       if s.kind == skParam and sfGenSym notin s.flags:
-        let x = actual.sons[s.position]
+        handleParam actual.sons[s.position]
-        if x.kind == nkArgList:
+      elif s.kind == skGenericParam:
-          for y in items(x): result.add(y)
+        handleParam actual.sons[s.owner.typ.len + s.position - 1]
-        else:
-          result.add copyTree(x)
       else:
         internalAssert sfGenSym in s.flags
         var x = PSym(idTableGet(c.mapping, s))
@@ -56,22 +61,45 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
 proc evalTemplateArgs(n: PNode, s: PSym): PNode =
   # if the template has zero arguments, it can be called without ``()``
   # `n` is then a nkSym or something similar
-  var a: int
+  var totalParams = case n.kind
-  case n.kind
+    of nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand, nkCallStrLit: <n.len
-  of nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand, nkCallStrLit:
+    else: 0
-    a = sonsLen(n)
+
-  else: a = 0
+  var
-  var f = s.typ.sonsLen
+    # XXX: Since immediate templates are not subjected to the
-  if a > f: globalError(n.info, errWrongNumberOfArguments)
+    # standard sigmatching algorithm, they will have a number
+    # of deficiencies when it comes to generic params:
+    # Type dependencies between the parameters won't be honoured
+    # and the bound generic symbols won't be resolvable within
+    # their bodies. We could try to fix this, but it may be
+    # wiser to just deprecate immediate templates and macros
+    # now that we have working untyped parameters.
+    genericParams = if sfImmediate in s.flags: 0
+                    else: s.ast[genericParamsPos].len
+    expectedRegularParams = <s.typ.len
+    givenRegularParams = totalParams - genericParams
+
+  if totalParams > expectedRegularParams + genericParams:
+    globalError(n.info, errWrongNumberOfArguments)
   
   result = newNodeI(nkArgList, n.info)
-  for i in countup(1, f - 1):
+  for i in 1 .. givenRegularParams:
-    var arg = if i < a: n.sons[i] else: copyTree(s.typ.n.sons[i].sym.ast)
+    result.addSon n.sons[i]
-    if arg == nil or arg.kind == nkEmpty:
+
+  # handle parameters with default values, which were
+  # not supplied by the user
+  for i in givenRegularParams+1 .. expectedRegularParams:
+    let default = s.typ.n.sons[i].sym.ast
+    internalAssert default != nil
+    if default.kind == nkEmpty:
       localError(n.info, errWrongNumberOfArguments)
       addSon(result, ast.emptyNode)
     else:
-      addSon(result, arg)
+      addSon(result, default.copyTree)
+  
+  # add any generic paramaters
+  for i in 1 .. genericParams:
+    result.addSon n.sons[givenRegularParams + i]
 
 var evalTemplateCounter* = 0
   # to prevent endless recursion in templates instantiation

compiler/installer.ini

@@ -6,7 +6,7 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;amd64;powerpc64;arm;sparc;mips;powerpc
+  linux: i386;amd64;powerpc64;arm;sparc;mips;mipsel;powerpc;powerpc64el;arm64
   macosx: i386;amd64;powerpc64
   solaris: i386;amd64;sparc
   freebsd: i386;amd64

compiler/jsgen.nim

@@ -729,14 +729,13 @@ proc genBreakStmt(p: PProc, n: PNode) =
   p.blocks[idx].id = abs(p.blocks[idx].id) # label is used
   addf(p.body, "break L$1;$n" | "goto ::L$1::;$n", [rope(p.blocks[idx].id)])
 
-proc genAsmStmt(p: PProc, n: PNode) =
+proc genAsmOrEmitStmt(p: PProc, n: PNode) =
   genLineDir(p, n)
-  assert(n.kind == nkAsmStmt)
   for i in countup(0, sonsLen(n) - 1):
     case n.sons[i].kind
     of nkStrLit..nkTripleStrLit: add(p.body, n.sons[i].strVal)
     of nkSym: add(p.body, mangleName(n.sons[i].sym))
-    else: internalError(n.sons[i].info, "jsgen: genAsmStmt()")
+    else: internalError(n.sons[i].info, "jsgen: genAsmOrEmitStmt()")
 
 proc genIf(p: PProc, n: PNode, r: var TCompRes) =
   var cond, stmt: TCompRes
@@ -1578,6 +1577,12 @@ proc genStmt(p: PProc, n: PNode) =
   gen(p, n, r)
   if r.res != nil: addf(p.body, "$#;$n", [r.res])
 
+proc genPragma(p: PProc, n: PNode) =
+  for it in n.sons:
+    case whichPragma(it)
+    of wEmit: genAsmOrEmitStmt(p, it.sons[1])
+    else: discard
+
 proc gen(p: PProc, n: PNode, r: var TCompRes) =
   r.typ = etyNone
   r.kind = resNone
@@ -1677,12 +1682,13 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
     if n.sons[0].kind != nkEmpty:
       genLineDir(p, n)
       gen(p, n.sons[0], r)
-  of nkAsmStmt: genAsmStmt(p, n)
+  of nkAsmStmt: genAsmOrEmitStmt(p, n)
   of nkTryStmt: genTry(p, n, r)
   of nkRaiseStmt: genRaiseStmt(p, n)
   of nkTypeSection, nkCommentStmt, nkIteratorDef, nkIncludeStmt,
      nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
-     nkFromStmt, nkTemplateDef, nkMacroDef, nkPragma: discard
+     nkFromStmt, nkTemplateDef, nkMacroDef: discard
+  of nkPragma: genPragma(p, n)
   of nkProcDef, nkMethodDef, nkConverterDef:
     var s = n.sons[namePos].sym
     if {sfExportc, sfCompilerProc} * s.flags == {sfExportc}:

compiler/lambdalifting.nim

@@ -184,7 +184,7 @@ proc addHiddenParam(routine: PSym, param: PSym) =
   var params = routine.ast.sons[paramsPos]
   # -1 is correct here as param.position is 0 based but we have at position 0
   # some nkEffect node:
-  param.position = params.len-1
+  param.position = routine.typ.n.len-1
   addSon(params, newSymNode(param))
   incl(routine.typ.flags, tfCapturesEnv)
   assert sfFromGeneric in param.flags

compiler/main.nim

@@ -116,7 +116,7 @@ proc interactivePasses =
   #incl(gGlobalOptions, optSafeCode)
   #setTarget(osNimrodVM, cpuNimrodVM)
   initDefines()
-  defineSymbol("nimrodvm")
+  defineSymbol("nimscript")
   when hasFFI: defineSymbol("nimffi")
   registerPass(verbosePass)
   registerPass(semPass)
@@ -356,6 +356,7 @@ proc mainCommand* =
     gGlobalOptions.incl(optCaasEnabled)
     msgs.gErrorMax = high(int)  # do not stop after first error
     serve(mainCommand)
+  of "nop": discard
   else:
     rawMessage(errInvalidCommandX, command)
 

compiler/modules.nim

@@ -85,6 +85,15 @@ proc resetAllModules* =
   resetPackageCache()
   # for m in cgenModules(): echo "CGEN MODULE FOUND"
 
+proc resetAllModulesHard* =
+  resetPackageCache()
+  gCompiledModules.setLen 0
+  gMemCacheData.setLen 0
+  magicsys.resetSysTypes()
+  # XXX
+  #gOwners = @[]
+  #rangeDestructorProc = nil
+
 proc checkDepMem(fileIdx: int32): TNeedRecompile =
   template markDirty =
     resetModule(fileIdx)
@@ -205,9 +214,8 @@ proc compileProject*(projectFileIdx = -1'i32) =
     compileSystemModule()
     discard compileModule(projectFile, {sfMainModule})
 
-var stdinModule: PSym
+proc makeModule*(filename: string): PSym =
-proc makeStdinModule*(): PSym =
+  result = newModule(fileInfoIdx filename)
-  if stdinModule == nil:
+  result.id = getID()
-    stdinModule = newModule(fileInfoIdx"stdin")
+
-    stdinModule.id = getID()
+proc makeStdinModule*(): PSym = makeModule"stdin"
-  result = stdinModule

compiler/msgs.nim

@@ -82,7 +82,7 @@ type
     errArrayExpectsTwoTypeParams, errInvalidVisibilityX, errInitHereNotAllowed,
     errXCannotBeAssignedTo, errIteratorNotAllowed, errXNeedsReturnType,
     errNoReturnTypeDeclared,
-    errInvalidCommandX, errXOnlyAtModuleScope,
+    errNoCommand, errInvalidCommandX, errXOnlyAtModuleScope,
     errXNeedsParamObjectType,
     errTemplateInstantiationTooNested, errInstantiationFrom,
     errInvalidIndexValueForTuple, errCommandExpectsFilename,
@@ -318,6 +318,7 @@ const
     errIteratorNotAllowed: "iterators can only be defined at the module\'s top level",
     errXNeedsReturnType: "$1 needs a return type",
     errNoReturnTypeDeclared: "no return type declared",
+    errNoCommand: "no command given",
     errInvalidCommandX: "invalid command: \'$1\'",
     errXOnlyAtModuleScope: "\'$1\' is only allowed at top level",
     errXNeedsParamObjectType: "'$1' needs a parameter that has an object type",

compiler/nim.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nim Compiler
-#        (c) Copyright 2013 Andreas Rumpf
+#        (c) Copyright 2015 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -16,7 +16,7 @@ when defined(gcc) and defined(windows):
 import
   commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes,
   extccomp, strutils, os, osproc, platform, main, parseopt, service,
-  nodejs
+  nodejs, scriptconfig
 
 when hasTinyCBackend:
   import tccgen
@@ -54,10 +54,17 @@ proc handleCmdLine() =
     else:
       gProjectPath = getCurrentDir()
     loadConfigs(DefaultConfig) # load all config files
+    let scriptFile = gProjectFull.changeFileExt("nims")
+    if fileExists(scriptFile):
+      runNimScript(scriptFile)
+      # 'nim foo.nims' means to just run the NimScript file and do nothing more:
+      if scriptFile == gProjectFull: return
     # now process command line arguments again, because some options in the
     # command line can overwite the config file's settings
     extccomp.initVars()
     processCmdLine(passCmd2, "")
+    if options.command == "":
+      rawMessage(errNoCommand, command)
     mainCommand()
     if optHints in gOptions and hintGCStats in gNotes: echo(GC_getStatistics())
     #echo(GC_getStatistics())

compiler/options.nim

@@ -210,7 +210,7 @@ proc removeTrailingDirSep*(path: string): string =
   else:
     result = path
 
-proc getGeneratedPath: string =
+proc getNimcacheDir*: string =
   result = if nimcacheDir.len > 0: nimcacheDir else: gProjectPath.shortenDir /
                                                          genSubDir
 
@@ -266,7 +266,7 @@ proc toGeneratedFile*(path, ext: string): string =
   ## converts "/home/a/mymodule.nim", "rod" to "/home/a/nimcache/mymodule.rod"
   var (head, tail) = splitPath(path)
   #if len(head) > 0: head = shortenDir(head & dirSep)
-  result = joinPath([getGeneratedPath(), changeFileExt(tail, ext)])
+  result = joinPath([getNimcacheDir(), changeFileExt(tail, ext)])
   #echo "toGeneratedFile(", path, ", ", ext, ") = ", result
 
 when noTimeMachine:
@@ -294,7 +294,7 @@ when noTimeMachine:
 proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string =
   var (head, tail) = splitPath(f)
   #if len(head) > 0: head = removeTrailingDirSep(shortenDir(head & dirSep))
-  var subdir = getGeneratedPath() # / head
+  var subdir = getNimcacheDir() # / head
   if createSubDir:
     try:
       createDir(subdir)

compiler/parampatterns.nim

@@ -178,13 +178,14 @@ type
     arDiscriminant,           # is a discriminant
     arStrange                 # it is a strange beast like 'typedesc[var T]'
 
-proc isAssignable*(owner: PSym, n: PNode): TAssignableResult =
+proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult =
   ## 'owner' can be nil!
   result = arNone
   case n.kind
   of nkSym:
-    # don't list 'skLet' here:
+    let kinds = if isUnsafeAddr: {skVar, skResult, skTemp, skParam, skLet}
-    if n.sym.kind in {skVar, skResult, skTemp}:
+                else: {skVar, skResult, skTemp}
+    if n.sym.kind in kinds:
       if owner != nil and owner.id == n.sym.owner.id and
           sfGlobal notin n.sym.flags:
         result = arLocalLValue
@@ -200,7 +201,7 @@ proc isAssignable*(owner: PSym, n: PNode): TAssignableResult =
         {tyVar, tyPtr, tyRef}:
       result = arLValue
     else:
-      result = isAssignable(owner, n.sons[0])
+      result = isAssignable(owner, n.sons[0], isUnsafeAddr)
     if result != arNone and sfDiscriminant in n.sons[1].sym.flags:
       result = arDiscriminant
   of nkBracketExpr:
@@ -208,23 +209,24 @@ proc isAssignable*(owner: PSym, n: PNode): TAssignableResult =
         {tyVar, tyPtr, tyRef}:
       result = arLValue
     else:
-      result = isAssignable(owner, n.sons[0])
+      result = isAssignable(owner, n.sons[0], isUnsafeAddr)
   of nkHiddenStdConv, nkHiddenSubConv, nkConv:
     # Object and tuple conversions are still addressable, so we skip them
     # XXX why is 'tyOpenArray' allowed here?
     if skipTypes(n.typ, abstractPtrs-{tyTypeDesc}).kind in
         {tyOpenArray, tyTuple, tyObject}:
-      result = isAssignable(owner, n.sons[1])
+      result = isAssignable(owner, n.sons[1], isUnsafeAddr)
     elif compareTypes(n.typ, n.sons[1].typ, dcEqIgnoreDistinct):
       # types that are equal modulo distinction preserve l-value:
-      result = isAssignable(owner, n.sons[1])
+      result = isAssignable(owner, n.sons[1], isUnsafeAddr)
   of nkHiddenDeref, nkDerefExpr, nkHiddenAddr:
     result = arLValue
   of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
-    result = isAssignable(owner, n.sons[0])
+    result = isAssignable(owner, n.sons[0], isUnsafeAddr)
   of nkCallKinds:
     # builtin slice keeps lvalue-ness:
-    if getMagic(n) == mSlice: result = isAssignable(owner, n.sons[1])
+    if getMagic(n) == mSlice:
+      result = isAssignable(owner, n.sons[1], isUnsafeAddr)
   else:
     discard
 

compiler/platform.nim

@@ -158,8 +158,8 @@ type
   TSystemCPU* = enum # Also add CPU for in initialization section and
                      # alias conditionals to condsyms (end of module).
     cpuNone, cpuI386, cpuM68k, cpuAlpha, cpuPowerpc, cpuPowerpc64,
-    cpuSparc, cpuVm, cpuIa64, cpuAmd64, cpuMips, cpuArm,
+    cpuPowerpc64el, cpuSparc, cpuVm, cpuIa64, cpuAmd64, cpuMips, cpuMipsel,
-    cpuJS, cpuNimrodVM, cpuAVR
+    cpuArm, cpuArm64, cpuJS, cpuNimrodVM, cpuAVR
 
 type
   TEndian* = enum
@@ -175,12 +175,15 @@ const
     (name: "alpha", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
     (name: "powerpc", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32),
     (name: "powerpc64", intSize: 64, endian: bigEndian, floatSize: 64,bit: 64),
+    (name: "powerpc64el", intSize: 64, endian: littleEndian, floatSize: 64,bit: 64),
     (name: "sparc", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32),
     (name: "vm", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
     (name: "ia64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
     (name: "amd64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
     (name: "mips", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32),
+    (name: "mipsel", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
     (name: "arm", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
+    (name: "arm64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
     (name: "js", intSize: 32, endian: bigEndian,floatSize: 64,bit: 32),
     (name: "nimrodvm", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32),
     (name: "avr", intSize: 16, endian: littleEndian, floatSize: 32, bit: 16)]

compiler/pragmas.nim

@@ -593,8 +593,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
                   validPragmas: TSpecialWords): bool =
   var it = n.sons[i]
   var key = if it.kind == nkExprColonExpr: it.sons[0] else: it
-  if key.kind == nkIdent:
+  if key.kind == nkBracketExpr:
-    var userPragma = strTableGet(c.userPragmas, key.ident)
+    processNote(c, it)
+    return
+  let ident = considerQuotedIdent(key)
+  var userPragma = strTableGet(c.userPragmas, ident)
   if userPragma != nil:
     inc c.instCounter
     if c.instCounter > 100:
@@ -605,7 +608,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
     n.sons[i] = userPragma.ast
     dec c.instCounter
   else:
-      var k = whichKeyword(key.ident)
+    var k = whichKeyword(ident)
     if k in validPragmas:
       case k
       of wExportc:
@@ -866,7 +869,6 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         noVal(it)
       else: invalidPragma(it)
     else: invalidPragma(it)
-  else: processNote(c, it)
 
 proc implicitPragmas*(c: PContext, sym: PSym, n: PNode,
                       validPragmas: TSpecialWords) =

compiler/rodread.nim

@@ -898,6 +898,8 @@ proc getBody*(s: PSym): PNode =
   ## it may perform an expensive reload operation. Otherwise it's a simple
   ## accessor.
   assert s.kind in routineKinds
+  # prevent crashes due to incorrect macro transformations (bug #2377)
+  if s.ast.isNil or bodyPos >= s.ast.len: return ast.emptyNode
   result = s.ast.sons[bodyPos]
   if result == nil:
     assert s.offset != 0

compiler/scriptconfig.nim

@@ -0,0 +1,119 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Implements the new configuration system for Nim. Uses Nim as a scripting
+## language.
+
+import
+  ast, modules, passes, passaux, condsyms,
+  options, nimconf, lists, sem, semdata, llstream, vm, vmdef, commands, msgs,
+  os, times
+
+# we support 'cmpIgnoreStyle' natively for efficiency:
+from strutils import cmpIgnoreStyle
+
+proc listDirs(a: VmArgs, filter: set[PathComponent]) =
+  let dir = getString(a, 0)
+  var result: seq[string] = @[]
+  for kind, path in walkDir(dir):
+    if kind in filter: result.add path
+  setResult(a, result)
+
+proc setupVM(module: PSym; scriptName: string): PEvalContext =
+  result = newCtx(module)
+  result.mode = emRepl
+  registerAdditionalOps(result)
+
+  # captured vars:
+  var errorMsg: string
+  var vthisDir = scriptName.splitFile.dir
+
+  template cbconf(name, body) {.dirty.} =
+    result.registerCallback "stdlib.system." & astToStr(name),
+      proc (a: VmArgs) =
+        body
+
+  template cbos(name, body) {.dirty.} =
+    result.registerCallback "stdlib.system." & astToStr(name),
+      proc (a: VmArgs) =
+        try:
+          body
+        except OSError:
+          errorMsg = getCurrentExceptionMsg()
+
+  # Idea: Treat link to file as a file, but ignore link to directory to prevent
+  # endless recursions out of the box.
+  cbos listFiles:
+    listDirs(a, {pcFile, pcLinkToFile})
+  cbos listDirs:
+    listDirs(a, {pcDir})
+  cbos removeDir:
+    os.removeDir getString(a, 0)
+  cbos removeFile:
+    os.removeFile getString(a, 0)
+  cbos createDir:
+    os.createDir getString(a, 0)
+  cbos getOsError:
+    setResult(a, errorMsg)
+  cbos setCurrentDir:
+    os.setCurrentDir getString(a, 0)
+  cbos getCurrentDir:
+    setResult(a, os.getCurrentDir())
+  cbos moveFile:
+    os.moveFile(getString(a, 0), getString(a, 1))
+  cbos getLastModificationTime:
+    setResult(a, toSeconds(getLastModificationTime(getString(a, 0))))
+
+  cbconf thisDir:
+    setResult(a, vthisDir)
+  cbconf put:
+    options.setConfigVar(getString(a, 0), getString(a, 1))
+  cbconf get:
+    setResult(a, options.getConfigVar(a.getString 0))
+  cbconf exists:
+    setResult(a, options.existsConfigVar(a.getString 0))
+  cbconf nimcacheDir:
+    setResult(a, options.getNimcacheDir())
+  cbconf paramStr:
+    setResult(a, os.paramStr(int a.getInt 0))
+  cbconf paramCount:
+    setResult(a, os.paramCount())
+  cbconf cmpIgnoreStyle:
+    setResult(a, strutils.cmpIgnoreStyle(a.getString 0, a.getString 1))
+  cbconf setCommand:
+    options.command = a.getString 0
+  cbconf getCommand:
+    setResult(a, options.command)
+  cbconf switch:
+    processSwitch(a.getString 0, a.getString 1, passPP, unknownLineInfo())
+
+
+proc runNimScript*(scriptName: string) =
+  passes.gIncludeFile = includeModule
+  passes.gImportModule = importModule
+  initDefines()
+
+  defineSymbol("nimscript")
+  defineSymbol("nimconfig")
+  registerPass(semPass)
+  registerPass(evalPass)
+
+  appendStr(searchPaths, options.libpath)
+
+  var m = makeModule(scriptName)
+  incl(m.flags, sfMainModule)
+  vm.globalCtx = setupVM(m, scriptName)
+
+  compileSystemModule()
+  processModule(m, llStreamOpen(scriptName, fmRead), nil)
+
+  # ensure we load 'system.nim' again for the real non-config stuff!
+  resetAllModulesHard()
+  vm.globalCtx = nil
+  initDefines()

compiler/sem.nim

@@ -171,11 +171,15 @@ proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym =
   result = newSym(kind, considerQuotedIdent(n), getCurrOwner(), n.info)
 
 proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
+  proc `$`(kind: TSymKind): string = substr(system.`$`(kind), 2).toLower
+
   # like newSymS, but considers gensym'ed symbols
   if n.kind == nkSym:
     # and sfGenSym in n.sym.flags:
     result = n.sym
-    internalAssert result.kind == kind
+    if result.kind != kind:
+      localError(n.info, "cannot use symbol of kind '" &
+                 $result.kind & "' as a '" & $kind & "'")
     # when there is a nested proc inside a template, semtmpl
     # will assign a wrong owner during the first pass over the
     # template; we must fix it here: see #909

compiler/semcall.nim

@@ -305,8 +305,22 @@ proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode =
     if containsGenericType(result.typ) or x.fauxMatch == tyUnknown:
       result.typ = newTypeS(x.fauxMatch, c)
     return
-  if finalCallee.ast.sons[genericParamsPos].kind != nkEmpty:
+  let gp = finalCallee.ast.sons[genericParamsPos]
+  if gp.kind != nkEmpty:
+    if x.calleeSym.kind notin {skMacro, skTemplate}:
       finalCallee = generateInstance(c, x.calleeSym, x.bindings, n.info)
+    else:
+      # For macros and templates, the resolved generic params
+      # are added as normal params.
+      for s in instantiateGenericParamList(c, gp, x.bindings):
+        case s.kind
+          of skConst:
+            x.call.add s.ast
+          of skType:
+            x.call.add newSymNode(s, n.info)
+          else:
+            internalAssert false
+  
   result = x.call
   instGenericConvertersSons(c, result, x)
   result.sons[0] = newSymNode(finalCallee, result.sons[0].info)

compiler/semdestruct.nim

@@ -177,6 +177,15 @@ proc instantiateDestructor(c: PContext, typ: PType): PType =
   else:
     return nil
 
+proc createDestructorCall(c: PContext, s: PSym): PNode =
+  let varTyp = s.typ
+  if varTyp == nil or sfGlobal in s.flags: return
+  let destructableT = instantiateDestructor(c, varTyp)
+  if destructableT != nil:
+    let call = semStmt(c, newNode(nkCall, s.info, @[
+      useSym(destructableT.destructor), useSym(s)]))
+    result = newNode(nkDefer, s.info, @[call])
+
 proc insertDestructors(c: PContext,
                        varSection: PNode): tuple[outer, inner: PNode] =
   # Accepts a var or let section.

compiler/semexprs.nim

@@ -597,8 +597,8 @@ proc skipObjConv(n: PNode): PNode =
   of nkObjUpConv, nkObjDownConv: result = n.sons[0]
   else: result = n
 
-proc isAssignable(c: PContext, n: PNode): TAssignableResult =
+proc isAssignable(c: PContext, n: PNode; isUnsafeAddr=false): TAssignableResult =
-  result = parampatterns.isAssignable(c.p.owner, n)
+  result = parampatterns.isAssignable(c.p.owner, n, isUnsafeAddr)
 
 proc newHiddenAddrTaken(c: PContext, n: PNode): PNode =
   if n.kind == nkHiddenDeref and not (gCmd == cmdCompileToCpp or
@@ -1700,7 +1700,7 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
   case s.magic # magics that need special treatment
   of mAddr:
     checkSonsLen(n, 2)
-    result = semAddr(c, n.sons[1])
+    result = semAddr(c, n.sons[1], s.name.s == "unsafeAddr")
   of mTypeOf:
     checkSonsLen(n, 2)
     result = semTypeOf(c, n.sons[1])
@@ -1720,6 +1720,8 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     result = newStrNodeT(renderTree(n[1], {renderNoComments}), n)
     result.typ = getSysType(tyString)
   of mParallel:
+    if not experimentalMode(c):
+      localError(n.info, "use the {.experimental.} pragma to enable 'parallel'")
     result = setMs(n, s)
     var x = n.lastSon
     if x.kind == nkDo: x = x.sons[bodyPos]
@@ -2259,7 +2261,10 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   of nkStaticStmt:
     result = semStaticStmt(c, n)
   of nkDefer:
-    localError(n.info, errGenerated, "'defer' not allowed in this context")
+    n.sons[0] = semExpr(c, n.sons[0])
+    if not n.sons[0].typ.isEmptyType and not implicitlyDiscardable(n.sons[0]):
+      localError(n.info, errGenerated, "'defer' takes a 'void' expression")
+    #localError(n.info, errGenerated, "'defer' not allowed in this context")
   else:
     localError(n.info, errInvalidExpressionX,
                renderTree(n, {renderNoComments}))

compiler/semfold.nim

@@ -307,12 +307,12 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mToU8: result = newIntNodeT(getInt(a) and 0x000000FF, n)
   of mToU16: result = newIntNodeT(getInt(a) and 0x0000FFFF, n)
   of mToU32: result = newIntNodeT(getInt(a) and 0x00000000FFFFFFFF'i64, n)
-  of mUnaryLt: result = newIntNodeT(getOrdValue(a) - 1, n)
+  of mUnaryLt: result = newIntNodeT(getOrdValue(a) |-| 1, n)
-  of mSucc: result = newIntNodeT(getOrdValue(a) + getInt(b), n)
+  of mSucc: result = newIntNodeT(getOrdValue(a) |+| getInt(b), n)
-  of mPred: result = newIntNodeT(getOrdValue(a) - getInt(b), n)
+  of mPred: result = newIntNodeT(getOrdValue(a) |-| getInt(b), n)
-  of mAddI: result = newIntNodeT(getInt(a) + getInt(b), n)
+  of mAddI: result = newIntNodeT(getInt(a) |+| getInt(b), n)
-  of mSubI: result = newIntNodeT(getInt(a) - getInt(b), n)
+  of mSubI: result = newIntNodeT(getInt(a) |-| getInt(b), n)
-  of mMulI: result = newIntNodeT(getInt(a) * getInt(b), n)
+  of mMulI: result = newIntNodeT(getInt(a) |*| getInt(b), n)
   of mMinI:
     if getInt(a) > getInt(b): result = newIntNodeT(getInt(b), n)
     else: result = newIntNodeT(getInt(a), n)
@@ -338,11 +338,11 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mDivI:
     let y = getInt(b)
     if y != 0:
-      result = newIntNodeT(getInt(a) div y, n)
+      result = newIntNodeT(`|div|`(getInt(a), y), n)
   of mModI:
     let y = getInt(b)
     if y != 0:
-      result = newIntNodeT(getInt(a) mod y, n)
+      result = newIntNodeT(`|mod|`(getInt(a), y), n)
   of mAddF64: result = newFloatNodeT(getFloat(a) + getFloat(b), n)
   of mSubF64: result = newFloatNodeT(getFloat(a) - getFloat(b), n)
   of mMulF64: result = newFloatNodeT(getFloat(a) * getFloat(b), n)

compiler/semgnrc.nim

@@ -57,7 +57,7 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
   of skUnknown:
     # Introduced in this pass! Leave it as an identifier.
     result = n
-  of skProc, skMethod, skIterators, skConverter:
+  of skProc, skMethod, skIterators, skConverter, skModule:
     result = symChoice(c, n, s, scOpen)
   of skTemplate:
     if macroToExpand(s):
@@ -141,11 +141,13 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
       elif s.name.id in ctx.toMixin:
         result = newDot(result, symChoice(c, n, s, scForceOpen))
       else:
-        let sym = semGenericStmtSymbol(c, n, s, ctx)
+        let syms = semGenericStmtSymbol(c, n, s, ctx)
-        if sym.kind == nkSym:
+        if syms.kind == nkSym:
-          result = newDot(result, symChoice(c, n, s, scForceOpen))
+          let choice = symChoice(c, n, s, scForceOpen)
+          choice.kind = nkClosedSymChoice
+          result = newDot(result, choice)
         else:
-          result = newDot(result, sym)
+          result = newDot(result, syms)
 
 proc addTempDecl(c: PContext; n: PNode; kind: TSymKind) =
   let s = newSymS(skUnknown, getIdentNode(n), c)
@@ -223,7 +225,7 @@ proc semGenericStmt(c: PContext, n: PNode,
       of skUnknown, skParam:
         # Leave it as an identifier.
         discard
-      of skProc, skMethod, skIterators, skConverter:
+      of skProc, skMethod, skIterators, skConverter, skModule:
         result.sons[0] = symChoice(c, fn, s, scOption)
         first = 1
       of skGenericParam:

compiler/seminst.nim

@@ -10,14 +10,10 @@
 # This module implements the instantiation of generic procs.
 # included from sem.nim
 
-proc instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable,
+iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable): PSym =
-                                 entry: var TInstantiation) =
+  internalAssert n.kind == nkGenericParams
-  if n.kind != nkGenericParams:
-    internalError(n.info, "instantiateGenericParamList; no generic params")
-  newSeq(entry.concreteTypes, n.len)
   for i, a in n.pairs:
-    if a.kind != nkSym:
+    internalAssert a.kind == nkSym
-      internalError(a.info, "instantiateGenericParamList; no symbol")
     var q = a.sym
     if q.typ.kind notin {tyTypeDesc, tyGenericParam, tyStatic, tyIter}+tyTypeClasses:
       continue
@@ -42,8 +38,7 @@ proc instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable,
       #t = ReplaceTypeVarsT(cl, t)
     s.typ = t
     if t.kind == tyStatic: s.ast = t.n
-    addDecl(c, s)
+    yield s
-    entry.concreteTypes[i] = t
 
 proc sameInstantiation(a, b: TInstantiation): bool =
   if a.concreteTypes.len == b.concreteTypes.len:
@@ -217,7 +212,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   ## The `pt` parameter is a type-unsafe mapping table used to link generic
   ## parameters to their concrete types within the generic instance.
   # no need to instantiate generic templates/macros:
-  if fn.kind in {skTemplate, skMacro}: return fn
+  internalAssert fn.kind notin {skMacro, skTemplate}
   # generates an instantiated proc
   if c.instCounter > 1000: internalError(fn.ast.info, "nesting too deep")
   inc(c.instCounter)
@@ -226,20 +221,27 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   # NOTE: for access of private fields within generics from a different module
   # we set the friend module:
   c.friendModules.add(getModule(fn))
-  #let oldScope = c.currentScope
+  let oldScope = c.currentScope
-  #c.currentScope = fn.scope
+  while not isTopLevel(c): c.currentScope = c.currentScope.parent
   result = copySym(fn, false)
   incl(result.flags, sfFromGeneric)
   result.owner = fn
   result.ast = n
   pushOwner(result)
+
   openScope(c)
-  internalAssert n.sons[genericParamsPos].kind != nkEmpty
+  let gp = n.sons[genericParamsPos]
+  internalAssert gp.kind != nkEmpty
   n.sons[namePos] = newSymNode(result)
   pushInfoContext(info)
   var entry = TInstantiation.new
   entry.sym = result
-  instantiateGenericParamList(c, n.sons[genericParamsPos], pt, entry[])
+  newSeq(entry.concreteTypes, gp.len)
+  var i = 0
+  for s in instantiateGenericParamList(c, gp, pt):
+    addDecl(c, s)
+    entry.concreteTypes[i] = s.typ
+    inc i
   pushProcCon(c, result)
   instantiateProcType(c, pt, result, info)
   n.sons[genericParamsPos] = ast.emptyNode
@@ -264,7 +266,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   popInfoContext()
   closeScope(c)           # close scope for parameters
   popOwner()
-  #c.currentScope = oldScope
+  c.currentScope = oldScope
   discard c.friendModules.pop()
   dec(c.instCounter)
   if result.kind == skMethod: finishMethod(c, result)

compiler/semmagic.nim

@@ -10,10 +10,10 @@
 # This include file implements the semantic checking for magics.
 # included from sem.nim
 
-proc semAddr(c: PContext; n: PNode): PNode =
+proc semAddr(c: PContext; n: PNode; isUnsafeAddr=false): PNode =
   result = newNodeI(nkAddr, n.info)
   let x = semExprWithType(c, n)
-  if isAssignable(c, x) notin {arLValue, arLocalLValue}:
+  if isAssignable(c, x, isUnsafeAddr) notin {arLValue, arLocalLValue}:
     localError(n.info, errExprHasNoAddress)
   result.add x
   result.typ = makePtrType(c, x.typ)
@@ -119,7 +119,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
   case n[0].sym.magic
   of mAddr:
     checkSonsLen(n, 2)
-    result = semAddr(c, n.sons[1])
+    result = semAddr(c, n.sons[1], n[0].sym.name.s == "unsafeAddr")
   of mTypeOf:
     checkSonsLen(n, 2)
     result = semTypeOf(c, n.sons[1])

compiler/semstmts.nim

@@ -369,6 +369,15 @@ proc addToVarSection(c: PContext; result: var PNode; orig, identDefs: PNode) =
   else:
     result.add identDefs
 
+proc addDefer(c: PContext; result: var PNode; s: PSym) =
+  let deferDestructorCall = createDestructorCall(c, s)
+  if deferDestructorCall != nil:
+    if result.kind != nkStmtList:
+      let oldResult = result
+      result = newNodeI(nkStmtList, result.info)
+      result.add oldResult
+    result.add deferDestructorCall
+
 proc isDiscardUnderscore(v: PSym): bool =
   if v.name.s == "_":
     v.flags.incl(sfGenSym)
@@ -469,6 +478,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
         if def.kind == nkPar: v.ast = def[j]
         v.typ = tup.sons[j]
         b.sons[j] = newSymNode(v)
+      addDefer(c, result, v)
       checkNilable(v)
       if sfCompileTime in v.flags: hasCompileTime = true
   if hasCompileTime: vm.setupCompileTimeVar(c.module, result)
@@ -1210,6 +1220,9 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   if n.sons[patternPos].kind != nkEmpty:
     c.patterns.add(s)
   if isAnon: result.typ = s.typ
+  if isTopLevel(c) and s.kind != skClosureIterator and
+      s.typ.callConv == ccClosure:
+    message(s.info, warnDeprecated, "top level '.closure' calling convention")
 
 proc determineType(c: PContext, s: PSym) =
   if s.typ != nil: return
@@ -1371,7 +1384,7 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
   for i in countup(0, length - 1):
     let k = n.sons[i].kind
     case k
-    of nkFinally, nkExceptBranch, nkDefer:
+    of nkFinally, nkExceptBranch:
       # stand-alone finally and except blocks are
       # transformed into regular try blocks:
       #
@@ -1424,21 +1437,13 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
         n.typ = n.sons[i].typ
         if not isEmptyType(n.typ): n.kind = nkStmtListExpr
       case n.sons[i].kind
-      of nkVarSection, nkLetSection:
-        let (outer, inner) = insertDestructors(c, n.sons[i])
-        if outer != nil:
-          n.sons[i] = outer
-          var rest = newNode(nkStmtList, n.info, n.sons[i+1 .. length-1])
-          inner.addSon(semStmtList(c, rest, flags))
-          n.sons.setLen(i+1)
-          return
       of LastBlockStmts:
         for j in countup(i + 1, length - 1):
           case n.sons[j].kind
           of nkPragma, nkCommentStmt, nkNilLit, nkEmpty: discard
           else: localError(n.sons[j].info, errStmtInvalidAfterReturn)
       else: discard
-  if result.len == 1:
+  if result.len == 1 and result.sons[0].kind != nkDefer:
     result = result.sons[0]
   when defined(nimfix):
     if result.kind == nkCommentStmt and not result.comment.isNil and

compiler/semtempl.nim

@@ -184,6 +184,21 @@ proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) =
   else:
     let ident = getIdentNode(c, n)
     if not isTemplParam(c, ident):
+      # fix #2670, consider:
+      #
+      # when b:
+      #    var a = "hi"
+      # else:
+      #    var a = 5
+      # echo a
+      #
+      # We need to ensure that both 'a' produce the same gensym'ed symbol.
+      # So we need only check the *current* scope.
+      let s = localSearchInScope(c.c, considerQuotedIdent ident)
+      if s != nil and s.owner == c.owner and sfGenSym in s.flags:
+        styleCheckUse(n.info, s)
+        replaceIdentBySym(n, newSymNode(s, n.info))
+      else:
         let local = newGenSym(k, ident, c)
         addPrelimDecl(c.c, local)
         styleCheckDef(n.info, local)

compiler/semtypes.nim

@@ -23,6 +23,9 @@ proc newConstraint(c: PContext, k: TTypeKind): PType =
 
 proc semEnum(c: PContext, n: PNode, prev: PType): PType =
   if n.sonsLen == 0: return newConstraint(c, tyEnum)
+  elif n.sonsLen == 1:
+    # don't create an empty tyEnum; fixes #3052
+    return errorType(c)
   var
     counter, x: BiggestInt
     e: PSym
@@ -505,8 +508,9 @@ proc semRecordCase(c: PContext, n: PNode, check: var IntSet, pos: var int,
   var typ = skipTypes(a.sons[0].typ, abstractVar-{tyTypeDesc})
   if not isOrdinalType(typ):
     localError(n.info, errSelectorMustBeOrdinal)
-  elif firstOrd(typ) < 0:
+  elif firstOrd(typ) != 0:
-    localError(n.info, errOrdXMustNotBeNegative, a.sons[0].sym.name.s)
+    localError(n.info, errGenerated, "low(" & $a.sons[0].sym.name.s &
+                                     ") must be 0 for discriminant")
   elif lengthOrd(typ) > 0x00007FFF:
     localError(n.info, errLenXinvalid, a.sons[0].sym.name.s)
   var chckCovered = true

compiler/transf.nim

@@ -16,6 +16,7 @@
 # * converts "continue" to "break"; disambiguates "break"
 # * introduces method dispatchers
 # * performs lambda lifting for closure support
+# * transforms 'defer' into a 'try finally' statement
 
 import
   intsets, strutils, lists, options, ast, astalgo, trees, treetab, msgs, os,
@@ -44,6 +45,7 @@ type
     inlining: int            # > 0 if we are in inlining context (copy vars)
     nestedProcs: int         # > 0 if we are in a nested proc
     contSyms, breakSyms: seq[PSym]  # to transform 'continue' and 'break'
+    deferDetected: bool
   PTransf = ref TTransfContext
 
 proc newTransNode(a: PNode): PTransNode {.inline.} =
@@ -680,6 +682,14 @@ proc commonOptimizations*(c: PSym, n: PNode): PNode =
       result = n
 
 proc transform(c: PTransf, n: PNode): PTransNode =
+  when false:
+    var oldDeferAnchor: PNode
+    if n.kind in {nkElifBranch, nkOfBranch, nkExceptBranch, nkElifExpr,
+                  nkElseExpr, nkElse, nkForStmt, nkWhileStmt, nkFinally,
+                  nkBlockStmt, nkBlockExpr}:
+      oldDeferAnchor = c.deferAnchor
+      c.deferAnchor = n
+
   case n.kind
   of nkSym:
     result = transformSym(c, n)
@@ -712,13 +722,36 @@ proc transform(c: PTransf, n: PNode): PTransNode =
     result = transformFor(c, n)
   of nkParForStmt:
     result = transformSons(c, n)
-  of nkCaseStmt: result = transformCase(c, n)
+  of nkCaseStmt:
+    result = transformCase(c, n)
+  of nkWhileStmt: result = transformWhile(c, n)
+  of nkBlockStmt, nkBlockExpr:
+    result = transformBlock(c, n)
+  of nkDefer:
+    c.deferDetected = true
+    result = transformSons(c, n)
+    when false:
+      let deferPart = newNodeI(nkFinally, n.info)
+      deferPart.add n.sons[0]
+      let tryStmt = newNodeI(nkTryStmt, n.info)
+      if c.deferAnchor.isNil:
+        tryStmt.add c.root
+        c.root = tryStmt
+        result = PTransNode(tryStmt)
+      else:
+        # modify the corresponding *action*, don't rely on nkStmtList:
+        let L = c.deferAnchor.len-1
+        tryStmt.add c.deferAnchor.sons[L]
+        c.deferAnchor.sons[L] = tryStmt
+        result = newTransNode(nkCommentStmt, n.info, 0)
+      tryStmt.addSon(deferPart)
+      # disable the original 'defer' statement:
+      n.kind = nkCommentStmt
   of nkContinueStmt:
     result = PTransNode(newNodeI(nkBreakStmt, n.info))
     var labl = c.contSyms[c.contSyms.high]
     add(result, PTransNode(newSymNode(labl)))
   of nkBreakStmt: result = transformBreak(c, n)
-  of nkWhileStmt: result = transformWhile(c, n)
   of nkCallKinds:
     result = transformCall(c, n)
   of nkAddr, nkHiddenAddr:
@@ -754,8 +787,6 @@ proc transform(c: PTransf, n: PNode): PTransNode =
       result = transformYield(c, n)
     else:
       result = transformSons(c, n)
-  of nkBlockStmt, nkBlockExpr:
-    result = transformBlock(c, n)
   of nkIdentDefs, nkConstDef:
     result = transformSons(c, n)
     # XXX comment handling really sucks:
@@ -764,6 +795,8 @@ proc transform(c: PTransf, n: PNode): PTransNode =
   of nkClosure: return PTransNode(n)
   else:
     result = transformSons(c, n)
+  when false:
+    if oldDeferAnchor != nil: c.deferAnchor = oldDeferAnchor
   var cnst = getConstExpr(c.module, PNode(result))
   # we inline constants if they are not complex constants:
   if cnst != nil and not dontInlineConstant(n, cnst):
@@ -785,12 +818,52 @@ proc openTransf(module: PSym, filename: string): PTransf =
   result.breakSyms = @[]
   result.module = module
 
+proc flattenStmts(n: PNode) =
+  var goOn = true
+  while goOn:
+    goOn = false
+    for i in 0..<n.len:
+      let it = n[i]
+      if it.kind in {nkStmtList, nkStmtListExpr}:
+        n.sons[i..i] = it.sons[0..<it.len]
+        goOn = true
+
+proc liftDeferAux(n: PNode) =
+  if n.kind in {nkStmtList, nkStmtListExpr}:
+    flattenStmts(n)
+    var goOn = true
+    while goOn:
+      goOn = false
+      let last = n.len-1
+      for i in 0..last:
+        if n.sons[i].kind == nkDefer:
+          let deferPart = newNodeI(nkFinally, n.sons[i].info)
+          deferPart.add n.sons[i].sons[0]
+          var tryStmt = newNodeI(nkTryStmt, n.sons[i].info)
+          var body = newNodeI(n.kind, n.sons[i].info)
+          if i < last:
+            body.sons = n.sons[(i+1)..last]
+          tryStmt.addSon(body)
+          tryStmt.addSon(deferPart)
+          n.sons[i] = tryStmt
+          n.sons.setLen(i+1)
+          n.typ = n.sons[i].typ
+          goOn = true
+          break
+  for i in 0..n.safeLen-1:
+    liftDeferAux(n.sons[i])
+
+template liftDefer(c, root) =
+  if c.deferDetected:
+    liftDeferAux(root)
+
 proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
   if nfTransf in n.flags or prc.kind in {skTemplate}:
     result = n
   else:
     var c = openTransf(module, "")
     result = processTransf(c, n, prc)
+    liftDefer(c, result)
     result = liftLambdas(prc, result)
     #if prc.kind == skClosureIterator:
     #  result = lambdalifting.liftIterator(prc, result)
@@ -805,6 +878,7 @@ proc transformStmt*(module: PSym, n: PNode): PNode =
   else:
     var c = openTransf(module, "")
     result = processTransf(c, n, module)
+    liftDefer(c, result)
     result = liftLambdasForTopLevel(module, result)
     incl(result.flags, nfTransf)
     when useEffectSystem: trackTopLevelStmt(module, result)
@@ -815,4 +889,5 @@ proc transformExpr*(module: PSym, n: PNode): PNode =
   else:
     var c = openTransf(module, "")
     result = processTransf(c, n, module)
+    liftDefer(c, result)
     incl(result.flags, nfTransf)

compiler/vm.nim

@@ -120,10 +120,10 @@ template decodeBx(k: expr) {.immediate, dirty.} =
 template move(a, b: expr) {.immediate, dirty.} = system.shallowCopy(a, b)
 # XXX fix minor 'shallowCopy' overloading bug in compiler
 
-proc createStrKeepNode(x: var TFullReg) =
+proc createStrKeepNode(x: var TFullReg; keepNode=true) =
   if x.node.isNil:
     x.node = newNode(nkStrLit)
-  elif x.node.kind == nkNilLit:
+  elif x.node.kind == nkNilLit and keepNode:
     when defined(useNodeIds):
       let id = x.node.id
     system.reset(x.node[])
@@ -385,6 +385,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     #if c.traceActive:
     #  echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra
     #  message(c.debug[pc], warnUser, "Trace")
+
     case instr.opcode
     of opcEof: return regs[ra]
     of opcRet:
@@ -407,8 +408,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       decodeB(rkInt)
       regs[ra].intVal = regs[rb].intVal
     of opcAsgnStr:
-      decodeB(rkNode)
+      decodeBC(rkNode)
-      createStrKeepNode regs[ra]
+      createStrKeepNode regs[ra], rc != 0
       regs[ra].node.strVal = regs[rb].node.strVal
     of opcAsgnFloat:
       decodeB(rkFloat)
@@ -431,7 +432,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         assert regs[rb].kind == rkNode
         let nb = regs[rb].node
         case nb.kind
-        of nkCharLit..nkInt64Lit:
+        of nkCharLit..nkUInt64Lit:
           ensureKind(rkInt)
           regs[ra].intVal = nb.intVal
         of nkFloatLit..nkFloat64Lit:
@@ -509,8 +510,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       of rkNode:
         if regs[rb].node.kind == nkNilLit:
           stackTrace(c, tos, pc, errNilAccess)
-        assert regs[rb].node.kind == nkRefTy
+        if regs[rb].node.kind == nkRefTy:
           regs[ra].node = regs[rb].node.sons[0]
+        else:
+          stackTrace(c, tos, pc, errGenerated, "limited VM support for 'ref'")
       else:
         stackTrace(c, tos, pc, errNilAccess)
     of opcWrDeref:
@@ -1404,7 +1407,7 @@ include vmops
 # storing&loading the 'globals' environment to get what a component system
 # requires.
 var
-  globalCtx: PCtx
+  globalCtx*: PCtx
 
 proc setupGlobalCtx(module: PSym) =
   if globalCtx.isNil:
@@ -1464,12 +1467,20 @@ proc evalStaticStmt*(module: PSym, e: PNode, prc: PSym) =
 proc setupCompileTimeVar*(module: PSym, n: PNode) =
   discard evalConstExprAux(module, nil, n, emStaticStmt)
 
-proc setupMacroParam(x: PNode): PNode =
+proc setupMacroParam(x: PNode, typ: PType): TFullReg =
-  result = x
+  case typ.kind
-  if result.kind in {nkHiddenSubConv, nkHiddenStdConv}: result = result.sons[1]
+  of tyStatic:
-  result = canonValue(result)
+    putIntoReg(result, x)
-  result.flags.incl nfIsRef
+  of tyTypeDesc:
-  result.typ = x.typ
+    putIntoReg(result, x)
+  else:
+    result.kind = rkNode
+    var n = x
+    if n.kind in {nkHiddenSubConv, nkHiddenStdConv}: n = n.sons[1]
+    n = n.canonValue
+    n.flags.incl nfIsRef
+    n.typ = x.typ
+    result.node = n
 
 var evalMacroCounter: int
 
@@ -1505,10 +1516,17 @@ proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode =
   # return value:
   tos.slots[0].kind = rkNode
   tos.slots[0].node = newNodeIT(nkEmpty, n.info, sym.typ.sons[0])
+
   # setup parameters:
-  for i in 1 .. < min(tos.slots.len, L):
+  for i in 1.. <sym.typ.len:
-    tos.slots[i].kind = rkNode
+    tos.slots[i] = setupMacroParam(n.sons[i], sym.typ.sons[i])
-    tos.slots[i].node = setupMacroParam(n.sons[i])
+
+  if sfImmediate notin sym.flags:
+    let gp = sym.ast[genericParamsPos]
+    for i in 0 .. <gp.len:
+      let idx = sym.typ.len + i
+      tos.slots[idx] = setupMacroParam(n.sons[idx], gp[i].sym.typ)
+
   # temporary storage:
   #for i in L .. <maxSlots: tos.slots[i] = newNode(nkEmpty)
   result = rawExecute(c, start, tos).regToNode

compiler/vmdeps.nim

@@ -30,7 +30,7 @@ proc opGorge*(cmd, input, cache: string): string =
       return
     var readSuccessful = false
     try:
-      var p = startProcess(cmd, options={poEvalCommand})
+      var p = startProcess(cmd, options={poEvalCommand, poStderrToStdout})
       if input.len != 0:
         p.inputStream.write(input)
         p.inputStream.close()
@@ -41,7 +41,7 @@ proc opGorge*(cmd, input, cache: string): string =
       if not readSuccessful: result = ""
   else:
     try:
-      var p = startProcess(cmd, options={poEvalCommand})
+      var p = startProcess(cmd, options={poEvalCommand, poStderrToStdout})
       if input.len != 0:
         p.inputStream.write(input)
         p.inputStream.close()

compiler/vmgen.nim

@@ -74,8 +74,9 @@ proc codeListing(c: PCtx, result: var string, start=0; last = -1) =
       result.addf("\t$#\tr$#, L$#", ($opc).substr(3), x.regA,
                   i+x.regBx-wordExcess)
     elif opc in {opcLdConst, opcAsgnConst}:
-      result.addf("\t$#\tr$#, $#", ($opc).substr(3), x.regA,
+      let idx = x.regBx-wordExcess
-        c.constants[x.regBx-wordExcess].renderTree)
+      result.addf("\t$#\tr$#, $# ($#)", ($opc).substr(3), x.regA,
+        c.constants[idx].renderTree, $idx)
     elif opc in {opcMarshalLoad, opcMarshalStore}:
       let y = c.code[i+1]
       result.addf("\t$#\tr$#, r$#, $#", ($opc).substr(3), x.regA, x.regB,
@@ -172,7 +173,8 @@ const
 proc bestEffort(c: PCtx): TLineInfo =
   (if c.prc == nil: c.module.info else: c.prc.sym.info)
 
-proc getTemp(cc: PCtx; typ: PType): TRegister =
+proc getTemp(cc: PCtx; tt: PType): TRegister =
+  let typ = tt.skipTypesOrNil({tyStatic})
   let c = cc.prc
   # we prefer the same slot kind here for efficiency. Unfortunately for
   # discardable return types we may not know the desired type. This can happen
@@ -184,7 +186,7 @@ proc getTemp(cc: PCtx; typ: PType): TRegister =
       return TRegister(i)
 
   # if register pressure is high, we re-use more aggressively:
-  if c.maxSlots >= HighRegisterPressure:
+  if c.maxSlots >= HighRegisterPressure and false:
     for i in 0 .. c.maxSlots-1:
       if not c.slots[i].inUse:
         c.slots[i] = (inUse: true, kind: k)
@@ -706,7 +708,7 @@ proc genConv(c: PCtx; n, arg: PNode; dest: var TDest; opc=opcConv) =
   if dest < 0: dest = c.getTemp(n.typ)
   c.gABC(n, opc, dest, tmp)
   c.gABx(n, opc, 0, genType(c, n.typ))
-  c.gABx(n, opc, 0, genType(c, arg.typ))
+  c.gABx(n, opc, 0, genType(c, arg.typ.skipTypes({tyStatic})))
   c.freeTemp(tmp)
 
 proc genCard(c: PCtx; n: PNode; dest: var TDest) =
@@ -1073,6 +1075,7 @@ const
     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64}
 
 proc fitsRegister*(t: PType): bool =
+  assert t != nil
   t.skipTypes(abstractInst-{tyTypeDesc}).kind in {
     tyRange, tyEnum, tyBool, tyInt..tyUInt64, tyChar}
 
@@ -1103,6 +1106,8 @@ proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
   # nkAddr we must not use 'unneededIndirection', but for deref we use it.
   if not isAddr and unneededIndirection(n.sons[0]):
     gen(c, n.sons[0], dest, newflags)
+    if gfAddrOf notin flags and fitsRegister(n.typ):
+      c.gABC(n, opcNodeToReg, dest, dest)
   elif isAddr and isGlobal(n.sons[0]):
     gen(c, n.sons[0], dest, flags+{gfAddrOf})
   else:
@@ -1110,6 +1115,7 @@ proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
     if dest < 0: dest = c.getTemp(n.typ)
     if not isAddr:
       gABC(c, n, opc, dest, tmp)
+      assert n.typ != nil
       if gfAddrOf notin flags and fitsRegister(n.typ):
         c.gABC(n, opcNodeToReg, dest, dest)
     elif c.prc.slots[tmp].kind >= slotTempUnknown:
@@ -1143,7 +1149,7 @@ proc whichAsgnOpc(n: PNode; opc: TOpcode): TOpcode = opc
 proc genAsgn(c: PCtx; dest: TDest; ri: PNode; requiresCopy: bool) =
   let tmp = c.genx(ri)
   assert dest >= 0
-  gABC(c, ri, whichAsgnOpc(ri), dest, tmp)
+  gABC(c, ri, whichAsgnOpc(ri), dest, tmp, 1-ord(requiresCopy))
   c.freeTemp(tmp)
 
 proc setSlot(c: PCtx; v: PSym) =
@@ -1177,7 +1183,10 @@ proc checkCanEval(c: PCtx; n: PNode) =
   let s = n.sym
   if {sfCompileTime, sfGlobal} <= s.flags: return
   if s.kind in {skVar, skTemp, skLet, skParam, skResult} and
-      not s.isOwnedBy(c.prc.sym) and s.owner != c.module:
+      not s.isOwnedBy(c.prc.sym) and s.owner != c.module and c.mode != emRepl:
+    cannotEval(n)
+  elif s.kind in {skProc, skConverter, skMethod,
+                  skIterator, skClosureIterator} and sfForward in s.flags:
     cannotEval(n)
 
 proc isTemp(c: PCtx; dest: TDest): bool =
@@ -1194,9 +1203,10 @@ proc preventFalseAlias(c: PCtx; n: PNode; opc: TOpcode;
   # opcLdObj et al really means "load address". We sometimes have to create a
   # copy in order to not introduce false aliasing:
   # mylocal = a.b  # needs a copy of the data!
+  assert n.typ != nil
   if needsAdditionalCopy(n):
     var cc = c.getTemp(n.typ)
-    c.gABC(n, whichAsgnOpc(n), cc, value)
+    c.gABC(n, whichAsgnOpc(n), cc, value, 0)
     c.gABC(n, opc, dest, idx, cc)
     c.freeTemp(cc)
   else:
@@ -1241,10 +1251,11 @@ proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) =
       internalAssert s.position > 0 or (s.position == 0 and
                                         s.kind in {skParam,skResult})
       var dest: TRegister = s.position + ord(s.kind == skParam)
+      assert le.typ != nil
       if needsAdditionalCopy(le) and s.kind in {skResult, skVar, skParam}:
         var cc = c.getTemp(le.typ)
         gen(c, ri, cc)
-        c.gABC(le, whichAsgnOpc(le), dest, cc)
+        c.gABC(le, whichAsgnOpc(le), dest, cc, 1)
         c.freeTemp(cc)
       else:
         gen(c, ri, dest)
@@ -1303,6 +1314,7 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
       if sfImportc in s.flags: c.importcSym(n.info, s)
       else: genGlobalInit(c, n, s)
     if dest < 0: dest = c.getTemp(n.typ)
+    assert s.typ != nil
     if gfAddrOf notin flags and fitsRegister(s.typ):
       var cc = c.getTemp(n.typ)
       c.gABx(n, opcLdGlobal, cc, s.position)
@@ -1426,6 +1438,7 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode =
     globalError(info, "cannot create null element for: " & $t.kind)
 
 proc ldNullOpcode(t: PType): TOpcode =
+  assert t != nil
   if fitsRegister(t): opcLdNullReg else: opcLdNull
 
 proc genVarSection(c: PCtx; n: PNode) =
@@ -1453,7 +1466,7 @@ proc genVarSection(c: PCtx; n: PNode) =
         if a.sons[2].kind != nkEmpty:
           let tmp = c.genx(a.sons[0], {gfAddrOf})
           let val = c.genx(a.sons[2])
-          c.preventFalseAlias(a, opcWrDeref, tmp, 0, val)
+          c.preventFalseAlias(a.sons[2], opcWrDeref, tmp, 0, val)
           c.freeTemp(val)
           c.freeTemp(tmp)
       else:
@@ -1461,13 +1474,15 @@ proc genVarSection(c: PCtx; n: PNode) =
         if a.sons[2].kind == nkEmpty:
           c.gABx(a, ldNullOpcode(s.typ), s.position, c.genType(s.typ))
         else:
+          assert s.typ != nil
           if not fitsRegister(s.typ):
             c.gABx(a, ldNullOpcode(s.typ), s.position, c.genType(s.typ))
           let le = a.sons[0]
+          assert le.typ != nil
           if not fitsRegister(le.typ) and s.kind in {skResult, skVar, skParam}:
             var cc = c.getTemp(le.typ)
             gen(c, a.sons[2], cc)
-            c.gABC(le, whichAsgnOpc(le), s.position.TRegister, cc)
+            c.gABC(le, whichAsgnOpc(le), s.position.TRegister, cc, 1)
             c.freeTemp(cc)
           else:
             gen(c, a.sons[2], s.position.TRegister)
@@ -1608,6 +1623,11 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
         c.gABx(n, opcLdConst, dest, lit)
     of skType:
       genTypeLit(c, s.typ, dest)
+    of skGenericParam:
+      if c.prc.sym.kind == skMacro:
+        genRdVar(c, n, dest, flags)
+      else:
+        internalError(n.info, "cannot generate code for: " & s.name.s)
     else:
       globalError(n.info, errGenerated, "cannot generate code for: " & s.name.s)
   of nkCallKinds:
@@ -1694,7 +1714,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     c.freeTemp(tmp1)
     c.freeTemp(tmp2)
     if dest >= 0:
-      gABC(c, n, whichAsgnOpc(n), dest, tmp0)
+      gABC(c, n, whichAsgnOpc(n), dest, tmp0, 1)
       c.freeTemp(tmp0)
     else:
       dest = tmp0
@@ -1758,6 +1778,14 @@ proc finalJumpTarget(c: PCtx; pc, diff: int) =
   c.code[pc] = ((oldInstr.uint32 and 0xffff'u32).uint32 or
                 uint32(diff+wordExcess) shl 16'u32).TInstr
 
+proc genGenericParams(c: PCtx; gp: PNode) =
+  var base = c.prc.maxSlots
+  for i in 0.. <gp.len:
+    var param = gp.sons[i].sym
+    param.position = base + i # XXX: fix this earlier; make it consistent with templates
+    c.prc.slots[base + i] = (inUse: true, kind: slotFixedLet)
+  c.prc.maxSlots = base + gp.len
+
 proc optimizeJumps(c: PCtx; start: int) =
   const maxIterations = 10
   for i in start .. <c.code.len:
@@ -1822,6 +1850,13 @@ proc genProc(c: PCtx; s: PSym): int =
     c.prc = p
     # iterate over the parameters and allocate space for them:
     genParams(c, s.typ.n)
+
+    # allocate additional space for any generically bound parameters
+    if s.kind == skMacro and
+       sfImmediate notin s.flags and
+       s.ast[genericParamsPos].kind != nkEmpty:
+      genGenericParams(c, s.ast[genericParamsPos])
+
     if tfCapturesEnv in s.typ.flags:
       #let env = s.ast.sons[paramsPos].lastSon.sym
       #assert env.position == 2

compiler/vmhooks.nim

@@ -30,6 +30,16 @@ proc setResult*(a: VmArgs; v: string) =
   s[a.ra].node = newNode(nkStrLit)
   s[a.ra].node.strVal = v
 
+proc setResult*(a: VmArgs; v: seq[string]) =
+  var s: seq[TFullReg]
+  move(s, cast[seq[TFullReg]](a.slots))
+  if s[a.ra].kind != rkNode:
+    myreset(s[a.ra])
+    s[a.ra].kind = rkNode
+  var n = newNode(nkBracket)
+  for x in v: n.add newStrNode(nkStrLit, x)
+  s[a.ra].node = n
+
 template getX(k, field) {.immediate, dirty.} =
   doAssert i < a.rc-1
   let s = cast[seq[TFullReg]](a.slots)

contributing.rst

@@ -66,12 +66,14 @@ Running tests
 You can run the tests with
 
 ::
+
   ./koch tests
 
 which will run a good subset of tests. Some tests may fail. If you
 only want to see the output of failing tests, go for
 
 ::
+
   ./koch tests --failing all
 
 You can also run only a single category of tests. A category is a subdirectory
@@ -79,6 +81,7 @@ in the ``tests`` directory. There are a couple of special categories; for a
 list of these, see ``tests/testament/categories.nim``, at the bottom.
 
 ::
+
   ./koch tests c lib
 
 Comparing tests
@@ -92,6 +95,7 @@ reference test. You'll also need to the commit id, because that's what
 the tester needs to know in order to compare the two.
 
 ::
+
   git checkout devel
   DEVEL_COMMIT=$(git rev-parse HEAD)
   ./koch tests
@@ -99,6 +103,7 @@ the tester needs to know in order to compare the two.
 Then switch over to your changes and run the tester again.
 
 ::
+
   git checkout your-changes
   ./koch tests
 
@@ -106,7 +111,8 @@ Then you can ask the tester to create a ``testresults.html`` which will
 tell you if any new tests passed/failed.
 
 ::
-  ./koch --print html $DEVEL_COMMIT
+
+  ./koch tests --print html $DEVEL_COMMIT
 
 
 Deprecation
@@ -216,3 +222,5 @@ General commit rules
       git diff --check --cached || exit $?
 
 3. Describe your commit and use your common sense.
+
+.. include:: docstyle.rst

doc/manual/pragmas.txt

@@ -511,3 +511,467 @@ Example:
     add "foo"
     add "bar"
 
+
+Implementation Specific Pragmas
+===============================
+
+This section describes additional pragmas that the current Nim implementation
+supports but which should not be seen as part of the language specification.
+
+
+Volatile pragma
+---------------
+The ``volatile`` pragma is for variables only. It declares the variable as
+``volatile``, whatever that means in C/C++ (its semantics are not well defined
+in C/C++).
+
+**Note**: This pragma will not exist for the LLVM backend.
+
+
+NoDecl pragma
+-------------
+The ``noDecl`` pragma can be applied to almost any symbol (variable, proc,
+type, etc.) and is sometimes useful for interoperability with C:
+It tells Nim that it should not generate a declaration for the symbol in
+the C code. For example:
+
+.. code-block:: Nim
+  var
+    EACCES {.importc, noDecl.}: cint # pretend EACCES was a variable, as
+                                     # Nim does not know its value
+
+However, the ``header`` pragma is often the better alternative.
+
+**Note**: This will not work for the LLVM backend.
+
+
+Header pragma
+-------------
+The ``header`` pragma is very similar to the ``noDecl`` pragma: It can be
+applied to almost any symbol and specifies that it should not be declared
+and instead the generated code should contain an ``#include``:
+
+.. code-block:: Nim
+  type
+    PFile {.importc: "FILE*", header: "<stdio.h>".} = distinct pointer
+      # import C's FILE* type; Nim will treat it as a new pointer type
+
+The ``header`` pragma always expects a string constant. The string contant
+contains the header file: As usual for C, a system header file is enclosed
+in angle brackets: ``<>``. If no angle brackets are given, Nim
+encloses the header file in ``""`` in the generated C code.
+
+**Note**: This will not work for the LLVM backend.
+
+
+IncompleteStruct pragma
+-----------------------
+The ``incompleteStruct`` pragma tells the compiler to not use the
+underlying C ``struct`` in a ``sizeof`` expression:
+
+.. code-block:: Nim
+  type
+    DIR* {.importc: "DIR", header: "<dirent.h>",
+           final, pure, incompleteStruct.} = object
+
+
+Compile pragma
+--------------
+The ``compile`` pragma can be used to compile and link a C/C++ source file
+with the project:
+
+.. code-block:: Nim
+  {.compile: "myfile.cpp".}
+
+**Note**: Nim computes a SHA1 checksum and only recompiles the file if it
+has changed. You can use the ``-f`` command line option to force recompilation
+of the file.
+
+
+Link pragma
+-----------
+The ``link`` pragma can be used to link an additional file with the project:
+
+.. code-block:: Nim
+  {.link: "myfile.o".}
+
+
+PassC pragma
+------------
+The ``passC`` pragma can be used to pass additional parameters to the C
+compiler like you would using the commandline switch ``--passC``:
+
+.. code-block:: Nim
+  {.passC: "-Wall -Werror".}
+
+Note that you can use ``gorge`` from the `system module <system.html>`_ to
+embed parameters from an external command at compile time:
+
+.. code-block:: Nim
+  {.passC: gorge("pkg-config --cflags sdl").}
+
+PassL pragma
+------------
+The ``passL`` pragma can be used to pass additional parameters to the linker
+like you would using the commandline switch ``--passL``:
+
+.. code-block:: Nim
+  {.passL: "-lSDLmain -lSDL".}
+
+Note that you can use ``gorge`` from the `system module <system.html>`_ to
+embed parameters from an external command at compile time:
+
+.. code-block:: Nim
+  {.passL: gorge("pkg-config --libs sdl").}
+
+
+Emit pragma
+-----------
+The ``emit`` pragma can be used to directly affect the output of the
+compiler's code generator. So it makes your code unportable to other code
+generators/backends. Its usage is highly discouraged! However, it can be
+extremely useful for interfacing with `C++`:idx: or `Objective C`:idx: code.
+
+Example:
+
+.. code-block:: Nim
+  {.emit: """
+  static int cvariable = 420;
+  """.}
+
+  {.push stackTrace:off.}
+  proc embedsC() =
+    var nimVar = 89
+    # use backticks to access Nim symbols within an emit section:
+    {.emit: """fprintf(stdout, "%d\n", cvariable + (int)`nimVar`);""".}
+  {.pop.}
+
+  embedsC()
+
+As can be seen from the example, to Nim symbols can be referred via backticks.
+Use two backticks to produce a single verbatim backtick.
+
+For a toplevel emit statement the section where in the generated C/C++ file
+the code should be emitted can be influenced via the
+prefixes ``/*TYPESECTION*/`` or ``/*VARSECTION*/`` or ``/*INCLUDESECTION*/``:
+
+.. code-block:: Nim
+  {.emit: """/*TYPESECTION*/
+  struct Vector3 {
+  public:
+    Vector3(): x(5) {}
+    Vector3(float x_): x(x_) {}
+    float x;
+  };
+  """.}
+
+  type Vector3 {.importcpp: "Vector3", nodecl} = object
+    x: cfloat
+
+  proc constructVector3(a: cfloat): Vector3 {.importcpp: "Vector3(@)", nodecl}
+
+
+ImportCpp pragma
+----------------
+
+**Note**: `c2nim <c2nim.html>`_ can parse a large subset of C++ and knows
+about the ``importcpp`` pragma pattern language. It is not necessary
+to know all the details described here.
+
+
+Similar to the `importc pragma for C <manual.html#importc-pragma>`_, the
+``importcpp`` pragma can be used to import `C++`:idx: methods or C++ symbols
+in general. The generated code then uses the C++ method calling
+syntax: ``obj->method(arg)``.  In combination with the ``header`` and ``emit``
+pragmas this allows *sloppy* interfacing with libraries written in C++:
+
+.. code-block:: Nim
+  # Horrible example of how to interface with a C++ engine ... ;-)
+
+  {.link: "/usr/lib/libIrrlicht.so".}
+
+  {.emit: """
+  using namespace irr;
+  using namespace core;
+  using namespace scene;
+  using namespace video;
+  using namespace io;
+  using namespace gui;
+  """.}
+
+  const
+    irr = "<irrlicht/irrlicht.h>"
+
+  type
+    IrrlichtDeviceObj {.final, header: irr,
+                        importcpp: "IrrlichtDevice".} = object
+    IrrlichtDevice = ptr IrrlichtDeviceObj
+
+  proc createDevice(): IrrlichtDevice {.
+    header: irr, importcpp: "createDevice(@)".}
+  proc run(device: IrrlichtDevice): bool {.
+    header: irr, importcpp: "#.run(@)".}
+
+The compiler needs to be told to generate C++ (command ``cpp``) for
+this to work. The conditional symbol ``cpp`` is defined when the compiler
+emits C++ code.
+
+
+Namespaces
+~~~~~~~~~~
+
+The *sloppy interfacing* example uses ``.emit`` to produce ``using namespace``
+declarations. It is usually much better to instead refer to the imported name
+via the ``namespace::identifier`` notation:
+
+.. code-block:: nim
+  type
+    IrrlichtDeviceObj {.final, header: irr,
+                        importcpp: "irr::IrrlichtDevice".} = object
+
+
+Importcpp for enums
+~~~~~~~~~~~~~~~~~~~
+
+When ``importcpp`` is applied to an enum type the numerical enum values are
+annotated with the C++ enum type, like in this example: ``((TheCppEnum)(3))``.
+(This turned out to be the simplest way to implement it.)
+
+
+Importcpp for procs
+~~~~~~~~~~~~~~~~~~~
+
+Note that the ``importcpp`` variant for procs uses a somewhat cryptic pattern
+language for maximum flexibility:
+
+- A hash ``#`` symbol is replaced by the first or next argument.
+- A dot following the hash ``#.`` indicates that the call should use C++'s dot
+  or arrow notation.
+- An at symbol ``@`` is replaced by the remaining arguments, separated by
+  commas.
+
+For example:
+
+.. code-block:: nim
+  proc cppMethod(this: CppObj, a, b, c: cint) {.importcpp: "#.CppMethod(@)".}
+  var x: ptr CppObj
+  cppMethod(x[], 1, 2, 3)
+
+Produces:
+
+.. code-block:: C
+  x->CppMethod(1, 2, 3)
+
+As a special rule to keep backwards compatibility with older versions of the
+``importcpp`` pragma, if there is no special pattern
+character (any of ``# ' @``) at all, C++'s
+dot or arrow notation is assumed, so the above example can also be written as:
+
+.. code-block:: nim
+  proc cppMethod(this: CppObj, a, b, c: cint) {.importcpp: "CppMethod".}
+
+Note that the pattern language naturally also covers C++'s operator overloading
+capabilities:
+
+.. code-block:: nim
+  proc vectorAddition(a, b: Vec3): Vec3 {.importcpp: "# + #".}
+  proc dictLookup(a: Dict, k: Key): Value {.importcpp: "#[#]".}
+
+
+- An apostrophe ``'`` followed by an integer ``i`` in the range 0..9
+  is replaced by the i'th parameter *type*. The 0th position is the result
+  type. This can be used to pass types to C++ function templates. Between
+  the ``'`` and the digit an asterisk can be used to get to the base type
+  of the type. (So it "takes away a star" from the type; ``T*`` becomes ``T``.)
+  Two stars can be used to get to the element type of the element type etc.
+
+For example:
+
+.. code-block:: nim
+
+  type Input {.importcpp: "System::Input".} = object
+  proc getSubsystem*[T](): ptr T {.importcpp: "SystemManager::getSubsystem<'*0>()", nodecl.}
+
+  let x: ptr Input = getSubsystem[Input]()
+
+Produces:
+
+.. code-block:: C
+  x = SystemManager::getSubsystem<System::Input>()
+
+
+- ``#@`` is a special case to support a ``cnew`` operation. It is required so
+  that the call expression is inlined directly, without going through a
+  temporary location. This is only required to circumvent a limitation of the
+  current code generator.
+
+For example C++'s ``new`` operator can be "imported" like this:
+
+.. code-block:: nim
+  proc cnew*[T](x: T): ptr T {.importcpp: "(new '*0#@)", nodecl.}
+
+  # constructor of 'Foo':
+  proc constructFoo(a, b: cint): Foo {.importcpp: "Foo(@)".}
+
+  let x = cnew constructFoo(3, 4)
+
+Produces:
+
+.. code-block:: C
+  x = new Foo(3, 4)
+
+However, depending on the use case ``new Foo`` can also be wrapped like this
+instead:
+
+.. code-block:: nim
+  proc newFoo(a, b: cint): ptr Foo {.importcpp: "new Foo(@)".}
+
+  let x = newFoo(3, 4)
+
+
+Wrapping constructors
+~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes a C++ class has a private copy constructor and so code like
+``Class c = Class(1,2);`` must not be generated but instead ``Class c(1,2);``.
+For this purpose the Nim proc that wraps a C++ constructor needs to be
+annotated with the `constructor`:idx: pragma. This pragma also helps to generate
+faster C++ code since construction then doesn't invoke the copy constructor:
+
+.. code-block:: nim
+  # a better constructor of 'Foo':
+  proc constructFoo(a, b: cint): Foo {.importcpp: "Foo(@)", constructor.}
+
+
+Wrapping destructors
+~~~~~~~~~~~~~~~~~~~~
+
+Since Nim generates C++ directly, any destructor is called implicitly by the
+C++ compiler at the scope exits. This means that often one can get away with
+not wrapping the destructor at all! However when it needs to be invoked
+explicitly, it needs to be wrapped. But the pattern language already provides
+everything that is required for that:
+
+.. code-block:: nim
+  proc destroyFoo(this: var Foo) {.importcpp: "#.~Foo()".}
+
+
+Importcpp for objects
+~~~~~~~~~~~~~~~~~~~~~
+
+Generic ``importcpp``'ed objects are mapped to C++ templates. This means that
+you can import C++'s templates rather easily without the need for a pattern
+language for object types:
+
+.. code-block:: nim
+  type
+    StdMap {.importcpp: "std::map", header: "<map>".} [K, V] = object
+  proc `[]=`[K, V](this: var StdMap[K, V]; key: K; val: V) {.
+    importcpp: "#[#] = #", header: "<map>".}
+
+  var x: StdMap[cint, cdouble]
+  x[6] = 91.4
+
+
+Produces:
+
+.. code-block:: C
+  std::map<int, double> x;
+  x[6] = 91.4;
+
+
+- If more precise control is needed, the apostrophe ``'`` can be used in the
+  supplied pattern to denote the concrete type parameters of the generic type.
+  See the usage of the apostrophe operator in proc patterns for more details.
+
+.. code-block:: nim
+
+  type
+    VectorIterator {.importcpp: "std::vector<'0>::iterator".} [T] = object
+
+  var x: VectorIterator[cint]
+
+
+Produces:
+
+.. code-block:: C
+
+  std::vector<int>::iterator x;
+
+
+ImportObjC pragma
+-----------------
+Similar to the `importc pragma for C <manual.html#importc-pragma>`_, the
+``importobjc`` pragma can be used to import `Objective C`:idx: methods.  The
+generated code then uses the Objective C method calling syntax: ``[obj method
+param1: arg]``.  In addition with the ``header`` and ``emit`` pragmas this
+allows *sloppy* interfacing with libraries written in Objective C:
+
+.. code-block:: Nim
+  # horrible example of how to interface with GNUStep ...
+
+  {.passL: "-lobjc".}
+  {.emit: """
+  #include <objc/Object.h>
+  @interface Greeter:Object
+  {
+  }
+
+  - (void)greet:(long)x y:(long)dummy;
+  @end
+
+  #include <stdio.h>
+  @implementation Greeter
+
+  - (void)greet:(long)x y:(long)dummy
+  {
+    printf("Hello, World!\n");
+  }
+  @end
+
+  #include <stdlib.h>
+  """.}
+
+  type
+    Id {.importc: "id", header: "<objc/Object.h>", final.} = distinct int
+
+  proc newGreeter: Id {.importobjc: "Greeter new", nodecl.}
+  proc greet(self: Id, x, y: int) {.importobjc: "greet", nodecl.}
+  proc free(self: Id) {.importobjc: "free", nodecl.}
+
+  var g = newGreeter()
+  g.greet(12, 34)
+  g.free()
+
+The compiler needs to be told to generate Objective C (command ``objc``) for
+this to work. The conditional symbol ``objc`` is defined when the compiler
+emits Objective C code.
+
+
+CodegenDecl pragma
+------------------
+
+The ``codegenDecl`` pragma can be used to directly influence Nim's code
+generator. It receives a format string that determines how the variable or
+proc is declared in the generated code:
+
+.. code-block:: nim
+  var
+    a {.codegenDecl: "$# progmem $#".}: int
+
+  proc myinterrupt() {.codegenDecl: "__interrupt $# $#$#".} =
+    echo "realistic interrupt handler"
+
+
+InjectStmt pragma
+-----------------
+
+The ``injectStmt`` pragma can be used to inject a statement before every
+other statement in the current module. It is only supposed to be used for
+debugging:
+
+.. code-block:: nim
+  {.injectStmt: gcInvariants().}
+
+  # ... complex code here that produces crashes ...
+

doc/manual/types.txt

@@ -588,7 +588,8 @@ can also be defined with indentation instead of ``[]``:
 
 Objects provide many features that tuples do not. Object provide inheritance
 and information hiding. Objects have access to their type at runtime, so that
-the ``of`` operator can be used to determine the object's type.
+the ``of`` operator can be used to determine the object's type. The ``of`` operator
+is similar to the ``instanceof`` operator in Java.
 
 .. code-block:: nim
   type

doc/nimc.txt

@@ -263,454 +263,6 @@ Nim manual. Some of the features here only make sense for the C code
 generator and are subject to change.
 
 
-NoDecl pragma
--------------
-The ``noDecl`` pragma can be applied to almost any symbol (variable, proc,
-type, etc.) and is sometimes useful for interoperability with C:
-It tells Nim that it should not generate a declaration for the symbol in
-the C code. For example:
-
-.. code-block:: Nim
-  var
-    EACCES {.importc, noDecl.}: cint # pretend EACCES was a variable, as
-                                     # Nim does not know its value
-
-However, the ``header`` pragma is often the better alternative.
-
-**Note**: This will not work for the LLVM backend.
-
-
-Header pragma
--------------
-The ``header`` pragma is very similar to the ``noDecl`` pragma: It can be
-applied to almost any symbol and specifies that it should not be declared
-and instead the generated code should contain an ``#include``:
-
-.. code-block:: Nim
-  type
-    PFile {.importc: "FILE*", header: "<stdio.h>".} = distinct pointer
-      # import C's FILE* type; Nim will treat it as a new pointer type
-
-The ``header`` pragma always expects a string constant. The string contant
-contains the header file: As usual for C, a system header file is enclosed
-in angle brackets: ``<>``. If no angle brackets are given, Nim
-encloses the header file in ``""`` in the generated C code.
-
-**Note**: This will not work for the LLVM backend.
-
-
-IncompleteStruct pragma
------------------------
-The ``incompleteStruct`` pragma tells the compiler to not use the
-underlying C ``struct`` in a ``sizeof`` expression:
-
-.. code-block:: Nim
-  type
-    DIR* {.importc: "DIR", header: "<dirent.h>",
-           final, pure, incompleteStruct.} = object
-
-
-Compile pragma
---------------
-The ``compile`` pragma can be used to compile and link a C/C++ source file
-with the project:
-
-.. code-block:: Nim
-  {.compile: "myfile.cpp".}
-
-**Note**: Nim computes a SHA1 checksum and only recompiles the file if it
-has changed. You can use the ``-f`` command line option to force recompilation
-of the file.
-
-
-Link pragma
------------
-The ``link`` pragma can be used to link an additional file with the project:
-
-.. code-block:: Nim
-  {.link: "myfile.o".}
-
-
-PassC pragma
-------------
-The ``passC`` pragma can be used to pass additional parameters to the C
-compiler like you would using the commandline switch ``--passC``:
-
-.. code-block:: Nim
-  {.passC: "-Wall -Werror".}
-
-Note that you can use ``gorge`` from the `system module <system.html>`_ to
-embed parameters from an external command at compile time:
-
-.. code-block:: Nim
-  {.passC: gorge("pkg-config --cflags sdl").}
-
-PassL pragma
-------------
-The ``passL`` pragma can be used to pass additional parameters to the linker
-like you would using the commandline switch ``--passL``:
-
-.. code-block:: Nim
-  {.passL: "-lSDLmain -lSDL".}
-
-Note that you can use ``gorge`` from the `system module <system.html>`_ to
-embed parameters from an external command at compile time:
-
-.. code-block:: Nim
-  {.passL: gorge("pkg-config --libs sdl").}
-
-
-Emit pragma
------------
-The ``emit`` pragma can be used to directly affect the output of the
-compiler's code generator. So it makes your code unportable to other code
-generators/backends. Its usage is highly discouraged! However, it can be
-extremely useful for interfacing with `C++`:idx: or `Objective C`:idx: code.
-
-Example:
-
-.. code-block:: Nim
-  {.emit: """
-  static int cvariable = 420;
-  """.}
-
-  {.push stackTrace:off.}
-  proc embedsC() =
-    var nimVar = 89
-    # use backticks to access Nim symbols within an emit section:
-    {.emit: """fprintf(stdout, "%d\n", cvariable + (int)`nimVar`);""".}
-  {.pop.}
-
-  embedsC()
-
-As can be seen from the example, to Nim symbols can be referred via backticks.
-Use two backticks to produce a single verbatim backtick.
-
-For a toplevel emit statement the section where in the generated C/C++ file
-the code should be emitted can be influenced via the
-prefixes ``/*TYPESECTION*/`` or ``/*VARSECTION*/`` or ``/*INCLUDESECTION*/``:
-
-.. code-block:: Nim
-  {.emit: """/*TYPESECTION*/
-  struct Vector3 {
-  public:
-    Vector3(): x(5) {}
-    Vector3(float x_): x(x_) {}
-    float x;
-  };
-  """.}
-
-  type Vector3 {.importcpp: "Vector3", nodecl} = object
-    x: cfloat
-
-  proc constructVector3(a: cfloat): Vector3 {.importcpp: "Vector3(@)", nodecl}
-
-
-ImportCpp pragma
-----------------
-
-**Note**: `c2nim <c2nim.html>`_ can parse a large subset of C++ and knows
-about the ``importcpp`` pragma pattern language. It is not necessary
-to know all the details described here.
-
-
-Similar to the `importc pragma for C <manual.html#importc-pragma>`_, the
-``importcpp`` pragma can be used to import `C++`:idx: methods or C++ symbols
-in general. The generated code then uses the C++ method calling
-syntax: ``obj->method(arg)``.  In combination with the ``header`` and ``emit``
-pragmas this allows *sloppy* interfacing with libraries written in C++:
-
-.. code-block:: Nim
-  # Horrible example of how to interface with a C++ engine ... ;-)
-
-  {.link: "/usr/lib/libIrrlicht.so".}
-
-  {.emit: """
-  using namespace irr;
-  using namespace core;
-  using namespace scene;
-  using namespace video;
-  using namespace io;
-  using namespace gui;
-  """.}
-
-  const
-    irr = "<irrlicht/irrlicht.h>"
-
-  type
-    IrrlichtDeviceObj {.final, header: irr,
-                        importcpp: "IrrlichtDevice".} = object
-    IrrlichtDevice = ptr IrrlichtDeviceObj
-
-  proc createDevice(): IrrlichtDevice {.
-    header: irr, importcpp: "createDevice(@)".}
-  proc run(device: IrrlichtDevice): bool {.
-    header: irr, importcpp: "#.run(@)".}
-
-The compiler needs to be told to generate C++ (command ``cpp``) for
-this to work. The conditional symbol ``cpp`` is defined when the compiler
-emits C++ code.
-
-
-Namespaces
-~~~~~~~~~~
-
-The *sloppy interfacing* example uses ``.emit`` to produce ``using namespace``
-declarations. It is usually much better to instead refer to the imported name
-via the ``namespace::identifier`` notation:
-
-.. code-block:: nim
-  type
-    IrrlichtDeviceObj {.final, header: irr,
-                        importcpp: "irr::IrrlichtDevice".} = object
-
-
-Importcpp for enums
-~~~~~~~~~~~~~~~~~~~
-
-When ``importcpp`` is applied to an enum type the numerical enum values are
-annotated with the C++ enum type, like in this example: ``((TheCppEnum)(3))``.
-(This turned out to be the simplest way to implement it.)
-
-
-Importcpp for procs
-~~~~~~~~~~~~~~~~~~~
-
-Note that the ``importcpp`` variant for procs uses a somewhat cryptic pattern
-language for maximum flexibility:
-
-- A hash ``#`` symbol is replaced by the first or next argument.
-- A dot following the hash ``#.`` indicates that the call should use C++'s dot
-  or arrow notation.
-- An at symbol ``@`` is replaced by the remaining arguments, separated by
-  commas.
-
-For example:
-
-.. code-block:: nim
-  proc cppMethod(this: CppObj, a, b, c: cint) {.importcpp: "#.CppMethod(@)".}
-  var x: ptr CppObj
-  cppMethod(x[], 1, 2, 3)
-
-Produces:
-
-.. code-block:: C
-  x->CppMethod(1, 2, 3)
-
-As a special rule to keep backwards compatibility with older versions of the
-``importcpp`` pragma, if there is no special pattern
-character (any of ``# ' @``) at all, C++'s
-dot or arrow notation is assumed, so the above example can also be written as:
-
-.. code-block:: nim
-  proc cppMethod(this: CppObj, a, b, c: cint) {.importcpp: "CppMethod".}
-
-Note that the pattern language naturally also covers C++'s operator overloading
-capabilities:
-
-.. code-block:: nim
-  proc vectorAddition(a, b: Vec3): Vec3 {.importcpp: "# + #".}
-  proc dictLookup(a: Dict, k: Key): Value {.importcpp: "#[#]".}
-
-
-- An apostrophe ``'`` followed by an integer ``i`` in the range 0..9
-  is replaced by the i'th parameter *type*. The 0th position is the result
-  type. This can be used to pass types to C++ function templates. Between
-  the ``'`` and the digit an asterisk can be used to get to the base type
-  of the type. (So it "takes away a star" from the type; ``T*`` becomes ``T``.)
-  Two stars can be used to get to the element type of the element type etc.
-
-For example:
-
-.. code-block:: nim
-
-  type Input {.importcpp: "System::Input".} = object
-  proc getSubsystem*[T](): ptr T {.importcpp: "SystemManager::getSubsystem<'*0>()", nodecl.}
-
-  let x: ptr Input = getSubsystem[Input]()
-
-Produces:
-
-.. code-block:: C
-  x = SystemManager::getSubsystem<System::Input>()
-
-
-- ``#@`` is a special case to support a ``cnew`` operation. It is required so
-  that the call expression is inlined directly, without going through a
-  temporary location. This is only required to circumvent a limitation of the
-  current code generator.
-
-For example C++'s ``new`` operator can be "imported" like this:
-
-.. code-block:: nim
-  proc cnew*[T](x: T): ptr T {.importcpp: "(new '*0#@)", nodecl.}
-
-  # constructor of 'Foo':
-  proc constructFoo(a, b: cint): Foo {.importcpp: "Foo(@)".}
-
-  let x = cnew constructFoo(3, 4)
-
-Produces:
-
-.. code-block:: C
-  x = new Foo(3, 4)
-
-However, depending on the use case ``new Foo`` can also be wrapped like this
-instead:
-
-.. code-block:: nim
-  proc newFoo(a, b: cint): ptr Foo {.importcpp: "new Foo(@)".}
-
-  let x = newFoo(3, 4)
-
-
-Wrapping constructors
-~~~~~~~~~~~~~~~~~~~~~
-
-Sometimes a C++ class has a private copy constructor and so code like
-``Class c = Class(1,2);`` must not be generated but instead ``Class c(1,2);``.
-For this purpose the Nim proc that wraps a C++ constructor needs to be
-annotated with the `constructor`:idx: pragma. This pragma also helps to generate
-faster C++ code since construction then doesn't invoke the copy constructor:
-
-.. code-block:: nim
-  # a better constructor of 'Foo':
-  proc constructFoo(a, b: cint): Foo {.importcpp: "Foo(@)", constructor.}
-
-
-Wrapping destructors
-~~~~~~~~~~~~~~~~~~~~
-
-Since Nim generates C++ directly, any destructor is called implicitly by the
-C++ compiler at the scope exits. This means that often one can get away with
-not wrapping the destructor at all! However when it needs to be invoked
-explicitly, it needs to be wrapped. But the pattern language already provides
-everything that is required for that:
-
-.. code-block:: nim
-  proc destroyFoo(this: var Foo) {.importcpp: "#.~Foo()".}
-
-
-Importcpp for objects
-~~~~~~~~~~~~~~~~~~~~~
-
-Generic ``importcpp``'ed objects are mapped to C++ templates. This means that
-you can import C++'s templates rather easily without the need for a pattern
-language for object types:
-
-.. code-block:: nim
-  type
-    StdMap {.importcpp: "std::map", header: "<map>".} [K, V] = object
-  proc `[]=`[K, V](this: var StdMap[K, V]; key: K; val: V) {.
-    importcpp: "#[#] = #", header: "<map>".}
-
-  var x: StdMap[cint, cdouble]
-  x[6] = 91.4
-
-
-Produces:
-
-.. code-block:: C
-  std::map<int, double> x;
-  x[6] = 91.4;
-
-
-- If more precise control is needed, the apostrophe ``'`` can be used in the
-  supplied pattern to denote the concrete type parameters of the generic type.
-  See the usage of the apostrophe operator in proc patterns for more details.
-
-.. code-block:: nim
-
-  type
-    VectorIterator {.importcpp: "std::vector<'0>::iterator".} [T] = object
-
-  var x: VectorIterator[cint]
-
-
-Produces:
-
-.. code-block:: C
-
-  std::vector<int>::iterator x;
-
-
-ImportObjC pragma
------------------
-Similar to the `importc pragma for C <manual.html#importc-pragma>`_, the
-``importobjc`` pragma can be used to import `Objective C`:idx: methods.  The
-generated code then uses the Objective C method calling syntax: ``[obj method
-param1: arg]``.  In addition with the ``header`` and ``emit`` pragmas this
-allows *sloppy* interfacing with libraries written in Objective C:
-
-.. code-block:: Nim
-  # horrible example of how to interface with GNUStep ...
-
-  {.passL: "-lobjc".}
-  {.emit: """
-  #include <objc/Object.h>
-  @interface Greeter:Object
-  {
-  }
-
-  - (void)greet:(long)x y:(long)dummy;
-  @end
-
-  #include <stdio.h>
-  @implementation Greeter
-
-  - (void)greet:(long)x y:(long)dummy
-  {
-    printf("Hello, World!\n");
-  }
-  @end
-
-  #include <stdlib.h>
-  """.}
-
-  type
-    Id {.importc: "id", header: "<objc/Object.h>", final.} = distinct int
-
-  proc newGreeter: Id {.importobjc: "Greeter new", nodecl.}
-  proc greet(self: Id, x, y: int) {.importobjc: "greet", nodecl.}
-  proc free(self: Id) {.importobjc: "free", nodecl.}
-
-  var g = newGreeter()
-  g.greet(12, 34)
-  g.free()
-
-The compiler needs to be told to generate Objective C (command ``objc``) for
-this to work. The conditional symbol ``objc`` is defined when the compiler
-emits Objective C code.
-
-
-CodegenDecl pragma
-------------------
-
-The ``codegenDecl`` pragma can be used to directly influence Nim's code
-generator. It receives a format string that determines how the variable or
-proc is declared in the generated code:
-
-.. code-block:: nim
-  var
-    a {.codegenDecl: "$# progmem $#".}: int
-
-  proc myinterrupt() {.codegenDecl: "__interrupt $# $#$#".} =
-    echo "realistic interrupt handler"
-
-
-InjectStmt pragma
------------------
-
-The ``injectStmt`` pragma can be used to inject a statement before every
-other statement in the current module. It is only supposed to be used for
-debugging:
-
-.. code-block:: nim
-  {.injectStmt: gcInvariants().}
-
-  # ... complex code here that produces crashes ...
-
-
 LineDir option
 --------------
 The ``lineDir`` option can be turned on or off. If turned on the
@@ -744,15 +296,6 @@ The *breakpoint* pragma was specially added for the sake of debugging with
 ENDB. See the documentation of `endb <endb.html>`_ for further information.
 
 
-Volatile pragma
----------------
-The ``volatile`` pragma is for variables only. It declares the variable as
-``volatile``, whatever that means in C/C++ (its semantics are not well defined
-in C/C++).
-
-**Note**: This pragma will not exist for the LLVM backend.
-
-
 DynlibOverride
 ==============
 

docstyle.rst

@@ -0,0 +1,140 @@
+Documentation Style
+===================
+
+General Guidelines
+------------------
+
+* Authors should document anything that is exported.
+* Within documentation, a period (`.`) should follow each sentence (or sentence fragment) in a comment block. The documentation may be limited to one sentence fragment, but if multiple sentences are within the documentation, each sentence after the first should be complete and in present tense.
+* Documentation is parsed as ReStructuredText (RST).
+* Inline code should be surrounded by double tick marks ("``````"). If you would like a character to immediately follow inline code (e.g., "``int8``s are great!"), escape the following character with a backslash (``\``). The preceding is typed as ``` ``int8``\s are great!```.
+
+Module-level documentation
+--------------------------
+
+Documentation of a module is placed at the top of the module itself. Each line of documentation begins with double hashes (``##``).
+Code samples are encouraged, and should follow the general RST syntax:
+
+.. code-block:: Nim
+
+  ## The ``universe`` module computes the answer to life, the universe, and everything.
+  ##
+  ## .. code-block:: Nim
+  ##  echo computeAnswerString() # "42"
+
+
+Within this top-level comment, you can indicate the authorship and copyright of the code, which will be featured in the produced documentation.
+
+.. code-block:: Nim
+
+  ## This is the best module ever. It provides answers to everything!
+  ##
+  ## :Author: Steve McQueen
+  ## :Copyright: 1965
+  ##
+
+Leave a space between the last line of top-level documentation and the beginning of Nim code (the imports, etc.).
+
+Procs, Templates, Macros, Converters, and Iterators
+---------------------------------------------------
+
+The documentation of a procedure should begin with a capital letter and should be in present tense. Variables referenced in the documentation should be surrounded by double tick marks (``````).
+
+.. code-block:: Nim
+
+  proc example1*(x: int) =
+    ## Prints the value of ``x``.
+    echo x
+
+Whenever an example of usage would be helpful to the user, you should include one within the documentation in RST format as below.
+
+.. code-block:: Nim
+
+  proc addThree*(x, y, z: int8): int =
+    ## Adds three ``int8`` values, treating them as unsigned and
+    ## truncating the result.
+    ##
+    ## .. code-block:: nim
+    ##  echo addThree(3, 125, 6) # -122
+    result = x +% y +% z
+
+The commands ``nim doc`` and ``nim doc2`` will then correctly syntax highlight the Nim code within the documentation.
+
+Types
+-----
+
+Exported types should also be documented. This documentation can also contain code samples, but those are better placed with the functions to which they refer.
+
+.. code-block:: Nim
+
+  type
+    NamedQueue*[T] = object ## Provides a linked data structure with names
+                            ## throughout. It is named for convenience. I'm making
+                            ## this comment long to show how you can, too.
+      name*: string ## The name of the item
+      val*: T ## Its value
+      next*: ref NamedQueue[T] ## The next item in the queue
+
+
+You have some flexibility when placing the documentation:
+
+.. code-block:: Nim
+
+  type
+    NamedQueue*[T] = object
+      ## Provides a linked data structure with names
+      ## throughout. It is named for convenience. I'm making
+      ## this comment long to show how you can, too.
+      name*: string ## The name of the item
+      val*: T ## Its value
+      next*: ref NamedQueue[T] ## The next item in the queue
+
+Make sure to place the documentation beside or within the object.
+
+.. code-block:: Nim
+
+  type
+    ## This documentation disappears because it annotates the ``type`` keyword
+    ## above, not ``NamedQueue``.
+    NamedQueue*[T] = object
+      name*: string ## This becomes the main documentation for the object, which
+                    ## is not what we want.
+      val*: T ## Its value
+      next*: ref NamedQueue[T] ## The next item in the queue
+
+Var, Let, and Const
+-------------------
+
+When declaring module-wide constants and values, documentation is encouraged. The placement of doc comments is similar to the ``type`` sections.
+
+.. code-block:: Nim
+
+  const
+    X* = 42 ## An awesome number.
+    SpreadArray* = [
+      [1,2,3],
+      [2,3,1],
+      [3,1,2],
+    ] ## Doc comment for ``SpreadArray``.
+
+Placement of comments in other areas is usually allowed, but will not become part of the documentation output and should therefore be prefaced by a single hash (``#``).
+
+.. code-block:: Nim
+
+  const
+    BadMathVals* = [
+      3.14, # pi
+      2.72, # e
+      0.58, # gamma
+    ] ## A bunch of badly rounded values.
+
+Nim supports Unicode in comments, so the above can be replaced with the following:
+
+.. code-block:: Nim
+
+  const
+    BadMathVals* = [
+      3.14, # π
+      2.72, # e
+      0.58, # γ
+    ] ## A bunch of badly rounded values (including π!).

lib/core/macros.nim

@@ -719,7 +719,13 @@ proc `$`*(node: NimNode): string {.compileTime.} =
 proc ident*(name: string): NimNode {.compileTime,inline.} = newIdentNode(name)
   ## Create a new ident node from a string
 
+iterator items*(n: NimNode): NimNode {.inline.} =
+  ## Iterates over the children of the NimNode ``n``.
+  for i in 0 ..< n.len:
+    yield n[i]
+
 iterator children*(n: NimNode): NimNode {.inline.} =
+  ## Iterates over the children of the NimNode ``n``.
   for i in 0 ..< n.len:
     yield n[i]
 

lib/nimbase.h

@@ -360,7 +360,7 @@ struct TFrame {
   FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = 0; nimFrame(&FR);
 
 #define nimfrs(proc, file, slots, length) \
-  struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; TVarSlot s[slots];} FR; \
+  struct {TFrame* prev;NCSTRING procname;NI line;NCSTRING filename; NI len; VarSlot s[slots];} FR; \
   FR.procname = proc; FR.filename = file; FR.line = 0; FR.len = length; nimFrame((TFrame*)&FR);
 
 #define nimln(n, file) \

lib/packages/docutils/rstgen.nim

@@ -755,7 +755,7 @@ proc renderTocEntries*(d: var RstGenerator, j: var int, lvl: int,
 
 proc renderImage(d: PDoc, n: PRstNode, result: var string) =
   template valid(s): expr =
-    s.len > 0 and allCharsInSet(s, {'/',':','%','_','\\','\128'..'\xFF'} +
+    s.len > 0 and allCharsInSet(s, {'.','/',':','%','_','\\','\128'..'\xFF'} +
                                    Digits + Letters + WhiteSpace)
 
   var options = ""

lib/posix/termios.nim

@@ -24,7 +24,6 @@ type
     c_oflag*: Cflag        # output mode flags
     c_cflag*: Cflag        # control mode flags
     c_lflag*: Cflag        # local mode flags
-    c_line*: cuchar         # line discipline
     c_cc*: array[NCCS, cuchar]  # control characters
 
 # cc characters

lib/pure/asyncdispatch.nim

@@ -124,7 +124,8 @@ export Port, SocketFlag
 ##
 ## * The effect system (``raises: []``) does not work with async procedures.
 ## * Can't await in a ``except`` body
-
+## * Forward declarations for async procs are broken,
+##   link includes workaround: https://github.com/nim-lang/Nim/issues/3182.
 
 # TODO: Check if yielded future is nil and throw a more meaningful exception
 
@@ -1412,9 +1413,9 @@ proc getName(node: NimNode): string {.compileTime.} =
   else:
     error("Unknown name.")
 
-macro async*(prc: stmt): stmt {.immediate.} =
+proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
-  ## Macro which processes async procedures into the appropriate
+  ## This macro transforms a single procedure into a closure iterator.
-  ## iterators and yield statements.
+  ## The ``async`` macro supports a stmtList holding multiple async procedures.
   if prc.kind notin {nnkProcDef, nnkLambda}:
       error("Cannot transform this node kind into an async proc." &
             " Proc definition or lambda node expected.")
@@ -1504,9 +1505,19 @@ macro async*(prc: stmt): stmt {.immediate.} =
   result[6] = outerProcBody
 
   #echo(treeRepr(result))
-  if prc[0].getName == "getAsync":
+  if prc[0].getName == "hubConnectionLoop":
     echo(toStrLit(result))
 
+macro async*(prc: stmt): stmt {.immediate.} =
+  ## Macro which processes async procedures into the appropriate
+  ## iterators and yield statements.
+  if prc.kind == nnkStmtList:
+    for oneProc in prc:
+      result = newStmtList()
+      result.add asyncSingleProc(oneProc)
+  else:
+    result = asyncSingleProc(prc)
+
 proc recvLine*(socket: AsyncFD): Future[string] {.async.} =
   ## Reads a line of data from ``socket``. Returned future will complete once
   ## a full line is read or an error occurs.

lib/pure/asynchttpserver.nim

@@ -126,8 +126,11 @@ proc parseHeader(line: string): tuple[key, value: string] =
   var i = 0
   i = line.parseUntil(result.key, ':')
   inc(i) # skip :
+  if i < len(line):
     i += line.skipWhiteSpace(i)
     i += line.parseUntil(result.value, {'\c', '\L'}, i)
+  else:
+    result.value = ""
 
 proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] =
   var i = protocol.skipIgnoreCase("HTTP/")

lib/pure/collections/tables.nim

@@ -226,6 +226,10 @@ proc `$`*[A, B](t: Table[A, B]): string =
   ## The `$` operator for hash tables.
   dollarImpl()
 
+proc hasKey*[A, B](t: TableRef[A, B], key: A): bool =
+  ## returns true iff `key` is in the table `t`.
+  result = t[].hasKey(key)
+
 template equalsImpl() =
   if s.counter == t.counter:
     # different insertion orders mean different 'data' seqs, so we have
@@ -293,10 +297,6 @@ proc hasKeyOrPut*[A, B](t: var TableRef[A, B], key: A, val: B): bool =
   ## returns true iff `key` is in the table, otherwise inserts `value`.
   t[].hasKeyOrPut(key, val)
 
-proc hasKey*[A, B](t: TableRef[A, B], key: A): bool =
-  ## returns true iff `key` is in the table `t`.
-  result = t[].hasKey(key)
-
 proc contains*[A, B](t: TableRef[A, B], key: A): bool =
   ## alias of `hasKey` for use with the `in` operator.
   return hasKey[A, B](t, key)

lib/pure/httpclient.nim

@@ -402,7 +402,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "",
 
   headers.add(" HTTP/1.1\c\L")
 
-  add(headers, "Host: " & r.hostname & "\c\L")
+  add(headers, "Host: " & parseUri(url).hostname & "\c\L")
   if userAgent != "":
     add(headers, "User-Agent: " & userAgent & "\c\L")
   if proxy != nil and proxy.auth != "":

lib/pure/json.nim

@@ -608,29 +608,29 @@ proc newJArray*(): JsonNode =
 proc getStr*(n: JsonNode, default: string = ""): string =
   ## Retrieves the string value of a `JString JsonNode`.
   ##
-  ## Returns ``default`` if ``n`` is not a ``JString``.
+  ## Returns ``default`` if ``n`` is not a ``JString``, or if ``n`` is nil.
-  if n.kind != JString: return default
+  if n.isNil or n.kind != JString: return default
   else: return n.str
 
 proc getNum*(n: JsonNode, default: BiggestInt = 0): BiggestInt =
   ## Retrieves the int value of a `JInt JsonNode`.
   ##
-  ## Returns ``default`` if ``n`` is not a ``JInt``.
+  ## Returns ``default`` if ``n`` is not a ``JInt``, or if ``n`` is nil.
-  if n.kind != JInt: return default
+  if n.isNil or n.kind != JInt: return default
   else: return n.num
 
 proc getFNum*(n: JsonNode, default: float = 0.0): float =
   ## Retrieves the float value of a `JFloat JsonNode`.
   ##
-  ## Returns ``default`` if ``n`` is not a ``JFloat``.
+  ## Returns ``default`` if ``n`` is not a ``JFloat``, or if ``n`` is nil.
-  if n.kind != JFloat: return default
+  if n.isNil or n.kind != JFloat: return default
   else: return n.fnum
 
 proc getBVal*(n: JsonNode, default: bool = false): bool =
   ## Retrieves the bool value of a `JBool JsonNode`.
   ##
-  ## Returns ``default`` if ``n`` is not a ``JBool``.
+  ## Returns ``default`` if ``n`` is not a ``JBool``, or if ``n`` is nil.
-  if n.kind != JBool: return default
+  if n.isNil or n.kind != JBool: return default
   else: return n.bval
 
 proc getFields*(n: JsonNode,
@@ -638,15 +638,15 @@ proc getFields*(n: JsonNode,
         seq[tuple[key: string, val: JsonNode]] =
   ## Retrieves the key, value pairs of a `JObject JsonNode`.
   ##
-  ## Returns ``default`` if ``n`` is not a ``JObject``.
+  ## Returns ``default`` if ``n`` is not a ``JObject``, or if ``n`` is nil.
-  if n.kind != JObject: return default
+  if n.isNil or n.kind != JObject: return default
   else: return n.fields
 
 proc getElems*(n: JsonNode, default: seq[JsonNode] = @[]): seq[JsonNode] =
   ## Retrieves the int value of a `JArray JsonNode`.
   ##
-  ## Returns ``default`` if ``n`` is not a ``JArray``.
+  ## Returns ``default`` if ``n`` is not a ``JArray``, or if ``n`` is nil.
-  if n.kind != JArray: return default
+  if n.isNil or n.kind != JArray: return default
   else: return n.elems
 
 proc `%`*(s: string): JsonNode =

lib/pure/math.nim

@@ -427,10 +427,12 @@ proc `^`*[T](x, y: T): T =
   var (x, y) = (x, y)
   result = 1
 
-  while y != 0:
+  while true:
     if (y and 1) != 0:
       result *= x
     y = y shr 1
+    if y == 0:
+      break
     x *= x
 
 proc gcd*[T](x, y: T): T =

lib/pure/memfiles.nim

@@ -11,6 +11,9 @@
 ##
 ## This module provides support for `memory mapped files`:idx:
 ## (Posix's `mmap`:idx:) on the different operating systems.
+##
+## It also provides some fast iterators over lines in text files (or
+## other "line-like", variable length, delimited records).
 
 when defined(windows):
   import winlean
@@ -245,3 +248,96 @@ proc close*(f: var MemFile) =
   
   if error: raiseOSError(lastErr)
 
+type MemSlice* = object  ## represent slice of a MemFile for iteration over delimited lines/records
+  data*: pointer
+  size*: int
+
+proc c_memcpy(a, b: pointer, n: int) {.importc: "memcpy", header: "<string.h>".}
+
+proc `$`*(ms: MemSlice): string {.inline.} =
+  ## Return a Nim string built from a MemSlice.
+  var buf = newString(ms.size)
+  c_memcpy(addr(buf[0]), ms.data, ms.size)
+  buf[ms.size] = '\0'
+  result = buf
+
+iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
+  ## Iterates over [optional `eat`] `delim`-delimited slices in MemFile `mfile`.
+  ##
+  ## Default parameters parse lines ending in either Unix(\\l) or Windows(\\r\\l)
+  ## style on on a line-by-line basis.  I.e., not every line needs the same ending.
+  ## Unlike readLine(File) & lines(File), archaic MacOS9 \\r-delimited lines
+  ## are not supported as a third option for each line.  Such archaic MacOS9
+  ## files can be handled by passing delim='\\r', eat='\\0', though.
+  ##
+  ## Delimiters are not part of the returned slice.  A final, unterminated line
+  ## or record is returned just like any other.
+  ##
+  ## Non-default delimiters can be passed to allow iteration over other sorts
+  ## of "line-like" variable length records.  Pass eat='\\0' to be strictly
+  ## `delim`-delimited. (Eating an optional prefix equal to '\\0' is not
+  ## supported.)
+  ##
+  ## This zero copy, memchr-limited interface is probably the fastest way to
+  ## iterate over line-like records in a file.  However, returned (data,size)
+  ## objects are not Nim strings, bounds checked Nim arrays, or even terminated
+  ## C strings.  So, care is required to access the data (e.g., think C mem*
+  ## functions, not str* functions).  Example:
+  ##
+  ## .. code-block:: nim
+  ##   var count = 0
+  ##   for slice in memSlices(memfiles.open("foo")):
+  ##     if slice.size > 0 and cast[cstring](slice.data)[0] != '#':
+  ##       inc(count)
+  ##   echo count
+
+  proc c_memchr(cstr: pointer, c: char, n: csize): pointer {.
+       importc: "memchr", header: "<string.h>" .}
+  proc `-!`(p, q: pointer): int {.inline.} = return cast[int](p) -% cast[int](q)
+  var ms: MemSlice
+  var ending: pointer
+  ms.data = mfile.mem
+  var remaining = mfile.size
+  while remaining > 0:
+    ending = c_memchr(ms.data, delim, remaining)
+    if ending == nil:                               # unterminated final slice
+      ms.size = remaining                           # Weird case..check eat?
+      yield ms
+      break
+    ms.size = ending -! ms.data                     # delim is NOT included
+    if eat != '\0' and ms.size > 0 and cast[cstring](ms.data)[ms.size - 1] == eat:
+      dec(ms.size)                                  # trim pre-delim char
+    yield ms
+    ms.data = cast[pointer](cast[int](ending) +% 1)     # skip delim
+    remaining = mfile.size - (ms.data -! mfile.mem)
+
+iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): TaintedString {.inline.} =
+  ## Replace contents of passed buffer with each new line, like
+  ## `readLine(File) <system.html#readLine,File,TaintedString>`_.
+  ## `delim`, `eat`, and delimiting logic is exactly as for
+  ## `memSlices <#memSlices>`_, but Nim strings are returned.  Example:
+  ##
+  ## .. code-block:: nim
+  ##   var buffer: TaintedString = ""
+  ##   for line in lines(memfiles.open("foo"), buffer):
+  ##     echo line
+
+  for ms in memSlices(mfile, delim, eat):
+    buf.setLen(ms.size)
+    c_memcpy(addr(buf[0]), ms.data, ms.size)
+    buf[ms.size] = '\0'
+    yield buf
+
+iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
+  ## Return each line in a file as a Nim string, like
+  ## `lines(File) <system.html#lines.i,File>`_.
+  ## `delim`, `eat`, and delimiting logic is exactly as for
+  ## `memSlices <#memSlices>`_, but Nim strings are returned.  Example:
+  ##
+  ## .. code-block:: nim
+  ##   for line in lines(memfiles.open("foo")):
+  ##     echo line
+
+  var buf = TaintedString(newStringOfCap(80))
+  for line in lines(mfile, buf, delim, eat):
+    yield buf

lib/pure/os.nim

@@ -1244,7 +1244,14 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect].} =
       while true:
         if not skipFindData(f) and
             (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) == 0'i32:
-          yield splitFile(pattern).dir / extractFilename(getFilename(f))
+          # Windows bug/gotcha: 't*.nim' matches 'tfoo.nims' -.- so we check
+          # that the file extensions have the same length ...
+          let ff = getFilename(f)
+          let dotPos = searchExtPos(pattern)
+          let idx = ff.len - pattern.len + dotPos
+          if dotPos < 0 or idx >= ff.len or ff[idx] == '.' or
+              pattern[dotPos+1] == '*':
+            yield splitFile(pattern).dir / extractFilename(ff)
         if findNextFile(res, f) == 0'i32: break
       findClose(res)
   else: # here we use glob

lib/pure/strutils.nim

@@ -168,7 +168,6 @@ proc cmpIgnoreStyle*(a, b: string): int {.noSideEffect,
     inc(i)
     inc(j)
 
-{.pop.}
 
 proc strip*(s: string, leading = true, trailing = true, chars: set[char] = Whitespace): string
   {.noSideEffect, rtl, extern: "nsuStrip".} =
@@ -1396,6 +1395,62 @@ proc format*(formatstr: string, a: varargs[string, `$`]): string {.noSideEffect,
 
 {.pop.}
 
+proc removeSuffix*(s: var string, chars: set[char] = Newlines) {.
+  rtl, extern: "nsuRemoveSuffixCharSet".} =
+  ## Removes the first matching character from the string (in-place) given a
+  ## set of characters. If the set of characters is only equal to `Newlines`
+  ## then it will remove both the newline and return feed.
+  ## .. code-block:: nim
+  ##   var
+  ##     userInput = "Hello World!\r\n"
+  ##     otherInput = "Hello!?!"
+  ##   userInput.removeSuffix
+  ##   userInput == "Hello World!"
+  ##   userInput.removeSuffix({'!', '?'})
+  ##   userInput == "Hello World"
+  ##   otherInput.removeSuffix({'!', '?'})
+  ##   otherInput == "Hello!?"
+
+  var last = len(s) - 1
+
+  if chars == Newlines:
+    if s[last] == '\10':
+      last -= 1
+
+    if s[last] == '\13':
+      last -= 1
+
+  else:
+    if s[last] in chars:
+      last -= 1
+
+  s.setLen(last + 1)
+
+proc removeSuffix*(s: var string, c: char) {.rtl, extern: "nsuRemoveSuffixChar".} =
+  ## Removes a single character (in-place) from a string.
+  ## .. code-block:: nim
+  ##   var
+  ##     table = "users"
+  ##   table.removeSuffix('s')
+  ##   table == "user"
+  removeSuffix(s, chars = {c})
+
+proc removeSuffix*(s: var string, suffix: string) {.
+  rtl, extern: "nsuRemoveSuffixString".} =
+  ## Remove the first matching suffix (in-place) from a string.
+  ## .. code-block:: nim
+  ##   var
+  ##     answers = "yeses"
+  ##   answers.removeSuffix("es")
+  ##   answers == "yes"
+
+  var newLen = s.len
+
+  if s.endsWith(suffix):
+    newLen -= len(suffix)
+
+  s.setLen(newLen)
+
 when isMainModule:
   doAssert align("abc", 4) == " abc"
   doAssert align("a", 0) == "a"

lib/pure/times.nim

@@ -26,11 +26,6 @@ type
   WeekDay* = enum ## represents a weekday
     dMon, dTue, dWed, dThu, dFri, dSat, dSun
 
-when not defined(JS):
-  var
-    timezone {.importc, header: "<time.h>".}: int
-    tzname {.importc, header: "<time.h>" .}: array[0..1, cstring]
-
 when defined(posix) and not defined(JS):
   type
     TimeImpl {.importc: "time_t", header: "<time.h>".} = int
@@ -49,6 +44,9 @@ when defined(posix) and not defined(JS):
   proc posix_gettimeofday(tp: var Timeval, unused: pointer = nil) {.
     importc: "gettimeofday", header: "<sys/time.h>".}
 
+  var
+    timezone {.importc, header: "<time.h>".}: int
+    tzname {.importc, header: "<time.h>" .}: array[0..1, cstring]
   # we also need tzset() to make sure that tzname is initialized
   proc tzset() {.importc, header: "<time.h>".}
   # calling tzset() implicitly to initialize tzname data.
@@ -60,12 +58,20 @@ elif defined(windows):
   when defined(vcc):
     # newest version of Visual C++ defines time_t to be of 64 bits
     type TimeImpl {.importc: "time_t", header: "<time.h>".} = int64
+    # visual c's c runtime exposes these under a different name
+    var
+      timezone {.importc: "_timezone", header: "<time.h>".}: int
+      tzname {.importc: "_tzname", header: "<time.h>"}: array[0..1, cstring]
   else:
     type TimeImpl {.importc: "time_t", header: "<time.h>".} = int32
+    var
+      timezone {.importc, header: "<time.h>".}: int
+      tzname {.importc, header: "<time.h>" .}: array[0..1, cstring]
 
   type
     Time* = distinct TimeImpl
 
+
 elif defined(JS):
   type
     Time* {.importc.} = object
@@ -542,11 +548,9 @@ elif defined(JS):
     ## get the milliseconds from the start of the program
     return int(getTime() - startMilsecs)
 
-  proc valueOf(time: Time): float {.importcpp: "getTime", tags:[]}
+  proc fromSeconds(since1970: float): Time = result = newDate(since1970 * 1000)
 
-  proc fromSeconds(since1970: float): Time = result = newDate(since1970)
+  proc toSeconds(time: Time): float = result = time.getTime() / 1000
-
-  proc toSeconds(time: Time): float = result = time.valueOf() / 1000
 
   proc getTimezone(): int = result = newDate().getTimezoneOffset()
 
@@ -1046,6 +1050,120 @@ proc parse*(value, layout: string): TimeInfo =
   info.weekday = getLocalTime(timeInfoToTime(info)).weekday
   return info
 
+# Leap year calculations are adapted from:
+# from http://www.codeproject.com/Articles/7358/Ultra-fast-Algorithms-for-Working-with-Leap-Years
+# The dayOfTheWeek procs are adapated from:
+# from http://stason.org/TULARC/society/calendars/2-5-What-day-of-the-week-was-2-August-1953.html
+
+# Note: for leap years, start date is assumed to be 1 AD.
+# counts the number of leap years up to January 1st of a given year.
+# Keep in mind that if specified year is a leap year, the leap day
+# has not happened before January 1st of that year.
+proc countLeapYears(yearSpan: int): int =
+  (((yearSpan - 1) / 4) - ((yearSpan - 1) / 100) + ((yearSpan - 1)/400)).int
+
+proc countDays(yearSpan: int): int =
+  (yearSpan - 1) * 365 + countLeapYears(yearSpan)
+
+proc countYears(daySpan: int): int =
+  # counts the number of years spanned by a given number of days.
+  ((daySpan - countLeapYears(daySpan div 365)) div 365)
+
+proc countYearsAndDays(daySpan: int): tuple[years: int, days: int] =
+  # counts the number of years spanned by a given number of days and the remainder as days.
+  let days = daySpan - countLeapYears(daySpan div 365)
+  result.years = days div 365
+  result.days = days mod 365
+
+const
+  secondsInMin = 60
+  secondsInHour = 60*60
+  secondsInDay = 60*60*24
+  epochStartYear = 1970
+
+proc getDayOfWeek*(day, month, year: int): WeekDay =
+  ## Returns the day of the week enum from day, month and year.
+  # Day & month start from one.
+  let
+    a = (14 - month) div 12
+    y = year - a
+    m = month + (12*a) - 2
+    d = (day + y + (y div 4) - (y div 100) + (y div 400) + (31*m) div 12) mod 7
+  # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. so we must correct
+  # for the WeekDay type.
+  if d == 0: return dSun
+  result = (d-1).WeekDay
+
+proc getDayOfWeekJulian*(day, month, year: int): WeekDay =
+  ## Returns the day of the week enum from day, month and year, according to the Julian calender.
+  # Day & month start from one.
+  let
+    a = (14 - month) div 12
+    y = year - a
+    m = month + (12*a) - 2
+    d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7
+  result = d.WeekDay
+
+proc timeToTimeInfo*(t: Time): TimeInfo =
+  ## Converts a Time to TimeInfo.
+  let
+    secs = t.toSeconds().int
+    daysSinceEpoch = secs div secondsInDay
+    (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch)
+    daySeconds = secs mod secondsInDay
+
+    y = yearsSinceEpoch + epochStartYear
+
+  var
+    mon = mJan
+    days = daysRemaining
+    daysInMonth = getDaysInMonth(mon, y)
+
+  # calculate month and day remainder
+  while days > daysInMonth and mon <= mDec:
+    days -= daysInMonth
+    mon.inc
+    daysInMonth = getDaysInMonth(mon, y)
+
+  let
+    yd = daysRemaining
+    m = mon  # month is zero indexed enum
+    md = days
+    # NB: month is zero indexed but dayOfWeek expects 1 indexed.
+    wd = getDayOfWeek(days, mon.int + 1, y).Weekday
+    h = daySeconds div secondsInHour + 1
+    mi = (daySeconds mod secondsInHour) div secondsInMin
+    s = daySeconds mod secondsInMin
+  result = TimeInfo(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s)
+
+proc timeToTimeInterval*(t: Time): TimeInterval =
+  ## Converts a Time to a TimeInterval.
+
+  let
+    secs = t.toSeconds().int
+    daysSinceEpoch = secs div secondsInDay
+    (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch)
+    daySeconds = secs mod secondsInDay
+
+  result.years = yearsSinceEpoch + epochStartYear
+
+  var
+    mon = mJan
+    days = daysRemaining
+    daysInMonth = getDaysInMonth(mon, result.years)
+
+  # calculate month and day remainder
+  while days > daysInMonth and mon <= mDec:
+    days -= daysInMonth
+    mon.inc
+    daysInMonth = getDaysInMonth(mon, result.years)
+
+  result.months = mon.int + 1 # month is 1 indexed int
+  result.days = days
+  result.hours = daySeconds div secondsInHour + 1
+  result.minutes = (daySeconds mod secondsInHour) div secondsInMin
+  result.seconds = daySeconds mod secondsInMin
+  # Milliseconds not available from Time
 
 when isMainModule:
   # $ date --date='@2147483647'
@@ -1066,7 +1184,8 @@ when isMainModule:
     " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") ==
     "27 27 Mon Monday 4 04 16 16 6 06 1 01 Jan January 29 29 P PM 5 75 975 1975 01975 0 00 00:00 UTC"
 
-  when not defined(JS) and sizeof(Time) == 8:
+  when not defined(JS):
+    when sizeof(Time) == 8:
       var t3 = getGMTime(fromSeconds(889067643645)) # Fri  7 Jun 19:20:45 BST 30143
       assert t3.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" &
         " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") ==
@@ -1131,3 +1250,18 @@ when isMainModule:
   assert "15:04:00" in $s.parse(f)
   when not defined(testing):
     echo "Kitchen: " & $s.parse(f)
+    var ti = timeToTimeInfo(getTime())
+    echo "Todays date after decoding: ", ti
+    var tint = timeToTimeInterval(getTime())
+    echo "Todays date after decoding to interval: ", tint
+  # checking dayOfWeek matches known days
+  assert getDayOfWeek(21, 9, 1900) == dFri
+  assert getDayOfWeek(1, 1, 1970) == dThu
+  assert getDayOfWeek(21, 9, 1970) == dMon
+  assert getDayOfWeek(1, 1, 2000) == dSat
+  assert getDayOfWeek(1, 1, 2021) == dFri
+  # Julian tests
+  assert getDayOfWeekJulian(21, 9, 1900) == dFri
+  assert getDayOfWeekJulian(21, 9, 1970) == dMon
+  assert getDayOfWeekJulian(1, 1, 2000) == dSat
+  assert getDayOfWeekJulian(1, 1, 2021) == dFri

lib/pure/unittest.nim

@@ -285,7 +285,7 @@ macro check*(conditions: stmt): stmt {.immediate.} =
 
     result = getAst(rewrite(checked, checked.lineinfo, checked.toStrLit))
 
-template require*(conditions: stmt): stmt {.immediate, dirty.} =
+template require*(conditions: stmt): stmt {.immediate.} =
   ## Same as `check` except any failed test causes the program to quit
   ## immediately. Any teardown statements are not executed and the failed
   ## test output is not generated.

lib/system.nim

@@ -153,6 +153,13 @@ proc `addr`*[T](x: var T): ptr T {.magic: "Addr", noSideEffect.} =
   ## Cannot be overloaded.
   discard
 
+proc unsafeAddr*[T](x: var T): ptr T {.magic: "Addr", noSideEffect.} =
+  ## Builtin 'addr' operator for taking the address of a memory location.
+  ## This works even for ``let`` variables or parameters for better interop
+  ## with C and so it is considered even more unsafe than the ordinary ``addr``.
+  ## Cannot be overloaded.
+  discard
+
 proc `type`*(x: expr): typeDesc {.magic: "TypeOf", noSideEffect.} =
   ## Builtin 'type' operator for accessing the type of an expression.
   ## Cannot be overloaded.
@@ -176,10 +183,19 @@ proc new*[T](a: var ref T) {.magic: "New", noSideEffect.}
   ## creates a new object of type ``T`` and returns a safe (traced)
   ## reference to it in ``a``.
 
-proc new*(T: typedesc): ref T =
+proc new*(T: typedesc): auto =
   ## creates a new object of type ``T`` and returns a safe (traced)
-  ## reference to it as result value
+  ## reference to it as result value.
-  new(result)
+  ##
+  ## When ``T`` is a ref type then the resulting type will be ``T``,
+  ## otherwise it will be ``ref T``.
+  when (T is ref):
+    var r: T
+  else:
+    var r: ref T
+  new(r)
+  return r
+
 
 proc internalNew*[T](a: var ref T) {.magic: "New", noSideEffect.}
   ## leaked implementation detail. Do not use.
@@ -331,7 +347,7 @@ const
 
 include "system/inclrtl"
 
-const NoFakeVars* = defined(NimrodVM) ## true if the backend doesn't support \
+const NoFakeVars* = defined(nimscript) ## true if the backend doesn't support \
   ## "fake variables" like 'var EBADF {.importc.}: cint'.
 
 const ArrayDummySize = when defined(cpu16): 10_000 else: 100_000_000
@@ -349,7 +365,7 @@ when not defined(JS):
       data: UncheckedCharArray
     NimString = ptr NimStringDesc
 
-when not defined(JS) and not defined(NimrodVM):
+when not defined(JS) and not defined(nimscript):
   template space(s: PGenericSeq): int {.dirty.} =
     s.reserved and not seqShallowFlag
 
@@ -481,6 +497,7 @@ type
     ## See the full `exception hierarchy`_.
   ObjectConversionError* = object of Exception ## \
     ## Raised if an object is converted to an incompatible object type.
+    ## You can use ``of`` operator to check if conversion will succeed.
     ##
     ## See the full `exception hierarchy`_.
   FloatingPointError* = object of Exception ## \
@@ -560,6 +577,9 @@ proc sizeof*[T](x: T): int {.magic: "SizeOf", noSideEffect.}
   ## that one never needs to know ``x``'s size. As a special semantic rule,
   ## ``x`` may also be a type identifier (``sizeof(int)`` is valid).
 
+when defined(nimtypedescfixed):
+  proc sizeof*(x: typedesc): int {.magic: "SizeOf", noSideEffect.}
+
 proc `<`*[T](x: Ordinal[T]): T {.magic: "UnaryLt", noSideEffect.}
   ## unary ``<`` that can be used for nice looking excluding ranges:
   ##
@@ -1150,7 +1170,8 @@ const
 
   hostCPU* {.magic: "HostCPU".}: string = ""
     ## a string that describes the host CPU. Possible values:
-    ## "i386", "alpha", "powerpc", "powerpc64", "sparc", "amd64", "mips", "arm".
+    ## "i386", "alpha", "powerpc", "powerpc64", "powerpc64el", "sparc",
+    ## "amd64", "mips", "mipsel", "arm", "arm64".
 
   seqShallowFlag = low(int)
 
@@ -1239,11 +1260,11 @@ template sysAssert(cond: bool, msg: string) =
       echo "[SYSASSERT] ", msg
       quit 1
 
-const hasAlloc = hostOS != "standalone" or not defined(nogc)
+const hasAlloc = (hostOS != "standalone" or not defined(nogc)) and not defined(nimscript)
 
-when not defined(JS) and not defined(nimrodVm) and hostOS != "standalone":
+when not defined(JS) and not defined(nimscript) and hostOS != "standalone":
   include "system/cgprocs"
-when not defined(JS) and not defined(nimrodVm) and hasAlloc:
+when not defined(JS) and not defined(nimscript) and hasAlloc:
   proc setStackBottom(theStackBottom: pointer) {.compilerRtl, noinline, benign.}
   proc addChar(s: NimString, c: char): NimString {.compilerProc, benign.}
 
@@ -1424,7 +1445,7 @@ proc substr*(s: string, first, last: int): string {.
   ## is used instead: This means ``substr`` can also be used to `cut`:idx:
   ## or `limit`:idx: a string's length.
 
-when not defined(nimrodVM):
+when not defined(nimscript):
   proc zeroMem*(p: pointer, size: Natural) {.importc, noDecl, benign.}
     ## overwrites the contents of the memory at ``p`` with the value 0.
     ## Exactly ``size`` bytes will be overwritten. Like any procedure
@@ -1482,7 +1503,7 @@ when not defined(nimrodVM):
       ## containing zero, so it is somewhat safer than ``createU``.
       ## The allocated memory belongs to its allocating thread!
       ## Use `createShared` to allocate from a shared heap.
-      cast[ptr T](alloc0(T.sizeof * size))
+      cast[ptr T](alloc0(sizeof(T) * size))
     proc realloc*(p: pointer, newSize: Natural): pointer {.noconv, rtl, tags: [],
                                                            benign.}
       ## grows or shrinks a given memory block. If p is **nil** then a new
@@ -1586,7 +1607,7 @@ proc `$`*(x: int64): string {.magic: "Int64ToStr", noSideEffect.}
   ## The stringify operator for an integer argument. Returns `x`
   ## converted to a decimal string.
 
-when not defined(NimrodVM):
+when not defined(nimscript):
   when not defined(JS) and hasAlloc:
     proc `$` *(x: uint64): string {.noSideEffect.}
       ## The stringify operator for an unsigned integer argument. Returns `x`
@@ -1654,7 +1675,7 @@ const
 
 # GC interface:
 
-when not defined(nimrodVM) and hasAlloc:
+when not defined(nimscript) and hasAlloc:
   proc getOccupiedMem*(): int {.rtl.}
     ## returns the number of bytes that are owned by the process and hold data.
 
@@ -1995,7 +2016,7 @@ proc `&` *[T](x: T, y: seq[T]): seq[T] {.noSideEffect.} =
   for i in 0..y.len-1:
     result[i+1] = y[i]
 
-when not defined(NimrodVM):
+when not defined(nimscript):
   when not defined(JS):
     proc seqToPtr[T](x: seq[T]): pointer {.inline, nosideeffect.} =
       result = cast[pointer](x)
@@ -2214,7 +2235,7 @@ when false:
 
 # ----------------- GC interface ---------------------------------------------
 
-when not defined(nimrodVM) and hasAlloc:
+when not defined(nimscript) and hasAlloc:
   proc GC_disable*() {.rtl, inl, benign.}
     ## disables the GC. If called n-times, n calls to `GC_enable` are needed to
     ## reactivate the GC. Note that in most circumstances one should only disable
@@ -2436,10 +2457,10 @@ else:
     if x < 0: -x else: x
 {.pop.}
 
-when not defined(JS): #and not defined(NimrodVM):
+when not defined(JS): #and not defined(nimscript):
   {.push stack_trace: off, profiler:off.}
 
-  when not defined(NimrodVM) and not defined(nogc):
+  when not defined(nimscript) and not defined(nogc):
     proc initGC()
     when not defined(boehmgc) and not defined(useMalloc) and not defined(gogc):
       proc initAllocator() {.inline.}
@@ -2467,13 +2488,19 @@ when not defined(JS): #and not defined(NimrodVM):
     strDesc.kind = tyString
     strDesc.flags = {ntfAcyclic}
 
+  when not defined(nimscript):
     include "system/ansi_c"
 
     proc cmp(x, y: string): int =
       result = int(c_strcmp(x, y))
+  else:
+    proc cmp(x, y: string): int =
+      if x < y: result = -1
+      elif x > y: result = 1
+      else: result = 0
 
   const pccHack = if defined(pcc): "_" else: "" # Hack for PCC
-  when not defined(NimrodVM):
+  when not defined(nimscript):
     when defined(windows):
       # work-around C's sucking abstraction:
       # BUGFIX: stdin and stdout should be binary files!
@@ -2515,6 +2542,7 @@ when not defined(JS): #and not defined(NimrodVM):
 
     {.deprecated: [TFile: File, TFileHandle: FileHandle, TFileMode: FileMode].}
 
+    when not defined(nimscript):
       # text file handling:
       var
         stdin* {.importc: "stdin", header: "<stdio.h>".}: File
@@ -2717,7 +2745,7 @@ when not defined(JS): #and not defined(NimrodVM):
         inc(i)
       dealloc(a)
 
-  when not defined(NimrodVM):
+  when not defined(nimscript):
     proc atomicInc*(memLoc: var int, x: int = 1): int {.inline,
       discardable, benign.}
       ## atomic increment of `memLoc`. Returns the value after the operation.
@@ -2744,11 +2772,11 @@ when not defined(JS): #and not defined(NimrodVM):
   when hasThreadSupport:
     include "system/syslocks"
     when hostOS != "standalone": include "system/threads"
-  elif not defined(nogc) and not defined(NimrodVM):
+  elif not defined(nogc) and not defined(nimscript):
     when not defined(useNimRtl) and not defined(createNimRtl): initStackBottom()
     when declared(initGC): initGC()
 
-  when not defined(NimrodVM):
+  when not defined(nimscript):
     proc setControlCHook*(hook: proc () {.noconv.} not nil)
       ## allows you to override the behaviour of your application when CTRL+C
       ## is pressed. Only one such hook is supported.
@@ -2777,9 +2805,9 @@ when not defined(JS): #and not defined(NimrodVM):
     {.pop.} # stack trace
   {.pop.} # stack trace
 
-  when hostOS != "standalone" and not defined(NimrodVM):
+  when hostOS != "standalone" and not defined(nimscript):
     include "system/dyncalls"
-  when not defined(NimrodVM):
+  when not defined(nimscript):
     include "system/sets"
 
     when defined(gogc):
@@ -2854,11 +2882,11 @@ when not defined(JS): #and not defined(NimrodVM):
       var res = TaintedString(newStringOfCap(80))
       while f.readLine(res): yield res
 
-  when not defined(NimrodVM) and hasAlloc:
+  when not defined(nimscript) and hasAlloc:
     include "system/assign"
     include "system/repr"
 
-  when hostOS != "standalone" and not defined(NimrodVM):
+  when hostOS != "standalone" and not defined(nimscript):
     proc getCurrentException*(): ref Exception {.compilerRtl, inl, benign.} =
       ## retrieves the current exception; if there is none, nil is returned.
       result = currException
@@ -2886,14 +2914,14 @@ when not defined(JS): #and not defined(NimrodVM):
       currException = exc
 
   {.push stack_trace: off, profiler:off.}
-  when defined(endb) and not defined(NimrodVM):
+  when defined(endb) and not defined(nimscript):
     include "system/debugger"
 
   when defined(profiler) or defined(memProfiler):
     include "system/profiler"
   {.pop.} # stacktrace
 
-  when not defined(NimrodVM):
+  when not defined(nimscript):
     proc likely*(val: bool): bool {.importc: "likely", nodecl, nosideeffect.}
       ## Hints the optimizer that `val` is likely going to be true.
       ##
@@ -2971,7 +2999,7 @@ elif defined(JS):
   when defined(JS):
     include "system/jssys"
     include "system/reprjs"
-  elif defined(NimrodVM):
+  elif defined(nimscript):
     proc cmp(x, y: string): int =
       if x == y: return 0
       if x < y: return -1
@@ -3284,7 +3312,7 @@ proc shallow*[T](s: var seq[T]) {.noSideEffect, inline.} =
   ## marks a sequence `s` as `shallow`:idx:. Subsequent assignments will not
   ## perform deep copies of `s`. This is only useful for optimization
   ## purposes.
-  when not defined(JS) and not defined(NimrodVM):
+  when not defined(JS) and not defined(nimscript):
     var s = cast[PGenericSeq](s)
     s.reserved = s.reserved or seqShallowFlag
 
@@ -3292,7 +3320,7 @@ proc shallow*(s: var string) {.noSideEffect, inline.} =
   ## marks a string `s` as `shallow`:idx:. Subsequent assignments will not
   ## perform deep copies of `s`. This is only useful for optimization
   ## purposes.
-  when not defined(JS) and not defined(NimrodVM):
+  when not defined(JS) and not defined(nimscript):
     var s = cast[PGenericSeq](s)
     s.reserved = s.reserved or seqShallowFlag
 
@@ -3381,7 +3409,7 @@ proc locals*(): RootObj {.magic: "Plugin", noSideEffect.} =
   ##   # -> B is 1
   discard
 
-when hasAlloc and not defined(NimrodVM) and not defined(JS):
+when hasAlloc and not defined(nimscript) and not defined(JS):
   proc deepCopy*[T](x: var T, y: T) {.noSideEffect, magic: "DeepCopy".} =
     ## performs a deep copy of `x`. This is also used by the code generator
     ## for the implementation of ``spawn``.
@@ -3423,3 +3451,6 @@ proc xlen*[T](x: seq[T]): int {.magic: "XLenSeq", noSideEffect.} =
   discard
 
 {.pop.} #{.push warning[GcMem]: off, warning[Uninit]: off.}
+
+when defined(nimconfig):
+  include "system/nimscript"

lib/system/debugger.nim

@@ -21,7 +21,7 @@ type
                            # only slots that are
                            # needed are allocated and not 10_000,
                            # except for the global data description.
-    f: Frame
+    f: TFrame
     slots: array[0..10_000, VarSlot]
 {.deprecated: [TVarSlot: VarSlot, TExtendedFrame: ExtendedFrame].}
 

lib/system/nimscript.nim

@@ -0,0 +1,152 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+
+# Nim's configuration system now uses Nim for scripting. This module provides
+# a few things that are required for this to work.
+
+template builtin = discard
+
+proc listDirs*(dir: string): seq[string] = builtin
+proc listFiles*(dir: string): seq[string] = builtin
+
+proc removeDir(dir: string) = builtin
+proc removeFile(dir: string) = builtin
+proc moveFile(src, dest: string) = builtin
+proc createDir(dir: string) = builtin
+proc getOsError: string = builtin
+proc setCurrentDir(dir: string) = builtin
+proc getCurrentDir(): string = builtin
+proc paramStr*(i: int): string = builtin
+proc paramCount*(): int = builtin
+
+proc switch*(key: string, val="") = builtin
+proc getCommand*(): string = builtin
+proc setCommand*(cmd: string) = builtin
+proc cmpIgnoreStyle(a, b: string): int = builtin
+
+proc strip(s: string): string =
+  var i = 0
+  while s[i] in {' ', '\c', '\L'}: inc i
+  result = s.substr(i)
+
+template `--`*(key, val: untyped) = switch(astToStr(key), strip astToStr(val))
+template `--`*(key: untyped) = switch(astToStr(key), "")
+
+type
+  ScriptMode* {.pure.} = enum
+    Silent,
+    Verbose,
+    Whatif
+
+var
+  mode*: ScriptMode ## Set this to influence how mkDir, rmDir, rmFile etc.
+                    ## behave
+
+template checkOsError =
+  let err = getOsError()
+  if err.len > 0: raise newException(OSError, err)
+
+template log(msg: string, body: untyped) =
+  if mode == ScriptMode.Verbose or mode == ScriptMode.Whatif:
+    echo "[NimScript] ", msg
+  if mode != ScriptMode.WhatIf:
+    body
+
+proc rmDir*(dir: string) {.raises: [OSError].} =
+  log "rmDir: " & dir:
+    removeDir dir
+    checkOsError()
+
+proc rmFile*(dir: string) {.raises: [OSError].} =
+  log "rmFile: " & dir:
+    removeFile dir
+    checkOsError()
+
+proc mkDir*(dir: string) {.raises: [OSError].} =
+  log "mkDir: " & dir:
+    createDir dir
+    checkOsError()
+
+proc mvFile*(`from`, to: string) {.raises: [OSError].} =
+  log "mvFile: " & `from` & ", " & to:
+    moveFile `from`, to
+    checkOsError()
+
+proc exec*(command: string, input = "", cache = "") =
+  ## Executes an external process.
+  log "exec: " & command:
+    echo staticExec(command, input, cache)
+
+proc put*(key, value: string) =
+  ## Sets a configuration 'key' like 'gcc.options.always' to its value.
+  builtin
+
+proc get*(key: string): string =
+  ## Retrieves a configuration 'key' like 'gcc.options.always'.
+  builtin
+
+proc exists*(key: string): bool =
+  ## Checks for the existance of a configuration 'key'
+  ## like 'gcc.options.always'.
+  builtin
+
+proc nimcacheDir*(): string =
+  ## Retrieves the location of 'nimcache'.
+  builtin
+
+proc thisDir*(): string =
+  ## Retrieves the location of the current ``nims`` script file.
+  builtin
+
+proc cd*(dir: string) {.raises: [OSError].} =
+  ## Changes the current directory.
+  ##
+  ## The change is permanent for the rest of the execution, since this is just
+  ## a shortcut for `os.setCurrentDir()
+  ## <http://nim-lang.org/os.html#setCurrentDir,string>`_ . Use the `withDir()
+  ## <#withDir>`_ template if you want to perform a temporary change only.
+  setCurrentDir(dir)
+  checkOsError()
+
+template withDir*(dir: string; body: untyped): untyped =
+  ## Changes the current directory temporarily.
+  ##
+  ## If you need a permanent change, use the `cd() <#cd>`_ proc. Usage example:
+  ##
+  ## .. code-block:: nimrod
+  ##   withDir "foo":
+  ##     # inside foo
+  ##   #back to last dir
+  var curDir = getCurrentDir()
+  try:
+    cd(dir)
+    body
+  finally:
+    cd(curDir)
+
+template `==?`(a, b: string): bool = cmpIgnoreStyle(a, b) == 0
+
+proc writeTask(name, desc: string) =
+  if desc.len > 0:
+    var spaces = " "
+    for i in 0 ..< 20 - name.len: spaces.add ' '
+    echo name, spaces, desc
+
+template task*(name: untyped; description: string; body: untyped): untyped =
+  ## Defines a task. Hidden tasks are supported via an empty description.
+  proc `name Task`() = body
+
+  let cmd = getCommand()
+  if cmd.len == 0 or cmd ==? "help" or cmd == "nop":
+    setCommand "nop"
+    writeTask(astToStr(name), description)
+  elif cmd ==? astToStr(name):
+    setCommand "nop"
+    `name Task`()

lib/system/platforms.nim

@@ -18,11 +18,14 @@ type
     alpha,                     ## Alpha processor
     powerpc,                   ## 32 bit PowerPC
     powerpc64,                 ## 64 bit PowerPC
+    powerpc64el,               ## Little Endian 64 bit PowerPC
     sparc,                     ## Sparc based processor
     ia64,                      ## Intel Itanium
     amd64,                     ## x86_64 (AMD64); 64 bit x86 compatible CPU
     mips,                      ## Mips based processor
+    mipsel,                    ## Little Endian Mips based processor
     arm,                       ## ARM based processor
+    arm64,                     ## ARM64 based processor
     vm,                        ## Some Virtual machine: Nim's VM or JavaScript
     avr                        ## AVR based processor
 
@@ -63,11 +66,14 @@ const
                elif defined(alpha): CpuPlatform.alpha
                elif defined(powerpc): CpuPlatform.powerpc
                elif defined(powerpc64): CpuPlatform.powerpc64
+               elif defined(powerpc64el): CpuPlatform.powerpc64el
                elif defined(sparc): CpuPlatform.sparc
                elif defined(ia64): CpuPlatform.ia64
                elif defined(amd64): CpuPlatform.amd64
                elif defined(mips): CpuPlatform.mips
+               elif defined(mipsel): CpuPlatform.mipsel
                elif defined(arm): CpuPlatform.arm
+               elif defined(arm64): CpuPlatform.arm64
                elif defined(vm): CpuPlatform.vm
                elif defined(avr): CpuPlatform.avr
                else: CpuPlatform.none

lib/system/repr.nim

@@ -256,7 +256,10 @@ when not defined(useNimRtl):
     of tyBool: add result, reprBool(cast[ptr bool](p)[])
     of tyChar: add result, reprChar(cast[ptr char](p)[])
     of tyString: reprStrAux(result, cast[ptr string](p)[])
-    of tyCString: reprStrAux(result, $(cast[ptr cstring](p)[]))
+    of tyCString:
+      let cs = cast[ptr cstring](p)[]
+      if cs.isNil: add result, "nil"
+      else: reprStrAux(result, $cs)
     of tyRange: reprAux(result, p, typ.base, cl)
     of tyProc, tyPointer:
       if cast[PPointer](p)[] == nil: add result, "nil"

lib/system/sets.nim

@@ -20,8 +20,8 @@ proc countBits32(n: int32): int {.compilerproc.} =
   result = ((v +% (v shr 4'i32) and 0xF0F0F0F'i32) *% 0x1010101'i32) shr 24'i32
 
 proc countBits64(n: int64): int {.compilerproc.} =
-  result = countBits32(toU32(n and 0xffff'i64)) +
+  result = countBits32(toU32(n and 0xffffffff'i64)) +
-           countBits32(toU32(n shr 16'i64))
+           countBits32(toU32(n shr 32'i64))
 
 proc cardSet(s: NimSet, len: int): int {.compilerproc.} =
   result = 0

lib/system/sysio.nim

@@ -182,7 +182,10 @@ proc readAllFile(file: File): string =
 proc readAll(file: File): TaintedString =
   # Separate handling needed because we need to buffer when we
   # don't know the overall length of the File.
+  when declared(stdin):
     let len = if file != stdin: rawFileSize(file) else: -1
+  else:
+    let len = rawFileSize(file)
   if len > 0:
     result = readAllFile(file, len).TaintedString
   else:
@@ -216,7 +219,7 @@ proc writeLine[Ty](f: File, x: varargs[Ty, `$`]) =
   for i in items(x): write(f, i)
   write(f, "\n")
 
-
+when declared(stdout):
   proc rawEcho(x: string) {.inline, compilerproc.} = write(stdout, x)
   proc rawEchoNL() {.inline, compilerproc.} = write(stdout, "\n")
 

tests/ccgbugs/tunsafeaddr.nim

@@ -0,0 +1,19 @@
+discard """
+  output: '''12'''
+"""
+
+{.emit: """
+long sum(long* a, long len) {
+  long i, result = 0;
+  for (i = 0; i < len; ++i) result += a[i];
+  return result;
+}
+""".}
+
+proc sum(a: ptr int; len: int): int {.importc, nodecl.}
+
+proc main =
+  let foo = [8, 3, 1]
+  echo sum(unsafeAddr foo[0], foo.len)
+
+main()

tests/destructor/tdestructor3.nim

@@ -0,0 +1,47 @@
+discard """
+  output: '''assign
+destroy
+destroy
+destroy Foo: 5
+5
+destroy Foo: 123
+123'''
+"""
+
+# bug #2821
+{.experimental.}
+
+type T = object
+
+proc `=`(lhs: var T, rhs: T) =
+    echo "assign"
+
+proc `=destroy`(v: var T) =
+    echo "destroy"
+
+block:
+    var v1 : T
+    var v2 : T = v1
+
+
+# bug #1632
+
+type
+  Foo = object of RootObj
+    x: int
+
+proc `=destroy`(a: var Foo) =
+  echo "destroy Foo: " & $a.x
+
+template toFooPtr(a: int{lit}): ptr Foo =
+  var temp = Foo(x:a)
+  temp.addr
+
+proc test(a: ptr Foo) =
+  echo a[].x
+
+proc main =
+  test(toFooPtr(5))
+  test(toFooPtr(123))
+
+main()

tests/exception/tdefer1.nim

@@ -1,6 +1,11 @@
 discard """
   output: '''hi
-hi'''
+hi
+1
+hi
+2
+B
+A'''
 """
 
 # bug #1742
@@ -16,3 +21,23 @@ import strutils
 let x = try: parseInt("133a")
         except: -1
         finally: echo "hi"
+
+
+template atFuncEnd =
+  defer:
+    echo "A"
+  defer:
+    echo "B"
+
+template testB(): expr =
+    let a = 0
+    defer: echo "hi" # Delete this line to make it work
+    a
+
+proc main =
+  atFuncEnd()
+  echo 1
+  let i = testB()
+  echo 2
+
+main()

tests/generics/mclosed_sym.nim

@@ -0,0 +1,10 @@
+
+type R* = object
+
+type Data*[T] = object
+  d*: T
+
+proc same(r:R, d:int) = echo "TEST2"
+
+proc doIt*(d:Data, r:R) =
+  r.same(1)      # Expecting this to invoke the local `same()` method

tests/generics/mmodule_same_as_proc.nim

@@ -0,0 +1,2 @@
+
+proc mmodule_same_as_proc*(x: string) = discard

tests/generics/tclosed_sym.nim

@@ -0,0 +1,11 @@
+discard """
+  output: "TEST2"
+"""
+
+# bug #2664
+
+import mclosed_sym
+
+proc same(r:R, d:int) = echo "TEST1"
+
+doIt(Data[int](d:123), R())

tests/generics/tdont_use_inner_scope.nim

@@ -0,0 +1,27 @@
+
+# bug #2752
+
+import future, sequtils
+
+proc myFilter[T](it: (iterator(): T), f: (proc(anything: T):bool)): (iterator(): T) =
+  iterator aNameWhichWillConflict(): T {.closure.}=
+    for x in it():
+      if f(x):
+        yield x
+  result = aNameWhichWillConflict
+
+
+iterator testIt():int {.closure.}=
+  yield -1
+  yield 2
+
+#let unusedVariable = myFilter(testIt, (x: int) => x > 0)
+
+proc onlyPos(it: (iterator(): int)): (iterator(): int)=
+  iterator aNameWhichWillConflict(): int {.closure.}=
+    var filtered = onlyPos(myFilter(it, (x:int) => x > 0))
+    for x in filtered():
+      yield x
+  result = aNameWhichWillConflict
+
+let x = onlyPos(testIt)

tests/generics/tmodule_same_as_proc.nim

@@ -0,0 +1,9 @@
+
+# bug #1965
+
+import mmodule_same_as_proc
+
+proc test[T](t: T) =
+  mmodule_same_as_proc"a"
+
+test(0)

tests/macros/tstaticparamsmacro.nim

@@ -0,0 +1,74 @@
+discard """
+  msg: '''letters
+aa
+bb
+numbers
+11
+22
+AST a 
+[(11, 22), (33, 44)]
+AST b 
+(e: [55, 66], f: [77, 88])
+55
+10
+20Test
+20
+'''
+"""
+
+import macros
+
+type
+  TConfig = tuple
+    letters: seq[string]
+    numbers:seq[int]
+
+const data: Tconfig = (@["aa", "bb"], @[11, 22])
+
+macro mymacro(data: static[TConfig]): stmt =
+  echo "letters"
+  for s in items(data.letters):
+    echo s
+  echo "numbers"
+  for n in items(data.numbers):
+    echo n
+
+mymacro(data)
+
+type
+  Ta = seq[tuple[c:int, d:int]]
+  Tb = tuple[e:seq[int], f:seq[int]]
+
+const
+  a : Ta = @[(11, 22), (33, 44)]
+  b : Tb = (@[55,66], @[77, 88])
+
+macro mA(data: static[Ta]): stmt =
+  echo "AST a \n", repr(data)
+
+macro mB(data: static[Tb]): stmt =
+  echo "AST b \n", repr(data)
+  echo data.e[0]
+
+mA(a)
+mB(b)
+
+type
+  Foo[N: static[int], Z: static[string]] = object
+
+macro staticIntMacro(f: static[int]): stmt = echo f
+staticIntMacro 10
+
+var
+  x: Foo[20, "Test"]
+
+macro genericMacro[N; Z: static[string]](f: Foo[N, Z], ll = 3, zz = 12): stmt =
+  echo N, Z
+
+genericMacro x
+
+template genericTemplate[N, Z](f: Foo[N, Z], ll = 3, zz = 12): int = N
+
+static:
+  echo genericTemplate(x)
+

tests/newconfig/tfoo.nim

@@ -0,0 +1,7 @@
+discard """
+  cmd: "nim default $file"
+  output: '''hello world!'''
+  msg: '''[NimScript] exec: gcc -v'''
+"""
+
+echo "hello world!"

tests/newconfig/tfoo.nims

@@ -0,0 +1,14 @@
+
+mode = ScriptMode.Whatif
+
+exec "gcc -v"
+
+--forceBuild
+
+task listDirs, "lists every subdirectory":
+  for x in listDirs("."):
+    echo "DIR ", x
+
+task default, "default target":
+  setCommand "c"
+

tests/parallel/nim.cfg

@@ -1 +1,2 @@
 threads:on
+--experimental

tests/pragmas/tsym_as_pragma.nim

@@ -0,0 +1,8 @@
+
+# bug #3171
+
+template newDataWindow(): stmt =
+    let eventClosure = proc (closure: pointer): bool {.closure, cdecl.} =
+        discard
+
+newDataWindow()

tests/stdlib/tmemlines.nim

@@ -0,0 +1,5 @@
+import memfiles
+var inp = memfiles.open("readme.txt")
+for line in lines(inp):
+  echo("#" & line & "#")
+close(inp)

tests/stdlib/tmemlinesBuf.nim

@@ -0,0 +1,6 @@
+import memfiles
+var inp = memfiles.open("readme.txt")
+var buffer: TaintedString = ""
+for line in lines(inp, buffer):
+  echo("#" & line & "#")
+close(inp)

tests/stdlib/tmemslices.nim

@@ -0,0 +1,6 @@
+import memfiles
+var inp = memfiles.open("readme.txt")
+for mem in memSlices(inp):
+  if mem.size > 3:
+    echo("#" & $mem & "#")
+close(inp)

tests/stdlib/tstrutil.nim

@@ -10,8 +10,48 @@ import
 proc testStrip() =
   write(stdout, strip("  ha  "))
 
+proc testRemoveSuffix =
+  var s = "hello\n\r"
+  s.removeSuffix
+  assert s == "hello\n"
+  s.removeSuffix
+  assert s == "hello"
+  s.removeSuffix
+  assert s == "hello"
+
+  s = "hello\n\n"
+  s.removeSuffix
+  assert s == "hello\n"
+
+  s = "hello\r"
+  s.removeSuffix
+  assert s == "hello"
+
+  s = "hello \n there"
+  s.removeSuffix
+  assert s == "hello \n there"
+
+  s = "hello"
+  s.removeSuffix("llo")
+  assert s == "he"
+  s.removeSuffix('e')
+  assert s == "h"
+
+  s = "hellos"
+  s.removeSuffix({'s','z'})
+  assert s == "hello"
+  s.removeSuffix({'l','o'})
+  assert s == "hell"
+
+  # Contrary to Chomp in other languages
+  # empty string does not change behaviour
+  s = "hello\r\n\r\n"
+  s.removeSuffix("")
+  assert s == "hello\r\n\r\n"
+
 proc main() =
   testStrip()
+  testRemoveSuffix()
   for p in split("/home/a1:xyz:/usr/bin", {':'}):
     write(stdout, p)
 

tests/stdlib/tunittest.nim

@@ -21,6 +21,11 @@ test "unittest typedescs":
   check(none(int) != some(1))
 
 
+test "unittest multiple requires":
+  require(true)
+  require(true)
+
+
 import math
 from strutils import parseInt
 proc defectiveRobot() =

tests/template/t_otemplates.nim

@@ -210,7 +210,7 @@ proc parse_complex_stmt(value, identifier: string, index: var int): NimNode {.co
         inc(index, read + 1)
 
         # Insert body into result
-        var stmtIndex = macros.high(result[resultIndex])
+        var stmtIndex = result[resultIndex].len-1
         result[resultIndex][stmtIndex] = body
 
         # Parse through EOL again & increment result index
@@ -245,7 +245,7 @@ proc parse_simple_statement(value: string, index: var int): NimNode {.compiletim
     inc(index, read + 1)
 
     # Insert body into result
-    var stmtIndex = macros.high(result)
+    var stmtIndex = result.len-1
     result[stmtIndex] = body
 
     # Parse through EOL again

tests/template/twhen_gensym.nim

@@ -0,0 +1,13 @@
+discard """
+  output: "hi"
+"""
+
+# bug #2670
+template testTemplate(b: bool): stmt =
+    when b:
+        var a = "hi"
+    else:
+        var a = 5
+    echo a
+
+testTemplate(true)

tests/template/twrongsymkind.nim

@@ -0,0 +1,20 @@
+discard """
+  errormsg: "cannot use symbol of kind 'var' as a 'param'"
+  line: 20
+"""
+
+# bug #3158
+
+type
+  MyData = object
+      x: int
+
+template newDataWindow(data: ref MyData): stmt =
+    proc testProc(data: ref MyData) =
+        echo "Hello, ", data.x
+    testProc(data)
+
+var d: ref MyData
+new(d)
+d.x = 10
+newDataWindow(d)

tests/vm/tforwardproc.nim

@@ -0,0 +1,17 @@
+discard """
+  errormsg: "cannot evaluate at compile time: initArray"
+  line: 11
+"""
+
+# bug #3066
+
+proc initArray(): array[10, int]
+
+const
+  someTable = initArray()
+
+proc initArray(): array[10, int] =
+  for f in 0..<10:
+    result[f] = 3
+
+when isMainModule: echo repr(someTable)

tests/vm/tstaticprintseq.nim

@@ -18,7 +18,8 @@ bb
 aa
 bb
 24
-2147483647 2147483647'''
+2147483647 2147483647
+5'''
 """
 
 const s = @[1,2,3]
@@ -80,3 +81,12 @@ static:
 static:
   var foo2 = int32.high
   echo foo2, " ", int32.high
+
+# bug #1329
+
+static:
+    var a: ref int
+    new(a)
+    a[] = 5
+
+    echo a[]

todo.txt

@@ -1,7 +1,6 @@
 version 0.11.4
 ==============
 
-- ``unsafeAddr``
 - document special cased varargs[untyped] and varargs[typed]
 
 - The remaining bugs of the lambda lifting pass that is responsible to enable
@@ -11,7 +10,6 @@ version 0.11.4
 - make '--implicitStatic:on' the default; then we can also clean up the
   'static[T]' mess in the compiler!
 
-- Mark the 'parallel' statement as experimental.
 - add "all threads are blocked" detection to 'spawn'
 - Deprecate ``immediate`` for templates and macros
 - make 'nil' work for 'add':
@@ -62,10 +60,8 @@ Bugs
 
 - VM: Pegs do not work at compile-time
 - VM: ptr/ref T cannot work in general
-- scopes are still broken for generic instantiation!
 - blocks can "export" an identifier but the CCG generates {} for them ...
 - ConcreteTypes in a 'case' means we don't check for duplicated case branches
-- typedesc matches a generic type T!
 
 
 version 0.9.x
@@ -80,8 +76,6 @@ version 0.9.x
 - implement 'bits' pragmas
 - we need a magic thisModule symbol
 - optimize 'genericReset'; 'newException' leads to code bloat
-- The 'do' notation might be trimmed so that its only purpose is to pass
-  multiple multi line constructs to a macro.
 
 
 version 0.9.X

web/news.txt

@@ -44,6 +44,13 @@ News
     been removed.
   - ``macros.high`` never worked and the manual says ``high`` cannot be
     overloaded, so we removed it with no deprecation cycle.
+  - To use the ``parallel`` statement you now have to
+    use the ``--experimental`` mode.
+  - Toplevel procs of calling convention ``closure`` never worked reliably
+    and are now deprecated and will be removed from the language. Instead you
+    have to insert type conversions
+    like ``(proc (a, b: int) {.closure.})(myToplevelProc)`` if necessary.
+
 
 
   Library additions
@@ -57,6 +64,11 @@ News
   Language Additions
   ------------------
 
+  - ``system.unsafeAddr`` can be used to access the address of a ``let``
+    variable or parameter for C interoperability. Since technically this
+    makes parameters and ``let`` variables mutable, it is considered even more
+    unsafe than the ordinary ``addr`` builtin.
+
 
   Bugfixes
   --------

web/question.txt

@@ -113,6 +113,7 @@ General FAQ
   - TextMate: Available in bundle installer (`Repository <https://github.com/textmate/nim.tmbundle>`_)
   - Sublime Text: Available via Package Control (`Repository <https://github.com/Varriount/NimLime>`_)
   - LiClipse: http://www.liclipse.com/ (Eclipse based plugin)
+  - Howl: Included
 
 
 .. container:: standout