Merge branch 'devel'

.gitignore

@@ -41,4 +41,4 @@ xcuserdata/
 /testresults.html
 /testresults.json
 testament.db
-/csources/
+/csources

.travis.yml

@@ -0,0 +1,35 @@
+sudo: false
+language: c
+os:
+  - linux
+addons:
+  apt:
+    packages:
+    - libcurl4-openssl-dev
+    - libsdl1.2-dev
+    - libgc-dev
+before_script:
+  - set -e
+  - wget http://flatassembler.net/fasm-1.71.39.tgz
+  - tar xvf fasm-1.71.39.tgz
+  - git clone --depth 1 https://github.com/nim-lang/csources.git
+  - cd csources
+  - sh build.sh
+  - cd ..
+  - sed -i -e 's,cc = gcc,cc = clang,' config/nim.cfg
+  - export PATH=$(pwd)/bin:$(pwd)/fasm:$PATH
+script:
+  - nim c koch
+  - ./koch boot
+  - ./koch boot -d:release
+  - nim e install_nimble.nims
+  - nimble update
+  - nimble install zip
+  - nimble install opengl
+  - nimble install sdl1
+  - nimble install jester
+  - nimble install niminst
+  - nim c --taintMode:on tests/testament/tester
+  - tests/testament/tester --pedantic all
+  - ./koch csource
+  - ./koch xz

appveyor.yml

@@ -0,0 +1,36 @@
+clone_depth: 5
+
+artifacts:
+  - path: bin\nim.exe
+
+platform:
+  - x64
+
+before_build:
+  - git log -1
+  - C:\msys64\usr\bin\bash -lc "pacman --noconfirm -S zlib-devel"
+  - appveyor DownloadFile http://nim-lang.org/download/dlls.zip
+  - 7z e dlls.zip -odlls
+  - del dlls\libcurl.dll
+  - appveyor DownloadFile http://flatassembler.net/fasmw17139.zip
+  - 7z e fasmw17139.zip -obin fasm.exe
+
+build_script:
+  - SET PATH=C:\msys64\mingw64\bin;dlls;bin;%PATH%
+  - gcc -v
+  - git clone -q --depth 1 https://github.com/nim-lang/csources
+  - cd csources
+  - build64.bat
+  - cd ..
+  - nim c koch
+  - koch boot
+  - koch boot -d:release
+
+before_test:
+  - nim e install_nimble.nims
+  - nimble update
+  - nimble install zip
+
+test_script:
+  - nim c --taintMode:on tests/testament/tester
+  - tests\testament\tester --pedantic all

bootstrap.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -e
+set -x
+
+if [ ! -e csources/.git ]; then
+	git clone --depth 1 git://github.com/nim-lang/csources.git csources
+fi
+
+cd "csources"
+sh build.sh
+cd ".."
+
+./bin/nim c koch
+./koch boot -d:release
+./koch geninstall
+
+set +x
+
+echo
+echo 'Install Nim using "./install.sh <dir>" or "sudo ./install.sh <dir>".'
+
+exit 0

build.sh

@@ -1,19 +0,0 @@
-#!/bin/sh
-set -e
-set -x
-
-if [ ! -d "csources" ]; then
-	git clone --depth 1 https://github.com/nim-lang/csources.git
-fi
-
-cd "csources"
-sh build.sh
-cd ".."
-
-./bin/nim c koch
-./koch boot -d:release
-
-cp -f install.sh.template install.sh
-chmod +x install.sh
-
-exit 0

compiler.nimble

@@ -1,6 +1,6 @@
 [Package]
 name = "compiler"
-version = "0.10.3"
+version = "0.11.3"
 author = "Andreas Rumpf"
 description = "Compiler package providing the compiler sources as a library."
 license = "MIT"
@@ -8,4 +8,4 @@ license = "MIT"
 InstallDirs = "doc, compiler"
 
 [Deps]
-Requires: "nim >= 0.10.3"
+Requires: "nim >= 0.11.3"

compiler/aliases.nim

@@ -146,7 +146,7 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
     # go down recursively; this is quite demanding:
     const
       Ix0Kinds = {nkDotExpr, nkBracketExpr, nkObjUpConv, nkObjDownConv,
-                  nkCheckedFieldExpr}
+                  nkCheckedFieldExpr, nkHiddenAddr}
       Ix1Kinds = {nkHiddenStdConv, nkHiddenSubConv, nkConv}
       DerefKinds = {nkHiddenDeref, nkDerefExpr}
     case b.kind

compiler/ast.nim

@@ -10,7 +10,7 @@
 # abstract syntax tree + symbol table
 
 import
-  msgs, hashes, nversion, options, strutils, crc, ropes, idents, lists,
+  msgs, hashes, nversion, options, strutils, securehash, ropes, idents, lists,
   intsets, idgen
 
 type
@@ -291,12 +291,13 @@ const
   sfNoForward* = sfRegister
     # forward declarations are not required (per module)
 
-  sfNoRoot* = sfBorrow # a local variable is provably no root so it doesn't
-                       # require RC ops
   sfCompileToCpp* = sfInfixCall       # compile the module as C++ code
   sfCompileToObjc* = sfNamedParamCall # compile the module as Objective-C code
   sfExperimental* = sfOverriden       # module uses the .experimental switch
   sfGoto* = sfOverriden               # var is used for 'goto' code generation
+  sfWrittenTo* = sfBorrow             # param is assigned to
+  sfEscapes* = sfProcvar              # param escapes
+  sfBase* = sfDiscriminant
 
 const
   # getting ready for the future expr/stmt merge
@@ -423,6 +424,7 @@ type
                 # but unfortunately it has measurable impact for compilation
                 # efficiency
     nfTransf,   # node has been transformed
+    nfNoRewrite # node should not be transformed anymore
     nfSem       # node has been checked for semantics
     nfLL        # node has gone through lambda lifting
     nfDotField  # the call can use a dot operator
@@ -475,6 +477,8 @@ type
                       # wildcard type.
     tfHasAsgn         # type has overloaded assignment operator
     tfBorrowDot       # distinct type borrows '.'
+    tfTriggersCompileTime # uses the NimNode type which make the proc
+                          # implicitly '.compiletime'
 
   TTypeFlags* = set[TTypeFlag]
 
@@ -521,6 +525,12 @@ const
   tfUnion* = tfNoSideEffect
   tfGcSafe* = tfThread
   tfObjHasKids* = tfEnumHasHoles
+  tfOldSchoolExprStmt* = tfVarargs # for now used to distinguish \
+    # 'varargs[expr]' from 'varargs[untyped]'. Eventually 'expr' will be
+    # deprecated and this mess can be cleaned up.
+  tfVoid* = tfVarargs # for historical reasons we conflated 'void' with
+                      # 'empty' ('@[]' has the type 'seq[empty]').
+  tfReturnsNew* = tfInheritable
   skError* = skUnknown
 
   # type flags that are essential for type equality:
@@ -529,40 +539,52 @@ const
 type
   TMagic* = enum # symbols that require compiler magic:
     mNone,
-    mDefined, mDefinedInScope, mCompiles,
+    mDefined, mDefinedInScope, mCompiles, mArrGet, mArrPut, mAsgn,
     mLow, mHigh, mSizeOf, mTypeTrait, mIs, mOf, mAddr, mTypeOf, mRoof, mPlugin,
     mEcho, mShallowCopy, mSlurp, mStaticExec,
     mParseExprToAst, mParseStmtToAst, mExpandToAst, mQuoteAst,
-    mUnaryLt, mInc, mDec, mOrd, mNew, mNewFinalize, mNewSeq, mLengthOpenArray,
+    mUnaryLt, mInc, mDec, mOrd,
-    mLengthStr, mLengthArray, mLengthSeq, mXLenStr, mXLenSeq,
+    mNew, mNewFinalize, mNewSeq,
+    mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq,
+    mXLenStr, mXLenSeq,
     mIncl, mExcl, mCard, mChr,
     mGCref, mGCunref,
-
+    mAddI, mSubI, mMulI, mDivI, mModI,
-    mAddI, mSubI, mMulI, mDivI, mModI, mAddI64, mSubI64, mMulI64,
+    mSucc, mPred,
-    mDivI64, mModI64, mSucc, mPred,
     mAddF64, mSubF64, mMulF64, mDivF64,
-
+    mShrI, mShlI, mBitandI, mBitorI, mBitxorI,
-    mShrI, mShlI, mBitandI, mBitorI, mBitxorI, mMinI, mMaxI,
+    mMinI, mMaxI,
-    mShrI64, mShlI64, mBitandI64, mBitorI64, mBitxorI64,
+    mMinF64, mMaxF64,
-    mMinF64, mMaxF64, mAddU, mSubU, mMulU,
+    mAddU, mSubU, mMulU, mDivU, mModU,
-    mDivU, mModU, mEqI, mLeI,
+    mEqI, mLeI, mLtI,
-    mLtI,
+    mEqF64, mLeF64, mLtF64,
-    mEqI64, mLeI64, mLtI64, mEqF64, mLeF64, mLtF64,
+    mLeU, mLtU,
-    mLeU, mLtU, mLeU64, mLtU64,
+    mLeU64, mLtU64,
-    mEqEnum, mLeEnum, mLtEnum, mEqCh, mLeCh, mLtCh, mEqB, mLeB, mLtB, mEqRef,
+    mEqEnum, mLeEnum, mLtEnum,
-    mEqUntracedRef, mLePtr, mLtPtr, mEqCString, mXor, mEqProc, mUnaryMinusI,
+    mEqCh, mLeCh, mLtCh,
-    mUnaryMinusI64, mAbsI, mAbsI64, mNot,
+    mEqB, mLeB, mLtB,
+    mEqRef, mEqUntracedRef, mLePtr, mLtPtr, mEqCString,
+    mXor, mEqProc,
+    mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot,
     mUnaryPlusI, mBitnotI,
-    mBitnotI64, mUnaryPlusF64, mUnaryMinusF64, mAbsF64, mZe8ToI, mZe8ToI64,
+    mUnaryPlusF64, mUnaryMinusF64, mAbsF64,
-    mZe16ToI, mZe16ToI64, mZe32ToI64, mZeIToI64, mToU8, mToU16, mToU32,
+    mZe8ToI, mZe8ToI64,
-    mToFloat, mToBiggestFloat, mToInt, mToBiggestInt, mCharToStr, mBoolToStr,
+    mZe16ToI, mZe16ToI64,
-    mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr,
+    mZe32ToI64, mZeIToI64,
-    mAnd, mOr, mEqStr, mLeStr, mLtStr, mEqSet, mLeSet, mLtSet, mMulSet,
+    mToU8, mToU16, mToU32,
-    mPlusSet, mMinusSet, mSymDiffSet, mConStrStr, mSlice,
+    mToFloat, mToBiggestFloat,
+    mToInt, mToBiggestInt,
+    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr,
+    mStrToStr, mEnumToStr,
+    mAnd, mOr,
+    mEqStr, mLeStr, mLtStr,
+    mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet, mSymDiffSet,
+    mConStrStr, mSlice,
     mDotDot, # this one is only necessary to give nice compile time warnings
     mFields, mFieldPairs, mOmpParFor,
     mAppendStrCh, mAppendStrStr, mAppendSeqElem,
-    mInRange, mInSet, mRepr, mExit, mSetLengthStr, mSetLengthSeq,
+    mInRange, mInSet, mRepr, mExit,
+    mSetLengthStr, mSetLengthSeq,
     mIsPartOf, mAstToStr, mParallel,
     mSwap, mIsNil, mArrToSeq, mCopyStr, mCopyStrLast,
     mNewString, mNewStringOfCap, mParseBiggestFloat,
@@ -584,42 +606,53 @@ type
     mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, mNLineInfo,
     mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, mIdentToStr,
     mNBindSym, mLocals, mNCallSite,
-    mEqIdent, mEqNimrodNode, mNHint, mNWarning, mNError,
+    mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl,
-    mInstantiationInfo, mGetTypeInfo, mNGenSym
+    mNHint, mNWarning, mNError,
+    mInstantiationInfo, mGetTypeInfo, mNGenSym,
+    mNimvm
 
 # things that we can evaluate safely at compile time, even if not asked for it:
 const
   ctfeWhitelist* = {mNone, mUnaryLt, mSucc,
     mPred, mInc, mDec, mOrd, mLengthOpenArray,
     mLengthStr, mLengthArray, mLengthSeq, mXLenStr, mXLenSeq,
+    mArrGet, mArrPut, mAsgn,
     mIncl, mExcl, mCard, mChr,
-    mAddI, mSubI, mMulI, mDivI, mModI, mAddI64, mSubI64, mMulI64,
+    mAddI, mSubI, mMulI, mDivI, mModI,
-    mDivI64, mModI64, mAddF64, mSubF64, mMulF64, mDivF64,
+    mAddF64, mSubF64, mMulF64, mDivF64,
-    mShrI, mShlI, mBitandI, mBitorI, mBitxorI, mMinI, mMaxI,
+    mShrI, mShlI, mBitandI, mBitorI, mBitxorI,
-    mShrI64, mShlI64, mBitandI64, mBitorI64, mBitxorI64,
+    mMinI, mMaxI,
-    mMinF64, mMaxF64, mAddU, mSubU, mMulU,
+    mMinF64, mMaxF64,
-    mDivU, mModU, mEqI, mLeI,
+    mAddU, mSubU, mMulU, mDivU, mModU,
-    mLtI,
+    mEqI, mLeI, mLtI,
-    mEqI64, mLeI64, mLtI64, mEqF64, mLeF64, mLtF64,
+    mEqF64, mLeF64, mLtF64,
-    mLeU, mLtU, mLeU64, mLtU64,
+    mLeU, mLtU,
-    mEqEnum, mLeEnum, mLtEnum, mEqCh, mLeCh, mLtCh, mEqB, mLeB, mLtB, mEqRef,
+    mLeU64, mLtU64,
-    mEqProc, mEqUntracedRef, mLePtr, mLtPtr, mEqCString, mXor, mUnaryMinusI,
+    mEqEnum, mLeEnum, mLtEnum,
-    mUnaryMinusI64, mAbsI, mAbsI64, mNot,
+    mEqCh, mLeCh, mLtCh,
-    mUnaryPlusI, mBitnotI,
+    mEqB, mLeB, mLtB,
-    mBitnotI64, mUnaryPlusF64, mUnaryMinusF64, mAbsF64, mZe8ToI, mZe8ToI64,
+    mEqRef, mEqProc, mEqUntracedRef, mLePtr, mLtPtr, mEqCString, mXor,
-    mZe16ToI, mZe16ToI64, mZe32ToI64, mZeIToI64, mToU8, mToU16, mToU32,
+    mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI,
-    mToFloat, mToBiggestFloat, mToInt, mToBiggestInt, mCharToStr, mBoolToStr,
+    mUnaryPlusF64, mUnaryMinusF64, mAbsF64,
-    mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr,
+    mZe8ToI, mZe8ToI64,
-    mAnd, mOr, mEqStr, mLeStr, mLtStr, mEqSet, mLeSet, mLtSet, mMulSet,
+    mZe16ToI, mZe16ToI64,
-    mPlusSet, mMinusSet, mSymDiffSet, mConStrStr,
+    mZe32ToI64, mZeIToI64,
-    mAppendStrCh, mAppendStrStr, mAppendSeqElem,
+    mToU8, mToU16, mToU32,
+    mToFloat, mToBiggestFloat,
+    mToInt, mToBiggestInt,
+    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr,
+    mStrToStr, mEnumToStr,
+    mAnd, mOr,
+    mEqStr, mLeStr, mLtStr,
+    mEqSet, mLeSet, mLtSet, mMulSet, mPlusSet, mMinusSet, mSymDiffSet,
+    mConStrStr, mAppendStrCh, mAppendStrStr, mAppendSeqElem,
     mInRange, mInSet, mRepr,
     mCopyStr, mCopyStrLast}
   # magics that require special semantic checking and
   # thus cannot be overloaded (also documented in the spec!):
   SpecialSemMagics* = {
     mDefined, mDefinedInScope, mCompiles, mLow, mHigh, mSizeOf, mIs, mOf,
-    mEcho, mShallowCopy, mExpandToAst, mParallel, mSpawn, mAstToStr}
+    mShallowCopy, mExpandToAst, mParallel, mSpawn, mAstToStr}
 
 type
   PNode* = ref TNode
@@ -679,6 +712,7 @@ type
     lfSingleUse               # no location yet and will only be used once
   TStorageLoc* = enum
     OnUnknown,                # location is unknown (stack, heap or static)
+    OnStatic,                 # in a static section
     OnStack,                  # location is on hardware stack
     OnHeap                    # location is on heap or global
                               # (reference counting needed)
@@ -704,6 +738,8 @@ type
     name*: Rope
     path*: PNode              # can be a string literal!
 
+  CompilesId* = int ## id that is used for the caching logic within
+                    ## ``system.compiles``. See the seminst module.
   TInstantiation* = object
     sym*: PSym
     concreteTypes*: seq[PType]
@@ -711,6 +747,7 @@ type
                               # needed in caas mode for purging the cache
                               # XXX: it's possible to switch to a
                               # simple ref count here
+    compilesId*: CompilesId
 
   PInstantiation* = ref TInstantiation
 
@@ -747,6 +784,7 @@ type
       tab*: TStrTable         # interface table for modules
     of skLet, skVar, skField, skForVar:
       guard*: PSym
+      bitsize*: int
     else: nil
     magic*: TMagic
     typ*: PType
@@ -842,7 +880,7 @@ type
     data*: TIdNodePairSeq
 
   TNodePair* = object
-    h*: THash                 # because it is expensive to compute!
+    h*: Hash                 # because it is expensive to compute!
     key*: PNode
     val*: int
 
@@ -947,6 +985,9 @@ proc add*(father, son: PNode) =
 proc `[]`*(n: PNode, i: int): PNode {.inline.} =
   result = n.sons[i]
 
+template `-|`*(b, s: expr): expr =
+  (if b >= 0: b else: s.len + b)
+
 # son access operators with support for negative indices
 template `{}`*(n: PNode, i: int): expr = n[i -| n]
 template `{}=`*(n: PNode, i: int, s: PNode): stmt =
@@ -1000,7 +1041,8 @@ proc newSym*(symKind: TSymKind, name: PIdent, owner: PSym,
   result.id = getID()
   when debugIds:
     registerId(result)
-  #if result.id < 2000:
+  #if result.id == 93289:
+  #  writeStacktrace()
   #  MessageOut(name.s & " has id: " & toString(result.id))
 
 var emptyNode* = newNode(nkEmpty)
@@ -1198,23 +1240,11 @@ proc newSons*(father: PType, length: int) =
   else:
     setLen(father.sons, length)
 
-proc sonsLen*(n: PType): int =
+proc sonsLen*(n: PType): int = n.sons.len
-  if isNil(n.sons): result = 0
+proc len*(n: PType): int = n.sons.len
-  else: result = len(n.sons)
+proc sonsLen*(n: PNode): int = n.sons.len
-
+proc lastSon*(n: PNode): PNode = n.sons[^1]
-proc len*(n: PType): int =
+proc lastSon*(n: PType): PType = n.sons[^1]
-  if isNil(n.sons): result = 0
-  else: result = len(n.sons)
-
-proc sonsLen*(n: PNode): int =
-  if isNil(n.sons): result = 0
-  else: result = len(n.sons)
-
-proc lastSon*(n: PNode): PNode =
-  result = n.sons[sonsLen(n) - 1]
-
-proc lastSon*(n: PType): PType =
-  result = n.sons[sonsLen(n) - 1]
 
 proc assignType*(dest, src: PType) =
   dest.kind = src.kind
@@ -1356,8 +1386,11 @@ proc propagateToOwner*(owner, elem: PType) =
       o2.flags.incl tfHasAsgn
       owner.flags.incl tfHasAsgn
 
+  if tfTriggersCompileTime in elem.flags:
+    owner.flags.incl tfTriggersCompileTime
+
   if owner.kind notin {tyProc, tyGenericInst, tyGenericBody,
-                       tyGenericInvocation}:
+                       tyGenericInvocation, tyPtr}:
     let elemB = elem.skipTypes({tyGenericInst})
     if elemB.isGCedMem or tfHasGCedMem in elemB.flags:
       # for simplicity, we propagate this flag even to generics. We then
@@ -1496,6 +1529,9 @@ proc getFloat*(a: PNode): BiggestFloat =
 proc getStr*(a: PNode): string =
   case a.kind
   of nkStrLit..nkTripleStrLit: result = a.strVal
+  of nkNilLit:
+    # let's hope this fixes more problems than it creates:
+    result = nil
   else:
     internalError(a.info, "getStr")
     result = ""
@@ -1553,6 +1589,14 @@ proc makeStmtList*(n: PNode): PNode =
     result = newNodeI(nkStmtList, n.info)
     result.add n
 
+proc skipStmtList*(n: PNode): PNode =
+  if n.kind in {nkStmtList, nkStmtListExpr}:
+    for i in 0 .. n.len-2:
+      if n[i].kind notin {nkEmpty, nkCommentStmt}: return n
+    result = n.lastSon
+  else:
+    result = n
+
 proc createMagic*(name: string, m: TMagic): PSym =
   result = newSym(skProc, getIdent(name), nil, unknownLineInfo())
   result.magic = m
@@ -1560,3 +1604,10 @@ proc createMagic*(name: string, m: TMagic): PSym =
 let
   opNot* = createMagic("not", mNot)
   opContains* = createMagic("contains", mInSet)
+
+when false:
+  proc containsNil*(n: PNode): bool =
+    # only for debugging
+    if n.isNil: return true
+    for i in 0 ..< n.safeLen:
+      if n[i].containsNil: return true

compiler/astalgo.nim

@@ -14,7 +14,7 @@
 import
   ast, hashes, intsets, strutils, options, msgs, ropes, idents, rodutils
 
-proc hashNode*(p: RootRef): THash
+proc hashNode*(p: RootRef): Hash
 proc treeToYaml*(n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope
   # Convert a tree into its YAML representation; this is used by the
   # YAML code generator and it is invaluable for debugging purposes.
@@ -49,7 +49,7 @@ proc strTableGet*(t: TStrTable, name: PIdent): PSym
 
 type
   TTabIter*{.final.} = object # consider all fields here private
-    h*: THash                 # current hash
+    h*: Hash                  # current hash
 
 proc initTabIter*(ti: var TTabIter, tab: TStrTable): PSym
 proc nextIter*(ti: var TTabIter, tab: TStrTable): PSym
@@ -65,7 +65,7 @@ proc nextIter*(ti: var TTabIter, tab: TStrTable): PSym
 
 type
   TIdentIter*{.final.} = object # iterator over all syms with same identifier
-    h*: THash                   # current hash
+    h*: Hash                    # current hash
     name*: PIdent
 
 
@@ -94,7 +94,7 @@ proc getSymFromList*(list: PNode, ident: PIdent, start: int = 0): PSym
 proc lookupInRecord*(n: PNode, field: PIdent): PSym
 proc getModule*(s: PSym): PSym
 proc mustRehash*(length, counter: int): bool
-proc nextTry*(h, maxHash: THash): THash {.inline.}
+proc nextTry*(h, maxHash: Hash): Hash {.inline.}
 
 # ------------- table[int, int] ---------------------------------------------
 const
@@ -196,7 +196,7 @@ proc getSymFromList(list: PNode, ident: PIdent, start: int = 0): PSym =
     else: internalError(list.info, "getSymFromList")
   result = nil
 
-proc hashNode(p: RootRef): THash =
+proc hashNode(p: RootRef): Hash =
   result = hash(cast[pointer](p))
 
 proc mustRehash(length, counter: int): bool =
@@ -452,7 +452,7 @@ proc debug(n: PSym) =
   elif n.kind == skUnknown:
     msgWriteln("skUnknown")
   else:
-    #writeln(stdout, $symToYaml(n, 0, 1))
+    #writeLine(stdout, $symToYaml(n, 0, 1))
     msgWriteln("$1_$2: $3, $4, $5, $6" % [
       n.name.s, $n.id, $flagsToStr(n.flags), $flagsToStr(n.loc.flags),
       $lineInfoToStr(n.info), $n.kind])
@@ -466,7 +466,7 @@ proc debug(n: PNode) =
 const
   EmptySeq = @[]
 
-proc nextTry(h, maxHash: THash): THash =
+proc nextTry(h, maxHash: Hash): Hash =
   result = ((5 * h) + 1) and maxHash
   # For any initial h in range(maxHash), repeating that maxHash times
   # generates each int in range(maxHash) exactly once (see any text on
@@ -474,7 +474,7 @@ proc nextTry(h, maxHash: THash): THash =
 
 proc objectSetContains(t: TObjectSet, obj: RootRef): bool =
   # returns true whether n is in t
-  var h: THash = hashNode(obj) and high(t.data) # start with real hash value
+  var h: Hash = hashNode(obj) and high(t.data) # start with real hash value
   while t.data[h] != nil:
     if t.data[h] == obj:
       return true
@@ -482,7 +482,7 @@ proc objectSetContains(t: TObjectSet, obj: RootRef): bool =
   result = false
 
 proc objectSetRawInsert(data: var TObjectSeq, obj: RootRef) =
-  var h: THash = hashNode(obj) and high(data)
+  var h: Hash = hashNode(obj) and high(data)
   while data[h] != nil:
     assert(data[h] != obj)
     h = nextTry(h, high(data))
@@ -503,7 +503,7 @@ proc objectSetIncl(t: var TObjectSet, obj: RootRef) =
 
 proc objectSetContainsOrIncl(t: var TObjectSet, obj: RootRef): bool =
   # returns true if obj is already in the string table:
-  var h: THash = hashNode(obj) and high(t.data)
+  var h: Hash = hashNode(obj) and high(t.data)
   while true:
     var it = t.data[h]
     if it == nil: break
@@ -520,7 +520,7 @@ proc objectSetContainsOrIncl(t: var TObjectSet, obj: RootRef): bool =
   result = false
 
 proc tableRawGet(t: TTable, key: RootRef): int =
-  var h: THash = hashNode(key) and high(t.data) # start with real hash value
+  var h: Hash = hashNode(key) and high(t.data) # start with real hash value
   while t.data[h].key != nil:
     if t.data[h].key == key:
       return h
@@ -529,7 +529,7 @@ proc tableRawGet(t: TTable, key: RootRef): int =
 
 proc tableSearch(t: TTable, key, closure: RootRef,
                  comparator: TCmpProc): RootRef =
-  var h: THash = hashNode(key) and high(t.data) # start with real hash value
+  var h: Hash = hashNode(key) and high(t.data) # start with real hash value
   while t.data[h].key != nil:
     if t.data[h].key == key:
       if comparator(t.data[h].val, closure):
@@ -544,7 +544,7 @@ proc tableGet(t: TTable, key: RootRef): RootRef =
   else: result = nil
 
 proc tableRawInsert(data: var TPairSeq, key, val: RootRef) =
-  var h: THash = hashNode(key) and high(data)
+  var h: Hash = hashNode(key) and high(data)
   while data[h].key != nil:
     assert(data[h].key != key)
     h = nextTry(h, high(data))
@@ -569,7 +569,7 @@ proc tablePut(t: var TTable, key, val: RootRef) =
     inc(t.counter)
 
 proc strTableContains(t: TStrTable, n: PSym): bool =
-  var h: THash = n.name.h and high(t.data) # start with real hash value
+  var h: Hash = n.name.h and high(t.data) # start with real hash value
   while t.data[h] != nil:
     if (t.data[h] == n):
       return true
@@ -577,7 +577,7 @@ proc strTableContains(t: TStrTable, n: PSym): bool =
   result = false
 
 proc strTableRawInsert(data: var TSymSeq, n: PSym) =
-  var h: THash = n.name.h and high(data)
+  var h: Hash = n.name.h and high(data)
   if sfImmediate notin n.flags:
     # fast path:
     while data[h] != nil:
@@ -606,7 +606,7 @@ proc strTableRawInsert(data: var TSymSeq, n: PSym) =
 
 proc symTabReplaceRaw(data: var TSymSeq, prevSym: PSym, newSym: PSym) =
   assert prevSym.name.h == newSym.name.h
-  var h: THash = prevSym.name.h and high(data)
+  var h: Hash = prevSym.name.h and high(data)
   while data[h] != nil:
     if data[h] == prevSym:
       data[h] = newSym
@@ -640,7 +640,7 @@ proc strTableIncl*(t: var TStrTable, n: PSym): bool {.discardable.} =
   # It is essential that `n` is written nevertheless!
   # This way the newest redefinition is picked by the semantic analyses!
   assert n.name != nil
-  var h: THash = n.name.h and high(t.data)
+  var h: Hash = n.name.h and high(t.data)
   var replaceSlot = -1
   while true:
     var it = t.data[h]
@@ -666,7 +666,7 @@ proc strTableIncl*(t: var TStrTable, n: PSym): bool {.discardable.} =
   result = false
 
 proc strTableGet(t: TStrTable, name: PIdent): PSym =
-  var h: THash = name.h and high(t.data)
+  var h: Hash = name.h and high(t.data)
   while true:
     result = t.data[h]
     if result == nil: break
@@ -694,7 +694,7 @@ proc nextIdentIter(ti: var TIdentIter, tab: TStrTable): PSym =
 
 proc nextIdentExcluding*(ti: var TIdentIter, tab: TStrTable,
                          excluding: IntSet): PSym =
-  var h: THash = ti.h and high(tab.data)
+  var h: Hash = ti.h and high(tab.data)
   var start = h
   result = tab.data[h]
   while result != nil:
@@ -743,7 +743,7 @@ proc hasEmptySlot(data: TIdPairSeq): bool =
   result = false
 
 proc idTableRawGet(t: TIdTable, key: int): int =
-  var h: THash
+  var h: Hash
   h = key and high(t.data)    # start with real hash value
   while t.data[h].key != nil:
     if t.data[h].key.id == key:
@@ -772,7 +772,7 @@ iterator pairs*(t: TIdTable): tuple[key: int, value: RootRef] =
       yield (t.data[i].key.id, t.data[i].val)
 
 proc idTableRawInsert(data: var TIdPairSeq, key: PIdObj, val: RootRef) =
-  var h: THash
+  var h: Hash
   h = key.id and high(data)
   while data[h].key != nil:
     assert(data[h].key.id != key.id)
@@ -805,7 +805,7 @@ iterator idTablePairs*(t: TIdTable): tuple[key: PIdObj, val: RootRef] =
     if not isNil(t.data[i].key): yield (t.data[i].key, t.data[i].val)
 
 proc idNodeTableRawGet(t: TIdNodeTable, key: PIdObj): int =
-  var h: THash
+  var h: Hash
   h = key.id and high(t.data) # start with real hash value
   while t.data[h].key != nil:
     if t.data[h].key.id == key.id:
@@ -824,7 +824,7 @@ proc idNodeTableGetLazy*(t: TIdNodeTable, key: PIdObj): PNode =
     result = idNodeTableGet(t, key)
 
 proc idNodeTableRawInsert(data: var TIdNodePairSeq, key: PIdObj, val: PNode) =
-  var h: THash
+  var h: Hash
   h = key.id and high(data)
   while data[h].key != nil:
     assert(data[h].key.id != key.id)
@@ -863,7 +863,7 @@ proc initIITable(x: var TIITable) =
   for i in countup(0, StartSize - 1): x.data[i].key = InvalidKey
 
 proc iiTableRawGet(t: TIITable, key: int): int =
-  var h: THash
+  var h: Hash
   h = key and high(t.data)    # start with real hash value
   while t.data[h].key != InvalidKey:
     if t.data[h].key == key: return h
@@ -876,7 +876,7 @@ proc iiTableGet(t: TIITable, key: int): int =
   else: result = InvalidKey
 
 proc iiTableRawInsert(data: var TIIPairSeq, key, val: int) =
-  var h: THash
+  var h: Hash
   h = key and high(data)
   while data[h].key != InvalidKey:
     assert(data[h].key != key)

compiler/ccgcalls.nim

@@ -159,6 +159,17 @@ proc genArgNoParam(p: BProc, n: PNode): Rope =
     initLocExprSingleUse(p, n, a)
     result = rdLoc(a)
 
+template genParamLoop(params) {.dirty.} =
+  if i < sonsLen(typ):
+    assert(typ.n.sons[i].kind == nkSym)
+    let paramType = typ.n.sons[i]
+    if not paramType.typ.isCompileTimeOnly:
+      if params != nil: add(params, ~", ")
+      add(params, genArg(p, ri.sons[i], paramType.sym, ri))
+  else:
+    if params != nil: add(params, ~", ")
+    add(params, genArgNoParam(p, ri.sons[i]))
+
 proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var op: TLoc
   # this is a hotspot in the compiler
@@ -170,13 +181,7 @@ proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   assert(sonsLen(typ) == sonsLen(typ.n))
   var length = sonsLen(ri)
   for i in countup(1, length - 1):
-    if ri.sons[i].typ.isCompileTimeOnly: continue
+    genParamLoop(params)
-    if params != nil: add(params, ~", ")
-    if i < sonsLen(typ):
-      assert(typ.n.sons[i].kind == nkSym)
-      add(params, genArg(p, ri.sons[i], typ.n.sons[i].sym, ri))
-    else:
-      add(params, genArgNoParam(p, ri.sons[i]))
   fixupCall(p, le, ri, d, op.r, params)
 
 proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
@@ -198,13 +203,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var length = sonsLen(ri)
   for i in countup(1, length - 1):
     assert(sonsLen(typ) == sonsLen(typ.n))
-    if ri.sons[i].typ.isCompileTimeOnly: continue
+    genParamLoop(pl)
-    if i < sonsLen(typ):
-      assert(typ.n.sons[i].kind == nkSym)
-      add(pl, genArg(p, ri.sons[i], typ.n.sons[i].sym, ri))
-    else:
-      add(pl, genArgNoParam(p, ri.sons[i]))
-    if i < length - 1: add(pl, ~", ")
 
   template genCallPattern {.dirty.} =
     lineF(p, cpsStmts, callPattern & ";$n", [op.r, pl, pl.addComma, rawProc])
@@ -241,13 +240,14 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
     genCallPattern()
 
 proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
-  if ri.sons[i].typ.isCompileTimeOnly:
+  if i < sonsLen(typ):
-    result = nil
-  elif i < sonsLen(typ):
     # 'var T' is 'T&' in C++. This means we ignore the request of
     # any nkHiddenAddr when it's a 'var T'.
-    assert(typ.n.sons[i].kind == nkSym)
+    let paramType = typ.n.sons[i]
-    if typ.sons[i].kind == tyVar and ri.sons[i].kind == nkHiddenAddr:
+    assert(paramType.kind == nkSym)
+    if paramType.typ.isCompileTimeOnly:
+      result = nil
+    elif typ.sons[i].kind == tyVar and ri.sons[i].kind == nkHiddenAddr:
       result = genArgNoParam(p, ri.sons[i][0])
     else:
       result = genArgNoParam(p, ri.sons[i]) #, typ.n.sons[i].sym)

compiler/ccgexprs.nim

@@ -221,7 +221,7 @@ proc optAsgnLoc(a: TLoc, t: PType, field: Rope): TLoc =
 
 proc genOptAsgnTuple(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
   let newflags =
-    if src.k == locData:
+    if src.s == OnStatic:
       flags + {needToCopy}
     elif tfShallow in dest.t.flags:
       flags - {needToCopy}
@@ -238,7 +238,7 @@ proc genOptAsgnObject(p: BProc, dest, src: TLoc, flags: TAssignmentFlags,
                       t: PNode) =
   if t == nil: return
   let newflags =
-    if src.k == locData:
+    if src.s == OnStatic:
       flags + {needToCopy}
     elif tfShallow in dest.t.flags:
       flags - {needToCopy}
@@ -287,13 +287,13 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
   of tyRef:
     genRefAssign(p, dest, src, flags)
   of tySequence:
-    if needToCopy notin flags and src.k != locData:
+    if needToCopy notin flags and src.s != OnStatic:
       genRefAssign(p, dest, src, flags)
     else:
       linefmt(p, cpsStmts, "#genericSeqAssign($1, $2, $3);$n",
               addrLoc(dest), rdLoc(src), genTypeInfo(p.module, dest.t))
   of tyString:
-    if needToCopy notin flags and src.k != locData:
+    if needToCopy notin flags and src.s != OnStatic:
       genRefAssign(p, dest, src, flags)
     else:
       if dest.s == OnStack or not usesNativeGC():
@@ -347,8 +347,8 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
     else:
       useStringh(p.module)
       linefmt(p, cpsStmts,
-           "memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($1));$n",
+           "memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n",
-           rdLoc(dest), rdLoc(src))
+           rdLoc(dest), rdLoc(src), getTypeDesc(p.module, ty))
   of tyOpenArray, tyVarargs:
     # open arrays are always on the stack - really? What if a sequence is
     # passed to an open array?
@@ -413,7 +413,7 @@ proc putDataIntoDest(p: BProc, d: var TLoc, t: PType, r: Rope) =
   var a: TLoc
   if d.k != locNone:
     # need to generate an assignment here
-    initLoc(a, locData, t, OnUnknown)
+    initLoc(a, locData, t, OnStatic)
     a.r = r
     if lfNoDeepCopy in d.flags: genAssignment(p, d, a, {})
     else: genAssignment(p, d, a, {needToCopy})
@@ -424,11 +424,11 @@ proc putDataIntoDest(p: BProc, d: var TLoc, t: PType, r: Rope) =
     d.t = t
     d.r = r
 
-proc putIntoDest(p: BProc, d: var TLoc, t: PType, r: Rope) =
+proc putIntoDest(p: BProc, d: var TLoc, t: PType, r: Rope; s=OnUnknown) =
   var a: TLoc
   if d.k != locNone:
     # need to generate an assignment here
-    initLoc(a, locExpr, t, OnUnknown)
+    initLoc(a, locExpr, t, s)
     a.r = r
     if lfNoDeepCopy in d.flags: genAssignment(p, d, a, {})
     else: genAssignment(p, d, a, {needToCopy})
@@ -503,13 +503,13 @@ proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
       "$# = #addInt($#, $#);$n", "$# = #subInt($#, $#);$n",
       "$# = #mulInt($#, $#);$n", "$# = #divInt($#, $#);$n",
       "$# = #modInt($#, $#);$n",
+      "$# = #addInt($#, $#);$n", "$# = #subInt($#, $#);$n"]
+    prc64: array[mAddI..mPred, string] = [
       "$# = #addInt64($#, $#);$n", "$# = #subInt64($#, $#);$n",
       "$# = #mulInt64($#, $#);$n", "$# = #divInt64($#, $#);$n",
       "$# = #modInt64($#, $#);$n",
-      "$# = #addInt($#, $#);$n", "$# = #subInt($#, $#);$n"]
+      "$# = #addInt64($#, $#);$n", "$# = #subInt64($#, $#);$n"]
     opr: array[mAddI..mPred, string] = [
-      "($#)($# + $#)", "($#)($# - $#)", "($#)($# * $#)",
-      "($#)($# / $#)", "($#)($# % $#)",
       "($#)($# + $#)", "($#)($# - $#)", "($#)($# * $#)",
       "($#)($# / $#)", "($#)($# % $#)",
       "($#)($# + $#)", "($#)($# - $#)"]
@@ -525,16 +525,16 @@ proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
     let res = opr[m] % [getTypeDesc(p.module, t), rdLoc(a), rdLoc(b)]
     putIntoDest(p, d, e.typ, res)
   else:
-    let res = binaryArithOverflowRaw(p, t, a, b, prc[m])
+    let res = binaryArithOverflowRaw(p, t, a, b,
+                                   if t.kind == tyInt64: prc64[m] else: prc[m])
     putIntoDest(p, d, e.typ, "($#)($#)" % [getTypeDesc(p.module, t), res])
 
 proc unaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
   const
-    opr: array[mUnaryMinusI..mAbsI64, string] = [
+    opr: array[mUnaryMinusI..mAbsI, string] = [
       mUnaryMinusI: "((NI$2)-($1))",
       mUnaryMinusI64: "-($1)",
-      mAbsI: "($1 > 0? ($1) : -($1))",
+      mAbsI: "($1 > 0? ($1) : -($1))"]
-      mAbsI64: "($1 > 0? ($1) : -($1))"]
   var
     a: TLoc
     t: PType
@@ -561,11 +561,6 @@ proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       "($4)($1 ^ $2)",      # BitxorI
       "(($1 <= $2) ? $1 : $2)", # MinI
       "(($1 >= $2) ? $1 : $2)", # MaxI
-      "($4)((NU64)($1) >> (NU64)($2))", # ShrI64
-      "($4)((NU64)($1) << (NU64)($2))", # ShlI64
-      "($4)($1 & $2)",            # BitandI64
-      "($4)($1 | $2)",            # BitorI64
-      "($4)($1 ^ $2)",            # BitxorI64
       "(($1 <= $2) ? $1 : $2)", # MinF64
       "(($1 >= $2) ? $1 : $2)", # MaxF64
       "($4)((NU$3)($1) + (NU$3)($2))", # AddU
@@ -576,9 +571,6 @@ proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
       "($1 == $2)",           # EqI
       "($1 <= $2)",           # LeI
       "($1 < $2)",            # LtI
-      "($1 == $2)",           # EqI64
-      "($1 <= $2)",           # LeI64
-      "($1 < $2)",            # LtI64
       "($1 == $2)",           # EqF64
       "($1 <= $2)",           # LeF64
       "($1 < $2)",            # LtF64
@@ -638,7 +630,6 @@ proc unaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     unArithTab: array[mNot..mToBiggestInt, string] = ["!($1)", # Not
       "$1",                   # UnaryPlusI
       "($3)((NU$2) ~($1))",   # BitnotI
-      "($3)((NU$2) ~($1))",   # BitnotI64
       "$1",                   # UnaryPlusF64
       "-($1)",                # UnaryMinusF64
       "($1 > 0? ($1) : -($1))", # AbsF64; BUGFIX: fabs() makes problems
@@ -694,7 +685,7 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) =
         d.s = OnUnknown
         if tfVarIsPtr notin typ.flags and p.module.compileToCpp and
             e.kind == nkHiddenDeref:
-          putIntoDest(p, d, e.typ, rdLoc(a))
+          putIntoDest(p, d, e.typ, rdLoc(a), a.s)
           return
       of tyPtr:
         d.s = OnUnknown         # BUGFIX!
@@ -703,7 +694,7 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) =
       let typ = skipTypes(a.t, abstractInst)
       if typ.kind == tyVar and tfVarIsPtr notin typ.flags and
            e.kind == nkHiddenDeref:
-        putIntoDest(p, d, e.typ, rdLoc(a))
+        putIntoDest(p, d, e.typ, rdLoc(a), a.s)
         return
     if enforceDeref and mt == ctPtrToArray:
       # we lie about the type for better C interop: 'ptr array[3,T]' is
@@ -711,23 +702,23 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) =
       # See tmissingderef. So we get rid of the deref instead. The codegen
       # ends up using 'memcpy' for the array assignment,
       # so the '&' and '*' cancel out:
-      putIntoDest(p, d, a.t.sons[0], rdLoc(a))
+      putIntoDest(p, d, a.t.sons[0], rdLoc(a), a.s)
     else:
-      putIntoDest(p, d, e.typ, "(*$1)" % [rdLoc(a)])
+      putIntoDest(p, d, e.typ, "(*$1)" % [rdLoc(a)], a.s)
 
 proc genAddr(p: BProc, e: PNode, d: var TLoc) =
   # careful  'addr(myptrToArray)' needs to get the ampersand:
   if e.sons[0].typ.skipTypes(abstractInst).kind in {tyRef, tyPtr}:
     var a: TLoc
     initLocExpr(p, e.sons[0], a)
-    putIntoDest(p, d, e.typ, "&" & a.r)
+    putIntoDest(p, d, e.typ, "&" & a.r, a.s)
     #Message(e.info, warnUser, "HERE NEW &")
   elif mapType(e.sons[0].typ) == ctArray or isCppRef(p, e.sons[0].typ):
     expr(p, e.sons[0], d)
   else:
     var a: TLoc
     initLocExpr(p, e.sons[0], a)
-    putIntoDest(p, d, e.typ, addrLoc(a))
+    putIntoDest(p, d, e.typ, addrLoc(a), a.s)
 
 template inheritLocation(d: var TLoc, a: TLoc) =
   if d.k == locNone: d.s = a.s
@@ -754,7 +745,18 @@ proc genTupleElem(p: BProc, e: PNode, d: var TLoc) =
   of nkIntLit..nkUInt64Lit: i = int(e.sons[1].intVal)
   else: internalError(e.info, "genTupleElem")
   addf(r, ".Field$1", [rope(i)])
-  putIntoDest(p, d, ty.sons[i], r)
+  putIntoDest(p, d, ty.sons[i], r, a.s)
+
+proc lookupFieldAgain(p: BProc, ty: PType; field: PSym; r: var Rope): PSym =
+  var ty = ty
+  assert r != nil
+  while ty != nil:
+    assert(ty.kind in {tyTuple, tyObject})
+    result = lookupInRecord(ty.n, field.name)
+    if result != nil: break
+    if not p.module.compileToCpp: add(r, ".Sup")
+    ty = getUniqueType(ty.sons[0])
+  if result == nil: internalError(field.info, "genCheckedRecordField")
 
 proc genRecordField(p: BProc, e: PNode, d: var TLoc) =
   var a: TLoc
@@ -765,25 +767,18 @@ proc genRecordField(p: BProc, e: PNode, d: var TLoc) =
     # we found a unique tuple type which lacks field information
     # so we use Field$i
     addf(r, ".Field$1", [rope(f.position)])
-    putIntoDest(p, d, f.typ, r)
+    putIntoDest(p, d, f.typ, r, a.s)
   else:
-    var field: PSym = nil
+    let field = lookupFieldAgain(p, ty, f, r)
-    while ty != nil:
-      if ty.kind notin {tyTuple, tyObject}:
-        internalError(e.info, "genRecordField")
-      field = lookupInRecord(ty.n, f.name)
-      if field != nil: break
-      if not p.module.compileToCpp: add(r, ".Sup")
-      ty = getUniqueType(ty.sons[0])
-    if field == nil: internalError(e.info, "genRecordField 2 ")
     if field.loc.r == nil: internalError(e.info, "genRecordField 3")
     addf(r, ".$1", [field.loc.r])
-    putIntoDest(p, d, field.typ, r)
+    putIntoDest(p, d, field.typ, r, a.s)
   #d.s = a.s
 
 proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc)
 
-proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) =
+proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym;
+                   origTy: PType) =
   var test, u, v: TLoc
   for i in countup(1, sonsLen(e) - 1):
     var it = e.sons[i]
@@ -795,8 +790,12 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) =
     assert(disc.kind == nkSym)
     initLoc(test, locNone, it.typ, OnStack)
     initLocExpr(p, it.sons[1], u)
-    initLoc(v, locExpr, disc.typ, OnUnknown)
+    var o = obj
-    v.r = "$1.$2" % [obj, disc.sym.loc.r]
+    let d = lookupFieldAgain(p, origTy, disc.sym, o)
+    initLoc(v, locExpr, d.typ, OnUnknown)
+    v.r = o
+    v.r.add(".")
+    v.r.add(d.loc.r)
     genInExprAux(p, it, u, v, test)
     let id = nodeTableTestOrSet(p.module.dataCache,
                                newStrNode(nkStrLit, field.name.s), gBackendId)
@@ -813,27 +812,16 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) =
 
 proc genCheckedRecordField(p: BProc, e: PNode, d: var TLoc) =
   if optFieldCheck in p.options:
-    var
+    var a: TLoc
-      a: TLoc
+    let ty = genRecordFieldAux(p, e.sons[0], d, a)
-      f, field: PSym
+    var r = rdLoc(a)
-      ty: PType
+    let f = e.sons[0].sons[1].sym
-      r: Rope
+    let field = lookupFieldAgain(p, ty, f, r)
-    ty = genRecordFieldAux(p, e.sons[0], d, a)
-    r = rdLoc(a)
-    f = e.sons[0].sons[1].sym
-    field = nil
-    while ty != nil:
-      assert(ty.kind in {tyTuple, tyObject})
-      field = lookupInRecord(ty.n, f.name)
-      if field != nil: break
-      if not p.module.compileToCpp: add(r, ".Sup")
-      ty = getUniqueType(ty.sons[0])
-    if field == nil: internalError(e.info, "genCheckedRecordField")
     if field.loc.r == nil:
       internalError(e.info, "genCheckedRecordField") # generate the checks:
-    genFieldCheck(p, e, r, field)
+    genFieldCheck(p, e, r, field, ty)
     add(r, rfmt(nil, ".$1", field.loc.r))
-    putIntoDest(p, d, field.typ, r)
+    putIntoDest(p, d, field.typ, r, a.s)
   else:
     genRecordField(p, e.sons[0], d)
 
@@ -860,7 +848,7 @@ proc genArrayElem(p: BProc, x, y: PNode, d: var TLoc) =
         localError(x.info, errIndexOutOfBounds)
   d.inheritLocation(a)
   putIntoDest(p, d, elemType(skipTypes(ty, abstractVar)),
-              rfmt(nil, "$1[($2)- $3]", rdLoc(a), rdCharLoc(b), first))
+              rfmt(nil, "$1[($2)- $3]", rdLoc(a), rdCharLoc(b), first), a.s)
 
 proc genCStringElem(p: BProc, x, y: PNode, d: var TLoc) =
   var a, b: TLoc
@@ -869,7 +857,7 @@ proc genCStringElem(p: BProc, x, y: PNode, d: var TLoc) =
   var ty = skipTypes(a.t, abstractVarRange)
   if d.k == locNone: d.s = a.s
   putIntoDest(p, d, elemType(skipTypes(ty, abstractVar)),
-              rfmt(nil, "$1[$2]", rdLoc(a), rdCharLoc(b)))
+              rfmt(nil, "$1[$2]", rdLoc(a), rdCharLoc(b)), a.s)
 
 proc genOpenArrayElem(p: BProc, x, y: PNode, d: var TLoc) =
   var a, b: TLoc
@@ -880,7 +868,7 @@ proc genOpenArrayElem(p: BProc, x, y: PNode, d: var TLoc) =
             rdLoc(b), rdLoc(a)) # BUGFIX: ``>=`` and not ``>``!
   if d.k == locNone: d.s = a.s
   putIntoDest(p, d, elemType(skipTypes(a.t, abstractVar)),
-              rfmt(nil, "$1[$2]", rdLoc(a), rdCharLoc(b)))
+              rfmt(nil, "$1[$2]", rdLoc(a), rdCharLoc(b)), a.s)
 
 proc genSeqElem(p: BProc, x, y: PNode, d: var TLoc) =
   var a, b: TLoc
@@ -903,7 +891,7 @@ proc genSeqElem(p: BProc, x, y: PNode, d: var TLoc) =
   if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}:
     a.r = rfmt(nil, "(*$1)", a.r)
   putIntoDest(p, d, elemType(skipTypes(a.t, abstractVar)),
-              rfmt(nil, "$1->data[$2]", rdLoc(a), rdCharLoc(b)))
+              rfmt(nil, "$1->data[$2]", rdLoc(a), rdCharLoc(b)), a.s)
 
 proc genBracketExpr(p: BProc; n: PNode; d: var TLoc) =
   var ty = skipTypes(n.sons[0].typ, abstractVarRange)
@@ -1061,9 +1049,9 @@ proc genSeqElemAppend(p: BProc, e: PNode, d: var TLoc) =
   #    seq = (typeof seq) incrSeq(&seq->Sup, sizeof(x));
   #    seq->data[seq->len-1] = x;
   let seqAppendPattern = if not p.module.compileToCpp:
-                           "$1 = ($2) #incrSeq(&($1)->Sup, sizeof($3));$n"
+                           "$1 = ($2) #incrSeqV2(&($1)->Sup, sizeof($3));$n"
                          else:
-                           "$1 = ($2) #incrSeq($1, sizeof($3));$n"
+                           "$1 = ($2) #incrSeqV2($1, sizeof($3));$n"
   var a, b, dest: TLoc
   initLocExpr(p, e.sons[1], a)
   initLocExpr(p, e.sons[2], b)
@@ -1073,8 +1061,9 @@ proc genSeqElemAppend(p: BProc, e: PNode, d: var TLoc) =
       getTypeDesc(p.module, skipTypes(e.sons[2].typ, abstractVar))])
   keepAlive(p, a)
   initLoc(dest, locExpr, b.t, OnHeap)
-  dest.r = rfmt(nil, "$1->data[$1->$2-1]", rdLoc(a), lenField(p))
+  dest.r = rfmt(nil, "$1->data[$1->$2]", rdLoc(a), lenField(p))
   genAssignment(p, dest, b, {needToCopy, afDestIsNil})
+  lineCg(p, cpsStmts, "++$1->$2;$n", rdLoc(a), lenField(p))
   gcUsage(e)
 
 proc genReset(p: BProc, n: PNode) =
@@ -1158,20 +1147,15 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) =
   else:
     constructLoc(p, tmp)
   discard getTypeDesc(p.module, t)
+  let ty = getUniqueType(t)
   for i in 1 .. <e.len:
     let it = e.sons[i]
     var tmp2: TLoc
     tmp2.r = r
-    var field: PSym = nil
+    let field = lookupFieldAgain(p, ty, it.sons[0].sym, tmp2.r)
-    var ty = getUniqueType(t)
+    if field.loc.r == nil: internalError(e.info, "genObjConstr")
-    while ty != nil:
-      field = lookupInRecord(ty.n, it.sons[0].sym.name)
-      if field != nil: break
-      if not p.module.compileToCpp: add(tmp2.r, ".Sup")
-      ty = getUniqueType(ty.sons[0])
-    if field == nil or field.loc.r == nil: internalError(e.info, "genObjConstr")
     if it.len == 3 and optFieldCheck in p.options:
-      genFieldCheck(p, it.sons[2], tmp2.r, field)
+      genFieldCheck(p, it.sons[2], r, field, ty)
     add(tmp2.r, ".")
     add(tmp2.r, field.loc.r)
     tmp2.k = locTemp
@@ -1280,7 +1264,7 @@ proc genOf(p: BProc, x: PNode, typ: PType, d: var TLoc) =
     r = rfmt(p.module, "(($1) && ($2))", nilCheck, genOfHelper(p, dest, r))
   else:
     r = rfmt(p.module, "($1)", genOfHelper(p, dest, r))
-  putIntoDest(p, d, getSysType(tyBool), r)
+  putIntoDest(p, d, getSysType(tyBool), r, a.s)
 
 proc genOf(p: BProc, n: PNode, d: var TLoc) =
   genOf(p, n.sons[1], n.sons[2].typ, d)
@@ -1292,45 +1276,47 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) =
   case t.kind
   of tyInt..tyInt64, tyUInt..tyUInt64:
     putIntoDest(p, d, e.typ,
-                ropecg(p.module, "#reprInt((NI64)$1)", [rdLoc(a)]))
+                ropecg(p.module, "#reprInt((NI64)$1)", [rdLoc(a)]), a.s)
   of tyFloat..tyFloat128:
-    putIntoDest(p, d, e.typ, ropecg(p.module, "#reprFloat($1)", [rdLoc(a)]))
+    putIntoDest(p, d, e.typ, ropecg(p.module, "#reprFloat($1)", [rdLoc(a)]), a.s)
   of tyBool:
-    putIntoDest(p, d, e.typ, ropecg(p.module, "#reprBool($1)", [rdLoc(a)]))
+    putIntoDest(p, d, e.typ, ropecg(p.module, "#reprBool($1)", [rdLoc(a)]), a.s)
   of tyChar:
-    putIntoDest(p, d, e.typ, ropecg(p.module, "#reprChar($1)", [rdLoc(a)]))
+    putIntoDest(p, d, e.typ, ropecg(p.module, "#reprChar($1)", [rdLoc(a)]), a.s)
   of tyEnum, tyOrdinal:
     putIntoDest(p, d, e.typ,
                 ropecg(p.module, "#reprEnum($1, $2)", [
-                rdLoc(a), genTypeInfo(p.module, t)]))
+                rdLoc(a), genTypeInfo(p.module, t)]), a.s)
   of tyString:
-    putIntoDest(p, d, e.typ, ropecg(p.module, "#reprStr($1)", [rdLoc(a)]))
+    putIntoDest(p, d, e.typ, ropecg(p.module, "#reprStr($1)", [rdLoc(a)]), a.s)
   of tySet:
     putIntoDest(p, d, e.typ, ropecg(p.module, "#reprSet($1, $2)", [
-                addrLoc(a), genTypeInfo(p.module, t)]))
+                addrLoc(a), genTypeInfo(p.module, t)]), a.s)
   of tyOpenArray, tyVarargs:
     var b: TLoc
     case a.t.kind
     of tyOpenArray, tyVarargs:
-      putIntoDest(p, b, e.typ, "$1, $1Len0" % [rdLoc(a)])
+      putIntoDest(p, b, e.typ, "$1, $1Len0" % [rdLoc(a)], a.s)
     of tyString, tySequence:
       putIntoDest(p, b, e.typ,
-                  "$1->data, $1->$2" % [rdLoc(a), lenField(p)])
+                  "$1->data, $1->$2" % [rdLoc(a), lenField(p)], a.s)
     of tyArray, tyArrayConstr:
       putIntoDest(p, b, e.typ,
-                  "$1, $2" % [rdLoc(a), rope(lengthOrd(a.t))])
+                  "$1, $2" % [rdLoc(a), rope(lengthOrd(a.t))], a.s)
     else: internalError(e.sons[0].info, "genRepr()")
     putIntoDest(p, d, e.typ,
         ropecg(p.module, "#reprOpenArray($1, $2)", [rdLoc(b),
-        genTypeInfo(p.module, elemType(t))]))
+        genTypeInfo(p.module, elemType(t))]), a.s)
   of tyCString, tyArray, tyArrayConstr, tyRef, tyPtr, tyPointer, tyNil,
      tySequence:
     putIntoDest(p, d, e.typ,
                 ropecg(p.module, "#reprAny($1, $2)", [
-                rdLoc(a), genTypeInfo(p.module, t)]))
+                rdLoc(a), genTypeInfo(p.module, t)]), a.s)
+  of tyEmpty:
+    localError(e.info, "'repr' doesn't support 'void' type")
   else:
     putIntoDest(p, d, e.typ, ropecg(p.module, "#reprAny($1, $2)",
-                                   [addrLoc(a), genTypeInfo(p.module, t)]))
+                                   [addrLoc(a), genTypeInfo(p.module, t)]), a.s)
   gcUsage(e)
 
 proc genGetTypeInfo(p: BProc, e: PNode, d: var TLoc) =
@@ -1492,11 +1478,11 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of 1, 2, 4, 8:
     case op
     of mIncl:
-      var ts = "NI" & $(size * 8)
+      var ts = "NU" & $(size * 8)
       binaryStmtInExcl(p, e, d,
           "$1 |= ((" & ts & ")1)<<(($2)%(sizeof(" & ts & ")*8));$n")
     of mExcl:
-      var ts = "NI" & $(size * 8)
+      var ts = "NU" & $(size * 8)
       binaryStmtInExcl(p, e, d, "$1 &= ~(((" & ts & ")1) << (($2) % (sizeof(" &
           ts & ")*8)));$n")
     of mCard:
@@ -1555,13 +1541,13 @@ proc genSomeCast(p: BProc, e: PNode, d: var TLoc) =
   let etyp = skipTypes(e.typ, abstractRange)
   if etyp.kind in ValueTypes and lfIndirect notin a.flags:
     putIntoDest(p, d, e.typ, "(*($1*) ($2))" %
-        [getTypeDesc(p.module, e.typ), addrLoc(a)])
+        [getTypeDesc(p.module, e.typ), addrLoc(a)], a.s)
   elif etyp.kind == tyProc and etyp.callConv == ccClosure:
     putIntoDest(p, d, e.typ, "(($1) ($2))" %
-        [getClosureType(p.module, etyp, clHalfWithEnv), rdCharLoc(a)])
+        [getClosureType(p.module, etyp, clHalfWithEnv), rdCharLoc(a)], a.s)
   else:
     putIntoDest(p, d, e.typ, "(($1) ($2))" %
-        [getTypeDesc(p.module, e.typ), rdCharLoc(a)])
+        [getTypeDesc(p.module, e.typ), rdCharLoc(a)], a.s)
 
 proc genCast(p: BProc, e: PNode, d: var TLoc) =
   const floatTypes = {tyFloat..tyFloat128}
@@ -1581,7 +1567,7 @@ proc genCast(p: BProc, e: PNode, d: var TLoc) =
     tmp.s = OnStack
     tmp.flags = {}
     expr(p, e.sons[1], tmp)
-    putIntoDest(p, d, e.typ, "LOC$#.dest" % [lbl])
+    putIntoDest(p, d, e.typ, "LOC$#.dest" % [lbl], tmp.s)
   else:
     # I prefer the shorter cast version for pointer types -> generate less
     # C code; plus it's the right thing to do for closures:
@@ -1591,16 +1577,17 @@ proc genRangeChck(p: BProc, n: PNode, d: var TLoc, magic: string) =
   var a: TLoc
   var dest = skipTypes(n.typ, abstractVar)
   # range checks for unsigned turned out to be buggy and annoying:
-  if optRangeCheck notin p.options or dest.kind in {tyUInt..tyUInt64}:
+  if optRangeCheck notin p.options or dest.skipTypes({tyRange}).kind in
+                                             {tyUInt..tyUInt64}:
     initLocExpr(p, n.sons[0], a)
     putIntoDest(p, d, n.typ, "(($1) ($2))" %
-        [getTypeDesc(p.module, dest), rdCharLoc(a)])
+        [getTypeDesc(p.module, dest), rdCharLoc(a)], a.s)
   else:
     initLocExpr(p, n.sons[0], a)
     putIntoDest(p, d, dest, ropecg(p.module, "(($1)#$5($2, $3, $4))", [
         getTypeDesc(p.module, dest), rdCharLoc(a),
         genLiteral(p, n.sons[1], dest), genLiteral(p, n.sons[2], dest),
-        rope(magic)]))
+        rope(magic)]), a.s)
 
 proc genConv(p: BProc, e: PNode, d: var TLoc) =
   let destType = e.typ.skipTypes({tyVar, tyGenericInst})
@@ -1612,13 +1599,13 @@ proc genConv(p: BProc, e: PNode, d: var TLoc) =
 proc convStrToCStr(p: BProc, n: PNode, d: var TLoc) =
   var a: TLoc
   initLocExpr(p, n.sons[0], a)
-  putIntoDest(p, d, skipTypes(n.typ, abstractVar), "$1->data" % [rdLoc(a)])
+  putIntoDest(p, d, skipTypes(n.typ, abstractVar), "$1->data" % [rdLoc(a)], a.s)
 
 proc convCStrToStr(p: BProc, n: PNode, d: var TLoc) =
   var a: TLoc
   initLocExpr(p, n.sons[0], a)
   putIntoDest(p, d, skipTypes(n.typ, abstractVar),
-              ropecg(p.module, "#cstrToNimstr($1)", [rdLoc(a)]))
+              ropecg(p.module, "#cstrToNimstr($1)", [rdLoc(a)]), a.s)
   gcUsage(n)
 
 proc genStrEquals(p: BProc, e: PNode, d: var TLoc) =
@@ -1661,7 +1648,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   case op
   of mOr, mAnd: genAndOr(p, e, d, op)
   of mNot..mToBiggestInt: unaryArith(p, e, d, op)
-  of mUnaryMinusI..mAbsI64: unaryArithOverflow(p, e, d, op)
+  of mUnaryMinusI..mAbsI: unaryArithOverflow(p, e, d, op)
   of mAddF64..mDivF64: binaryFloatArith(p, e, d, op)
   of mShrI..mXor: binaryArith(p, e, d, op)
   of mEqProc: genEqProc(p, e, d)
@@ -1746,6 +1733,8 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mEcho: genEcho(p, e[1].skipConv)
   of mArrToSeq: genArrToSeq(p, e, d)
   of mNLen..mNError, mSlurp..mQuoteAst:
+    echo "from here ", p.prc.name.s, " ", p.prc.info
+    writestacktrace()
     localError(e.info, errXMustBeCompileTime, e.sons[0].sym.name.s)
   of mSpawn:
     let n = lowerings.wrapProcForSpawn(p.module.module, e, e.typ, nil, nil)
@@ -1768,7 +1757,7 @@ proc handleConstExpr(p: BProc, n: PNode, d: var TLoc): bool =
     var t = getUniqueType(n.typ)
     discard getTypeDesc(p.module, t) # so that any fields are initialized
     var id = nodeTableTestOrSet(p.module.dataCache, n, gBackendId)
-    fillLoc(d, locData, t, "TMP" & rope(id), OnHeap)
+    fillLoc(d, locData, t, "TMP" & rope(id), OnStatic)
     if id == gBackendId:
       # expression not found in the cache:
       inc(gBackendId)
@@ -1807,7 +1796,7 @@ proc genSetConstr(p: BProc, e: PNode, d: var TLoc) =
                [rdLoc(d), rdSetElemLoc(a, e.typ)])
     else:
       # small set
-      var ts = "NI" & $(getSize(e.typ) * 8)
+      var ts = "NU" & $(getSize(e.typ) * 8)
       lineF(p, cpsStmts, "$1 = 0;$n", [rdLoc(d)])
       for i in countup(0, sonsLen(e) - 1):
         if e.sons[i].kind == nkRange:
@@ -1815,13 +1804,13 @@ proc genSetConstr(p: BProc, e: PNode, d: var TLoc) =
           initLocExpr(p, e.sons[i].sons[0], a)
           initLocExpr(p, e.sons[i].sons[1], b)
           lineF(p, cpsStmts, "for ($1 = $3; $1 <= $4; $1++) $n" &
-              "$2 |=(1<<((" & ts & ")($1)%(sizeof(" & ts & ")*8)));$n", [
+              "$2 |=((" & ts & ")(1)<<(($1)%(sizeof(" & ts & ")*8)));$n", [
               rdLoc(idx), rdLoc(d), rdSetElemLoc(a, e.typ),
               rdSetElemLoc(b, e.typ)])
         else:
           initLocExpr(p, e.sons[i], a)
           lineF(p, cpsStmts,
-               "$1 |=(1<<((" & ts & ")($2)%(sizeof(" & ts & ")*8)));$n",
+               "$1 |=((" & ts & ")(1)<<(($2)%(sizeof(" & ts & ")*8)));$n",
                [rdLoc(d), rdSetElemLoc(a, e.typ)])
 
 proc genTupleConstr(p: BProc, n: PNode, d: var TLoc) =
@@ -1854,7 +1843,7 @@ proc genClosure(p: BProc, n: PNode, d: var TLoc) =
     var tmp = "LOC" & rope(p.labels)
     addf(p.module.s[cfsData], "NIM_CONST $1 $2 = $3;$n",
         [getTypeDesc(p.module, n.typ), tmp, genConstExpr(p, n)])
-    putIntoDest(p, d, n.typ, tmp)
+    putIntoDest(p, d, n.typ, tmp, OnStatic)
   else:
     var tmp, a, b: TLoc
     initLocExpr(p, n.sons[0], a)
@@ -1908,10 +1897,10 @@ proc upConv(p: BProc, n: PNode, d: var TLoc) =
               r, genTypeInfo(p.module, dest))
   if n.sons[0].typ.kind != tyObject:
     putIntoDest(p, d, n.typ,
-                "(($1) ($2))" % [getTypeDesc(p.module, n.typ), rdLoc(a)])
+                "(($1) ($2))" % [getTypeDesc(p.module, n.typ), rdLoc(a)], a.s)
   else:
     putIntoDest(p, d, n.typ, "(*($1*) ($2))" %
-                             [getTypeDesc(p.module, dest), addrLoc(a)])
+                             [getTypeDesc(p.module, dest), addrLoc(a)], a.s)
 
 proc downConv(p: BProc, n: PNode, d: var TLoc) =
   if p.module.compileToCpp:
@@ -1943,9 +1932,9 @@ proc downConv(p: BProc, n: PNode, d: var TLoc) =
         linefmt(p, cpsStmts, "$1 = &$2;$n", rdLoc(d), r)
       else:
         r = "&" & r
-        putIntoDest(p, d, n.typ, r)
+        putIntoDest(p, d, n.typ, r, a.s)
     else:
-      putIntoDest(p, d, n.typ, r)
+      putIntoDest(p, d, n.typ, r, a.s)
 
 proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) =
   var t = getUniqueType(n.typ)
@@ -1960,7 +1949,7 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) =
          [getTypeDesc(p.module, t), tmp, genConstExpr(p, n)])
 
   if d.k == locNone:
-    fillLoc(d, locData, t, tmp, OnHeap)
+    fillLoc(d, locData, t, tmp, OnStatic)
   else:
     putDataIntoDest(p, d, t, tmp)
 
@@ -1978,6 +1967,9 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
         genProc(p.module, sym)
       putLocIntoDest(p, d, sym.loc)
     of skProc, skConverter, skIterators:
+      if sfCompileTime in sym.flags:
+        localError(n.info, "request to generate code for .compileTime proc: " &
+           sym.name.s)
       genProc(p.module, sym)
       if sym.loc.r == nil or sym.loc.t == nil:
         internalError(n.info, "expr: proc not init " & sym.name.s)
@@ -1987,7 +1979,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
         if sfGlobal in sym.flags: genVarPrototype(p.module, sym)
         putLocIntoDest(p, d, sym.loc)
       elif isSimpleConst(sym.typ):
-        putIntoDest(p, d, n.typ, genLiteral(p, sym.ast, sym.typ))
+        putIntoDest(p, d, n.typ, genLiteral(p, sym.ast, sym.typ), OnStatic)
       else:
         genComplexConst(p, sym, d)
     of skEnumField:
@@ -2074,6 +2066,9 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
   of nkStmtList:
     for i in countup(0, sonsLen(n) - 1): genStmts(p, n.sons[i])
   of nkIfExpr, nkIfStmt: genIf(p, n, d)
+  of nkWhen:
+    # This should be a "when nimvm" node.
+    expr(p, n.sons[1].sons[0], d)
   of nkObjDownConv: downConv(p, n, d)
   of nkObjUpConv: upConv(p, n, d)
   of nkChckRangeF: genRangeChck(p, n, d, "chckRangeF")
@@ -2128,7 +2123,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       # due to a bug/limitation in the lambda lifting, unused inner procs
       # are not transformed correctly. We work around this issue (#411) here
       # by ensuring it's no inner proc (owner is a module):
-      if prc.skipGenericOwner.kind == skModule:
+      if prc.skipGenericOwner.kind == skModule and sfCompileTime notin prc.flags:
         if (optDeadCodeElim notin gGlobalOptions and
             sfDeadCodeElim notin getModule(prc).flags) or
             ({sfExportc, sfCompilerProc} * prc.flags == {sfExportc}) or
@@ -2150,7 +2145,7 @@ proc genNamedConstExpr(p: BProc, n: PNode): Rope =
 proc genConstSimpleList(p: BProc, n: PNode): Rope =
   var length = sonsLen(n)
   result = rope("{")
-  for i in countup(0, length - 2):
+  for i in countup(ord(n.kind == nkObjConstr), length - 2):
     addf(result, "$1,$n", [genNamedConstExpr(p, n.sons[i])])
   if length > 0: add(result, genNamedConstExpr(p, n.sons[length - 1]))
   addf(result, "}$n", [])

compiler/ccgstmts.nim

@@ -102,7 +102,7 @@ proc assignLabel(b: var TBlock): Rope {.inline.} =
 proc blockBody(b: var TBlock): Rope =
   result = b.sections[cpsLocals]
   if b.frameLen > 0:
-    result.addf("F.len+=$1;$n", [b.frameLen.rope])
+    result.addf("FR.len+=$1;$n", [b.frameLen.rope])
   result.add(b.sections[cpsInit])
   result.add(b.sections[cpsStmts])
 
@@ -123,7 +123,7 @@ proc endBlock(p: BProc) =
       ~"}$n"
   let frameLen = p.blocks[topBlock].frameLen
   if frameLen > 0:
-    blockEnd.addf("F.len-=$1;$n", [frameLen.rope])
+    blockEnd.addf("FR.len-=$1;$n", [frameLen.rope])
   endBlock(p, blockEnd)
 
 proc genSimpleBlock(p: BProc, stmts: PNode) {.inline.} =
@@ -1002,8 +1002,10 @@ proc genAsmStmt(p: BProc, t: PNode) =
 proc determineSection(n: PNode): TCFileSection =
   result = cfsProcHeaders
   if n.len >= 1 and n.sons[0].kind in {nkStrLit..nkTripleStrLit}:
-    if n.sons[0].strVal.startsWith("/*TYPESECTION*/"): result = cfsTypes
+    let sec = n.sons[0].strVal
-    elif n.sons[0].strVal.startsWith("/*VARSECTION*/"): result = cfsVars
+    if sec.startsWith("/*TYPESECTION*/"): result = cfsTypes
+    elif sec.startsWith("/*VARSECTION*/"): result = cfsVars
+    elif sec.startsWith("/*INCLUDESECTION*/"): result = cfsHeaders
 
 proc genEmit(p: BProc, t: PNode) =
   var s = genAsmOrEmitStmt(p, t.sons[1])
@@ -1099,6 +1101,9 @@ proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
     genGotoVar(p, e.sons[1])
   elif not fieldDiscriminantCheckNeeded(p, e):
     var a: TLoc
+    if e[0].kind in {nkDerefExpr, nkHiddenDeref}:
+      genDeref(p, e[0], a, enforceDeref=true)
+    else:
       initLocExpr(p, e.sons[0], a)
     if fastAsgn: incl(a.flags, lfNoDeepCopy)
     assert(a.t != nil)

compiler/ccgtypes.nim

@@ -323,7 +323,8 @@ proc paramStorageLoc(param: PSym): TStorageLoc =
     result = OnUnknown
 
 proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
-                   check: var IntSet, declareEnvironment=true) =
+                   check: var IntSet, declareEnvironment=true;
+                   weakDep=false) =
   params = nil
   if (t.sons[0] == nil) or isInvalidReturnType(t.sons[0]):
     rettype = ~"void"
@@ -341,6 +342,8 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
       add(params, ~"*")
       incl(param.loc.flags, lfIndirect)
       param.loc.s = OnUnknown
+    elif weakDep:
+      add(params, getTypeDescWeak(m, param.typ, check))
     else:
       add(params, getTypeDescAux(m, param.typ, check))
     add(params, ~" ")
@@ -438,6 +441,8 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
       elif fieldType.kind == tySequence:
         # we need to use a weak dependency here for trecursive_table.
         addf(result, "$1 $2;$n", [getTypeDescWeak(m, field.loc.t, check), sname])
+      elif field.bitsize != 0:
+        addf(result, "$1 $2:$3;$n", [getTypeDescAux(m, field.loc.t, check), sname, rope($field.bitsize)])
       else:
         # don't use fieldType here because we need the
         # tyGenericInst for C++ template support
@@ -577,7 +582,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope =
     result = getTypeName(t)
     idTablePut(m.typeCache, t, result)
     var rettype, desc: Rope
-    genProcParams(m, t, rettype, desc, check)
+    genProcParams(m, t, rettype, desc, check, true, true)
     if not isImportedType(t):
       if t.callConv != ccClosure: # procedure vars may need a closure!
         addf(m.s[cfsTypes], "typedef $1_PTR($2, $3) $4;$n",

compiler/ccgutils.nim

@@ -99,7 +99,10 @@ proc getUniqueType*(key: PType): PType =
       gCanonicalTypes[k] = key
       result = key
   of tyTypeDesc, tyTypeClasses, tyGenericParam, tyFromExpr, tyFieldAccessor:
-    internalError("getUniqueType")
+    if key.sym != nil:
+      internalError(key.sym.info, "metatype not eliminated")
+    else:
+      internalError("metatype not eliminated")
   of tyDistinct:
     if key.deepCopy != nil: result = key
     else: result = getUniqueType(lastSon(key))

compiler/cgen.nim

@@ -10,12 +10,11 @@
 ## This module implements the C code generator.
 
 import
-  ast, astalgo, hashes, trees, platform, magicsys, extccomp,
+  ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets,
-  options, intsets,
+  nversion, nimsets, msgs, securehash, bitsets, idents, lists, types,
-  nversion, nimsets, msgs, crc, bitsets, idents, lists, types, ccgutils, os,
+  ccgutils, os, ropes, math, passes, rodread, wordrecg, treetab, cgmeth,
-  ropes, math, passes, rodread, wordrecg, treetab, cgmeth, condsyms,
+  condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases,
-  rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings,
+  lowerings, semparallel
-  semparallel
 
 import strutils except `%` # collides with ropes.`%`
 
@@ -376,7 +375,7 @@ proc localDebugInfo(p: BProc, s: PSym) =
   var a = "&" & s.loc.r
   if s.kind == skParam and ccgIntroducedPtr(s): a = s.loc.r
   lineF(p, cpsInit,
-       "F.s[$1].address = (void*)$3; F.s[$1].typ = $4; F.s[$1].name = $2;$n",
+       "FR.s[$1].address = (void*)$3; FR.s[$1].typ = $4; FR.s[$1].name = $2;$n",
        [p.maxFrameLen.rope, makeCString(normalize(s.name.s)), a,
         genTypeInfo(p.module, s.loc.t)])
   inc(p.maxFrameLen)
@@ -506,8 +505,7 @@ proc loadDynamicLib(m: BModule, lib: PLib) =
     if lib.path.kind in {nkStrLit..nkTripleStrLit}:
       var s: TStringSeq = @[]
       libCandidates(lib.path.strVal, s)
-      if gVerbosity >= 2:
+      rawMessage(hintDependency, lib.path.strVal)
-        msgWriteln("Dependency: " & lib.path.strVal)
       var loadlib: Rope = nil
       for i in countup(0, high(s)):
         inc(m.labels)
@@ -599,7 +597,7 @@ proc cgsym(m: BModule, name: string): Rope =
     of skProc, skMethod, skConverter, skIterators: genProc(m, sym)
     of skVar, skResult, skLet: genVarPrototype(m, sym)
     of skType: discard getTypeDesc(m, sym.typ)
-    else: internalError("cgsym: " & name)
+    else: internalError("cgsym: " & name & ": " & $sym.kind)
   else:
     # we used to exclude the system module from this check, but for DLL
     # generation support this sloppyness leads to hard to detect bugs, so
@@ -611,10 +609,12 @@ proc generateHeaders(m: BModule) =
   add(m.s[cfsHeaders], tnl & "#include \"nimbase.h\"" & tnl)
   var it = PStrEntry(m.headerFiles.head)
   while it != nil:
-    if it.data[0] notin {'\"', '<'}:
+    if it.data[0] == '#':
-      addf(m.s[cfsHeaders], "$N#include \"$1\"$N", [rope(it.data)])
+      add(m.s[cfsHeaders], rope(it.data.replace('`', '"') & tnl))
+    elif it.data[0] notin {'\"', '<'}:
+      addf(m.s[cfsHeaders], "#include \"$1\"$N", [rope(it.data)])
     else:
-      addf(m.s[cfsHeaders], "$N#include $1$N", [rope(it.data)])
+      addf(m.s[cfsHeaders], "#include $1$N", [rope(it.data)])
     it = PStrEntry(it.next)
 
 proc retIsNotVoid(s: PSym): bool =
@@ -623,7 +623,7 @@ proc retIsNotVoid(s: PSym): bool =
 proc initFrame(p: BProc, procname, filename: Rope): Rope =
   discard cgsym(p.module, "nimFrame")
   if p.maxFrameLen > 0:
-    discard cgsym(p.module, "TVarSlot")
+    discard cgsym(p.module, "VarSlot")
     result = rfmt(nil, "\tnimfrs($1, $2, $3, $4)$N",
                   procname, filename, p.maxFrameLen.rope,
                   p.blocks[0].frameLen.rope)
@@ -676,8 +676,11 @@ proc genProcAux(m: BModule, prc: PSym) =
   closureSetup(p, prc)
   genStmts(p, prc.getBody) # modifies p.locals, p.init, etc.
   var generatedProc: Rope
+  if sfNoReturn in prc.flags:
+    if hasDeclspec in extccomp.CC[extccomp.cCompiler].props:
+      header = "__declspec(noreturn) " & header
   if sfPure in prc.flags:
-    if hasNakedDeclspec in extccomp.CC[extccomp.cCompiler].props:
+    if hasDeclspec in extccomp.CC[extccomp.cCompiler].props:
       header = "__declspec(naked) " & header
     generatedProc = rfmt(nil, "$N$1 {$n$2$3$4}$N$N",
                          header, p.s(cpsLocals), p.s(cpsInit), p.s(cpsStmts))
@@ -718,10 +721,14 @@ proc genProcPrototype(m: BModule, sym: PSym) =
                         getTypeDesc(m, sym.loc.t), mangleDynLibProc(sym)))
   elif not containsOrIncl(m.declaredProtos, sym.id):
     var header = genProcHeader(m, sym)
+    if sfNoReturn in sym.flags and hasDeclspec in extccomp.CC[cCompiler].props:
+      header = "__declspec(noreturn) " & header
     if sym.typ.callConv != ccInline and crossesCppBoundary(m, sym):
       header = "extern \"C\" " & header
-    if sfPure in sym.flags and hasNakedAttribute in CC[cCompiler].props:
+    if sfPure in sym.flags and hasAttribute in CC[cCompiler].props:
       header.add(" __attribute__((naked))")
+    if sfNoReturn in sym.flags and hasAttribute in CC[cCompiler].props:
+      header.add(" __attribute__((noreturn))")
     add(m.s[cfsProcHeaders], rfmt(nil, "$1;$n", header))
 
 proc genProcNoForward(m: BModule, prc: PSym) =
@@ -753,7 +760,7 @@ proc requestConstImpl(p: BProc, sym: PSym) =
   var m = p.module
   useHeader(m, sym)
   if sym.loc.k == locNone:
-    fillLoc(sym.loc, locData, sym.typ, mangleName(sym), OnUnknown)
+    fillLoc(sym.loc, locData, sym.typ, mangleName(sym), OnStatic)
   if lfNoDecl in sym.loc.flags: return
   # declare implementation:
   var q = findPendingModule(m, sym)
@@ -808,7 +815,7 @@ proc genVarPrototype(m: BModule, sym: PSym) =
   genVarPrototypeAux(m, sym)
 
 proc addIntTypes(result: var Rope) {.inline.} =
-  addf(result, "#define NIM_INTBITS $1", [
+  addf(result, "#define NIM_INTBITS $1" & tnl, [
     platform.CPU[targetCPU].intSize.rope])
 
 proc getCopyright(cfile: string): Rope =
@@ -846,14 +853,14 @@ proc genMainProc(m: BModule) =
     # functions, which might otherwise merge their stack frames.
     PreMainBody =
       "void PreMainInner() {$N" &
-      "\tsystemInit();$N" &
+      "\tsystemInit000();$N" &
       "$1" &
       "$2" &
       "$3" &
       "}$N$N" &
       "void PreMain() {$N" &
       "\tvoid (*volatile inner)();$N" &
-      "\tsystemDatInit();$N" &
+      "\tsystemDatInit000();$N" &
       "\tinner = PreMainInner;$N" &
       "$4$5" &
       "\t(*inner)();$N" &
@@ -943,7 +950,7 @@ proc genMainProc(m: BModule) =
     gBreakpoints.add(m.genFilenames)
 
   let initStackBottomCall =
-    if platform.targetOS == osStandalone: "".rope
+    if platform.targetOS == osStandalone or gSelectedGC == gcNone: "".rope
     else: ropecg(m, "\t#initStackBottomWith((void *)&inner);$N")
   inc(m.labels)
   appcg(m, m.s[cfsProcs], PreMainBody, [
@@ -967,8 +974,8 @@ proc getSomeInitName(m: PSym, suffix: string): Rope =
   result.add m.name.s
   result.add suffix
 
-proc getInitName(m: PSym): Rope = getSomeInitName(m, "Init")
+proc getInitName(m: PSym): Rope = getSomeInitName(m, "Init000")
-proc getDatInitName(m: PSym): Rope = getSomeInitName(m, "DatInit")
+proc getDatInitName(m: PSym): Rope = getSomeInitName(m, "DatInit000")
 
 proc registerModuleToMain(m: PSym) =
   var
@@ -1010,7 +1017,7 @@ proc genInitCode(m: BModule) =
       var procname = makeCString(m.module.name.s)
       add(prc, initFrame(m.initProc, procname, m.module.info.quotedFilename))
     else:
-      add(prc, ~"\tTFrame F; F.len = 0;$N")
+      add(prc, ~"\tTFrame F; FR.len = 0;$N")
 
   add(prc, genSectionStart(cpsInit))
   add(prc, m.preInitProc.s(cpsInit))
@@ -1103,7 +1110,7 @@ proc rawNewModule(module: PSym, filename: string): BModule =
 
 proc nullify[T](arr: var T) =
   for i in low(arr)..high(arr):
-    arr[i] = nil
+    arr[i] = Rope(nil)
 
 proc resetModule*(m: BModule) =
   # between two compilations in CAAS mode, we can throw
@@ -1322,4 +1329,3 @@ proc cgenWriteModules* =
   if generatedHeader != nil: writeHeader(generatedHeader)
 
 const cgenPass* = makePass(myOpen, myOpenCached, myProcess, myClose)
-

compiler/cgmeth.nim

@@ -47,8 +47,10 @@ proc methodCall*(n: PNode): PNode =
 var
   gMethods: seq[tuple[methods: TSymSeq, dispatcher: PSym]] = @[]
 
-proc sameMethodBucket(a, b: PSym): bool = 
+type
-  result = false
+  MethodResult = enum No, Invalid, Yes
+
+proc sameMethodBucket(a, b: PSym): MethodResult =
   if a.name.id != b.name.id: return
   if sonsLen(a.typ) != sonsLen(b.typ):
     return                    # check for return type:
@@ -64,13 +66,16 @@ proc sameMethodBucket(a, b: PSym): bool =
         bb = bb.lastSon
       else:
         break
-    if sameType(aa, bb) or
+    if sameType(aa, bb):
-        (aa.kind == tyObject) and (bb.kind == tyObject) and
+      if aa.kind == tyObject and result != Invalid: result = Yes
-        (inheritanceDiff(bb, aa) < 0):
+    elif aa.kind == tyObject and bb.kind == tyObject:
-      discard
+      let diff = inheritanceDiff(bb, aa)
+      if diff < 0:
+        if result != Invalid: result = Yes
+      elif diff != high(int):
+        result = Invalid
     else:
-      return
+      return No
-  result = true
 
 proc attachDispatcher(s: PSym, dispatcher: PNode) =
   var L = s.ast.len-1
@@ -133,18 +138,31 @@ proc fixupDispatcher(meth, disp: PSym) =
 
 proc methodDef*(s: PSym, fromCache: bool) =
   var L = len(gMethods)
+  var witness: PSym
   for i in countup(0, L - 1):
     var disp = gMethods[i].dispatcher
-    if sameMethodBucket(disp, s):
+    case sameMethodBucket(disp, s)
+    of Yes:
       add(gMethods[i].methods, s)
       attachDispatcher(s, lastSon(disp.ast))
       fixupDispatcher(s, disp)
       when useEffectSystem: checkMethodEffects(disp, s)
+      if sfBase in s.flags and gMethods[i].methods[0] != s:
+        # already exists due to forwarding definition?
+        localError(s.info, "method is not a base")
       return
+    of No: discard
+    of Invalid:
+      if witness.isNil: witness = gMethods[i].methods[0]
   # create a new dispatcher:
   add(gMethods, (methods: @[s], dispatcher: createDispatcher(s)))
   if fromCache:
     internalError(s.info, "no method dispatcher found")
+  if witness != nil:
+    localError(s.info, "invalid declaration order; cannot attach '" & s.name.s &
+                       "' to method defined here: " & $witness.info)
+  elif sfBase notin s.flags:
+    message(s.info, warnUseBase)
 
 proc relevantCol(methods: TSymSeq, col: int): bool =
   # returns true iff the position is relevant
@@ -214,7 +232,7 @@ proc genDispatcher(methods: TSymSeq, relevantCols: IntSet): PSym =
                            curr.typ.sons[col], false))
     var ret: PNode
     if base.typ.sons[0] != nil:
-      var a = newNodeI(nkAsgn, base.info)
+      var a = newNodeI(nkFastAsgn, base.info)
       addSon(a, newSymNode(base.ast.sons[resultPos].sym))
       addSon(a, call)
       ret = newNodeI(nkReturnStmt, base.info)
@@ -239,4 +257,3 @@ proc generateMethodDispatchers*(): PNode =
     sortBucket(gMethods[bucket].methods, relevantCols)
     addSon(result,
            newSymNode(genDispatcher(gMethods[bucket].methods, relevantCols)))
-

compiler/commands.nim

@@ -17,11 +17,12 @@ template bootSwitch(name, expr, userString: expr): expr =
   const name = if expr: " " & userString else: ""
 
 bootSwitch(usedRelease, defined(release), "-d:release")
-bootSwitch(usedGnuReadline, defined(useGnuReadline), "-d:useGnuReadline")
+bootSwitch(usedGnuReadline, defined(useLinenoise), "-d:useLinenoise")
 bootSwitch(usedNoCaas, defined(noCaas), "-d:noCaas")
 bootSwitch(usedBoehm, defined(boehmgc), "--gc:boehm")
 bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep")
 bootSwitch(usedGenerational, defined(gcgenerational), "--gc:generational")
+bootSwitch(usedGoGC, defined(gogc), "--gc:go")
 bootSwitch(usedNoGC, defined(nogc), "--gc:none")
 
 import
@@ -64,14 +65,15 @@ proc getCommandLineDesc(): string =
 
 proc helpOnError(pass: TCmdLinePass) =
   if pass == passCmd1:
-    msgWriteln(getCommandLineDesc())
+    msgWriteln(getCommandLineDesc(), {msgStdout})
     msgQuit(0)
 
 proc writeAdvancedUsage(pass: TCmdLinePass) =
   if pass == passCmd1:
     msgWriteln(`%`(HelpMessage, [VersionAsString,
                                  platform.OS[platform.hostOS].name,
-                                 CPU[platform.hostCPU].name]) & AdvancedUsage)
+                                 CPU[platform.hostCPU].name]) & AdvancedUsage,
+               {msgStdout})
     msgQuit(0)
 
 proc writeVersionInfo(pass: TCmdLinePass) =
@@ -86,7 +88,7 @@ proc writeVersionInfo(pass: TCmdLinePass) =
 
     msgWriteln("active boot switches:" & usedRelease & usedAvoidTimeMachine &
       usedTinyC & usedGnuReadline & usedNativeStacktrace & usedNoCaas &
-      usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedNoGC)
+      usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedGoGC & usedNoGC)
     msgQuit(0)
 
 var
@@ -94,7 +96,7 @@ var
 
 proc writeCommandLineUsage() =
   if not helpWritten:
-    msgWriteln(getCommandLineDesc())
+    msgWriteln(getCommandLineDesc(), {msgStdout})
     helpWritten = true
 
 proc addPrefix(switch: string): string =
@@ -127,6 +129,18 @@ proc processOnOffSwitch(op: TOptions, arg: string, pass: TCmdLinePass,
   of wOff: gOptions = gOptions - op
   else: localError(info, errOnOrOffExpectedButXFound, arg)
 
+proc processOnOffSwitchOrList(op: TOptions, arg: string, pass: TCmdLinePass,
+                              info: TLineInfo): bool =
+  result = false
+  case whichKeyword(arg)
+  of wOn: gOptions = gOptions + op
+  of wOff: gOptions = gOptions - op
+  else:
+    if arg == "list":
+      result = true
+    else:
+      localError(info, errOnOffOrListExpectedButXFound, arg)
+
 proc processOnOffSwitchG(op: TGlobalOptions, arg: string, pass: TCmdLinePass,
                          info: TLineInfo) =
   case whichKeyword(arg)
@@ -140,6 +154,10 @@ proc expectArg(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
 proc expectNoArg(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
   if arg != "": localError(info, errCmdLineNoArgExpected, addPrefix(switch))
 
+var
+  enableNotes: TNoteKinds
+  disableNotes: TNoteKinds
+
 proc processSpecificNote(arg: string, state: TSpecialWord, pass: TCmdLinePass,
                          info: TLineInfo; orig: string) =
   var id = ""  # arg = "X]:on|off"
@@ -161,8 +179,12 @@ proc processSpecificNote(arg: string, state: TSpecialWord, pass: TCmdLinePass,
     if x >= 0: n = TNoteKind(x + ord(warnMin))
     else: localError(info, "unknown warning: " & id)
   case whichKeyword(substr(arg, i))
-  of wOn: incl(gNotes, n)
+  of wOn:
-  of wOff: excl(gNotes, n)
+    incl(gNotes, n)
+    incl(enableNotes, n)
+  of wOff:
+    excl(gNotes, n)
+    incl(disableNotes, n)
   else: localError(info, errOnOrOffExpectedButXFound, arg)
 
 proc processCompile(filename: string) =
@@ -181,6 +203,7 @@ proc testCompileOptionArg*(switch, arg: string, info: TLineInfo): bool =
     of "v2":           result = gSelectedGC == gcV2
     of "markandsweep": result = gSelectedGC == gcMarkAndSweep
     of "generational": result = gSelectedGC == gcGenerational
+    of "go":           result = gSelectedGC == gcGo
     of "none":         result = gSelectedGC == gcNone
     else: localError(info, errNoneBoehmRefcExpectedButXFound, arg)
   of "opt":
@@ -229,7 +252,8 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool =
   of "experimental": result = gExperimentalMode
   else: invalidCmdLineOption(passCmd1, switch, info)
 
-proc processPath(path: string, notRelativeToProj = false): string =
+proc processPath(path: string, notRelativeToProj = false,
+                               cfginfo = unknownLineInfo()): string =
   let p = if notRelativeToProj or os.isAbsolute(path) or
               '$' in path or path[0] == '.':
             path
@@ -239,6 +263,7 @@ proc processPath(path: string, notRelativeToProj = false): string =
     "nim", getPrefixDir(),
     "lib", libpath,
     "home", removeTrailingDirSep(os.getHomeDir()),
+    "config", cfginfo.toFullPath().splitFile().dir,
     "projectname", options.gProjectName,
     "projectpath", options.gProjectPath])
 
@@ -281,7 +306,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
   case switch.normalize
   of "path", "p":
     expectArg(switch, arg, pass, info)
-    addPath(processPath(arg), info)
+    addPath(processPath(arg, cfginfo=info), info)
   of "nimblepath", "babelpath":
     # keep the old name for compat
     if pass in {passCmd2, passPP} and not options.gNoNimblePath:
@@ -363,14 +388,19 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
     of "generational":
       gSelectedGC = gcGenerational
       defineSymbol("gcgenerational")
+    of "go":
+      gSelectedGC = gcGo
+      defineSymbol("gogc")
     of "none":
       gSelectedGC = gcNone
       defineSymbol("nogc")
     else: localError(info, errNoneBoehmRefcExpectedButXFound, arg)
-  of "warnings", "w": processOnOffSwitch({optWarns}, arg, pass, info)
+  of "warnings", "w":
+    if processOnOffSwitchOrList({optWarns}, arg, pass, info): listWarnings()
   of "warning": processSpecificNote(arg, wWarning, pass, info, switch)
   of "hint": processSpecificNote(arg, wHint, pass, info, switch)
-  of "hints": processOnOffSwitch({optHints}, arg, pass, info)
+  of "hints":
+    if processOnOffSwitchOrList({optHints}, arg, pass, info): listHints()
   of "threadanalysis": processOnOffSwitchG({optThreadAnalysis}, arg, pass, info)
   of "stacktrace": processOnOffSwitch({optStackTrace}, arg, pass, info)
   of "linetrace": processOnOffSwitch({optLineTrace}, arg, pass, info)
@@ -405,6 +435,8 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
   of "linedir": processOnOffSwitch({optLineDir}, arg, pass, info)
   of "assertions", "a": processOnOffSwitch({optAssert}, arg, pass, info)
   of "deadcodeelim": processOnOffSwitchG({optDeadCodeElim}, arg, pass, info)
+  of "reportconceptfailures":
+    processOnOffSwitchG({optReportConceptFailures}, arg, pass, info)
   of "threads":
     processOnOffSwitchG({optThreads}, arg, pass, info)
     #if optThreads in gGlobalOptions: incl(gNotes, warnGcUnsafe)
@@ -501,6 +533,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
   of "verbosity":
     expectArg(switch, arg, pass, info)
     gVerbosity = parseInt(arg)
+    gNotes = NotesVerbosity[gVerbosity]
+    incl(gNotes, enableNotes)
+    excl(gNotes, disableNotes)
   of "parallelbuild":
     expectArg(switch, arg, pass, info)
     gNumberOfProcessors = parseInt(arg)
@@ -530,6 +565,7 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
   of "genscript":
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optGenScript)
+  of "colors": processOnOffSwitchG({optUseColors}, arg, pass, info)
   of "lib":
     expectArg(switch, arg, pass, info)
     libpath = processPath(arg, notRelativeToProj=true)
@@ -579,6 +615,10 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
   of "experimental":
     expectNoArg(switch, arg, pass, info)
     gExperimentalMode = true
+  of "assembler":
+    cAssembler = nameToCC(arg)
+    if cAssembler notin cValidAssemblers:
+      localError(info, errGenerated, "'$1' is not a valid assembler." % [arg])
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(switch, arg)
     else: invalidCmdLineOption(pass, switch, info)
@@ -608,11 +648,18 @@ proc processSwitch*(pass: TCmdLinePass; p: OptParser) =
 proc processArgument*(pass: TCmdLinePass; p: OptParser;
                       argsCount: var int): bool =
   if argsCount == 0:
+    # nim filename.nims  is the same as "nim e filename.nims":
+    if p.key.endswith(".nims"):
+      options.command = "e"
+      options.gProjectName = unixToNativePath(p.key)
+      arguments = cmdLineRest(p)
+      result = true
+    elif pass != passCmd2:
       options.command = p.key
   else:
     if pass == passCmd1: options.commandArgs.add p.key
     if argsCount == 1:
-      # support UNIX style filenames anywhere for portable build scripts:
+      # support UNIX style filenames everywhere for portable build scripts:
       options.gProjectName = unixToNativePath(p.key)
       arguments = cmdLineRest(p)
       result = true

compiler/condsyms.nim

@@ -88,3 +88,8 @@ proc initDefines*() =
   defineSymbol("nimalias")
   defineSymbol("nimlocks")
   defineSymbol("nimnode")
+  defineSymbol("nimnomagic64")
+  defineSymbol("nimvarargstyped")
+  defineSymbol("nimtypedescfixed")
+  defineSymbol("nimKnowsNimvm")
+  defineSymbol("nimArrIdx")

compiler/crc.nim

@@ -1,147 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2012 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-import 
-  strutils
-
-type 
-  TCrc32* = int32
-
-const 
-  InitCrc32* = TCrc32(- 1)
-  InitAdler32* = int32(1)
-
-proc updateCrc32*(val: int8, crc: TCrc32): TCrc32 {.inline.}
-proc updateCrc32*(val: char, crc: TCrc32): TCrc32 {.inline.}
-proc crcFromBuf*(buf: pointer, length: int): TCrc32
-proc strCrc32*(s: string): TCrc32
-proc crcFromFile*(filename: string): TCrc32
-proc updateAdler32*(adler: int32, buf: pointer, length: int): int32
-# implementation
-
-type 
-  TCRC_TabEntry = int
-
-const 
-  crc32table: array[0..255, TCRC_TabEntry] = [0, 1996959894, - 301047508, 
-    - 1727442502, 124634137, 1886057615, - 379345611, - 1637575261, 249268274, 
-    2044508324, - 522852066, - 1747789432, 162941995, 2125561021, - 407360249, 
-    - 1866523247, 498536548, 1789927666, - 205950648, - 2067906082, 450548861, 
-    1843258603, - 187386543, - 2083289657, 325883990, 1684777152, - 43845254, 
-    - 1973040660, 335633487, 1661365465, - 99664541, - 1928851979, 997073096, 
-    1281953886, - 715111964, - 1570279054, 1006888145, 1258607687, - 770865667, 
-    - 1526024853, 901097722, 1119000684, - 608450090, - 1396901568, 853044451, 
-    1172266101, - 589951537, - 1412350631, 651767980, 1373503546, - 925412992, 
-    - 1076862698, 565507253, 1454621731, - 809855591, - 1195530993, 671266974, 
-    1594198024, - 972236366, - 1324619484, 795835527, 1483230225, - 1050600021, 
-    - 1234817731, 1994146192, 31158534, - 1731059524, - 271249366, 1907459465, 
-    112637215, - 1614814043, - 390540237, 2013776290, 251722036, - 1777751922, 
-    - 519137256, 2137656763, 141376813, - 1855689577, - 429695999, 1802195444, 
-    476864866, - 2056965928, - 228458418, 1812370925, 453092731, - 2113342271, 
-    - 183516073, 1706088902, 314042704, - 1950435094, - 54949764, 1658658271, 
-    366619977, - 1932296973, - 69972891, 1303535960, 984961486, - 1547960204, 
-    - 725929758, 1256170817, 1037604311, - 1529756563, - 740887301, 1131014506, 
-    879679996, - 1385723834, - 631195440, 1141124467, 855842277, - 1442165665, 
-    - 586318647, 1342533948, 654459306, - 1106571248, - 921952122, 1466479909, 
-    544179635, - 1184443383, - 832445281, 1591671054, 702138776, - 1328506846, 
-    - 942167884, 1504918807, 783551873, - 1212326853, - 1061524307, - 306674912, 
-    - 1698712650, 62317068, 1957810842, - 355121351, - 1647151185, 81470997, 
-    1943803523, - 480048366, - 1805370492, 225274430, 2053790376, - 468791541, 
-    - 1828061283, 167816743, 2097651377, - 267414716, - 2029476910, 503444072, 
-    1762050814, - 144550051, - 2140837941, 426522225, 1852507879, - 19653770, 
-    - 1982649376, 282753626, 1742555852, - 105259153, - 1900089351, 397917763, 
-    1622183637, - 690576408, - 1580100738, 953729732, 1340076626, - 776247311, 
-    - 1497606297, 1068828381, 1219638859, - 670225446, - 1358292148, 906185462, 
-    1090812512, - 547295293, - 1469587627, 829329135, 1181335161, - 882789492, 
-    - 1134132454, 628085408, 1382605366, - 871598187, - 1156888829, 570562233, 
-    1426400815, - 977650754, - 1296233688, 733239954, 1555261956, - 1026031705, 
-    - 1244606671, 752459403, 1541320221, - 1687895376, - 328994266, 1969922972, 
-    40735498, - 1677130071, - 351390145, 1913087877, 83908371, - 1782625662, 
-    - 491226604, 2075208622, 213261112, - 1831694693, - 438977011, 2094854071, 
-    198958881, - 2032938284, - 237706686, 1759359992, 534414190, - 2118248755, 
-    - 155638181, 1873836001, 414664567, - 2012718362, - 15766928, 1711684554, 
-    285281116, - 1889165569, - 127750551, 1634467795, 376229701, - 1609899400, 
-    - 686959890, 1308918612, 956543938, - 1486412191, - 799009033, 1231636301, 
-    1047427035, - 1362007478, - 640263460, 1088359270, 936918000, - 1447252397, 
-    - 558129467, 1202900863, 817233897, - 1111625188, - 893730166, 1404277552, 
-    615818150, - 1160759803, - 841546093, 1423857449, 601450431, - 1285129682, 
-    - 1000256840, 1567103746, 711928724, - 1274298825, - 1022587231, 1510334235, 
-    755167117]
-
-proc updateCrc32(val: int8, crc: TCrc32): TCrc32 = 
-  result = TCrc32(crc32table[(int(crc) xor (int(val) and 0x000000FF)) and
-      0x000000FF]) xor (crc shr TCrc32(8))
-
-proc updateCrc32(val: char, crc: TCrc32): TCrc32 = 
-  result = updateCrc32(toU8(ord(val)), crc)
-
-proc strCrc32(s: string): TCrc32 = 
-  result = InitCrc32
-  for i in countup(0, len(s) - 1): result = updateCrc32(s[i], result)
-  
-proc `><`*(c: TCrc32, s: string): TCrc32 = 
-  result = c
-  for i in 0..len(s)-1: result = updateCrc32(s[i], result)  
-  
-type 
-  TByteArray = array[0..10000000, int8]
-  PByteArray = ref TByteArray
-
-proc crcFromBuf(buf: pointer, length: int): TCrc32 = 
-  var p = cast[PByteArray](buf)
-  result = InitCrc32
-  for i in countup(0, length - 1): result = updateCrc32(p[i], result)
-  
-proc crcFromFile(filename: string): TCrc32 = 
-  const 
-    bufSize = 8000 # don't use 8K for the memory allocator!
-  var 
-    bin: File
-  result = InitCrc32
-  if not open(bin, filename): 
-    return                    # not equal if file does not exist
-  var buf = alloc(bufSize)
-  var p = cast[PByteArray](buf)
-  while true: 
-    var readBytes = readBuffer(bin, buf, bufSize)
-    for i in countup(0, readBytes - 1): result = updateCrc32(p[i], result)
-    if readBytes != bufSize: break 
-  dealloc(buf)
-  close(bin)
-
-const 
-  base = int32(65521) # largest prime smaller than 65536 
-                      # NMAX = 5552; original code with unsigned 32 bit integer 
-                      # NMAX is the largest n 
-                      # such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 
-  nmax = 3854 # code with signed 32 bit integer 
-              # NMAX is the largest n such that 
-              # 255n(n+1)/2 + (n+1)(BASE-1) <= 2^31-1 
-              # The penalty is the time loss in the extra MOD-calls. 
-
-proc updateAdler32(adler: int32, buf: pointer, length: int): int32 = 
-  var 
-    s1, s2: int32
-    L, k, b: int
-  s1 = adler and int32(0x0000FFFF)
-  s2 = (adler shr int32(16)) and int32(0x0000FFFF)
-  L = length
-  b = 0
-  while (L > 0): 
-    if L < nmax: k = L
-    else: k = nmax
-    dec(L, k)
-    while (k > 0): 
-      s1 = s1 +% int32((cast[cstring](buf))[b])
-      s2 = s2 +% s1
-      inc(b)
-      dec(k)
-    s1 = `%%`(s1, base)
-    s2 = `%%`(s2, base)
-  result = (s2 shl int32(16)) or s1

compiler/docgen.nim

@@ -18,7 +18,7 @@ import
 
 type
   TSections = array[TSymKind, Rope]
-  TDocumentor = object of rstgen.TRstGenerator
+  TDocumentor = object of rstgen.RstGenerator
     modDesc: Rope           # module description
     id: int                  # for generating IDs
     toc, section: TSections
@@ -29,7 +29,7 @@ type
   PDoc* = ref TDocumentor ## Alias to type less.
 
 proc compilerMsgHandler(filename: string, line, col: int,
-                        msgKind: rst.TMsgKind, arg: string) {.procvar.} =
+                        msgKind: rst.MsgKind, arg: string) {.procvar.} =
   # translate msg kind:
   var k: msgs.TMsgKind
   case msgKind
@@ -53,7 +53,7 @@ proc docgenFindFile(s: string): string {.procvar.} =
 
 proc parseRst(text, filename: string,
               line, column: int, hasToc: var bool,
-              rstOptions: TRstParseOptions): PRstNode =
+              rstOptions: RstParseOptions): PRstNode =
   result = rstParse(text, filename, line, column, hasToc, rstOptions,
                     docgenFindFile, compilerMsgHandler)
 
@@ -75,7 +75,7 @@ proc newDocumentor*(filename: string, config: StringTableRef): PDoc =
   ga('send', 'pageview');
 
 </script>
-    """ % [config["doc.googleAnalytics"]]
+    """ % [config.getOrDefault"doc.googleAnalytics"]
   else:
     result.analytics = ""
 
@@ -158,7 +158,7 @@ proc genRecComment(d: PDoc, n: PNode): Rope =
   if n == nil: return nil
   result = genComment(d, n).rope
   if result == nil:
-    if n.kind notin {nkEmpty..nkNilLit}:
+    if n.kind notin {nkEmpty..nkNilLit, nkEnumTy}:
       for i in countup(0, len(n)-1):
         result = genRecComment(d, n.sons[i])
         if result != nil: return

compiler/evaltempl.nim

@@ -25,16 +25,21 @@ proc copyNode(ctx: TemplCtx, a, b: PNode): PNode =
   if ctx.instLines: result.info = b.info
 
 proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
+  template handleParam(param) =
+    let x = param
+    if x.kind == nkArgList:
+      for y in items(x): result.add(y)
+    else:
+      result.add copyTree(x)
+
   case templ.kind
   of nkSym:
     var s = templ.sym
     if s.owner.id == c.owner.id:
       if s.kind == skParam and sfGenSym notin s.flags:
-        let x = actual.sons[s.position]
+        handleParam actual.sons[s.position]
-        if x.kind == nkArgList:
+      elif s.kind == skGenericParam:
-          for y in items(x): result.add(y)
+        handleParam actual.sons[s.owner.typ.len + s.position - 1]
-        else:
-          result.add copyTree(x)
       else:
         internalAssert sfGenSym in s.flags
         var x = PSym(idTableGet(c.mapping, s))
@@ -56,20 +61,44 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
 proc evalTemplateArgs(n: PNode, s: PSym): PNode =
   # if the template has zero arguments, it can be called without ``()``
   # `n` is then a nkSym or something similar
-  var a: int
+  var totalParams = case n.kind
-  case n.kind
+    of nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand, nkCallStrLit: n.len-1
-  of nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand, nkCallStrLit:
+    else: 0
-    a = sonsLen(n)
+
-  else: a = 0
+  var
-  var f = s.typ.sonsLen
+    # XXX: Since immediate templates are not subjected to the
-  if a > f: globalError(n.info, errWrongNumberOfArguments)
+    # standard sigmatching algorithm, they will have a number
+    # of deficiencies when it comes to generic params:
+    # Type dependencies between the parameters won't be honoured
+    # and the bound generic symbols won't be resolvable within
+    # their bodies. We could try to fix this, but it may be
+    # wiser to just deprecate immediate templates and macros
+    # now that we have working untyped parameters.
+    genericParams = if sfImmediate in s.flags: 0
+                    else: s.ast[genericParamsPos].len
+    expectedRegularParams = <s.typ.len
+    givenRegularParams = totalParams - genericParams
+
+  if totalParams > expectedRegularParams + genericParams:
+    globalError(n.info, errWrongNumberOfArguments)
 
   result = newNodeI(nkArgList, n.info)
-  for i in countup(1, f - 1):
+  for i in 1 .. givenRegularParams:
-    var arg = if i < a: n.sons[i] else: copyTree(s.typ.n.sons[i].sym.ast)
+    result.addSon n.sons[i]
-    if arg == nil or arg.kind == nkEmpty:
+
+  # handle parameters with default values, which were
+  # not supplied by the user
+  for i in givenRegularParams+1 .. expectedRegularParams:
+    let default = s.typ.n.sons[i].sym.ast
+    if default.isNil or default.kind == nkEmpty:
       localError(n.info, errWrongNumberOfArguments)
-    addSon(result, arg)
+      addSon(result, ast.emptyNode)
+    else:
+      addSon(result, default.copyTree)
+
+  # add any generic paramaters
+  for i in 1 .. genericParams:
+    result.addSon n.sons[givenRegularParams + i]
 
 var evalTemplateCounter* = 0
   # to prevent endless recursion in templates instantiation

compiler/extccomp.nim

@@ -13,12 +13,13 @@
 # nim files.
 
 import
-  lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs, crc
+  lists, ropes, os, strutils, osproc, platform, condsyms, options, msgs,
+  securehash, streams
 
 type
   TSystemCC* = enum
     ccNone, ccGcc, ccLLVM_Gcc, ccCLang, ccLcc, ccBcc, ccDmc, ccWcc, ccVcc,
-    ccTcc, ccPcc, ccUcc, ccIcl
+    ccTcc, ccPcc, ccUcc, ccIcl, asmFasm
   TInfoCCProp* = enum         # properties of the C compiler:
     hasSwitchRange,           # CC allows ranges in switch statements (GNU C)
     hasComputedGoto,          # CC has computed goto (GNU C extension)
@@ -26,8 +27,8 @@ type
     hasAssume,                # CC has __assume (Visual C extension)
     hasGcGuard,               # CC supports GC_GUARD to keep stack roots
     hasGnuAsm,                # CC's asm uses the absurd GNU assembler syntax
-    hasNakedDeclspec,         # CC has __declspec(naked)
+    hasDeclspec,              # CC has __declspec(X)
-    hasNakedAttribute         # CC has __attribute__((naked))
+    hasAttribute,             # CC has __attribute__((X))
   TInfoCCProps* = set[TInfoCCProp]
   TInfoCC* = tuple[
     name: string,        # the short name of the compiler
@@ -85,7 +86,7 @@ compiler gcc:
     structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name
     packedPragma: "__attribute__((__packed__))",
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm,
-            hasNakedAttribute})
+            hasAttribute})
 
 # LLVM Frontend for GCC/G++
 compiler llvmGcc:
@@ -122,12 +123,12 @@ compiler vcc:
     includeCmd: " /I",
     linkDirCmd: " /LIBPATH:",
     linkLibCmd: " $1.lib",
-    debug: " /GZ /Zi ",
+    debug: " /RTC1 /Z7 ",
     pic: "",
     asmStmtFrmt: "__asm{$n$1$n}$n",
     structStmtFmt: "$3$n$1 $2",
     packedPragma: "#pragma pack(1)",
-    props: {hasCpp, hasAssume, hasNakedDeclspec})
+    props: {hasCpp, hasAssume, hasDeclspec})
 
 # Intel C/C++ Compiler
 compiler icl:
@@ -317,6 +318,31 @@ compiler ucc:
     packedPragma: "", # XXX: not supported yet
     props: {})
 
+# fasm assembler
+compiler fasm:
+  result = (
+    name: "fasm",
+    objExt: "o",
+    optSpeed: "",
+    optSize: "",
+    compilerExe: "fasm",
+    cppCompiler: "fasm",
+    compileTmpl: "$file $objfile",
+    buildGui: "",
+    buildDll: "",
+    buildLib: "",
+    linkerExe: "",
+    linkTmpl: "",
+    includeCmd: "",
+    linkDirCmd: "",
+    linkLibCmd: "",
+    debug: "",
+    pic: "",
+    asmStmtFrmt: "",
+    structStmtFmt: "",
+    packedPragma: "",
+    props: {})
+
 const
   CC*: array[succ(low(TSystemCC))..high(TSystemCC), TInfoCC] = [
     gcc(),
@@ -330,17 +356,22 @@ const
     tcc(),
     pcc(),
     ucc(),
-    icl()]
+    icl(),
+    fasm()]
 
   hExt* = ".h"
 
 var
   cCompiler* = ccGcc # the used compiler
+  cAssembler* = ccNone
   gMixedMode*: bool  # true if some module triggered C++ codegen
   cIncludes*: seq[string] = @[]   # directories to search for included files
   cLibs*: seq[string] = @[]       # directories to search for lib files
   cLinkedLibs*: seq[string] = @[] # libraries to link
 
+const
+  cValidAssemblers* = {asmFasm}
+
 # implementation
 
 proc libNameTmpl(): string {.inline.} =
@@ -428,7 +459,7 @@ proc resetCompilationLists* =
   initLinkedList(toCompile)
   ## XXX: we must associate these with their originating module
   # when the module is loaded/unloaded it adds/removes its items
-  # That's because we still need to CRC check the external files
+  # That's because we still need to hash check the external files
   # Maybe we can do that in checkDep on the other hand?
   initLinkedList(externalToCompile)
   initLinkedList(toLink)
@@ -437,17 +468,13 @@ proc addFileToLink*(filename: string) =
   prependStr(toLink, filename)
   # BUGFIX: was ``appendStr``
 
-proc execWithEcho(cmd: string, prettyCmd = ""): int =
+proc execWithEcho(cmd: string, msg = hintExecuting): int =
-  if optListCmd in gGlobalOptions or gVerbosity > 0:
+  rawMessage(msg, cmd)
-    if prettyCmd != "":
-      msgWriteln(prettyCmd)
-    else:
-      msgWriteln(cmd)
   result = execCmd(cmd)
 
-proc execExternalProgram*(cmd: string, prettyCmd = "") =
+proc execExternalProgram*(cmd: string, msg = hintExecuting) =
-  if execWithEcho(cmd, prettyCmd) != 0:
+  if execWithEcho(cmd, msg) != 0:
-    rawMessage(errExecutionOfProgramFailed, "")
+    rawMessage(errExecutionOfProgramFailed, cmd)
 
 proc generateScript(projectFile: string, script: Rope) =
   let (dir, name, ext) = splitFile(projectFile)
@@ -530,6 +557,21 @@ proc getLinkerExe(compiler: TSystemCC): string =
 
 proc getCompileCFileCmd*(cfilename: string, isExternal = false): string =
   var c = cCompiler
+  if cfilename.endswith(".asm"):
+    var customAssembler = getConfigVar("assembler")
+    if customAssembler.len > 0:
+      c = nameToCC(customAssembler)
+    else:
+      if targetCPU == cpuI386 or targetCPU == cpuAmd64:
+        c = asmFasm
+      else:
+        c = ccNone
+
+    if c == ccNone:
+      rawMessage(errExternalAssemblerNotFound, "")
+    elif c notin cValidAssemblers:
+      rawMessage(errExternalAssemblerNotValid, customAssembler)
+
   var options = cFileSpecificOptions(cfilename)
   var exe = getConfigVar(c, ".exe")
   if exe.len == 0: exe = c.getCompilerExe
@@ -572,32 +614,30 @@ proc getCompileCFileCmd*(cfilename: string, isExternal = false): string =
     "nim", quoteShell(getPrefixDir()),
     "lib", quoteShell(libpath)])
 
-proc footprint(filename: string): TCrc32 =
+proc footprint(filename: string): SecureHash =
-  # note, '><' further modifies a crc value with a string.
+  result = secureHash(
-  result = crcFromFile(filename) ><
+    $secureHashFile(filename) &
-      platform.OS[targetOS].name ><
+    platform.OS[targetOS].name &
-      platform.CPU[targetCPU].name ><
+    platform.CPU[targetCPU].name &
-      extccomp.CC[extccomp.cCompiler].name ><
+    extccomp.CC[extccomp.cCompiler].name &
-      getCompileCFileCmd(filename, true)
+    getCompileCFileCmd(filename, true))
 
 proc externalFileChanged(filename: string): bool =
   if gCmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM}:
     return false
 
-  var crcFile = toGeneratedFile(filename.withPackageName, "crc")
+  var hashFile = toGeneratedFile(filename.withPackageName, "sha1")
-  var currentCrc = int(footprint(filename))
+  var currentHash = footprint(filename)
   var f: File
-  if open(f, crcFile, fmRead):
+  if open(f, hashFile, fmRead):
-    var line = newStringOfCap(40)
+    let oldHash = parseSecureHash(f.readLine())
-    if not f.readLine(line): line = "0"
     close(f)
-    var oldCrc = parseInt(line)
+    result = oldHash != currentHash
-    result = oldCrc != currentCrc
   else:
     result = true
   if result:
-    if open(f, crcFile, fmWrite):
+    if open(f, hashFile, fmWrite):
-      f.writeln($currentCrc)
+      f.writeLine($currentHash)
       close(f)
 
 proc addExternalFileToCompile*(filename: string) =
@@ -632,6 +672,12 @@ proc callCCompiler*(projectfile: string) =
   var prettyCmds: TStringSeq = @[]
   let prettyCb = proc (idx: int) =
     echo prettyCmds[idx]
+  let runCb = proc (idx: int, p: Process) =
+    let exitCode = p.peekExitCode
+    if exitCode != 0:
+      rawMessage(errGenerated, "execution of an external compiler program '" &
+        cmds[idx] & "' failed with exit code: " & $exitCode & "\n\n" &
+        p.outputStream.readAll.strip)
   compileCFile(toCompile, script, cmds, prettyCmds, false)
   compileCFile(externalToCompile, script, cmds, prettyCmds, true)
   if optCompileOnly notin gGlobalOptions:
@@ -640,22 +686,19 @@ proc callCCompiler*(projectfile: string) =
     if gNumberOfProcessors <= 1:
       for i in countup(0, high(cmds)):
         res = execWithEcho(cmds[i])
-        if res != 0: rawMessage(errExecutionOfProgramFailed, [])
+        if res != 0: rawMessage(errExecutionOfProgramFailed, cmds[i])
     elif optListCmd in gGlobalOptions or gVerbosity > 1:
-      res = execProcesses(cmds, {poEchoCmd, poUsePath, poParentStreams},
+      res = execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams},
-                          gNumberOfProcessors)
+                          gNumberOfProcessors, afterRunEvent=runCb)
     elif gVerbosity == 1:
-      res = execProcesses(cmds, {poUsePath, poParentStreams},
+      res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams},
-                          gNumberOfProcessors, prettyCb)
+                          gNumberOfProcessors, prettyCb, afterRunEvent=runCb)
     else:
-      res = execProcesses(cmds, {poUsePath, poParentStreams},
+      res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams},
-                          gNumberOfProcessors)
+                          gNumberOfProcessors, afterRunEvent=runCb)
     if res != 0:
       if gNumberOfProcessors <= 1:
-        rawMessage(errExecutionOfProgramFailed, [])
+        rawMessage(errExecutionOfProgramFailed, cmds.join())
-      else:
-        rawMessage(errGenerated, " execution of an external program failed; " &
-                   "rerun with --parallelBuild:1 to see the error message")
   if optNoLinking notin gGlobalOptions:
     # call the linker:
     var it = PStrEntry(toLink.head)
@@ -668,7 +711,8 @@ proc callCCompiler*(projectfile: string) =
       it = PStrEntry(it.next)
 
     if optGenStaticLib in gGlobalOptions:
-      linkCmd = CC[c].buildLib % ["libfile", (libNameTmpl() % gProjectName),
+      let name = splitFile(gProjectName).name
+      linkCmd = CC[c].buildLib % ["libfile", (libNameTmpl() % name),
                                   "objfiles", objfiles]
     else:
       var linkerExe = getConfigVar(c, ".linkerexe")
@@ -703,10 +747,8 @@ proc callCCompiler*(projectfile: string) =
           "nim", quoteShell(getPrefixDir()),
           "lib", quoteShell(libpath)])
     if optCompileOnly notin gGlobalOptions:
-      if gVerbosity == 1:
+      execExternalProgram(linkCmd,
-        execExternalProgram(linkCmd, "[Linking]")
+                          if gVerbosity > 1: hintExecuting else: hintLinking)
-      else:
-        execExternalProgram(linkCmd)
   else:
     linkCmd = ""
   if optGenScript in gGlobalOptions:

compiler/filter_tmpl.nim

@@ -67,9 +67,9 @@ proc parseLine(p: var TTmplParser) =
     keyw: string
   j = 0
   while p.x[j] == ' ': inc(j)
-  if (p.x[0] == p.nimDirective) and (p.x[0 + 1] == '!'): 
+  if p.x[0] == p.nimDirective and p.x[1] in {'?', '!'}:
     newLine(p)
-  elif (p.x[j] == p.nimDirective): 
+  elif p.x[j] == p.nimDirective:
     newLine(p)
     inc(j)
     while p.x[j] == ' ': inc(j)

compiler/forloops.nim

@@ -12,9 +12,9 @@
 import ast, astalgo
 
 const
-  someCmp = {mEqI, mEqI64, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
+  someCmp = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
-    mEqUntracedRef, mLeI, mLeI64, mLeF64, mLeU, mLeU64, mLeEnum,
+    mEqUntracedRef, mLeI, mLeF64, mLeU, mLeU64, mLeEnum,
-    mLeCh, mLeB, mLePtr, mLtI, mLtI64, mLtF64, mLtU, mLtU64, mLtEnum, 
+    mLeCh, mLeB, mLePtr, mLtI, mLtF64, mLtU, mLtU64, mLtEnum,
     mLtCh, mLtB, mLtPtr}
 
 proc isCounter(s: PSym): bool {.inline.} =

compiler/guards.nim

@@ -13,13 +13,13 @@ import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents,
   saturate
 
 const
-  someEq = {mEqI, mEqI64, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
+  someEq = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
     mEqUntracedRef, mEqStr, mEqSet, mEqCString}
 
   # set excluded here as the semantics are vastly different:
-  someLe = {mLeI, mLeI64, mLeF64, mLeU, mLeU64, mLeEnum,
+  someLe = {mLeI, mLeF64, mLeU, mLeU64, mLeEnum,
             mLeCh, mLeB, mLePtr, mLeStr}
-  someLt = {mLtI, mLtI64, mLtF64, mLtU, mLtU64, mLtEnum,
+  someLt = {mLtI, mLtF64, mLtU, mLtU64, mLtEnum,
             mLtCh, mLtB, mLtPtr, mLtStr}
 
   someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq,
@@ -30,13 +30,14 @@ const
   someHigh = {mHigh}
   # we don't list unsigned here because wrap around semantics suck for
   # proving anything:
-  someAdd = {mAddI, mAddI64, mAddF64, mSucc}
+  someAdd = {mAddI, mAddF64, mSucc}
-  someSub = {mSubI, mSubI64, mSubF64, mPred}
+  someSub = {mSubI, mSubF64, mPred}
-  someMul = {mMulI, mMulI64, mMulF64}
+  someMul = {mMulI, mMulF64}
-  someDiv = {mDivI, mDivI64, mDivF64}
+  someDiv = {mDivI, mDivF64}
-  someMod = {mModI, mModI64}
+  someMod = {mModI}
   someMax = {mMaxI, mMaxF64}
   someMin = {mMinI, mMinF64}
+  someBinaryOp = someAdd+someSub+someMul+someMax+someMin
 
 proc isValue(n: PNode): bool = n.kind in {nkCharLit..nkNilLit}
 proc isLocation(n: PNode): bool = not n.isValue
@@ -165,11 +166,21 @@ proc `|+|`(a, b: PNode): PNode =
   if a.kind in {nkCharLit..nkUInt64Lit}: result.intVal = a.intVal |+| b.intVal
   else: result.floatVal = a.floatVal + b.floatVal
 
+proc `|-|`(a, b: PNode): PNode =
+  result = copyNode(a)
+  if a.kind in {nkCharLit..nkUInt64Lit}: result.intVal = a.intVal |-| b.intVal
+  else: result.floatVal = a.floatVal - b.floatVal
+
 proc `|*|`(a, b: PNode): PNode =
   result = copyNode(a)
   if a.kind in {nkCharLit..nkUInt64Lit}: result.intVal = a.intVal |*| b.intVal
   else: result.floatVal = a.floatVal * b.floatVal
 
+proc `|div|`(a, b: PNode): PNode =
+  result = copyNode(a)
+  if a.kind in {nkCharLit..nkUInt64Lit}: result.intVal = a.intVal div b.intVal
+  else: result.floatVal = a.floatVal / b.floatVal
+
 proc negate(a, b, res: PNode): PNode =
   if b.kind in {nkCharLit..nkUInt64Lit} and b.intVal != low(BiggestInt):
     var b = copyNode(b)
@@ -213,10 +224,16 @@ proc reassociation(n: PNode): PNode =
     if result[2].isValue and
         result[1].getMagic in someAdd and result[1][2].isValue:
       result = opAdd.buildCall(result[1][1], result[1][2] |+| result[2])
+      if result[2].intVal == 0:
+        result = result[1]
   of someMul:
     if result[2].isValue and
         result[1].getMagic in someMul and result[1][2].isValue:
-      result = opAdd.buildCall(result[1][1], result[1][2] |*| result[2])
+      result = opMul.buildCall(result[1][1], result[1][2] |*| result[2])
+      if result[2].intVal == 1:
+        result = result[1]
+      elif result[2].intVal == 0:
+        result = zero()
   else: discard
 
 proc pred(n: PNode): PNode =
@@ -234,7 +251,7 @@ proc canon*(n: PNode): PNode =
       result.sons[i] = canon(n.sons[i])
   elif n.kind == nkSym and n.sym.kind == skLet and
       n.sym.ast.getMagic in (someEq + someAdd + someMul + someMin +
-      someMax + someHigh + {mUnaryLt} + someSub + someLen):
+      someMax + someHigh + {mUnaryLt} + someSub + someLen + someDiv):
     result = n.sym.ast.copyTree
   else:
     result = n
@@ -248,7 +265,7 @@ proc canon*(n: PNode): PNode =
     # high == len+(-1)
     result = opAdd.buildCall(opLen.buildCall(result[1]), minusOne())
   of mUnaryLt:
-    result = buildCall(opAdd, result[1], newIntNode(nkIntLit, -1))
+    result = buildCall(opAdd, result[1], minusOne())
   of someSub:
     # x - 4  -->  x + (-4)
     result = negate(result[1], result[2], result)
@@ -294,6 +311,16 @@ proc canon*(n: PNode): PNode =
         if plus != nil and not isLetLocation(x, true):
           result = buildCall(result[0].sym, plus, y[1])
       else: discard
+    elif x.isValue and y.getMagic in someAdd and y[2].isValue:
+      # 0 <= a.len + 3
+      # -3 <= a.len
+      result.sons[1] = x |-| y[2]
+      result.sons[2] = y[1]
+    elif x.isValue and y.getMagic in someSub and y[2].isValue:
+      # 0 <= a.len - 3
+      # 3 <= a.len
+      result.sons[1] = x |+| y[2]
+      result.sons[2] = y[1]
   else: discard
 
 proc `+@`*(a: PNode; b: BiggestInt): PNode =
@@ -313,6 +340,9 @@ proc usefulFact(n: PNode): PNode =
     if isLetLocation(n.sons[1], true) or isLetLocation(n.sons[2], true):
       # XXX algebraic simplifications!  'i-1 < a.len' --> 'i < a.len+1'
       result = n
+    elif n[1].getMagic in someLen or n[2].getMagic in someLen:
+      # XXX Rethink this whole idea of 'usefulFact' for semparallel
+      result = n
   of mIsNil:
     if isLetLocation(n.sons[1], false) or isVar(n.sons[1]):
       result = n
@@ -366,8 +396,8 @@ proc usefulFact(n: PNode): PNode =
 type
   TModel* = seq[PNode] # the "knowledge base"
 
-proc addFact*(m: var TModel, n: PNode) =
+proc addFact*(m: var TModel, nn: PNode) =
-  let n = usefulFact(n)
+  let n = usefulFact(nn)
   if n != nil: m.add n
 
 proc addFactNeg*(m: var TModel, n: PNode) =
@@ -697,10 +727,57 @@ proc simpleSlice*(a, b: PNode): BiggestInt =
   else:
     result = -1
 
+
+template isMul(x): expr = x.getMagic in someMul
+template isDiv(x): expr = x.getMagic in someDiv
+template isAdd(x): expr = x.getMagic in someAdd
+template isSub(x): expr = x.getMagic in someSub
+template isVal(x): expr = x.kind in {nkCharLit..nkUInt64Lit}
+template isIntVal(x, y): expr = x.intVal == y
+
+import macros
+
+macro `=~`(x: PNode, pat: untyped): bool =
+  proc m(x, pat, conds: NimNode) =
+    case pat.kind
+    of nnkInfix:
+      case $pat[0]
+      of "*": conds.add getAst(isMul(x))
+      of "/": conds.add getAst(isDiv(x))
+      of "+": conds.add getAst(isAdd(x))
+      of "-": conds.add getAst(isSub(x))
+      else:
+        error("invalid pattern")
+      m(newTree(nnkBracketExpr, x, newLit(1)), pat[1], conds)
+      m(newTree(nnkBracketExpr, x, newLit(2)), pat[2], conds)
+    of nnkPar:
+      if pat.len == 1:
+        m(x, pat[0], conds)
+      else:
+        error("invalid pattern")
+    of nnkIdent:
+      let c = newTree(nnkStmtListExpr, newLetStmt(pat, x))
+      conds.add c
+      if ($pat)[^1] == 'c': c.add(getAst(isVal(pat)))
+      else: c.add bindSym"true"
+    of nnkIntLit:
+      conds.add(getAst(isIntVal(pat.intVal)))
+    else:
+      error("invalid pattern")
+
+  var conds = newTree(nnkBracket)
+  m(x, pat, conds)
+  result = nestList(!"and", conds)
+
+
+proc isMinusOne(n: PNode): bool =
+  n.kind in {nkCharLit..nkUInt64Lit} and n.intVal == -1
+
 proc pleViaModel(model: TModel; aa, bb: PNode): TImplication
 
 proc ple(m: TModel; a, b: PNode): TImplication =
   template `<=?`(a,b): expr = ple(m,a,b) == impYes
+  template `>=?`(a,b): expr = ple(m, nkIntLit.newIntNode(b), a) == impYes
 
   #   0 <= 3
   if a.isValue and b.isValue:
@@ -721,6 +798,7 @@ proc ple(m: TModel; a, b: PNode): TImplication =
     if a.intVal <= 0: return impYes
 
   #   x <= y+c  if 0 <= c and x <= y
+  #   x <= y+(-c)  if c <= 0  and y >= x
   if b.getMagic in someAdd and zero() <=? b[2] and a <=? b[1]: return impYes
 
   #   x+c <= y  if c <= 0 and x <= y
@@ -730,10 +808,44 @@ proc ple(m: TModel; a, b: PNode): TImplication =
   if b.getMagic in someMul:
     if a <=? b[1] and one() <=? b[2] and zero() <=? b[1]: return impYes
 
+
+  if a.getMagic in someMul and a[2].isValue and a[1].getMagic in someDiv and
+      a[1][2].isValue:
+    # simplify   (x div 4) * 2 <= y   to  x div (c div d)  <= y
+    if ple(m, buildCall(opDiv, a[1][1], `|div|`(a[1][2], a[2])), b) == impYes:
+      return impYes
+
+  # x*3 + x == x*4. It follows that:
+  # x*3 + y <= x*4  if  y <= x  and 3 <= 4
+  if a =~ x*dc + y and b =~ x2*ec:
+    if sameTree(x, x2):
+      let ec1 = opAdd.buildCall(ec, minusOne())
+      if x >=? 1 and ec >=? 1 and dc >=? 1 and dc <=? ec1 and y <=? x:
+        return impYes
+  elif a =~ x*dc and b =~ x2*ec + y:
+    #echo "BUG cam ehrer e ", a, " <=? ", b
+    if sameTree(x, x2):
+      let ec1 = opAdd.buildCall(ec, minusOne())
+      if x >=? 1 and ec >=? 1 and dc >=? 1 and dc <=? ec1 and y <=? zero():
+        return impYes
+
+  #  x+c <= x+d  if c <= d. Same for *, - etc.
+  if a.getMagic in someBinaryOp and a.getMagic == b.getMagic:
+    if sameTree(a[1], b[1]) and a[2] <=? b[2]: return impYes
+    elif sameTree(a[2], b[2]) and a[1] <=? b[1]: return impYes
+
   #   x div c <= y   if   1 <= c  and  0 <= y  and x <= y:
   if a.getMagic in someDiv:
     if one() <=? a[2] and zero() <=? b and a[1] <=? b: return impYes
 
+    #  x div c <= x div d  if d <= c
+    if b.getMagic in someDiv:
+      if sameTree(a[1], b[1]) and b[2] <=? a[2]: return impYes
+
+    # x div z <= x - 1   if  z <= x
+    if a[2].isValue and b.getMagic in someAdd and b[2].isMinusOne:
+      if a[2] <=? a[1] and sameTree(a[1], b[1]): return impYes
+
   # slightly subtle:
   # x <= max(y, z)  iff x <= y or x <= z
   # note that 'x <= max(x, z)' is a special case of the above rule
@@ -769,11 +881,19 @@ proc pleViaModelRec(m: var TModel; a, b: PNode): TImplication =
   for i in 0..m.high:
     let fact = m[i]
     if fact != nil and fact.getMagic in someLe:
-      # x <= y implies a <= b  if  a <= x and y <= b
-      let x = fact[1]
-      let y = fact[2]
       # mark as used:
       m[i] = nil
+      # i <= len-100
+      # i <=? len-1
+      # --> true  if  (len-100) <= (len-1)
+      let x = fact[1]
+      let y = fact[2]
+      if sameTree(x, a) and y.getMagic in someAdd and b.getMagic in someAdd and
+         sameTree(y[1], b[1]):
+        if ple(m, b[2], y[2]) == impYes:
+          return impYes
+
+      # x <= y implies a <= b  if  a <= x and y <= b
       if ple(m, a, x) == impYes:
         if ple(m, y, b) == impYes:
           return impYes

compiler/idents.nim

@@ -12,7 +12,7 @@
 # id. This module is essential for the compiler's performance.
 
 import
-  hashes, strutils
+  hashes, strutils, etcpriv
 
 type
   TIdObj* = object of RootObj
@@ -23,7 +23,7 @@ type
   TIdent*{.acyclic.} = object of TIdObj
     s*: string
     next*: PIdent             # for hash-table chaining
-    h*: THash                 # hash value of s
+    h*: Hash                 # hash value of s
 
 var firstCharIsCS*: bool = true
 var buckets*: array[0..4096 * 2 - 1, PIdent]
@@ -37,6 +37,8 @@ proc cmpIgnoreStyle(a, b: cstring, blen: int): int =
   while j < blen:
     while a[i] == '_': inc(i)
     while b[j] == '_': inc(j)
+    while isMagicIdentSeparatorRune(a, i): inc(i, magicIdentSeparatorRuneByteWidth)
+    while isMagicIdentSeparatorRune(b, j): inc(j, magicIdentSeparatorRuneByteWidth)
     # tolower inlined:
     var aa = a[i]
     var bb = b[j]
@@ -65,7 +67,7 @@ proc cmpExact(a, b: cstring, blen: int): int =
 
 var wordCounter = 1
 
-proc getIdent*(identifier: cstring, length: int, h: THash): PIdent =
+proc getIdent*(identifier: cstring, length: int, h: Hash): PIdent =
   var idx = h and high(buckets)
   result = buckets[idx]
   var last: PIdent = nil
@@ -99,7 +101,7 @@ proc getIdent*(identifier: string): PIdent =
   result = getIdent(cstring(identifier), len(identifier),
                     hashIgnoreStyle(identifier))
 
-proc getIdent*(identifier: string, h: THash): PIdent = 
+proc getIdent*(identifier: string, h: Hash): PIdent =
   result = getIdent(cstring(identifier), len(identifier), h)
 
 proc identEq*(id: PIdent, name: string): bool =

compiler/idgen.nim

@@ -48,8 +48,8 @@ proc toGid(f: string): string =
 
 proc saveMaxIds*(project: string) =
   var f = open(project.toGid, fmWrite)
-  f.writeln($gFrontEndId)
+  f.writeLine($gFrontEndId)
-  f.writeln($gBackendId)
+  f.writeLine($gBackendId)
   f.close()
 
 proc loadMaxIds*(project: string) =

compiler/importer.nim

@@ -121,7 +121,7 @@ proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
       if s.kind != skEnumField:
         if s.kind notin ExportableSymKinds:
           internalError(s.info, "importAllSymbols: " & $s.kind)
-        if exceptSet.empty or s.name.id notin exceptSet:
+        if exceptSet.isNil or s.name.id notin exceptSet:
           rawImportSymbol(c, s)
     s = nextIter(i, fromMod.tab)
 
@@ -138,7 +138,7 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet) =
       let s = a.sym
       if s.kind == skModule:
         importAllSymbolsExcept(c, s, exceptSet)
-      elif exceptSet.empty or s.name.id notin exceptSet:
+      elif exceptSet.isNil or s.name.id notin exceptSet:
         rawImportSymbol(c, s)
   of nkExportExceptStmt:
     localError(n.info, errGenerated, "'export except' not implemented")

compiler/installer.ini

@@ -6,7 +6,7 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;amd64;powerpc64;arm;sparc;mips;powerpc
+  linux: i386;amd64;powerpc64;arm;sparc;mips;mipsel;powerpc;powerpc64el;arm64
   macosx: i386;amd64;powerpc64
   solaris: i386;amd64;sparc
   freebsd: i386;amd64
@@ -99,9 +99,13 @@ Files: "lib/pure/concurrency/*.nim"
 Files: "lib/pure/unidecode/*.nim"
 Files: "lib/pure/concurrency/*.cfg"
 Files: "lib/impure/*.nim"
+Files: "lib/impure/nre/private/*.nim"
 Files: "lib/wrappers/*.nim"
 
 Files: "lib/wrappers/readline/*.nim"
+Files: "lib/wrappers/linenoise/*.nim"
+Files: "lib/wrappers/linenoise/*.c"
+Files: "lib/wrappers/linenoise/*.h"
 Files: "lib/wrappers/sdl/*.nim"
 Files: "lib/wrappers/zip/*.nim"
 Files: "lib/wrappers/zip/libzip_all.c"
@@ -111,6 +115,9 @@ Files: "lib/posix/*.nim"
 Files: "lib/js/*.nim"
 Files: "lib/packages/docutils/*.nim"
 
+Files: "lib/deprecated/core/*.nim"
+Files: "lib/deprecated/pure/*.nim"
+Files: "lib/deprecated/pure/*.cfg"
 
 [Other]
 Files: "examples/*.nim"
@@ -203,7 +210,11 @@ Files: "tests/stdlib/*.nim"
 Files: "tests/system/*.nim"
 Files: "tests/template/*.nim"
 Files: "tests/testament/*.nim"
-Files: "tests/testdata/*.nim"
+Files: "tests/testdata/*.csv"
+Files: "tests/testdata/*.html"
+Files: "tests/testdata/*.json"
+Files: "tests/testdata/*.txt"
+Files: "tests/testdata/*.xml"
 Files: "tests/threads/*.nim"
 Files: "tests/threads/*.cfg"
 Files: "tests/trmacros/*.nim"
@@ -236,6 +247,7 @@ BinPath: r"bin;dist\mingw\bin;dist"
 ;           Section | dir | zipFile | size hint (in KB) | url | exe start menu entry
 Download: r"Documentation|doc|docs.zip|13824|http://nim-lang.org/download/docs-${version}.zip|overview.html"
 Download: r"C Compiler (MingW)|dist|mingw.zip|82944|http://nim-lang.org/download/${mingw}.zip"
+Download: r"Support DLL's|bin|nim_dlls.zip|479|http://nim-lang.org/download/dlls.zip"
 Download: r"Aporia IDE|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.3.0.zip|aporia-0.3.0\bin\aporia.exe"
 ; for now only NSIS supports optional downloads
 

compiler/jsgen.nim

@@ -30,8 +30,8 @@ implements the required case distinction.
 
 
 import
-  ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp,
+  ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options,
-  options, nversion, nimsets, msgs, crc, bitsets, idents, lists, types, os,
+  nversion, nimsets, msgs, securehash, bitsets, idents, lists, types, os,
   times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils,
   intsets, cgmeth, lowerings
 
@@ -136,7 +136,7 @@ proc mapType(typ: PType): TJSTypeKind =
       result = etyBaseIndex
   of tyPointer:
     # treat a tyPointer like a typed pointer to an array of bytes
-    result = etyInt
+    result = etyBaseIndex
   of tyRange, tyDistinct, tyOrdinal, tyConst, tyMutable, tyIter, tyProxy:
     result = mapType(t.sons[0])
   of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyChar: result = etyInt
@@ -197,10 +197,10 @@ proc isSimpleExpr(n: PNode): bool =
   elif n.isAtom:
     result = true
 
-proc getTemp(p: PProc): Rope =
+proc getTemp(p: PProc, defineInLocals: bool = true): Rope =
   inc(p.unique)
   result = "Tmp$1" % [rope(p.unique)]
-  addf(p.locals, "var $1;$n" | "local $1;$n", [result])
+  if defineInLocals: addf(p.locals, "var $1;$n" | "local $1;$n", [result])
 
 proc genAnd(p: PProc, a, b: PNode, r: var TCompRes) =
   assert r.kind == resNone
@@ -258,11 +258,6 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["mulInt", "", "mulInt($1, $2)", "($1 * $2)"], # MulI
     ["divInt", "", "divInt($1, $2)", "Math.floor($1 / $2)"], # DivI
     ["modInt", "", "modInt($1, $2)", "Math.floor($1 % $2)"], # ModI
-    ["addInt64", "", "addInt64($1, $2)", "($1 + $2)"], # AddI64
-    ["subInt64", "", "subInt64($1, $2)", "($1 - $2)"], # SubI64
-    ["mulInt64", "", "mulInt64($1, $2)", "($1 * $2)"], # MulI64
-    ["divInt64", "", "divInt64($1, $2)", "Math.floor($1 / $2)"], # DivI64
-    ["modInt64", "", "modInt64($1, $2)", "Math.floor($1 % $2)"], # ModI64
     ["addInt", "", "addInt($1, $2)", "($1 + $2)"], # Succ
     ["subInt", "", "subInt($1, $2)", "($1 - $2)"], # Pred
     ["", "", "($1 + $2)", "($1 + $2)"], # AddF64
@@ -276,11 +271,6 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 ^ $2)", "($1 ^ $2)"], # BitxorI
     ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinI
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxI
-    ["", "", "($1 >>> $2)", "($1 >>> $2)"], # ShrI64
-    ["", "", "($1 << $2)", "($1 << $2)"], # ShlI64
-    ["", "", "($1 & $2)", "($1 & $2)"], # BitandI64
-    ["", "", "($1 | $2)", "($1 | $2)"], # BitorI64
-    ["", "", "($1 ^ $2)", "($1 ^ $2)"], # BitxorI64
     ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinF64
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxF64
     ["addU", "addU", "addU($1, $2)", "addU($1, $2)"], # addU
@@ -291,9 +281,6 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 == $2)", "($1 == $2)"], # EqI
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeI
     ["", "", "($1 < $2)", "($1 < $2)"], # LtI
-    ["", "", "($1 == $2)", "($1 == $2)"], # EqI64
-    ["", "", "($1 <= $2)", "($1 <= $2)"], # LeI64
-    ["", "", "($1 < $2)", "($1 < $2)"], # LtI64
     ["", "", "($1 == $2)", "($1 == $2)"], # EqF64
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeF64
     ["", "", "($1 < $2)", "($1 < $2)"], # LtF64
@@ -320,11 +307,9 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["negInt", "", "negInt($1)", "-($1)"], # UnaryMinusI
     ["negInt64", "", "negInt64($1)", "-($1)"], # UnaryMinusI64
     ["absInt", "", "absInt($1)", "Math.abs($1)"], # AbsI
-    ["absInt64", "", "absInt64($1)", "Math.abs($1)"], # AbsI64
     ["", "", "!($1)", "!($1)"], # Not
     ["", "", "+($1)", "+($1)"], # UnaryPlusI
     ["", "", "~($1)", "~($1)"], # BitnotI
-    ["", "", "~($1)", "~($1)"], # BitnotI64
     ["", "", "+($1)", "+($1)"], # UnaryPlusF64
     ["", "", "-($1)", "-($1)"], # UnaryMinusF64
     ["", "", "Math.abs($1)", "Math.abs($1)"], # AbsF64
@@ -357,11 +342,6 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["mulInt", "", "mulInt($1, $2)", "($1 * $2)"], # MulI
     ["divInt", "", "divInt($1, $2)", "Math.floor($1 / $2)"], # DivI
     ["modInt", "", "modInt($1, $2)", "Math.floor($1 % $2)"], # ModI
-    ["addInt64", "", "addInt64($1, $2)", "($1 + $2)"], # AddI64
-    ["subInt64", "", "subInt64($1, $2)", "($1 - $2)"], # SubI64
-    ["mulInt64", "", "mulInt64($1, $2)", "($1 * $2)"], # MulI64
-    ["divInt64", "", "divInt64($1, $2)", "Math.floor($1 / $2)"], # DivI64
-    ["modInt64", "", "modInt64($1, $2)", "Math.floor($1 % $2)"], # ModI64
     ["addInt", "", "addInt($1, $2)", "($1 + $2)"], # Succ
     ["subInt", "", "subInt($1, $2)", "($1 - $2)"], # Pred
     ["", "", "($1 + $2)", "($1 + $2)"], # AddF64
@@ -375,11 +355,6 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 ^ $2)", "($1 ^ $2)"], # BitxorI
     ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinI
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxI
-    ["", "", "($1 >>> $2)", "($1 >>> $2)"], # ShrI64
-    ["", "", "($1 << $2)", "($1 << $2)"], # ShlI64
-    ["", "", "($1 & $2)", "($1 & $2)"], # BitandI64
-    ["", "", "($1 | $2)", "($1 | $2)"], # BitorI64
-    ["", "", "($1 ^ $2)", "($1 ^ $2)"], # BitxorI64
     ["nimMin", "nimMin", "nimMin($1, $2)", "nimMin($1, $2)"], # MinF64
     ["nimMax", "nimMax", "nimMax($1, $2)", "nimMax($1, $2)"], # MaxF64
     ["addU", "addU", "addU($1, $2)", "addU($1, $2)"], # addU
@@ -390,9 +365,6 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["", "", "($1 == $2)", "($1 == $2)"], # EqI
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeI
     ["", "", "($1 < $2)", "($1 < $2)"], # LtI
-    ["", "", "($1 == $2)", "($1 == $2)"], # EqI64
-    ["", "", "($1 <= $2)", "($1 <= $2)"], # LeI64
-    ["", "", "($1 < $2)", "($1 < $2)"], # LtI64
     ["", "", "($1 == $2)", "($1 == $2)"], # EqF64
     ["", "", "($1 <= $2)", "($1 <= $2)"], # LeF64
     ["", "", "($1 < $2)", "($1 < $2)"], # LtF64
@@ -419,11 +391,9 @@ const # magic checked op; magic unchecked op; checked op; unchecked op
     ["negInt", "", "negInt($1)", "-($1)"], # UnaryMinusI
     ["negInt64", "", "negInt64($1)", "-($1)"], # UnaryMinusI64
     ["absInt", "", "absInt($1)", "Math.abs($1)"], # AbsI
-    ["absInt64", "", "absInt64($1)", "Math.abs($1)"], # AbsI64
     ["", "", "not ($1)", "not ($1)"], # Not
     ["", "", "+($1)", "+($1)"], # UnaryPlusI
     ["", "", "~($1)", "~($1)"], # BitnotI
-    ["", "", "~($1)", "~($1)"], # BitnotI64
     ["", "", "+($1)", "+($1)"], # UnaryPlusF64
     ["", "", "-($1)", "-($1)"], # UnaryMinusF64
     ["", "", "Math.abs($1)", "Math.abs($1)"], # AbsF64
@@ -535,12 +505,12 @@ proc moveInto(p: PProc, src: var TCompRes, dest: TCompRes) =
 proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   # code to generate:
   #
-  #  var sp = {prev: excHandler, exc: null};
+  #  ++excHandler;
-  #  excHandler = sp;
   #  try {
   #    stmts;
-  #    TMP = e
+  #  } catch (EXC) {
-  #  } catch (e) {
+  #    var prevJSError = lastJSError; lastJSError = EXC;
+  #    --excHandler;
   #    if (e.typ && e.typ == NTI433 || e.typ == NTI2321) {
   #      stmts;
   #    } else if (e.typ && e.typ == NTI32342) {
@@ -548,35 +518,41 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
   #    } else {
   #      stmts;
   #    }
+  #    lastJSError = prevJSError;
   #  } finally {
   #    stmts;
-  #    excHandler = excHandler.prev;
   #  }
   genLineDir(p, n)
   if not isEmptyType(n.typ):
     r.kind = resVal
     r.res = getTemp(p)
   inc(p.unique)
+  var i = 1
+  var length = sonsLen(n)
+  var catchBranchesExist = length > 1 and n.sons[i].kind == nkExceptBranch
+  if catchBranchesExist:
+    add(p.body, "++excHandler;" & tnl)
   var safePoint = "Tmp$1" % [rope(p.unique)]
   addf(p.body,
-       "var $1 = {prev: excHandler, exc: null};$nexcHandler = $1;$n" |
+       "" |
        "local $1 = pcall(",
        [safePoint])
   if optStackTrace in p.options: add(p.body, "framePtr = F;" & tnl)
   addf(p.body, "try {$n" | "function()$n", [])
-  var length = sonsLen(n)
   var a: TCompRes
   gen(p, n.sons[0], a)
   moveInto(p, a, r)
-  var i = 1
+  var generalCatchBranchExists = false
-  if p.target == targetJS and length > 1 and n.sons[i].kind == nkExceptBranch:
+  if p.target == targetJS and catchBranchesExist:
-    addf(p.body, "} catch (EXC) {$n  lastJSError = EXC;$n", [])
+    addf(p.body, "} catch (EXC) {$n var prevJSError = lastJSError;$n" &
+        " lastJSError = EXC;$n --excHandler;$n", [])
   elif p.target == targetLua:
     addf(p.body, "end)$n", [])
   while i < length and n.sons[i].kind == nkExceptBranch:
     let blen = sonsLen(n.sons[i])
     if blen == 1:
       # general except section:
+      generalCatchBranchExists = true
       if i > 1: addf(p.body, "else {$n" | "else$n", [])
       gen(p, n.sons[i].sons[0], a)
       moveInto(p, a, r)
@@ -588,17 +564,22 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) =
         if n.sons[i].sons[j].kind != nkType:
           internalError(n.info, "genTryStmt")
         if orExpr != nil: add(orExpr, "||" | " or ")
-        addf(orExpr, "isObj($1.exc.m_type, $2)",
+        addf(orExpr, "isObj(lastJSError.m_type, $1)",
-             [safePoint, genTypeInfo(p, n.sons[i].sons[j].typ)])
+             [genTypeInfo(p, n.sons[i].sons[j].typ)])
       if i > 1: add(p.body, "else ")
-      addf(p.body, "if ($1.exc && ($2)) {$n" | "if $1.exc and ($2) then$n",
+      addf(p.body, "if (lastJSError && ($2)) {$n" | "if $1.exc and ($2) then$n",
         [safePoint, orExpr])
       gen(p, n.sons[i].sons[blen - 1], a)
       moveInto(p, a, r)
       addf(p.body, "}$n" | "end$n", [])
     inc(i)
   if p.target == targetJS:
-    add(p.body, "} finally {" & tnl & "excHandler = excHandler.prev;" & tnl)
+    if catchBranchesExist:
+      if not generalCatchBranchExists:
+        useMagic(p, "reraiseException")
+        add(p.body, "else {" & tnl & "reraiseException();" & tnl & "}" & tnl)
+      add(p.body, "lastJSError = prevJSError;" & tnl)
+    add(p.body, "} finally {" & tnl)
   if i < length and n.sons[i].kind == nkFinally:
     genStmt(p, n.sons[i].sons[0])
   if p.target == targetJS:
@@ -748,14 +729,13 @@ proc genBreakStmt(p: PProc, n: PNode) =
   p.blocks[idx].id = abs(p.blocks[idx].id) # label is used
   addf(p.body, "break L$1;$n" | "goto ::L$1::;$n", [rope(p.blocks[idx].id)])
 
-proc genAsmStmt(p: PProc, n: PNode) =
+proc genAsmOrEmitStmt(p: PProc, n: PNode) =
   genLineDir(p, n)
-  assert(n.kind == nkAsmStmt)
   for i in countup(0, sonsLen(n) - 1):
     case n.sons[i].kind
     of nkStrLit..nkTripleStrLit: add(p.body, n.sons[i].strVal)
     of nkSym: add(p.body, mangleName(n.sons[i].sym))
-    else: internalError(n.sons[i].info, "jsgen: genAsmStmt()")
+    else: internalError(n.sons[i].info, "jsgen: genAsmOrEmitStmt()")
 
 proc genIf(p: PProc, n: PNode, r: var TCompRes) =
   var cond, stmt: TCompRes
@@ -810,18 +790,35 @@ proc needsNoCopy(y: PNode): bool =
 proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
   var a, b: TCompRes
   gen(p, x, a)
+
+  let xtyp = mapType(x.typ)
+
+  if x.kind == nkHiddenDeref and x.sons[0].kind == nkCall and xtyp != etyObject:
+    gen(p, x.sons[0], a)
+    let tmp = p.getTemp(false)
+    addf(p.body, "var $1 = $2;$n", [tmp, a.rdLoc])
+    a.res = "$1[0][$1[1]]" % [tmp]
+  else:
+    gen(p, x, a)
+
   gen(p, y, b)
-  case mapType(x.typ)
+
+  case xtyp
   of etyObject:
-    if needsNoCopy(y) or noCopyNeeded:
+    if (needsNoCopy(y) and needsNoCopy(x)) or noCopyNeeded:
       addf(p.body, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
-      addf(p.body, "$1 = nimCopy($2, $3);$n",
+      addf(p.body, "nimCopy($1, $2, $3);$n",
            [a.res, b.res, genTypeInfo(p, y.typ)])
   of etyBaseIndex:
     if a.typ != etyBaseIndex or b.typ != etyBaseIndex:
+      if y.kind == nkCall:
+        let tmp = p.getTemp(false)
+        addf(p.body, "var $1 = $4; $2 = $1[0]; $3 = $1[1];$n", [tmp, a.address, a.res, b.rdLoc])
+      else:
         internalError(x.info, "genAsgn")
+    else:
       addf(p.body, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res])
   else:
     addf(p.body, "$1 = $2;$n", [a.res, b.res])
@@ -838,11 +835,9 @@ proc genSwap(p: PProc, n: PNode) =
   var a, b: TCompRes
   gen(p, n.sons[1], a)
   gen(p, n.sons[2], b)
-  inc(p.unique)
+  var tmp = p.getTemp(false)
-  var tmp = "Tmp$1" % [rope(p.unique)]
   if mapType(skipTypes(n.sons[1].typ, abstractVar)) == etyBaseIndex:
-    inc(p.unique)
+    let tmp2 = p.getTemp(false)
-    let tmp2 = "Tmp$1" % [rope(p.unique)]
     if a.typ != etyBaseIndex or b.typ != etyBaseIndex:
       internalError(n.info, "genSwap")
     addf(p.body, "var $1 = $2; $2 = $3; $3 = $1;$n" |
@@ -880,7 +875,7 @@ proc genFieldAccess(p: PProc, n: PNode, r: var TCompRes) =
   if skipTypes(n.sons[0].typ, abstractVarRange).kind == tyTuple:
     r.res = "$1.Field$2" % [r.res, getFieldPosition(n.sons[1]).rope]
   else:
-    if n.sons[1].kind != nkSym: internalError(n.sons[1].info, "genFieldAddr")
+    if n.sons[1].kind != nkSym: internalError(n.sons[1].info, "genFieldAccess")
     var f = n.sons[1].sym
     if f.loc.r == nil: f.loc.r = mangleName(f)
     r.res = "$1.$2" % [r.res, f.loc.r]
@@ -970,18 +965,35 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) =
   of nkCheckedFieldExpr:
     genCheckedFieldAddr(p, n, r)
   of nkDotExpr:
+    if mapType(n.typ) == etyBaseIndex:
       genFieldAddr(p, n.sons[0], r)
+    else:
+      genFieldAccess(p, n.sons[0], r)
   of nkBracketExpr:
     var ty = skipTypes(n.sons[0].typ, abstractVarRange)
-    if ty.kind in {tyRef, tyPtr}: ty = skipTypes(ty.lastSon, abstractVarRange)
+    if ty.kind in MappedToObject:
-    case ty.kind
+      gen(p, n.sons[0], r)
+    else:
+      let kindOfIndexedExpr = skipTypes(n.sons[0].sons[0].typ, abstractVarRange).kind
+      case kindOfIndexedExpr
       of tyArray, tyArrayConstr, tyOpenArray, tySequence, tyString, tyCString,
-       tyVarargs, tyChar:
+          tyVarargs:
         genArrayAddr(p, n.sons[0], r)
       of tyTuple:
         genFieldAddr(p, n.sons[0], r)
-    else: internalError(n.sons[0].info, "expr(nkBracketExpr, " & $ty.kind & ')')
+      else: internalError(n.sons[0].info, "expr(nkBracketExpr, " & $kindOfIndexedExpr & ')')
-  else: internalError(n.sons[0].info, "genAddr")
+  of nkObjDownConv:
+    gen(p, n.sons[0], r)
+  else: internalError(n.sons[0].info, "genAddr: " & $n.sons[0].kind)
+
+proc genProcForSymIfNeeded(p: PProc, s: PSym) =
+  if not p.g.generatedSyms.containsOrIncl(s.id):
+    let newp = genProc(p, s)
+    var owner = p
+    while owner != nil and owner.prc != s.owner:
+      owner = owner.up
+    if owner != nil: add(owner.locals, newp)
+    else: add(p.g.code, newp)
 
 proc genSym(p: PProc, n: PNode, r: var TCompRes) =
   var s = n.sym
@@ -1018,13 +1030,8 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
       discard
     elif sfForward in s.flags:
       p.g.forwarded.add(s)
-    elif not p.g.generatedSyms.containsOrIncl(s.id):
+    else:
-      let newp = genProc(p, s)
+      genProcForSymIfNeeded(p, s)
-      var owner = p
-      while owner != nil and owner.prc != s.owner:
-        owner = owner.up
-      if owner != nil: add(owner.locals, newp)
-      else: add(p.g.code, newp)
   else:
     if s.loc.r == nil:
       internalError(n.info, "symbol has no generated name: " & s.name.s)
@@ -1037,10 +1044,15 @@ proc genDeref(p: PProc, n: PNode, r: var TCompRes) =
   else:
     var a: TCompRes
     gen(p, n.sons[0], a)
-    if a.typ != etyBaseIndex: internalError(n.info, "genDeref")
+    if a.typ == etyBaseIndex:
       r.res = "$1[$2]" % [a.address, a.res]
+    elif n.sons[0].kind == nkCall:
+      let tmp = p.getTemp
+      r.res = "($1 = $2, $1[0][$1[1]])" % [tmp, a.res]
+    else:
+      internalError(n.info, "genDeref")
 
-proc genArg(p: PProc, n: PNode, r: var TCompRes) =
+proc genArgNoParam(p: PProc, n: PNode, r: var TCompRes) =
   var a: TCompRes
   gen(p, n, a)
   if a.typ == etyBaseIndex:
@@ -1050,13 +1062,42 @@ proc genArg(p: PProc, n: PNode, r: var TCompRes) =
   else:
     add(r.res, a.res)
 
+proc genArg(p: PProc, n: PNode, param: PSym, r: var TCompRes) =
+  var a: TCompRes
+  gen(p, n, a)
+  if skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs} and
+      a.typ == etyBaseIndex:
+    add(r.res, "$1[$2]" % [a.address, a.res])
+  elif a.typ == etyBaseIndex:
+    add(r.res, a.address)
+    add(r.res, ", ")
+    add(r.res, a.res)
+  else:
+    add(r.res, a.res)
+
+
 proc genArgs(p: PProc, n: PNode, r: var TCompRes) =
   add(r.res, "(")
+  var hasArgs = false
+
+  var typ = skipTypes(n.sons[0].typ, abstractInst)
+  assert(typ.kind == tyProc)
+  assert(sonsLen(typ) == sonsLen(typ.n))
+
   for i in countup(1, sonsLen(n) - 1):
     let it = n.sons[i]
-    if it.typ.isCompileTimeOnly: continue
+    var paramType : PNode = nil
-    if i > 1: add(r.res, ", ")
+    if i < sonsLen(typ):
-    genArg(p, it, r)
+      assert(typ.n.sons[i].kind == nkSym)
+      paramType = typ.n.sons[i]
+      if paramType.typ.isCompileTimeOnly: continue
+
+    if hasArgs: add(r.res, ", ")
+    if paramType.isNil:
+      genArgNoParam(p, it, r)
+    else:
+      genArg(p, it, paramType.sym, r)
+    hasArgs = true
   add(r.res, ")")
   r.kind = resExpr
 
@@ -1080,11 +1121,12 @@ proc genInfixCall(p: PProc, n: PNode, r: var TCompRes) =
   add(r.res, "(")
   for i in countup(2, sonsLen(n) - 1):
     if i > 2: add(r.res, ", ")
-    genArg(p, n.sons[i], r)
+    genArgNoParam(p, n.sons[i], r)
   add(r.res, ")")
   r.kind = resExpr
 
 proc genEcho(p: PProc, n: PNode, r: var TCompRes) =
+  useMagic(p, "toJSStr") # Used in rawEcho
   useMagic(p, "rawEcho")
   add(r.res, "rawEcho(")
   let n = n[1].skipConv
@@ -1093,7 +1135,7 @@ proc genEcho(p: PProc, n: PNode, r: var TCompRes) =
     let it = n.sons[i]
     if it.typ.isCompileTimeOnly: continue
     if i > 0: add(r.res, ", ")
-    genArg(p, it, r)
+    genArgNoParam(p, it, r)
   add(r.res, ")")
   r.kind = resExpr
 
@@ -1102,24 +1144,32 @@ proc putToSeq(s: string, indirect: bool): Rope =
   if indirect: result = "[$1]" % [result]
 
 proc createVar(p: PProc, typ: PType, indirect: bool): Rope
-proc createRecordVarAux(p: PProc, rec: PNode, c: var int): Rope =
+proc createRecordVarAux(p: PProc, rec: PNode, excludedFieldIDs: IntSet, output: var Rope) =
-  result = nil
   case rec.kind
   of nkRecList:
     for i in countup(0, sonsLen(rec) - 1):
-      add(result, createRecordVarAux(p, rec.sons[i], c))
+      createRecordVarAux(p, rec.sons[i], excludedFieldIDs, output)
   of nkRecCase:
-    add(result, createRecordVarAux(p, rec.sons[0], c))
+    createRecordVarAux(p, rec.sons[0], excludedFieldIDs, output)
     for i in countup(1, sonsLen(rec) - 1):
-      add(result, createRecordVarAux(p, lastSon(rec.sons[i]), c))
+      createRecordVarAux(p, lastSon(rec.sons[i]), excludedFieldIDs, output)
   of nkSym:
-    if c > 0: add(result, ", ")
+    if rec.sym.id notin excludedFieldIDs:
-    add(result, mangleName(rec.sym))
+      if output.len > 0: output.add(", ")
-    add(result, ": ")
+      output.add(mangleName(rec.sym))
-    add(result, createVar(p, rec.sym.typ, false))
+      output.add(": ")
-    inc(c)
+      output.add(createVar(p, rec.sym.typ, false))
   else: internalError(rec.info, "createRecordVarAux")
 
+proc createObjInitList(p: PProc, typ: PType, excludedFieldIDs: IntSet, output: var Rope) =
+  var t = typ
+  if tfFinal notin t.flags or t.sons[0] != nil:
+    if output.len > 0: output.add(", ")
+    addf(output, "m_type: $1" | "m_type = $#", [genTypeInfo(p, t)])
+  while t != nil:
+    createRecordVarAux(p, t.n, excludedFieldIDs, output)
+    t = t.sons[0]
+
 proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   var t = skipTypes(typ, abstractInst)
   case t.kind
@@ -1160,15 +1210,9 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
     add(result, "}")
     if indirect: result = "[$1]" % [result]
   of tyObject:
-    result = rope("{")
+    var initList : Rope
-    var c = 0
+    createObjInitList(p, t, initIntSet(), initList)
-    if tfFinal notin t.flags or t.sons[0] != nil:
+    result = "{$1}" % [initList]
-      inc(c)
-      addf(result, "m_type: $1" | "m_type = $#", [genTypeInfo(p, t)])
-    while t != nil:
-      add(result, createRecordVarAux(p, t.n, c))
-      t = t.sons[0]
-    add(result, "}")
     if indirect: result = "[$1]" % [result]
   of tyVar, tyPtr, tyRef:
     if mapType(t) == etyBaseIndex:
@@ -1197,7 +1241,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
         s = a.res
       else:
         useMagic(p, "nimCopy")
-        s = "nimCopy($1, $2)" % [a.res, genTypeInfo(p, n.typ)]
+        s = "nimCopy(null, $1, $2)" % [a.res, genTypeInfo(p, n.typ)]
     of etyBaseIndex:
       if (a.typ != etyBaseIndex): internalError(n.info, "genVarInit")
       if {sfAddrTaken, sfGlobal} * v.flags != {}:
@@ -1389,6 +1433,12 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
   of mCopyStrLast: ternaryExpr(p, n, r, "", "($1.slice($2, ($3)+1).concat(0))")
   of mNewString: unaryExpr(p, n, r, "mnewString", "mnewString($1)")
   of mNewStringOfCap: unaryExpr(p, n, r, "mnewString", "mnewString(0)")
+  of mDotDot:
+    genProcForSymIfNeeded(p, n.sons[0].sym)
+    genCall(p, n, r)
+  of mParseBiggestFloat:
+    useMagic(p, "nimParseBiggestFloat")
+    genCall(p, n, r)
   else:
     genCall(p, n, r)
     #else internalError(e.info, 'genMagic: ' + magicToStr[op]);
@@ -1434,19 +1484,22 @@ proc genTupleConstr(p: PProc, n: PNode, r: var TCompRes) =
   r.res.add("}")
 
 proc genObjConstr(p: PProc, n: PNode, r: var TCompRes) =
-  # XXX inheritance?
   var a: TCompRes
-  r.res = rope("{")
   r.kind = resExpr
+  var initList : Rope
+  var fieldIDs = initIntSet()
   for i in countup(1, sonsLen(n) - 1):
-    if i > 1: add(r.res, ", ")
+    if i > 1: add(initList, ", ")
     var it = n.sons[i]
     internalAssert it.kind == nkExprColonExpr
     gen(p, it.sons[1], a)
     var f = it.sons[0].sym
     if f.loc.r == nil: f.loc.r = mangleName(f)
-    addf(r.res, "$#: $#" | "$# = $#" , [f.loc.r, a.res])
+    fieldIDs.incl(f.id)
-  r.res.add("}")
+    addf(initList, "$#: $#" | "$# = $#" , [f.loc.r, a.res])
+  let t = skipTypes(n.typ, abstractInst + skipPtrs)
+  createObjInitList(p, t, fieldIDs, initList)
+  r.res = "{$1}" % [initList]
 
 proc genConv(p: PProc, n: PNode, r: var TCompRes) =
   var dest = skipTypes(n.typ, abstractVarRange)
@@ -1551,6 +1604,9 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
         mangleName(resultSym),
         createVar(p, resultSym.typ, isIndirect(resultSym))]
     gen(p, prc.ast.sons[resultPos], a)
+    if mapType(resultSym.typ) == etyBaseIndex:
+      returnStmt = "return [$#, $#];$n" % [a.address, a.res]
+    else:
       returnStmt = "return $#;$n" % [a.res]
   genStmt(p, prc.getBody)
   result = ("function $#($#) {$n$#$#$#$#}$n" |
@@ -1565,6 +1621,12 @@ proc genStmt(p: PProc, n: PNode) =
   gen(p, n, r)
   if r.res != nil: addf(p.body, "$#;$n", [r.res])
 
+proc genPragma(p: PProc, n: PNode) =
+  for it in n.sons:
+    case whichPragma(it)
+    of wEmit: genAsmOrEmitStmt(p, it.sons[1])
+    else: discard
+
 proc gen(p: PProc, n: PNode, r: var TCompRes) =
   r.typ = etyNone
   r.kind = resNone
@@ -1648,6 +1710,9 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
       gen(p, lastSon(n), r)
   of nkBlockStmt, nkBlockExpr: genBlock(p, n, r)
   of nkIfStmt, nkIfExpr: genIf(p, n, r)
+  of nkWhen:
+    # This is "when nimvm" node
+    gen(p, n.sons[1].sons[0], r)
   of nkWhileStmt: genWhileStmt(p, n)
   of nkVarSection, nkLetSection: genVarStmt(p, n)
   of nkConstSection: discard
@@ -1664,12 +1729,13 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
     if n.sons[0].kind != nkEmpty:
       genLineDir(p, n)
       gen(p, n.sons[0], r)
-  of nkAsmStmt: genAsmStmt(p, n)
+  of nkAsmStmt: genAsmOrEmitStmt(p, n)
   of nkTryStmt: genTry(p, n, r)
   of nkRaiseStmt: genRaiseStmt(p, n)
   of nkTypeSection, nkCommentStmt, nkIteratorDef, nkIncludeStmt,
      nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
-     nkFromStmt, nkTemplateDef, nkMacroDef, nkPragma: discard
+     nkFromStmt, nkTemplateDef, nkMacroDef: discard
+  of nkPragma: genPragma(p, n)
   of nkProcDef, nkMethodDef, nkConverterDef:
     var s = n.sons[namePos].sym
     if {sfExportc, sfCompilerProc} * s.flags == {sfExportc}:
@@ -1691,7 +1757,7 @@ proc genHeader(): Rope =
   result = ("/* Generated by the Nim Compiler v$1 */$n" &
             "/*   (c) 2015 Andreas Rumpf */$n$n" &
             "var framePtr = null;$n" &
-            "var excHandler = null;$n" &
+            "var excHandler = 0;$n" &
             "var lastJSError = null;$n") %
            [rope(VersionAsString)]
 

compiler/jstypes.nim

@@ -116,13 +116,12 @@ proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
          [name, genTypeInfo(p, typ.sons[0])])
 
 proc genTypeInfo(p: PProc, typ: PType): Rope =
-  var t = typ
+  let t = typ.skipTypes({tyGenericInst})
-  if t.kind == tyGenericInst: t = lastSon(t)
   result = "NTI$1" % [rope(t.id)]
   if containsOrIncl(p.g.typeInfoGenerated, t.id): return
   case t.kind
   of tyDistinct:
-    result = genTypeInfo(p, typ.sons[0])
+    result = genTypeInfo(p, t.sons[0])
   of tyPointer, tyProc, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
@@ -134,14 +133,14 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
     addf(p.g.typeInfo, "$1.base = $2;$n",
-         [result, genTypeInfo(p, typ.lastSon)])
+         [result, genTypeInfo(p, t.lastSon)])
   of tyArrayConstr, tyArray:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
     addf(p.g.typeInfo, "$1.base = $2;$n",
-         [result, genTypeInfo(p, typ.sons[1])])
+         [result, genTypeInfo(p, t.sons[1])])
   of tyEnum: genEnumInfo(p, t, result)
   of tyObject: genObjectInfo(p, t, result)
   of tyTuple: genTupleInfo(p, t, result)

compiler/lambdalifting.nim

@@ -184,7 +184,7 @@ proc addHiddenParam(routine: PSym, param: PSym) =
   var params = routine.ast.sons[paramsPos]
   # -1 is correct here as param.position is 0 based but we have at position 0
   # some nkEffect node:
-  param.position = params.len-1
+  param.position = routine.typ.n.len-1
   addSon(params, newSymNode(param))
   incl(routine.typ.flags, tfCapturesEnv)
   assert sfFromGeneric in param.flags
@@ -859,11 +859,17 @@ proc transformOuterProc(o: POuterContext, n: PNode; it: TIter): PNode =
         return indirectAccess(newSymNode(it.closureParam), local, n.info)
 
     if local.kind == skClosureIterator:
+      # bug #3354; allow for
+      #iterator iter(): int {.closure.}=
+      #  s.add(iter)
+      #  yield 1
+
+      #if local == o.fn or local == it.fn:
+      #  message(n.info, errRecursiveDependencyX, local.name.s)
+
       # consider: [i1, i2, i1]  Since we merged the iterator's closure
       # with the captured owning variables, we need to generate the
       # closure generation code again:
-      if local == o.fn or local == it.fn:
-        message(n.info, errRecursiveDependencyX, local.name.s)
       # XXX why doesn't this work?
       var closure = PEnv(idTableGet(o.lambdasToEnv, local))
       if closure.isNil:
@@ -946,7 +952,11 @@ proc transformOuterProc(o: POuterContext, n: PNode; it: TIter): PNode =
 proc liftLambdas*(fn: PSym, body: PNode): PNode =
   # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs
   # the transformation even when compiling to JS ...
-  if body.kind == nkEmpty or gCmd == cmdCompileToJS or
+
+  # However we can do lifting for the stuff which is *only* compiletime.
+  let isCompileTime = sfCompileTime in fn.flags or fn.kind == skMacro
+
+  if body.kind == nkEmpty or (gCmd == cmdCompileToJS and not isCompileTime) or
       fn.skipGenericOwner.kind != skModule:
     # ignore forward declaration:
     result = body
@@ -1012,7 +1022,9 @@ proc liftForLoop*(body: PNode): PNode =
         ...
     """
   var L = body.len
-  internalAssert body.kind == nkForStmt and body[L-2].kind in nkCallKinds
+  if not (body.kind == nkForStmt and body[L-2].kind in nkCallKinds):
+    localError(body.info, "ignored invalid for loop")
+    return body
   var call = body[L-2]
 
   result = newNodeI(nkStmtList, body.info)

compiler/lexer.nim

@@ -17,7 +17,7 @@
 
 import
   hashes, options, msgs, strutils, platform, idents, nimlexbase, llstream,
-  wordrecg
+  wordrecg, etcpriv
 
 const
   MaxLineLength* = 80         # lines longer than this lead to a warning
@@ -140,10 +140,12 @@ proc isKeyword*(kind: TTokType): bool =
 proc isNimIdentifier*(s: string): bool =
   if s[0] in SymStartChars:
     var i = 1
-    while i < s.len:
+    var sLen = s.len
+    while i < sLen:
       if s[i] == '_':
         inc(i)
-        if s[i] notin SymChars: return
+      elif isMagicIdentSeparatorRune(cstring s, i):
+        inc(i, magicIdentSeparatorRuneByteWidth)
       if s[i] notin SymChars: return
       inc(i)
     result = true
@@ -229,23 +231,6 @@ proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
   var info = newLineInfo(L.fileIdx, L.lineNumber, pos - L.lineStart)
   L.dispMessage(info, msg, arg)
 
-proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]) =
-  var pos = L.bufpos              # use registers for pos, buf
-  var buf = L.buf
-  while true:
-    if buf[pos] in chars:
-      add(tok.literal, buf[pos])
-      inc(pos)
-    else:
-      break
-    if buf[pos] == '_':
-      if buf[pos+1] notin chars:
-        lexMessage(L, errInvalidToken, "_")
-        break
-      add(tok.literal, '_')
-      inc(pos)
-  L.bufpos = pos
-
 proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
   result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)
 
@@ -268,136 +253,200 @@ proc unsafeParseUInt(s: string, b: var BiggestInt, start = 0): int =
     result = i - start
 {.pop.} # overflowChecks
 
+
+template eatChar(L: var TLexer, t: var TToken, replacementChar: char) =
+  add(t.literal, replacementChar)
+  inc(L.bufpos)
+
+template eatChar(L: var TLexer, t: var TToken) =
+  add(t.literal, L.buf[L.bufpos])
+  inc(L.bufpos)
+
 proc getNumber(L: var TLexer): TToken =
+  proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]) =
+    var pos = L.bufpos              # use registers for pos, buf
+    var buf = L.buf
+    while true:
+      if buf[pos] in chars:
+        add(tok.literal, buf[pos])
+        inc(pos)
+      else:
+        break
+      if buf[pos] == '_':
+        if buf[pos+1] notin chars:
+          lexMessage(L, errInvalidToken, "_")
+          break
+        add(tok.literal, '_')
+        inc(pos)
+    L.bufpos = pos
+
+  proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) =
+    var pos = L.bufpos              # use registers for pos, buf
+    var buf = L.buf
+    while buf[pos] in chars:
+      add(tok.literal, buf[pos])
+      inc(pos)
+    L.bufpos = pos
+
+  proc lexMessageLitNum(L: var TLexer, msg: TMsgKind, startpos: int) =
+    # Used to get slightly human friendlier err messages.
+    # Note: the erroneous 'O' char in the character set is intentional
+    const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O',
+      'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'}
+    var msgPos = L.bufpos
+    var t: TToken
+    t.literal = ""
+    L.bufpos = startpos # Use L.bufpos as pos because of matchChars
+    matchChars(L, t, literalishChars)
+    # We must verify +/- specifically so that we're not past the literal
+    if  L.buf[L.bufpos] in {'+', '-'} and
+        L.buf[L.bufpos - 1] in {'e', 'E'}:
+      add(t.literal, L.buf[L.bufpos])
+      inc(L.bufpos)
+      matchChars(L, t, literalishChars)
+    if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
+      inc(L.bufpos)
+      add(t.literal, L.buf[L.bufpos])
+      matchChars(L, t, {'0'..'9'})
+    L.bufpos = msgPos
+    lexMessage(L, msg, t.literal)
+
   var
-    pos, endpos: int
+    startpos, endpos: int
     xi: BiggestInt
-  # get the base:
+    isBase10 = true
+  const
+    baseCodeChars = {'X', 'x', 'o', 'c', 'C', 'b', 'B'}
+    literalishChars = baseCodeChars + {'A'..'F', 'a'..'f', '0'..'9', '_', '\''}
+    floatTypes = {tkFloatLit, tkFloat32Lit, tkFloat64Lit, tkFloat128Lit}
   result.tokType = tkIntLit   # int literal until we know better
   result.literal = ""
-  result.base = base10        # BUGFIX
+  result.base = base10
-  pos = L.bufpos     # make sure the literal is correct for error messages:
+  startpos = L.bufpos
-  var eallowed = false
+
-  if L.buf[pos] == '0' and L.buf[pos+1] in {'X', 'x'}:
+  # First stage: find out base, make verifications, build token literal string
-    matchUnderscoreChars(L, result, {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x'})
+  if L.buf[L.bufpos] == '0' and L.buf[L.bufpos + 1] in baseCodeChars + {'O'}:
+    isBase10 = false
+    eatChar(L, result, '0')
+    case L.buf[L.bufpos]
+    of 'O':
+      lexMessageLitNum(L, errInvalidNumberOctalCode, startpos)
+    of 'x', 'X':
+      eatChar(L, result, 'x')
+      matchUnderscoreChars(L, result, {'0'..'9', 'a'..'f', 'A'..'F'})
+    of 'o', 'c', 'C':
+      eatChar(L, result, 'c')
+      matchUnderscoreChars(L, result, {'0'..'7'})
+    of 'b', 'B':
+      eatChar(L, result, 'b')
+      matchUnderscoreChars(L, result, {'0'..'1'})
+    else:
+      internalError(getLineInfo(L), "getNumber")
   else:
-    matchUnderscoreChars(L, result, {'0'..'9', 'b', 'B', 'o', 'c', 'C'})
-    eallowed = true
-  if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
-    add(result.literal, '.')
-    inc(L.bufpos)
     matchUnderscoreChars(L, result, {'0'..'9'})
-    eallowed = true
+    if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
-  if eallowed and L.buf[L.bufpos] in {'e', 'E'}:
+      result.tokType = tkFloat64Lit
-    add(result.literal, 'e')
+      eatChar(L, result, '.')
-    inc(L.bufpos)
+      matchUnderscoreChars(L, result, {'0'..'9'})
+    if L.buf[L.bufpos] in {'e', 'E'}:
+      result.tokType = tkFloat64Lit
+      eatChar(L, result, 'e')
       if L.buf[L.bufpos] in {'+', '-'}:
-      add(result.literal, L.buf[L.bufpos])
+        eatChar(L, result)
-      inc(L.bufpos)
       matchUnderscoreChars(L, result, {'0'..'9'})
   endpos = L.bufpos
-  if L.buf[endpos] in {'\'', 'f', 'F', 'i', 'I', 'u', 'U'}:
+
-    if L.buf[endpos] == '\'': inc(endpos)
+  # Second stage, find out if there's a datatype suffix and handle it
-    L.bufpos = pos            # restore position
+  var postPos = endpos
-    case L.buf[endpos]
+  if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
+    if L.buf[postPos] == '\'':
+      inc(postPos)
+
+    case L.buf[postPos]
     of 'f', 'F':
-      inc(endpos)
+      inc(postPos)
-      if (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'):
+      if (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
         result.tokType = tkFloat32Lit
-        inc(endpos, 2)
+        inc(postPos, 2)
-      elif (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'):
+      elif (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
         result.tokType = tkFloat64Lit
-        inc(endpos, 2)
+        inc(postPos, 2)
-      elif (L.buf[endpos] == '1') and
+      elif (L.buf[postPos] == '1') and
-           (L.buf[endpos + 1] == '2') and
+           (L.buf[postPos + 1] == '2') and
-           (L.buf[endpos + 2] == '8'):
+           (L.buf[postPos + 2] == '8'):
         result.tokType = tkFloat128Lit
-        inc(endpos, 3)
+        inc(postPos, 3)
-      else:
+      else:   # "f" alone defaults to float32
-        lexMessage(L, errInvalidNumber, result.literal & "'f" & L.buf[endpos])
+        result.tokType = tkFloat32Lit
+    of 'd', 'D':  # ad hoc convenience shortcut for f64
+      inc(postPos)
+      result.tokType = tkFloat64Lit
     of 'i', 'I':
-      inc(endpos)
+      inc(postPos)
-      if (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'):
+      if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
         result.tokType = tkInt64Lit
-        inc(endpos, 2)
+        inc(postPos, 2)
-      elif (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'):
+      elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
         result.tokType = tkInt32Lit
-        inc(endpos, 2)
+        inc(postPos, 2)
-      elif (L.buf[endpos] == '1') and (L.buf[endpos + 1] == '6'):
+      elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
         result.tokType = tkInt16Lit
-        inc(endpos, 2)
+        inc(postPos, 2)
-      elif (L.buf[endpos] == '8'):
+      elif (L.buf[postPos] == '8'):
         result.tokType = tkInt8Lit
-        inc(endpos)
+        inc(postPos)
       else:
-        lexMessage(L, errInvalidNumber, result.literal & "'i" & L.buf[endpos])
+        lexMessageLitNum(L, errInvalidNumber, startpos)
     of 'u', 'U':
-      inc(endpos)
+      inc(postPos)
-      if (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'):
+      if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
         result.tokType = tkUInt64Lit
-        inc(endpos, 2)
+        inc(postPos, 2)
-      elif (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'):
+      elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
         result.tokType = tkUInt32Lit
-        inc(endpos, 2)
+        inc(postPos, 2)
-      elif (L.buf[endpos] == '1') and (L.buf[endpos + 1] == '6'):
+      elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
         result.tokType = tkUInt16Lit
-        inc(endpos, 2)
+        inc(postPos, 2)
-      elif (L.buf[endpos] == '8'):
+      elif (L.buf[postPos] == '8'):
         result.tokType = tkUInt8Lit
-        inc(endpos)
+        inc(postPos)
       else:
         result.tokType = tkUIntLit
-    else: lexMessage(L, errInvalidNumber, result.literal & "'" & L.buf[endpos])
     else:
-    L.bufpos = pos            # restore position
+      lexMessageLitNum(L, errInvalidNumber, startpos)
+
+  # Is there still a literalish char awaiting? Then it's an error!
+  if  L.buf[postPos] in literalishChars or
+     (L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}):
+    lexMessageLitNum(L, errInvalidNumber, startpos)
+
+  # Third stage, extract actual number
+  L.bufpos = startpos            # restore position
+  var pos: int = startpos
   try:
-    if (L.buf[pos] == '0') and
+    if (L.buf[pos] == '0') and (L.buf[pos + 1] in baseCodeChars):
-        (L.buf[pos + 1] in {'x', 'X', 'b', 'B', 'o', 'O', 'c', 'C'}):
       inc(pos, 2)
-      xi = 0                  # it may be a base prefix
+      xi = 0                  # it is a base prefix
-      case L.buf[pos - 1]     # now look at the optional type suffix:
+
+      case L.buf[pos - 1]
       of 'b', 'B':
         result.base = base2
-        while true:
+        while pos < endpos:
-          case L.buf[pos]
+          if L.buf[pos] != '_':
-          of '2'..'9', '.':
-            lexMessage(L, errInvalidNumber, result.literal)
-            inc(pos)
-          of '_':
-            if L.buf[pos+1] notin {'0'..'1'}:
-              lexMessage(L, errInvalidToken, "_")
-              break
-            inc(pos)
-          of '0', '1':
             xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
           inc(pos)
-          else: break
       of 'o', 'c', 'C':
         result.base = base8
-        while true:
+        while pos < endpos:
-          case L.buf[pos]
+          if L.buf[pos] != '_':
-          of '8'..'9', '.':
-            lexMessage(L, errInvalidNumber, result.literal)
-            inc(pos)
-          of '_':
-            if L.buf[pos+1] notin {'0'..'7'}:
-              lexMessage(L, errInvalidToken, "_")
-              break
-            inc(pos)
-          of '0'..'7':
             xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0'))
           inc(pos)
-          else: break
-      of 'O':
-        lexMessage(L, errInvalidNumber, result.literal)
       of 'x', 'X':
         result.base = base16
-        while true:
+        while pos < endpos:
           case L.buf[pos]
           of '_':
-            if L.buf[pos+1] notin {'0'..'9', 'a'..'f', 'A'..'F'}:
-              lexMessage(L, errInvalidToken, "_")
-              break
             inc(pos)
           of '0'..'9':
             xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0'))
@@ -408,28 +457,48 @@ proc getNumber(L: var TLexer): TToken =
           of 'A'..'F':
             xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10)
             inc(pos)
-          else: break
+          else:
-      else: internalError(getLineInfo(L), "getNumber")
+            break
+      else:
+        internalError(getLineInfo(L), "getNumber")
+
       case result.tokType
       of tkIntLit, tkInt64Lit: result.iNumber = xi
       of tkInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi))))
-      of tkInt16Lit: result.iNumber = BiggestInt(toU16(int(xi)))
+      of tkInt16Lit: result.iNumber = BiggestInt(int16(toU16(int(xi))))
-      of tkInt32Lit: result.iNumber = BiggestInt(toU32(xi))
+      of tkInt32Lit: result.iNumber = BiggestInt(int32(toU32(int64(xi))))
       of tkUIntLit, tkUInt64Lit: result.iNumber = xi
-      of tkUInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi))))
+      of tkUInt8Lit: result.iNumber = BiggestInt(uint8(toU8(int(xi))))
-      of tkUInt16Lit: result.iNumber = BiggestInt(toU16(int(xi)))
+      of tkUInt16Lit: result.iNumber = BiggestInt(uint16(toU16(int(xi))))
-      of tkUInt32Lit: result.iNumber = BiggestInt(toU32(xi))
+      of tkUInt32Lit: result.iNumber = BiggestInt(uint32(toU32(int64(xi))))
       of tkFloat32Lit:
         result.fNumber = (cast[PFloat32](addr(xi)))[]
         # note: this code is endian neutral!
         # XXX: Test this on big endian machine!
       of tkFloat64Lit: result.fNumber = (cast[PFloat64](addr(xi)))[]
       else: internalError(getLineInfo(L), "getNumber")
-    elif isFloatLiteral(result.literal) or (result.tokType == tkFloat32Lit) or
+
-        (result.tokType == tkFloat64Lit):
+      # Bounds checks. Non decimal literals are allowed to overflow the range of
+      # the datatype as long as their pattern don't overflow _bitwise_, hence
+      # below checks of signed sizes against uint*.high is deliberate:
+      # (0x80'u8 = 128, 0x80'i8 = -128, etc == OK)
+      if result.tokType notin floatTypes:
+        let outOfRange = case result.tokType:
+        of tkUInt8Lit, tkUInt16Lit, tkUInt32Lit: result.iNumber != xi
+        of tkInt8Lit: (xi > BiggestInt(uint8.high))
+        of tkInt16Lit: (xi > BiggestInt(uint16.high))
+        of tkInt32Lit: (xi > BiggestInt(uint32.high))
+        else: false
+
+        if outOfRange:
+          #echo "out of range num: ", result.iNumber, " vs ", xi
+          lexMessageLitNum(L, errNumberOutOfRange, startpos)
+
+    else:
+      case result.tokType
+      of floatTypes:
         result.fNumber = parseFloat(result.literal)
-      if result.tokType == tkIntLit: result.tokType = tkFloatLit
+      of tkUint64Lit:
-    elif result.tokType == tkUint64Lit:
         xi = 0
         let len = unsafeParseUInt(result.literal, xi)
         if len != result.literal.len or len == 0:
@@ -437,22 +506,32 @@ proc getNumber(L: var TLexer): TToken =
         result.iNumber = xi
       else:
         result.iNumber = parseBiggestInt(result.literal)
-      if (result.iNumber < low(int32)) or (result.iNumber > high(int32)):
+
+      # Explicit bounds checks
+      let outOfRange = case result.tokType:
+      of tkInt8Lit: (result.iNumber < int8.low or result.iNumber > int8.high)
+      of tkUInt8Lit: (result.iNumber < BiggestInt(uint8.low) or
+                      result.iNumber > BiggestInt(uint8.high))
+      of tkInt16Lit: (result.iNumber < int16.low or result.iNumber > int16.high)
+      of tkUInt16Lit: (result.iNumber < BiggestInt(uint16.low) or
+                      result.iNumber > BiggestInt(uint16.high))
+      of tkInt32Lit: (result.iNumber < int32.low or result.iNumber > int32.high)
+      of tkUInt32Lit: (result.iNumber < BiggestInt(uint32.low) or
+                      result.iNumber > BiggestInt(uint32.high))
+      else: false
+
+      if outOfRange: lexMessageLitNum(L, errNumberOutOfRange, startpos)
+
+    # Promote int literal to int64? Not always necessary, but more consistent
     if result.tokType == tkIntLit:
+      if (result.iNumber < low(int32)) or (result.iNumber > high(int32)):
         result.tokType = tkInt64Lit
-        elif result.tokType in {tkInt8Lit, tkInt16Lit, tkInt32Lit}:
+
-          lexMessage(L, errNumberOutOfRange, result.literal)
-      elif result.tokType == tkInt8Lit and
-          (result.iNumber < int8.low or result.iNumber > int8.high):
-        lexMessage(L, errNumberOutOfRange, result.literal)
-      elif result.tokType == tkInt16Lit and
-          (result.iNumber < int16.low or result.iNumber > int16.high):
-        lexMessage(L, errNumberOutOfRange, result.literal)
   except ValueError:
-    lexMessage(L, errInvalidNumber, result.literal)
+    lexMessageLitNum(L, errInvalidNumber, startpos)
   except OverflowError, RangeError:
-    lexMessage(L, errNumberOutOfRange, result.literal)
+    lexMessageLitNum(L, errNumberOutOfRange, startpos)
-  L.bufpos = endpos
+  L.bufpos = postPos
 
 proc handleHexChar(L: var TLexer, xi: var int) =
   case L.buf[L.bufpos]
@@ -625,23 +704,34 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
   inc(L.bufpos)               # skip '
 
 proc getSymbol(L: var TLexer, tok: var TToken) =
-  var h: THash = 0
+  var h: Hash = 0
   var pos = L.bufpos
   var buf = L.buf
   while true:
     var c = buf[pos]
     case c
     of 'a'..'z', '0'..'9', '\x80'..'\xFF':
+      if  c == '\226' and
+          buf[pos+1] == '\128' and
+          buf[pos+2] == '\147':  # It's a 'magic separator' en-dash Unicode
+        if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars:
+          lexMessage(L, errInvalidToken, "–")
+          break
+        inc(pos, magicIdentSeparatorRuneByteWidth)
+      else:
         h = h !& ord(c)
+        inc(pos)
     of 'A'..'Z':
       c = chr(ord(c) + (ord('a') - ord('A'))) # toLower()
       h = h !& ord(c)
+      inc(pos)
     of '_':
       if buf[pos+1] notin SymChars:
         lexMessage(L, errInvalidToken, "_")
         break
-    else: break
       inc(pos)
+
+    else: break
   h = !$h
   tok.ident = getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
   L.bufpos = pos
@@ -652,7 +742,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
     tok.tokType = TTokType(tok.ident.id + ord(tkSymbol))
 
 proc endOperator(L: var TLexer, tok: var TToken, pos: int,
-                 hash: THash) {.inline.} =
+                 hash: Hash) {.inline.} =
   var h = !$hash
   tok.ident = getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
   if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr
@@ -662,7 +752,7 @@ proc endOperator(L: var TLexer, tok: var TToken, pos: int,
 proc getOperator(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
   var buf = L.buf
-  var h: THash = 0
+  var h: Hash = 0
   while true:
     var c = buf[pos]
     if c notin OpChars: break

compiler/llstream.nim

@@ -12,7 +12,8 @@
 import
   strutils
 
-when not defined(windows) and defined(useGnuReadline):
+# support '-d:useGnuReadline' for backwards compatibility:
+when not defined(windows) and (defined(useGnuReadline) or defined(useLinenoise)):
   import rdstdin
 
 type

compiler/lookups.nim

@@ -116,7 +116,7 @@ proc errorSym*(c: PContext, n: PNode): PSym =
   result.typ = errorType(c)
   incl(result.flags, sfDiscardable)
   # pretend it's imported from some unknown module to prevent cascading errors:
-  if gCmd != cmdInteractive and c.inCompilesContext == 0:
+  if gCmd != cmdInteractive and c.compilesContextId == 0:
     c.importTable.addSym(result)
 
 type

compiler/lowerings.nim

@@ -63,6 +63,52 @@ proc lowerTupleUnpacking*(n: PNode; owner: PSym): PNode =
     if n.sons[i].kind == nkSym: v.addVar(n.sons[i])
     result.add newAsgnStmt(n.sons[i], newTupleAccess(tempAsNode, i))
 
+proc newTupleAccessRaw*(tup: PNode, i: int): PNode =
+  result = newNodeI(nkBracketExpr, tup.info)
+  addSon(result, copyTree(tup))
+  var lit = newNodeI(nkIntLit, tup.info)
+  lit.intVal = i
+  addSon(result, lit)
+
+proc lowerTupleUnpackingForAsgn*(n: PNode; owner: PSym): PNode =
+  let value = n.lastSon
+  result = newNodeI(nkStmtList, n.info)
+
+  var temp = newSym(skTemp, getIdent(genPrefix), owner, value.info)
+  var v = newNodeI(nkLetSection, value.info)
+  let tempAsNode = newIdentNode(getIdent(genPrefix & $temp.id), value.info)
+
+  var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
+  vpart.sons[0] = tempAsNode
+  vpart.sons[1] = ast.emptyNode
+  vpart.sons[2] = value
+  addSon(v, vpart)
+  result.add(v)
+
+  let lhs = n.sons[0]
+  for i in 0 .. lhs.len-1:
+    result.add newAsgnStmt(lhs.sons[i], newTupleAccessRaw(tempAsNode, i))
+
+proc lowerSwap*(n: PNode; owner: PSym): PNode =
+  result = newNodeI(nkStmtList, n.info)
+  # note: cannot use 'skTemp' here cause we really need the copy for the VM :-(
+  var temp = newSym(skVar, getIdent(genPrefix), owner, n.info)
+  temp.typ = n.sons[1].typ
+  incl(temp.flags, sfFromGeneric)
+
+  var v = newNodeI(nkVarSection, n.info)
+  let tempAsNode = newSymNode(temp)
+
+  var vpart = newNodeI(nkIdentDefs, v.info, 3)
+  vpart.sons[0] = tempAsNode
+  vpart.sons[1] = ast.emptyNode
+  vpart.sons[2] = n[1]
+  addSon(v, vpart)
+
+  result.add(v)
+  result.add newFastAsgnStmt(n[1], n[2])
+  result.add newFastAsgnStmt(n[2], tempAsNode)
+
 proc createObj*(owner: PSym, info: TLineInfo): PType =
   result = newType(tyObject, owner)
   rawAddSon(result, nil)
@@ -167,7 +213,7 @@ proc genDeref*(n: PNode): PNode =
   result.add n
 
 proc callCodegenProc*(name: string, arg1: PNode;
-                      arg2, arg3: PNode = nil): PNode =
+                      arg2, arg3, optionalArgs: PNode = nil): PNode =
   result = newNodeI(nkCall, arg1.info)
   let sym = magicsys.getCompilerProc(name)
   if sym == nil:
@@ -177,6 +223,9 @@ proc callCodegenProc*(name: string, arg1: PNode;
     result.add arg1
     if arg2 != nil: result.add arg2
     if arg3 != nil: result.add arg3
+    if optionalArgs != nil:
+      for i in 1..optionalArgs.len-3:
+        result.add optionalArgs[i]
     result.typ = sym.typ.sons[0]
 
 proc callProc(a: PNode): PNode =
@@ -483,7 +532,7 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType;
                        barrier, dest: PNode = nil): PNode =
   # if 'barrier' != nil, then it is in a 'parallel' section and we
   # generate quite different code
-  let n = spawnExpr[1]
+  let n = spawnExpr[^2]
   let spawnKind = spawnResult(retType, barrier!=nil)
   case spawnKind
   of srVoid:
@@ -569,7 +618,7 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType;
     fvField = newDotExpr(scratchObj, field)
     fvAsExpr = indirectAccess(castExpr, field, n.info)
     # create flowVar:
-    result.add newFastAsgnStmt(fvField, callProc(spawnExpr[2]))
+    result.add newFastAsgnStmt(fvField, callProc(spawnExpr[^1]))
     if barrier == nil:
       result.add callCodegenProc("nimFlowVarCreateSemaphore", fvField)
 
@@ -584,7 +633,7 @@ proc wrapProcForSpawn*(owner: PSym; spawnExpr: PNode; retType: PType;
   let wrapper = createWrapperProc(fn, threadParam, argsParam,
                                   varSection, varInit, call,
                                   barrierAsExpr, fvAsExpr, spawnKind)
-  result.add callCodegenProc("nimSpawn", wrapper.newSymNode,
+  result.add callCodegenProc("nimSpawn" & $spawnExpr.len, wrapper.newSymNode,
-                             genAddrOf(scratchObj.newSymNode))
+                             genAddrOf(scratchObj.newSymNode), nil, spawnExpr)
 
   if spawnKind == srFlowVar: result.add fvField

compiler/magicsys.nim

@@ -14,22 +14,14 @@ import
 
 var systemModule*: PSym
 
-proc registerSysType*(t: PType)
-  # magic symbols in the system module:
-proc getSysType*(kind: TTypeKind): PType
-proc getCompilerProc*(name: string): PSym
-proc registerCompilerProc*(s: PSym)
-proc finishSystem*(tab: TStrTable)
-proc getSysSym*(name: string): PSym
-# implementation
-
 var
   gSysTypes: array[TTypeKind, PType]
   compilerprocs: TStrTable
+  exposed: TStrTable
 
 proc nilOrSysInt*: PType = gSysTypes[tyInt]
 
-proc registerSysType(t: PType) = 
+proc registerSysType*(t: PType) =
   if gSysTypes[t.kind] == nil: gSysTypes[t.kind] = t
 
 proc newSysType(kind: TTypeKind, size: int): PType =
@@ -37,7 +29,7 @@ proc newSysType(kind: TTypeKind, size: int): PType =
   result.size = size
   result.align = size.int16
 
-proc getSysSym(name: string): PSym = 
+proc getSysSym*(name: string): PSym =
   result = strTableGet(systemModule.tab, getIdent(name))
   if result == nil:
     rawMessage(errSystemNeeds, name)
@@ -61,7 +53,7 @@ proc getSysMagic*(name: string, m: TMagic): PSym =
 proc sysTypeFromName*(name: string): PType =
   result = getSysSym(name).typ
 
-proc getSysType(kind: TTypeKind): PType = 
+proc getSysType*(kind: TTypeKind): PType =
   result = gSysTypes[kind]
   if result == nil:
     case kind
@@ -97,6 +89,7 @@ var
 proc resetSysTypes* =
   systemModule = nil
   initStrTable(compilerprocs)
+  initStrTable(exposed)
   for i in low(gSysTypes)..high(gSysTypes):
     gSysTypes[i] = nil
 
@@ -163,8 +156,8 @@ proc setIntLitType*(result: PNode) =
       result.typ = getSysType(tyInt64)
   else: internalError(result.info, "invalid int size")
 
-proc getCompilerProc(name: string): PSym = 
+proc getCompilerProc*(name: string): PSym =
-  var ident = getIdent(name, hashIgnoreStyle(name))
+  let ident = getIdent(name)
   result = strTableGet(compilerprocs, ident)
   if result == nil:
     result = strTableGet(rodCompilerprocs, ident)
@@ -173,9 +166,22 @@ proc getCompilerProc(name: string): PSym =
       if result.kind == skStub: loadStub(result)
       if result.kind == skAlias: result = result.owner
 
-proc registerCompilerProc(s: PSym) =
+proc registerCompilerProc*(s: PSym) =
   strTableAdd(compilerprocs, s)
 
-proc finishSystem(tab: TStrTable) = discard
+proc registerNimScriptSymbol*(s: PSym) =
+  # Nimscript symbols must be al unique:
+  let conflict = strTableGet(exposed, s.name)
+  if conflict == nil:
+    strTableAdd(exposed, s)
+  else:
+    localError(s.info, "symbol conflicts with other .exportNims symbol at: " &
+      $conflict.info)
+
+proc getNimScriptSymbol*(name: string): PSym =
+  strTableGet(exposed, getIdent(name))
+
+proc resetNimScriptSymbols*() = initStrTable(exposed)
 
 initStrTable(compilerprocs)
+initStrTable(exposed)

compiler/main.nim

@@ -15,7 +15,7 @@ import
   wordrecg, sem, semdata, idents, passes, docgen, extccomp,
   cgen, jsgen, json, nversion,
   platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen,
-  tables, docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists
+  docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists
 
 from magicsys import systemModule, resetSysTypes
 
@@ -63,14 +63,14 @@ proc commandCompileToC =
   compileProject()
   cgenWriteModules()
   if gCmd != cmdRun:
-    extccomp.callCCompiler(if gProjectName == "-": "stdinfile" else: changeFileExt(gProjectFull, ""))
+    extccomp.callCCompiler(changeFileExt(gProjectFull, ""))
 
   if isServing:
     # caas will keep track only of the compilation commands
     lastCaasCmd = curCaasCmd
     resetCgenModules()
     for i in 0 .. <gMemCacheData.len:
-      gMemCacheData[i].crcStatus = crcCached
+      gMemCacheData[i].hashStatus = hashCached
       gMemCacheData[i].needsRecompile = Maybe
 
       # XXX: clean these global vars
@@ -116,7 +116,7 @@ proc interactivePasses =
   #incl(gGlobalOptions, optSafeCode)
   #setTarget(osNimrodVM, cpuNimrodVM)
   initDefines()
-  defineSymbol("nimrodvm")
+  defineSymbol("nimscript")
   when hasFFI: defineSymbol("nimffi")
   registerPass(verbosePass)
   registerPass(semPass)
@@ -190,7 +190,6 @@ proc resetMemory =
   resetRopeCache()
   resetSysTypes()
   gOwners = @[]
-  rangeDestructorProc = nil
   for i in low(buckets)..high(buckets):
     buckets[i] = nil
   idAnon = nil
@@ -237,7 +236,7 @@ proc mainCommand* =
   when SimulateCaasMemReset:
     gGlobalOptions.incl(optCaasEnabled)
 
-  # In "nimrod serve" scenario, each command must reset the registered passes
+  # In "nim serve" scenario, each command must reset the registered passes
   clearPasses()
   gLastCmdTime = epochTime()
   appendStr(searchPaths, options.libpath)
@@ -318,11 +317,12 @@ proc mainCommand* =
         (key: "lib_paths", val: libpaths)
       ]
 
-      outWriteln($dumpdata)
+      msgWriteln($dumpdata, {msgStdout, msgSkipHook})
     else:
-      outWriteln("-- list of currently defined symbols --")
+      msgWriteln("-- list of currently defined symbols --",
-      for s in definedSymbolNames(): outWriteln(s)
+                 {msgStdout, msgSkipHook})
-      outWriteln("-- end of list --")
+      for s in definedSymbolNames(): msgWriteln(s, {msgStdout, msgSkipHook})
+      msgWriteln("-- end of list --", {msgStdout, msgSkipHook})
 
       for it in iterSearchPath(searchPaths): msgWriteln(it)
   of "check":
@@ -337,7 +337,7 @@ proc mainCommand* =
     wantMainModule()
     commandScan()
     msgWriteln("Beware: Indentation tokens depend on the parser\'s state!")
-  of "i":
+  of "secret":
     gCmd = cmdInteractive
     commandInteractive()
   of "e":
@@ -356,12 +356,14 @@ proc mainCommand* =
     gGlobalOptions.incl(optCaasEnabled)
     msgs.gErrorMax = high(int)  # do not stop after first error
     serve(mainCommand)
+  of "nop", "help":
+    # prevent the "success" message:
+    gCmd = cmdDump
   else:
     rawMessage(errInvalidCommandX, command)
 
-  if (msgs.gErrorCounter == 0 and
+  if msgs.gErrorCounter == 0 and
-      gCmd notin {cmdInterpret, cmdRun, cmdDump} and
+     gCmd notin {cmdInterpret, cmdRun, cmdDump}:
-      gVerbosity > 0):
     rawMessage(hintSuccessX, [$gLinesCompiled,
                formatFloat(epochTime() - gLastCmdTime, ffDecimal, 3),
                formatSize(getTotalMem()),
@@ -379,3 +381,4 @@ proc mainCommand* =
   when SimulateCaasMemReset:
     resetMemory()
 
+  resetAttributes()

compiler/modules.nim

@@ -10,19 +10,19 @@
 ## implements the module handling
 
 import
-  ast, astalgo, magicsys, crc, rodread, msgs, cgendata, sigmatch, options, 
+  ast, astalgo, magicsys, securehash, rodread, msgs, cgendata, sigmatch, options,
-  idents, os, lexer, idgen, passes, syntaxes
+  idents, os, lexer, idgen, passes, syntaxes, llstream
 
 type
   TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled
-  TCrcStatus* = enum crcNotTaken, crcCached, crcHasChanged, crcNotChanged
+  THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged
 
   TModuleInMemory* = object
     compiledAt*: float
-    crc*: TCrc32
+    hash*: SecureHash
     deps*: seq[int32] ## XXX: slurped files are currently not tracked
     needsRecompile*: TNeedRecompile
-    crcStatus*: TCrcStatus
+    hashStatus*: THashStatus
 
 var
   gCompiledModules: seq[PSym] = @[]
@@ -34,36 +34,36 @@ proc getModule(fileIdx: int32): PSym =
   if fileIdx >= 0 and fileIdx < gCompiledModules.len:
     result = gCompiledModules[fileIdx]
 
-template crc(x: PSym): expr =
+template hash(x: PSym): expr =
-  gMemCacheData[x.position].crc
+  gMemCacheData[x.position].hash
 
-proc crcChanged(fileIdx: int32): bool =
+proc hashChanged(fileIdx: int32): bool =
   internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len
 
   template updateStatus =
-    gMemCacheData[fileIdx].crcStatus = if result: crcHasChanged
+    gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged
-                                       else: crcNotChanged
+                                       else: hashNotChanged
-    # echo "TESTING CRC: ", fileIdx.toFilename, " ", result
+    # echo "TESTING Hash: ", fileIdx.toFilename, " ", result
 
-  case gMemCacheData[fileIdx].crcStatus:
+  case gMemCacheData[fileIdx].hashStatus:
-  of crcHasChanged:
+  of hashHasChanged:
     result = true
-  of crcNotChanged:
+  of hashNotChanged:
     result = false
-  of crcCached:
+  of hashCached:
-    let newCrc = crcFromFile(fileIdx.toFilename)
+    let newHash = secureHashFile(fileIdx.toFullPath)
-    result = newCrc != gMemCacheData[fileIdx].crc
+    result = newHash != gMemCacheData[fileIdx].hash
-    gMemCacheData[fileIdx].crc = newCrc
+    gMemCacheData[fileIdx].hash = newHash
     updateStatus()
-  of crcNotTaken:
+  of hashNotTaken:
-    gMemCacheData[fileIdx].crc = crcFromFile(fileIdx.toFilename)
+    gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
     result = true
     updateStatus()
 
-proc doCRC(fileIdx: int32) =
+proc doHash(fileIdx: int32) =
-  if gMemCacheData[fileIdx].crcStatus == crcNotTaken:
+  if gMemCacheData[fileIdx].hashStatus == hashNotTaken:
-    # echo "FIRST CRC: ", fileIdx.ToFilename
+    # echo "FIRST Hash: ", fileIdx.ToFilename
-    gMemCacheData[fileIdx].crc = crcFromFile(fileIdx.toFilename)
+    gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath)
 
 proc addDep(x: PSym, dep: int32) =
   growCache gMemCacheData, dep
@@ -78,6 +78,13 @@ proc resetModule*(fileIdx: int32) =
   if fileIdx <% cgendata.gModules.len:
     cgendata.gModules[fileIdx] = nil
 
+proc resetModule*(module: PSym) =
+  let conflict = getModule(module.position.int32)
+  if conflict == nil: return
+  doAssert conflict == module
+  resetModule(module.position.int32)
+  initStrTable(module.tab)
+
 proc resetAllModules* =
   for i in 0..gCompiledModules.high:
     if gCompiledModules[i] != nil:
@@ -85,6 +92,14 @@ proc resetAllModules* =
   resetPackageCache()
   # for m in cgenModules(): echo "CGEN MODULE FOUND"
 
+proc resetAllModulesHard* =
+  resetPackageCache()
+  gCompiledModules.setLen 0
+  gMemCacheData.setLen 0
+  magicsys.resetSysTypes()
+  # XXX
+  #gOwners = @[]
+
 proc checkDepMem(fileIdx: int32): TNeedRecompile =
   template markDirty =
     resetModule(fileIdx)
@@ -94,7 +109,7 @@ proc checkDepMem(fileIdx: int32): TNeedRecompile =
     return gMemCacheData[fileIdx].needsRecompile
 
   if optForceFullMake in gGlobalOptions or
-     crcChanged(fileIdx):
+     hashChanged(fileIdx):
        markDirty
 
   if gMemCacheData[fileIdx].deps != nil:
@@ -116,7 +131,7 @@ proc newModule(fileIdx: int32): PSym =
   result.kind = skModule
   let filename = fileIdx.toFullPath
   result.name = getIdent(splitFile(filename).name)
-  if result.name.s != "-" and not isNimIdentifier(result.name.s):
+  if not isNimIdentifier(result.name.s):
     rawMessage(errInvalidModuleName, result.name.s)
 
   result.info = newLineInfo(fileIdx, 1, 1)
@@ -148,11 +163,14 @@ proc compileModule*(fileIdx: int32, flags: TSymFlags): PSym =
         return
     else:
       result.id = getID()
+    if sfMainModule in flags and gProjectIsStdin:
+      processModule(result, llStreamOpen(stdin), rd)
+    else:
       processModule(result, nil, rd)
     if optCaasEnabled in gGlobalOptions:
       gMemCacheData[fileIdx].compiledAt = gLastCmdTime
       gMemCacheData[fileIdx].needsRecompile = Recompiled
-      doCRC fileIdx
+      doHash fileIdx
   else:
     if checkDepMem(fileIdx) == Yes:
       result = compileModule(fileIdx, flags)
@@ -171,7 +189,7 @@ proc includeModule*(s: PSym, fileIdx: int32): PNode {.procvar.} =
   if optCaasEnabled in gGlobalOptions:
     growCache gMemCacheData, fileIdx
     addDep(s, fileIdx)
-    doCRC(fileIdx)
+    doHash(fileIdx)
 
 proc `==^`(a, b: string): bool =
   try:
@@ -202,9 +220,8 @@ proc compileProject*(projectFileIdx = -1'i32) =
     compileSystemModule()
     discard compileModule(projectFile, {sfMainModule})
 
-var stdinModule: PSym
+proc makeModule*(filename: string): PSym =
-proc makeStdinModule*(): PSym =
+  result = newModule(fileInfoIdx filename)
-  if stdinModule == nil:
+  result.id = getID()
-    stdinModule = newModule(fileInfoIdx"stdin")
+
-    stdinModule.id = getID()
+proc makeStdinModule*(): PSym = makeModule"stdin"
-  result = stdinModule

compiler/msgs.nim

@@ -8,7 +8,7 @@
 #
 
 import
-  options, strutils, os, tables, ropes, platform
+  options, strutils, os, tables, ropes, platform, terminal, macros
 
 type
   TMsgKind* = enum
@@ -17,10 +17,9 @@ type
     errIntLiteralExpected, errInvalidCharacterConstant,
     errClosingTripleQuoteExpected, errClosingQuoteExpected,
     errTabulatorsAreNotAllowed, errInvalidToken, errLineTooLong,
-    errInvalidNumber, errNumberOutOfRange, errNnotAllowedInCharacter,
+    errInvalidNumber, errInvalidNumberOctalCode, errNumberOutOfRange,
-    errClosingBracketExpected, errMissingFinalQuote, errIdentifierExpected,
+    errNnotAllowedInCharacter, errClosingBracketExpected, errMissingFinalQuote,
-    errNewlineExpected,
+    errIdentifierExpected, errNewlineExpected, errInvalidModuleName,
-    errInvalidModuleName,
     errOperatorExpected, errTokenExpected, errStringAfterIncludeExpected,
     errRecursiveDependencyX, errOnOrOffExpected, errNoneSpeedOrSizeExpected,
     errInvalidPragma, errUnknownPragma, errInvalidDirectiveX,
@@ -31,11 +30,14 @@ type
     errStmtInvalidAfterReturn, errStmtExpected, errInvalidLabel,
     errInvalidCmdLineOption, errCmdLineArgExpected, errCmdLineNoArgExpected,
     errInvalidVarSubstitution, errUnknownVar, errUnknownCcompiler,
-    errOnOrOffExpectedButXFound, errNoneBoehmRefcExpectedButXFound,
+    errOnOrOffExpectedButXFound, errOnOffOrListExpectedButXFound,
+    errNoneBoehmRefcExpectedButXFound,
     errNoneSpeedOrSizeExpectedButXFound, errGuiConsoleOrLibExpectedButXFound,
     errUnknownOS, errUnknownCPU, errGenOutExpectedButXFound,
     errArgsNeedRunOption, errInvalidMultipleAsgn, errColonOrEqualsExpected,
-    errExprExpected, errUndeclaredIdentifier, errUseQualifier, errTypeExpected,
+    errExprExpected, errUndeclaredIdentifier, errUndeclaredField,
+    errUndeclaredRoutine, errUseQualifier,
+    errTypeExpected,
     errSystemNeeds, errExecutionOfProgramFailed, errNotOverloadable,
     errInvalidArgForX, errStmtHasNoEffect, errXExpectsTypeOrValue,
     errXExpectsArrayType, errIteratorCannotBeInstantiated, errExprXAmbiguous,
@@ -80,7 +82,7 @@ type
     errArrayExpectsTwoTypeParams, errInvalidVisibilityX, errInitHereNotAllowed,
     errXCannotBeAssignedTo, errIteratorNotAllowed, errXNeedsReturnType,
     errNoReturnTypeDeclared,
-    errInvalidCommandX, errXOnlyAtModuleScope,
+    errNoCommand, errInvalidCommandX, errXOnlyAtModuleScope,
     errXNeedsParamObjectType,
     errTemplateInstantiationTooNested, errInstantiationFrom,
     errInvalidIndexValueForTuple, errCommandExpectsFilename,
@@ -106,6 +108,8 @@ type
     errCannotInferReturnType,
     errGenericLambdaNotAllowed,
     errCompilerDoesntSupportTarget,
+    errExternalAssemblerNotFound,
+    errExternalAssemblerNotValid,
     errUser,
     warnCannotOpenFile,
     warnOctalEscape, warnXIsNeverRead, warnXmightNotBeenInit,
@@ -114,7 +118,7 @@ type
     warnUnknownSubstitutionX, warnLanguageXNotSupported,
     warnFieldXNotSupported, warnCommentXIgnored,
     warnNilStatement, warnTypelessParam,
-    warnDifferentHeaps, warnWriteToForeignHeap, warnUnsafeCode,
+    warnUseBase, warnWriteToForeignHeap, warnUnsafeCode,
     warnEachIdentIsTuple, warnShadowIdent,
     warnProveInit, warnProveField, warnProveIndex, warnGcUnsafe, warnGcUnsafe2,
     warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed,
@@ -124,6 +128,8 @@ type
     hintConvFromXtoItselfNotNeeded, hintExprAlwaysX, hintQuitCalled,
     hintProcessing, hintCodeBegin, hintCodeEnd, hintConf, hintPath,
     hintConditionAlwaysTrue, hintName, hintPattern,
+    hintExecuting, hintLinking, hintDependency,
+    hintSource, hintStackTrace, hintGCStats,
     hintUser
 
 const
@@ -143,6 +149,7 @@ const
     errInvalidToken: "invalid token: $1",
     errLineTooLong: "line too long",
     errInvalidNumber: "$1 is not a valid number",
+    errInvalidNumberOctalCode: "$1 is not a valid number; did you mean octal? Then use one of '0o', '0c' or '0C'.",
     errNumberOutOfRange: "number $1 out of valid range",
     errNnotAllowedInCharacter: "\\n not allowed in character literal",
     errClosingBracketExpected: "closing ']' expected, but end of file reached",
@@ -179,6 +186,7 @@ const
     errUnknownVar: "unknown variable: \'$1\'",
     errUnknownCcompiler: "unknown C compiler: \'$1\'",
     errOnOrOffExpectedButXFound: "\'on\' or \'off\' expected, but \'$1\' found",
+    errOnOffOrListExpectedButXFound: "\'on\', \'off\' or \'list\' expected, but \'$1\' found",
     errNoneBoehmRefcExpectedButXFound: "'none', 'boehm' or 'refc' expected, but '$1' found",
     errNoneSpeedOrSizeExpectedButXFound: "'none', 'speed' or 'size' expected, but '$1' found",
     errGuiConsoleOrLibExpectedButXFound: "'gui', 'console' or 'lib' expected, but '$1' found",
@@ -190,10 +198,12 @@ const
     errColonOrEqualsExpected: "\':\' or \'=\' expected, but found \'$1\'",
     errExprExpected: "expression expected, but found \'$1\'",
     errUndeclaredIdentifier: "undeclared identifier: \'$1\'",
+    errUndeclaredField: "undeclared field: \'$1\'",
+    errUndeclaredRoutine: "attempting to call undeclared routine: \'$1\'",
     errUseQualifier: "ambiguous identifier: \'$1\' -- use a qualifier",
     errTypeExpected: "type expected",
     errSystemNeeds: "system module needs \'$1\'",
-    errExecutionOfProgramFailed: "execution of an external program failed",
+    errExecutionOfProgramFailed: "execution of an external program failed: '$1'",
     errNotOverloadable: "overloaded \'$1\' leads to ambiguous calls",
     errInvalidArgForX: "invalid argument for \'$1\'",
     errStmtHasNoEffect: "statement has no effect",
@@ -308,6 +318,7 @@ const
     errIteratorNotAllowed: "iterators can only be defined at the module\'s top level",
     errXNeedsReturnType: "$1 needs a return type",
     errNoReturnTypeDeclared: "no return type declared",
+    errNoCommand: "no command given",
     errInvalidCommandX: "invalid command: \'$1\'",
     errXOnlyAtModuleScope: "\'$1\' is only allowed at top level",
     errXNeedsParamObjectType: "'$1' needs a parameter that has an object type",
@@ -362,55 +373,63 @@ const
                                 "it is used as an operand to another routine and the types " &
                                 "of the generic paramers can be inferred from the expected signature.",
     errCompilerDoesntSupportTarget: "The current compiler \'$1\' doesn't support the requested compilation target",
+    errExternalAssemblerNotFound: "External assembler not found",
+    errExternalAssemblerNotValid: "External assembler '$1' is not a valid assembler",
     errUser: "$1",
-    warnCannotOpenFile: "cannot open \'$1\' [CannotOpenFile]",
+    warnCannotOpenFile: "cannot open \'$1\'",
-    warnOctalEscape: "octal escape sequences do not exist; leading zero is ignored [OctalEscape]",
+    warnOctalEscape: "octal escape sequences do not exist; leading zero is ignored",
-    warnXIsNeverRead: "\'$1\' is never read [XIsNeverRead]",
+    warnXIsNeverRead: "\'$1\' is never read",
-    warnXmightNotBeenInit: "\'$1\' might not have been initialized [XmightNotBeenInit]",
+    warnXmightNotBeenInit: "\'$1\' might not have been initialized",
-    warnDeprecated: "$1 is deprecated [Deprecated]",
+    warnDeprecated: "$1 is deprecated",
-    warnConfigDeprecated: "config file '$1' is deprecated [ConfigDeprecated]",
+    warnConfigDeprecated: "config file '$1' is deprecated",
-    warnSmallLshouldNotBeUsed: "\'l\' should not be used as an identifier; may look like \'1\' (one) [SmallLshouldNotBeUsed]",
+    warnSmallLshouldNotBeUsed: "\'l\' should not be used as an identifier; may look like \'1\' (one)",
-    warnUnknownMagic: "unknown magic \'$1\' might crash the compiler [UnknownMagic]",
+    warnUnknownMagic: "unknown magic \'$1\' might crash the compiler",
-    warnRedefinitionOfLabel: "redefinition of label \'$1\' [RedefinitionOfLabel]",
+    warnRedefinitionOfLabel: "redefinition of label \'$1\'",
-    warnUnknownSubstitutionX: "unknown substitution \'$1\' [UnknownSubstitutionX]",
+    warnUnknownSubstitutionX: "unknown substitution \'$1\'",
-    warnLanguageXNotSupported: "language \'$1\' not supported [LanguageXNotSupported]",
+    warnLanguageXNotSupported: "language \'$1\' not supported",
-    warnFieldXNotSupported: "field \'$1\' not supported [FieldXNotSupported]",
+    warnFieldXNotSupported: "field \'$1\' not supported",
-    warnCommentXIgnored: "comment \'$1\' ignored [CommentXIgnored]",
+    warnCommentXIgnored: "comment \'$1\' ignored",
-    warnNilStatement: "'nil' statement is deprecated; use an empty 'discard' statement instead [NilStmt]",
+    warnNilStatement: "'nil' statement is deprecated; use an empty 'discard' statement instead",
-    warnTypelessParam: "'$1' has no type. Typeless parameters are deprecated; only allowed for 'template' [TypelessParam]",
+    warnTypelessParam: "'$1' has no type. Typeless parameters are deprecated; only allowed for 'template'",
-    warnDifferentHeaps: "possible inconsistency of thread local heaps [DifferentHeaps]",
+    warnUseBase: "use {.base.} for base methods; baseless methods are deprecated",
-    warnWriteToForeignHeap: "write to foreign heap [WriteToForeignHeap]",
+    warnWriteToForeignHeap: "write to foreign heap",
-    warnUnsafeCode: "unsafe code: '$1' [UnsafeCode]",
+    warnUnsafeCode: "unsafe code: '$1'",
-    warnEachIdentIsTuple: "each identifier is a tuple [EachIdentIsTuple]",
+    warnEachIdentIsTuple: "each identifier is a tuple",
-    warnShadowIdent: "shadowed identifier: '$1' [ShadowIdent]",
+    warnShadowIdent: "shadowed identifier: '$1'",
-    warnProveInit: "Cannot prove that '$1' is initialized. This will become a compile time error in the future. [ProveInit]",
+    warnProveInit: "Cannot prove that '$1' is initialized. This will become a compile time error in the future.",
-    warnProveField: "cannot prove that field '$1' is accessible [ProveField]",
+    warnProveField: "cannot prove that field '$1' is accessible",
-    warnProveIndex: "cannot prove index '$1' is valid [ProveIndex]",
+    warnProveIndex: "cannot prove index '$1' is valid",
-    warnGcUnsafe: "not GC-safe: '$1' [GcUnsafe]",
+    warnGcUnsafe: "not GC-safe: '$1'",
     warnGcUnsafe2: "$1",
-    warnUninit: "'$1' might not have been initialized [Uninit]",
+    warnUninit: "'$1' might not have been initialized",
-    warnGcMem: "'$1' uses GC'ed memory [GcMem]",
+    warnGcMem: "'$1' uses GC'ed memory",
-    warnDestructor: "usage of a type with a destructor in a non destructible context. This will become a compile time error in the future. [Destructor]",
+    warnDestructor: "usage of a type with a destructor in a non destructible context. This will become a compile time error in the future.",
-    warnLockLevel: "$1 [LockLevel]",
+    warnLockLevel: "$1",
-    warnResultShadowed: "Special variable 'result' is shadowed. [ResultShadowed]",
+    warnResultShadowed: "Special variable 'result' is shadowed.",
-    warnUser: "$1 [User]",
+    warnUser: "$1",
-    hintSuccess: "operation successful [Success]",
+    hintSuccess: "operation successful",
-    hintSuccessX: "operation successful ($# lines compiled; $# sec total; $#; $#) [SuccessX]",
+    hintSuccessX: "operation successful ($# lines compiled; $# sec total; $#; $#)",
-    hintLineTooLong: "line too long [LineTooLong]",
+    hintLineTooLong: "line too long",
-    hintXDeclaredButNotUsed: "\'$1\' is declared but not used [XDeclaredButNotUsed]",
+    hintXDeclaredButNotUsed: "\'$1\' is declared but not used",
-    hintConvToBaseNotNeeded: "conversion to base object is not needed [ConvToBaseNotNeeded]",
+    hintConvToBaseNotNeeded: "conversion to base object is not needed",
-    hintConvFromXtoItselfNotNeeded: "conversion from $1 to itself is pointless [ConvFromXtoItselfNotNeeded]",
+    hintConvFromXtoItselfNotNeeded: "conversion from $1 to itself is pointless",
-    hintExprAlwaysX: "expression evaluates always to \'$1\' [ExprAlwaysX]",
+    hintExprAlwaysX: "expression evaluates always to \'$1\'",
-    hintQuitCalled: "quit() called [QuitCalled]",
+    hintQuitCalled: "quit() called",
-    hintProcessing: "$1 [Processing]",
+    hintProcessing: "$1",
-    hintCodeBegin: "generated code listing: [CodeBegin]",
+    hintCodeBegin: "generated code listing:",
-    hintCodeEnd: "end of listing [CodeEnd]",
+    hintCodeEnd: "end of listing",
-    hintConf: "used config file \'$1\' [Conf]",
+    hintConf: "used config file \'$1\'",
-    hintPath: "added path: '$1' [Path]",
+    hintPath: "added path: '$1'",
-    hintConditionAlwaysTrue: "condition is always true: '$1' [CondTrue]",
+    hintConditionAlwaysTrue: "condition is always true: '$1'",
-    hintName: "name should be: '$1' [Name]",
+    hintName: "name should be: '$1'",
-    hintPattern: "$1 [Pattern]",
+    hintPattern: "$1",
-    hintUser: "$1 [User]"]
+    hintExecuting: "$1",
+    hintLinking: "",
+    hintDependency: "$1",
+    hintSource: "$1",
+    hintStackTrace: "$1",
+    hintGCStats: "$1",
+    hintUser: "$1"]
 
 const
   WarningsToStr*: array[0..30, string] = ["CannotOpenFile", "OctalEscape",
@@ -420,15 +439,16 @@ const
     "RedefinitionOfLabel", "UnknownSubstitutionX",
     "LanguageXNotSupported", "FieldXNotSupported",
     "CommentXIgnored", "NilStmt",
-    "TypelessParam", "DifferentHeaps", "WriteToForeignHeap",
+    "TypelessParam", "UseBase", "WriteToForeignHeap",
     "UnsafeCode", "EachIdentIsTuple", "ShadowIdent",
     "ProveInit", "ProveField", "ProveIndex", "GcUnsafe", "GcUnsafe2", "Uninit",
     "GcMem", "Destructor", "LockLevel", "ResultShadowed", "User"]
 
-  HintsToStr*: array[0..16, string] = ["Success", "SuccessX", "LineTooLong",
+  HintsToStr*: array[0..22, string] = ["Success", "SuccessX", "LineTooLong",
     "XDeclaredButNotUsed", "ConvToBaseNotNeeded", "ConvFromXtoItselfNotNeeded",
     "ExprAlwaysX", "QuitCalled", "Processing", "CodeBegin", "CodeEnd", "Conf",
-    "Path", "CondTrue", "Name", "Pattern",
+    "Path", "CondTrue", "Name", "Pattern", "Exec", "Link", "Dependency",
+    "Source", "StackTrace", "GCStats",
     "User"]
 
 const
@@ -479,6 +499,30 @@ type
   ERecoverableError* = object of ValueError
   ESuggestDone* = object of Exception
 
+const
+  NotesVerbosity*: array[0..3, TNoteKinds] = [
+    {low(TNoteKind)..high(TNoteKind)} - {warnShadowIdent, warnUninit,
+                                         warnProveField, warnProveIndex,
+                                         warnGcUnsafe,
+                                         hintSuccessX, hintPath, hintConf,
+                                         hintProcessing,
+                                         hintDependency,
+                                         hintExecuting, hintLinking,
+                                         hintCodeBegin, hintCodeEnd,
+                                         hintSource, hintStackTrace,
+                                         hintGCStats},
+    {low(TNoteKind)..high(TNoteKind)} - {warnShadowIdent, warnUninit,
+                                         warnProveField, warnProveIndex,
+                                         warnGcUnsafe,
+                                         hintPath, hintConf,
+                                         hintDependency,
+                                         hintExecuting,
+                                         hintCodeBegin, hintCodeEnd,
+                                         hintSource, hintStackTrace,
+                                         hintGCStats},
+    {low(TNoteKind)..high(TNoteKind)} - {hintStackTrace},
+    {low(TNoteKind)..high(TNoteKind)}]
+
 const
   InvalidFileIDX* = int32(-1)
 
@@ -567,9 +611,7 @@ proc raiseRecoverableError*(msg: string) {.noinline, noreturn.} =
 proc sourceLine*(i: TLineInfo): Rope
 
 var
-  gNotes*: TNoteKinds = {low(TNoteKind)..high(TNoteKind)} -
+  gNotes*: TNoteKinds = NotesVerbosity[1] # defaults to verbosity of 1
-                        {warnShadowIdent, warnUninit,
-                         warnProveField, warnProveIndex, warnGcUnsafe}
   gErrorCounter*: int = 0     # counts the number of errors
   gHintCounter*: int = 0
   gWarnCounter*: int = 0
@@ -589,8 +631,11 @@ var
 
 proc suggestWriteln*(s: string) =
   if eStdOut in errorOutputs:
-    if isNil(writelnHook): writeln(stdout, s)
+    if isNil(writelnHook):
-    else: writelnHook(s)
+      writeLine(stdout, s)
+      flushFile(stdout)
+    else:
+      writelnHook(s)
 
 proc msgQuit*(x: int8) = quit x
 proc msgQuit*(x: string) = quit x
@@ -601,13 +646,17 @@ proc suggestQuit*() =
 # this format is understood by many text editors: it is the same that
 # Borland and Freepascal use
 const
-  PosErrorFormat* = "$1($2, $3) Error: $4"
+  PosFormat    = "$1($2, $3) "
-  PosWarningFormat* = "$1($2, $3) Warning: $4"
+  KindFormat   = " [$1]"
-  PosHintFormat* = "$1($2, $3) Hint: $4"
+  KindColor    = fgCyan
-  PosContextFormat = "$1($2, $3) Info: $4"
+  ErrorTitle   = "Error: "
-  RawErrorFormat* = "Error: $1"
+  ErrorColor   = fgRed
-  RawWarningFormat* = "Warning: $1"
+  WarningTitle = "Warning: "
-  RawHintFormat* = "Hint: $1"
+  WarningColor = fgYellow
+  HintTitle    = "Hint: "
+  HintColor    = fgGreen
+  InfoTitle    = "Info: "
+  InfoColor    = fgCyan
 
 proc getInfoContextLen*(): int = return msgContext.len
 proc setInfoContextLen*(L: int) = setLen(msgContext, L)
@@ -678,21 +727,79 @@ proc `??`* (info: TLineInfo, filename: string): bool =
 
 var gTrackPos*: TLineInfo
 
-proc outWriteln*(s: string) =
+type
-  ## Writes to stdout. Always.
+  MsgFlag* = enum  ## flags altering msgWriteln behavior
-  if eStdOut in errorOutputs: writeln(stdout, s)
+    msgStdout,     ## force writing to stdout, even stderr is default
+    msgSkipHook    ## skip message hook even if it is present
+  MsgFlags* = set[MsgFlag]
 
-proc msgWriteln*(s: string) =
+proc msgWriteln*(s: string, flags: MsgFlags = {}) =
-  ## Writes to stdout. If --stdout option is given, writes to stderr instead.
+  ## Writes given message string to stderr by default.
+  ## If ``--stdout`` option is given, writes to stdout instead. If message hook
+  ## is present, then it is used to output message rather than stderr/stdout.
+  ## This behavior can be altered by given optional flags.
 
   #if gCmd == cmdIdeTools and optCDebug notin gGlobalOptions: return
 
-  if not isNil(writelnHook):
+  if not isNil(writelnHook) and msgSkipHook notin flags:
     writelnHook(s)
-  elif optStdout in gGlobalOptions:
+  elif optStdout in gGlobalOptions or msgStdout in flags:
-    if eStdErr in errorOutputs: writeln(stderr, s)
+    if eStdOut in errorOutputs:
+      writeLine(stdout, s)
+      flushFile(stdout)
   else:
-    if eStdOut in errorOutputs: writeln(stdout, s)
+    if eStdErr in errorOutputs:
+      writeLine(stderr, s)
+      # On Windows stderr is fully-buffered when piped, regardless of C std.
+      when defined(windows):
+        flushFile(stderr)
+
+macro callIgnoringStyle(theProc: typed, first: typed,
+                        args: varargs[expr]): stmt =
+  let typForegroundColor = bindSym"ForegroundColor".getType
+  let typBackgroundColor = bindSym"BackgroundColor".getType
+  let typStyle = bindSym"Style".getType
+  let typTerminalCmd = bindSym"TerminalCmd".getType
+  result = newCall(theProc)
+  if first.kind != nnkNilLit: result.add(first)
+  for arg in children(args[0][1]):
+    if arg.kind == nnkNilLit: continue
+    let typ = arg.getType
+    if typ.kind != nnkEnumTy or
+       typ != typForegroundColor and
+       typ != typBackgroundColor and
+       typ != typStyle and
+       typ != typTerminalCmd:
+      result.add(arg)
+
+macro callStyledWriteLineStderr(args: varargs[expr]): stmt =
+  result = newCall(bindSym"styledWriteLine")
+  result.add(bindSym"stderr")
+  for arg in children(args[0][1]):
+    result.add(arg)
+
+template callWritelnHook(args: varargs[string, `$`]) =
+  var s = ""
+  for arg in args:
+    s.add arg
+  writelnHook s
+
+template styledMsgWriteln*(args: varargs[expr]) =
+  if not isNil(writelnHook):
+    callIgnoringStyle(callWritelnHook, nil, args)
+  elif optStdout in gGlobalOptions:
+    if eStdOut in errorOutputs:
+      callIgnoringStyle(writeLine, stdout, args)
+      flushFile(stdout)
+  else:
+    if eStdErr in errorOutputs:
+      if optUseColors in gGlobalOptions:
+        callStyledWriteLineStderr(args)
+      else:
+        callIgnoringStyle(writeLine, stderr, args)
+      # On Windows stderr is fully-buffered when piped, regardless of C std.
+      when defined(windows):
+        flushFile(stderr)
 
 proc coordToStr(coord: int): string =
   if coord == -1: result = "???"
@@ -710,11 +817,11 @@ type
 
 proc handleError(msg: TMsgKind, eh: TErrorHandling, s: string) =
   template quit =
-    if defined(debug) or gVerbosity >= 3 or msg == errInternal:
+    if defined(debug) or msg == errInternal or hintStackTrace in gNotes:
       if stackTraceAvailable() and isNil(writelnHook):
         writeStackTrace()
       else:
-        msgWriteln("No stack traceback available\nTo create a stacktrace, rerun compilation with ./koch temp " & options.command & " <file>")
+        styledMsgWriteln(fgRed, "No stack traceback available\nTo create a stacktrace, rerun compilation with ./koch temp " & options.command & " <file>")
     quit 1
 
   if msg >= fatalMin and msg <= fatalMax:
@@ -736,61 +843,85 @@ proc writeContext(lastinfo: TLineInfo) =
   var info = lastinfo
   for i in countup(0, len(msgContext) - 1):
     if msgContext[i] != lastinfo and msgContext[i] != info:
-      msgWriteln(PosContextFormat % [toMsgFilename(msgContext[i]),
+      styledMsgWriteln(styleBright,
+                       PosFormat % [toMsgFilename(msgContext[i]),
                                     coordToStr(msgContext[i].line),
-                                     coordToStr(msgContext[i].col+1),
+                                    coordToStr(msgContext[i].col+1)],
-                                     getMessageStr(errInstantiationFrom, "")])
+                       resetStyle,
+                       getMessageStr(errInstantiationFrom, ""))
     info = msgContext[i]
 
 proc ignoreMsgBecauseOfIdeTools(msg: TMsgKind): bool =
   msg >= errGenerated and gCmd == cmdIdeTools and optIdeDebug notin gGlobalOptions
 
 proc rawMessage*(msg: TMsgKind, args: openArray[string]) =
-  var frmt: string
+  var
+    title: string
+    color: ForegroundColor
+    kind:  string
   case msg
   of errMin..errMax:
     writeContext(unknownLineInfo())
-    frmt = RawErrorFormat
+    title = ErrorTitle
+    color = ErrorColor
   of warnMin..warnMax:
     if optWarns notin gOptions: return
     if msg notin gNotes: return
     writeContext(unknownLineInfo())
-    frmt = RawWarningFormat
+    title = WarningTitle
+    color = WarningColor
+    kind = WarningsToStr[ord(msg) - ord(warnMin)]
     inc(gWarnCounter)
   of hintMin..hintMax:
     if optHints notin gOptions: return
     if msg notin gNotes: return
-    frmt = RawHintFormat
+    title = HintTitle
+    color = HintColor
+    kind = HintsToStr[ord(msg) - ord(hintMin)]
     inc(gHintCounter)
-  let s = `%`(frmt, `%`(msgKindToString(msg), args))
+  let s = `%`(msgKindToString(msg), args)
   if not ignoreMsgBecauseOfIdeTools(msg):
-    msgWriteln(s)
+    if kind != nil:
+      styledMsgWriteln(color, title, resetStyle, s,
+                       KindColor, `%`(KindFormat, kind))
+    else:
+      styledMsgWriteln(color, title, resetStyle, s)
   handleError(msg, doAbort, s)
 
 proc rawMessage*(msg: TMsgKind, arg: string) =
   rawMessage(msg, [arg])
 
+proc resetAttributes* =
+  if {optUseColors, optStdout} * gGlobalOptions == {optUseColors}:
+    terminal.resetAttributes(stderr)
+
 proc writeSurroundingSrc(info: TLineInfo) =
   const indent = "  "
   msgWriteln(indent & $info.sourceLine)
   msgWriteln(indent & spaces(info.col) & '^')
 
 proc formatMsg*(info: TLineInfo, msg: TMsgKind, arg: string): string =
-  let frmt = case msg
+  let title = case msg
-             of warnMin..warnMax: PosWarningFormat
+              of warnMin..warnMax: WarningTitle
-             of hintMin..hintMax: PosHintFormat
+              of hintMin..hintMax: HintTitle
-             else: PosErrorFormat
+              else: ErrorTitle
-  result = frmt % [toMsgFilename(info), coordToStr(info.line),
+  result = PosFormat % [toMsgFilename(info), coordToStr(info.line),
-                   coordToStr(info.col+1), getMessageStr(msg, arg)]
+                        coordToStr(info.col+1)] &
+           title &
+           getMessageStr(msg, arg)
 
 proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
                eh: TErrorHandling) =
-  var frmt: string
+  var
-  var ignoreMsg = false
+    title: string
+    color: ForegroundColor
+    kind:  string
+    ignoreMsg = false
   case msg
   of errMin..errMax:
     writeContext(info)
-    frmt = PosErrorFormat
+    title = ErrorTitle
+    color = ErrorColor
     # we try to filter error messages so that not two error message
     # in the same file and line are produced:
     #ignoreMsg = lastError == info and eh != doAbort
@@ -798,20 +929,29 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
   of warnMin..warnMax:
     ignoreMsg = optWarns notin gOptions or msg notin gNotes
     if not ignoreMsg: writeContext(info)
-    frmt = PosWarningFormat
+    title = WarningTitle
+    color = WarningColor
+    kind = WarningsToStr[ord(msg) - ord(warnMin)]
     inc(gWarnCounter)
   of hintMin..hintMax:
     ignoreMsg = optHints notin gOptions or msg notin gNotes
-    frmt = PosHintFormat
+    title = HintTitle
+    color = HintColor
+    kind = HintsToStr[ord(msg) - ord(hintMin)]
     inc(gHintCounter)
   # NOTE: currently line info line numbers start with 1,
   # but column numbers start with 0, however most editors expect
   # first column to be 1, so we need to +1 here
-  let s = frmt % [toMsgFilename(info), coordToStr(info.line),
+  let x = PosFormat % [toMsgFilename(info), coordToStr(info.line),
-                  coordToStr(info.col+1), getMessageStr(msg, arg)]
+                       coordToStr(info.col+1)]
+  let s = getMessageStr(msg, arg)
   if not ignoreMsg and not ignoreMsgBecauseOfIdeTools(msg):
-    msgWriteln(s)
+    if kind != nil:
-    if optPrintSurroundingSrc and msg in errMin..errMax:
+      styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s,
+                       KindColor, `%`(KindFormat, kind))
+    else:
+      styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s)
+    if msg in errMin..errMax and hintSource in gNotes:
       info.writeSurroundingSrc
   handleError(msg, eh, s)
 
@@ -882,3 +1022,22 @@ ropes.errorHandler = proc (err: RopesError, msg: string, useWarning: bool) =
   of rCannotOpenFile:
     rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile, msg)
 
+proc listWarnings*() =
+  msgWriteln("Warnings:")
+  for warn in warnMin..warnMax:
+    msgWriteln("  [$1] $2" % [
+      if warn in gNotes: "x" else: " ",
+      msgs.WarningsToStr[ord(warn) - ord(warnMin)]
+    ])
+
+proc listHints*() =
+  msgWriteln("Hints:")
+  for hint in hintMin..hintMax:
+    msgWriteln("  [$1] $2" % [
+      if hint in gNotes: "x" else: " ",
+      msgs.HintsToStr[ord(hint) - ord(hintMin)]
+    ])
+
+# enable colors by default on terminals
+if terminal.isatty(stderr):
+  incl(gGlobalOptions, optUseColors)

compiler/nim.cfg

@@ -17,5 +17,4 @@ define:useStdoutAsStdmsg
 
 cs:partial
 #define:useNodeIds
-symbol:nimfix
 #gc:markAndSweep

compiler/nim.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nim Compiler
-#        (c) Copyright 2013 Andreas Rumpf
+#        (c) Copyright 2015 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -16,7 +16,7 @@ when defined(gcc) and defined(windows):
 import
   commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes,
   extccomp, strutils, os, osproc, platform, main, parseopt, service,
-  nodejs
+  nodejs, scriptconfig
 
 when hasTinyCBackend:
   import tccgen
@@ -38,31 +38,44 @@ proc handleCmdLine() =
   else:
     # Process command line arguments:
     processCmdLine(passCmd1, "")
-    if gProjectName != "":
+    if gProjectName == "-":
+      gProjectName = "stdinfile"
+      gProjectFull = "stdinfile"
+      gProjectPath = getCurrentDir()
+      gProjectIsStdin = true
+    elif gProjectName != "":
       try:
         gProjectFull = canonicalizePath(gProjectName)
       except OSError:
         gProjectFull = gProjectName
-      var p = splitFile(gProjectFull)
+      let p = splitFile(gProjectFull)
       gProjectPath = p.dir
       gProjectName = p.name
     else:
       gProjectPath = getCurrentDir()
     loadConfigs(DefaultConfig) # load all config files
+    let scriptFile = gProjectFull.changeFileExt("nims")
+    if fileExists(scriptFile):
+      runNimScript(scriptFile)
+      # 'nim foo.nims' means to just run the NimScript file and do nothing more:
+      if scriptFile == gProjectFull: return
+    elif fileExists(gProjectPath / "config.nims"):
+      # directory wide NimScript file
+      runNimScript(gProjectPath / "config.nims")
     # now process command line arguments again, because some options in the
     # command line can overwite the config file's settings
     extccomp.initVars()
     processCmdLine(passCmd2, "")
+    if options.command == "":
+      rawMessage(errNoCommand, command)
     mainCommand()
-    if gVerbosity >= 2: echo(GC_getStatistics())
+    if optHints in gOptions and hintGCStats in gNotes: echo(GC_getStatistics())
     #echo(GC_getStatistics())
     if msgs.gErrorCounter == 0:
       when hasTinyCBackend:
         if gCmd == cmdRun:
           tccgen.run(commands.arguments)
       if optRun in gGlobalOptions:
-        if gProjectName == "-":
-          gProjectFull = "stdinfile"
         if gCmd == cmdCompileToJS:
           var ex: string
           if options.outFile.len > 0:

compiler/nimblecmd.nim

@@ -48,7 +48,7 @@ proc addPackage(packages: StringTableRef, p: string) =
   let name = p.substr(0, x-1)
   if x < p.len:
     let version = p.substr(x+1)
-    if packages[name] <. version:
+    if packages.getOrDefault(name) <. version:
       packages[name] = version
   else:
     packages[name] = latest
@@ -60,7 +60,7 @@ iterator chosen(packages: StringTableRef): string =
 
 proc addNimblePath(p: string, info: TLineInfo) =
   if not contains(options.searchPaths, p):
-    if gVerbosity >= 1: message(info, hintPath, p)
+    message(info, hintPath, p)
     lists.prependStr(options.lazyPaths, p)
 
 proc addPathWithNimFiles(p: string, info: TLineInfo) =

compiler/nimconf.nim

@@ -211,7 +211,7 @@ proc readConfigFile(filename: string) =
     while tok.tokType != tkEof: parseAssignment(L, tok)
     if len(condStack) > 0: lexMessage(L, errTokenExpected, "@end")
     closeLexer(L)
-    if gVerbosity >= 1: rawMessage(hintConf, filename)
+    rawMessage(hintConf, filename)
 
 proc getUserConfigPath(filename: string): string =
   result = joinPath(getConfigDir(), filename)
@@ -226,15 +226,7 @@ proc getSystemConfigPath(filename: string): string =
     if not existsFile(result): result = "/etc/" & filename
 
 proc loadConfigs*(cfg: string) =
-  # set default value (can be overwritten):
+  setDefaultLibpath()
-  if libpath == "": 
-    # choose default libpath:
-    var prefix = getPrefixDir()
-    when defined(posix):
-      if prefix == "/usr": libpath = "/usr/lib/nim"
-      elif prefix == "/usr/local": libpath = "/usr/local/lib/nim"
-      else: libpath = joinPath(prefix, "lib")
-    else: libpath = joinPath(prefix, "lib")
 
   if optSkipConfigFile notin gGlobalOptions:
     readConfigFile(getSystemConfigPath(cfg))

compiler/nimfix/nimfix.nim

@@ -10,8 +10,10 @@
 ## Nimfix is a tool that helps to convert old-style Nimrod code to Nim code.
 
 import strutils, os, parseopt
-import options, commands, modules, sem, passes, passaux, pretty, msgs, nimconf,
+import compiler/options, compiler/commands, compiler/modules, compiler/sem,
-  extccomp, condsyms, lists
+  compiler/passes, compiler/passaux, compiler/nimfix/pretty,
+  compiler/msgs, compiler/nimconf,
+  compiler/extccomp, compiler/condsyms, compiler/lists
 
 const Usage = """
 Nimfix - Tool to patch Nim code
@@ -80,7 +82,7 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
 
 proc handleCmdLine() =
   if paramCount() == 0:
-    stdout.writeln(Usage)
+    stdout.writeLine(Usage)
   else:
     processCmdLine(passCmd1, "")
     if gProjectName != "":

compiler/nimfix/pretty.nim

@@ -11,8 +11,10 @@
 ## to convert Nim code into a consistent style.
 
 import
-  strutils, os, options, ast, astalgo, msgs, ropes, idents,
+  strutils, os, intsets, strtabs
-  intsets, strtabs, semdata, prettybase
+
+import compiler/options, compiler/ast, compiler/astalgo, compiler/msgs,
+  compiler/semdata, compiler/nimfix/prettybase, compiler/ropes, compiler/idents
 
 type
   StyleCheck* {.pure.} = enum None, Warn, Auto

compiler/nimfix/prettybase.nim

@@ -7,7 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
-import ast, msgs, strutils, idents, lexbase, streams
+import strutils, lexbase, streams
+import compiler/ast, compiler/msgs, compiler/idents
 from os import splitFile
 
 type

compiler/nimsuggest/nimsuggest.nim

@@ -7,192 +7,6 @@
 #    distribution, for details about the copyright.
 #
 
-## Nimsuggest is a tool that helps to give editors IDE like capabilities.
+## Nimsuggest has been moved to https://github.com/nim-lang/nimsuggest
 
-import strutils, os, parseopt, parseUtils
+{.error: "This project has moved to the following repo: https://github.com/nim-lang/nimsuggest".}
-import options, commands, modules, sem, passes, passaux, msgs, nimconf,
-  extccomp, condsyms, lists, net, rdstdin
-
-const Usage = """
-Nimsuggest - Tool to give every editor IDE like capabilities for Nim
-Usage:
-  nimsuggest [options] projectfile.nim
-
-Options:
-  --port:PORT             port, by default 6000
-  --address:HOST          binds to that address, by default ""
-  --stdin                 read commands from stdin and write results to
-                          stdout instead of using sockets
-
-The server then listens to the connection and takes line-based commands.
-
-In addition, all command line options of Nim that do not affect code generation
-are supported.
-"""
-
-var
-  gPort = 6000.Port
-  gAddress = ""
-  gUseStdin: bool
-
-const
-  seps = {':', ';', ' ', '\t'}
-  Help = "usage: sug|con|def|use file.nim[;dirtyfile.nim]:line:col\n"&
-         "type 'quit' to quit\n" &
-         "type 'debug' to toggle debug mode on/off\n" &
-         "type 'terse' to toggle terse mode on/off"
-
-proc parseQuoted(cmd: string; outp: var string; start: int): int =
-  var i = start
-  i += skipWhitespace(cmd, i)
-  if cmd[i] == '"':
-    i += parseUntil(cmd, outp, '"', i+1)+2
-  else:
-    i += parseUntil(cmd, outp, seps, i)
-  result = i
-
-proc action(cmd: string) =
-  template toggle(sw) =
-    if sw in gGlobalOptions:
-      excl(gGlobalOptions, sw)
-    else:
-      incl(gGlobalOptions, sw)
-    return
-
-  template err() =
-    echo Help
-    return
-
-  var opc = ""
-  var i = parseIdent(cmd, opc, 0)
-  case opc.normalize
-  of "sug": gIdeCmd = ideSug
-  of "con": gIdeCmd = ideCon
-  of "def": gIdeCmd = ideDef
-  of "use":
-    modules.resetAllModules()
-    gIdeCmd = ideUse
-  of "quit": quit()
-  of "debug": toggle optIdeDebug
-  of "terse": toggle optIdeTerse
-  else: err()
-  var dirtyfile = ""
-  var orig = ""
-  i = parseQuoted(cmd, orig, i)
-  if cmd[i] == ';':
-    i = parseQuoted(cmd, dirtyfile, i+1)
-  i += skipWhile(cmd, seps, i)
-  var line = -1
-  var col = 0
-  i += parseInt(cmd, line, i)
-  i += skipWhile(cmd, seps, i)
-  i += parseInt(cmd, col, i)
-
-  var isKnownFile = true
-  if orig.len == 0: err()
-  let dirtyIdx = orig.fileInfoIdx(isKnownFile)
-
-  if dirtyfile.len != 0: msgs.setDirtyFile(dirtyIdx, dirtyfile)
-  else: msgs.setDirtyFile(dirtyIdx, nil)
-
-  resetModule dirtyIdx
-  if dirtyIdx != gProjectMainIdx:
-    resetModule gProjectMainIdx
-  gTrackPos = newLineInfo(dirtyIdx, line, col-1)
-  #echo dirtyfile, gDirtyBufferIdx, " project ", gProjectMainIdx
-  gErrorCounter = 0
-  if not isKnownFile:
-    compileProject(dirtyIdx)
-  else:
-    compileProject()
-
-proc serve() =
-  # do not stop after the first error:
-  msgs.gErrorMax = high(int)
-  if gUseStdin:
-    echo Help
-    var line = ""
-    while readLineFromStdin("> ", line):
-      action line
-      echo ""
-      flushFile(stdout)
-  else:
-    var server = newSocket()
-    server.bindAddr(gPort, gAddress)
-    var inp = "".TaintedString
-    server.listen()
-
-    while true:
-      var stdoutSocket = newSocket()
-      msgs.writelnHook = proc (line: string) =
-        stdoutSocket.send(line & "\c\L")
-
-      accept(server, stdoutSocket)
-
-      stdoutSocket.readLine(inp)
-      action inp.string
-
-      stdoutSocket.send("\c\L")
-      stdoutSocket.close()
-
-proc mainCommand =
-  registerPass verbosePass
-  registerPass semPass
-  gCmd = cmdIdeTools
-  incl gGlobalOptions, optCaasEnabled
-  isServing = true
-  wantMainModule()
-  appendStr(searchPaths, options.libpath)
-  if gProjectFull.len != 0:
-    # current path is always looked first for modules
-    prependStr(searchPaths, gProjectPath)
-
-  serve()
-
-proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
-  var p = parseopt.initOptParser(cmd)
-  while true:
-    parseopt.next(p)
-    case p.kind
-    of cmdEnd: break
-    of cmdLongoption, cmdShortOption:
-      case p.key.normalize
-      of "port": gPort = parseInt(p.val).Port
-      of "address": gAddress = p.val
-      of "stdin": gUseStdin = true
-      else: processSwitch(pass, p)
-    of cmdArgument:
-      options.gProjectName = unixToNativePath(p.key)
-      # if processArgument(pass, p, argsCount): break
-
-proc handleCmdLine() =
-  if paramCount() == 0:
-    stdout.writeln(Usage)
-  else:
-    processCmdLine(passCmd1, "")
-    if gProjectName != "":
-      try:
-        gProjectFull = canonicalizePath(gProjectName)
-      except OSError:
-        gProjectFull = gProjectName
-      var p = splitFile(gProjectFull)
-      gProjectPath = p.dir
-      gProjectName = p.name
-    else:
-      gProjectPath = getCurrentDir()
-    loadConfigs(DefaultConfig) # load all config files
-    # now process command line arguments again, because some options in the
-    # command line can overwite the config file's settings
-    extccomp.initVars()
-    processCmdLine(passCmd2, "")
-    mainCommand()
-
-when false:
-  proc quitCalled() {.noconv.} =
-    writeStackTrace()
-
-  addQuitProc(quitCalled)
-
-condsyms.initDefines()
-defineSymbol "nimsuggest"
-handleCmdline()

compiler/nimsuggest/nimsuggest.nim.cfg

@@ -1,17 +0,0 @@
-# Special configuration file for the Nim project
-
-gc:markAndSweep
-
-hint[XDeclaredButNotUsed]:off
-path:"$projectPath/../.."
-
-path:"$lib/packages/docutils"
-path:"../../compiler"
-
-define:useStdoutAsStdmsg
-define:nimsuggest
-
-cs:partial
-#define:useNodeIds
-define:booting
-#define:noDocgen

compiler/nodejs.nim

@@ -4,3 +4,5 @@ proc findNodeJs*(): string =
   result = findExe("nodejs")
   if result == "":
     result = findExe("node")
+  if result == "":
+    result = findExe("iojs")

compiler/options.nim

@@ -13,6 +13,7 @@ import
 const
   hasTinyCBackend* = defined(tinyc)
   useEffectSystem* = true
+  useWriteTracking* = false
   hasFFI* = defined(useFFI)
   newScopeForIf* = true
   useCaas* = not defined(noCaas)
@@ -39,7 +40,7 @@ type                          # please make sure we have under 32 options
   TGlobalOption* = enum       # **keep binary compatible**
     gloptNone, optForceFullMake, optDeadCodeElim,
     optListCmd, optCompileOnly, optNoLinking,
-    optSafeCode,              # only allow safe code
+    optReportConceptFailures, # report 'compiles' or 'concept' matching failures
     optCDebug,                # turn on debugging information
     optGenDynLib,             # generate a dynamic library
     optGenStaticLib,          # generate a static library
@@ -54,6 +55,7 @@ type                          # please make sure we have under 32 options
     optSkipUserConfigFile,    # skip the users's config file
     optSkipParentConfigFiles, # skip parent dir's config files
     optNoMain,                # do not generate a "main" proc
+    optUseColors,             # use colors for hints, warnings, and errors
     optThreads,               # support for multi-threading
     optStdout,                # output to stdout
     optThreadAnalysis,        # thread analysis pass
@@ -81,13 +83,13 @@ type                          # please make sure we have under 32 options
     cmdRun                    # run the project via TCC backend
   TStringSeq* = seq[string]
   TGCMode* = enum             # the selected GC
-    gcNone, gcBoehm, gcMarkAndSweep, gcRefc, gcV2, gcGenerational
+    gcNone, gcBoehm, gcGo, gcMarkAndSweep, gcRefc, gcV2, gcGenerational
 
-  TIdeCmd* = enum
+  IdeCmd* = enum
-    ideNone, ideSug, ideCon, ideDef, ideUse
+    ideNone, ideSug, ideCon, ideDef, ideUse, ideDus
 
 var
-  gIdeCmd*: TIdeCmd
+  gIdeCmd*: IdeCmd
 
 const
   ChecksOptions* = {optObjCheck, optFieldCheck, optRangeCheck, optNilCheck,
@@ -126,9 +128,6 @@ template compilationCachePresent*: expr =
 template optPreserveOrigSource*: expr =
   optEmbedOrigSrc in gGlobalOptions
 
-template optPrintSurroundingSrc*: expr =
-  gVerbosity >= 2
-
 const
   genSubDir* = "nimcache"
   NimExt* = "nim"
@@ -145,10 +144,12 @@ const
 var
   gConfigVars* = newStringTable(modeStyleInsensitive)
   gDllOverrides = newStringTable(modeCaseInsensitive)
+  gPrefixDir* = "" # Overrides the default prefix dir in getPrefixDir proc.
   libpath* = ""
   gProjectName* = "" # holds a name like 'nimrod'
   gProjectPath* = "" # holds a path like /home/alice/projects/nimrod/compiler/
   gProjectFull* = "" # projectPath/projectName
+  gProjectIsStdin* = false # whether we're compiling from stdin
   gProjectMainIdx*: int32 # the canonical path id of the main module
   nimcacheDir* = ""
   command* = "" # the main command (e.g. cc, check, scan, etc)
@@ -172,7 +173,7 @@ proc existsConfigVar*(key: string): bool =
   result = hasKey(gConfigVars, key)
 
 proc getConfigVar*(key: string): string =
-  result = gConfigVars[key]
+  result = gConfigVars.getOrDefault key
 
 proc setConfigVar*(key, val: string) =
   gConfigVars[key] = val
@@ -182,9 +183,25 @@ proc getOutFile*(filename, ext: string): string =
   else: result = changeFileExt(filename, ext)
 
 proc getPrefixDir*(): string =
-  ## gets the application directory
+  ## Gets the prefix dir, usually the parent directory where the binary resides.
+  ##
+  ## This is overrided by some tools (namely nimsuggest) via the ``gPrefixDir``
+  ## global.
+  if gPrefixDir != "": result = gPrefixDir
+  else:
     result = splitPath(getAppDir()).head
 
+proc setDefaultLibpath*() =
+  # set default value (can be overwritten):
+  if libpath == "":
+    # choose default libpath:
+    var prefix = getPrefixDir()
+    when defined(posix):
+      if prefix == "/usr": libpath = "/usr/lib/nim"
+      elif prefix == "/usr/local": libpath = "/usr/local/lib/nim"
+      else: libpath = joinPath(prefix, "lib")
+    else: libpath = joinPath(prefix, "lib")
+
 proc canonicalizePath*(path: string): string =
   when not FileSystemCaseSensitive: result = path.expandFilename.toLower
   else: result = path.expandFilename
@@ -205,7 +222,7 @@ proc removeTrailingDirSep*(path: string): string =
   else:
     result = path
 
-proc getGeneratedPath: string =
+proc getNimcacheDir*: string =
   result = if nimcacheDir.len > 0: nimcacheDir else: gProjectPath.shortenDir /
                                                          genSubDir
 
@@ -261,7 +278,7 @@ proc toGeneratedFile*(path, ext: string): string =
   ## converts "/home/a/mymodule.nim", "rod" to "/home/a/nimcache/mymodule.rod"
   var (head, tail) = splitPath(path)
   #if len(head) > 0: head = shortenDir(head & dirSep)
-  result = joinPath([getGeneratedPath(), changeFileExt(tail, ext)])
+  result = joinPath([getNimcacheDir(), changeFileExt(tail, ext)])
   #echo "toGeneratedFile(", path, ", ", ext, ") = ", result
 
 when noTimeMachine:
@@ -289,14 +306,14 @@ when noTimeMachine:
 proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string =
   var (head, tail) = splitPath(f)
   #if len(head) > 0: head = removeTrailingDirSep(shortenDir(head & dirSep))
-  var subdir = getGeneratedPath() # / head
+  var subdir = getNimcacheDir() # / head
   if createSubDir:
     try:
       createDir(subdir)
       when noTimeMachine:
         excludeDirFromTimeMachine(subdir)
     except OSError:
-      writeln(stdout, "cannot create directory: " & subdir)
+      writeLine(stdout, "cannot create directory: " & subdir)
       quit(1)
   result = joinPath(subdir, tail)
   #echo "completeGeneratedFilePath(", f, ") = ", result
@@ -325,6 +342,9 @@ proc rawFindFile2(f: string): string =
   result = ""
 
 proc findFile*(f: string): string {.procvar.} =
+  if f.isAbsolute:
+    result = if f.existsFile: f else: ""
+  else:
     result = f.rawFindFile
     if result.len == 0:
       result = f.toLower.rawFindFile
@@ -395,3 +415,20 @@ template cnimdbg*: expr = p.module.module.fileIdx == gProjectMainIdx
 template pnimdbg*: expr = p.lex.fileIdx == gProjectMainIdx
 template lnimdbg*: expr = L.fileIdx == gProjectMainIdx
 
+proc parseIdeCmd*(s: string): IdeCmd =
+  case s:
+  of "sug": ideSug
+  of "con": ideCon
+  of "def": ideDef
+  of "use": ideUse
+  of "dus": ideDus
+  else: ideNone
+
+proc `$`*(c: IdeCmd): string =
+  case c:
+  of ideSug: "sug"
+  of ideCon: "con"
+  of ideDef: "def"
+  of ideUse: "use"
+  of ideDus: "dus"
+  of ideNone: "none"

compiler/parampatterns.nim

@@ -178,13 +178,14 @@ type
     arDiscriminant,           # is a discriminant
     arStrange                 # it is a strange beast like 'typedesc[var T]'
 
-proc isAssignable*(owner: PSym, n: PNode): TAssignableResult =
+proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult =
   ## 'owner' can be nil!
   result = arNone
   case n.kind
   of nkSym:
-    # don't list 'skLet' here:
+    let kinds = if isUnsafeAddr: {skVar, skResult, skTemp, skParam, skLet}
-    if n.sym.kind in {skVar, skResult, skTemp}:
+                else: {skVar, skResult, skTemp}
+    if n.sym.kind in kinds:
       if owner != nil and owner.id == n.sym.owner.id and
           sfGlobal notin n.sym.flags:
         result = arLocalLValue
@@ -200,7 +201,7 @@ proc isAssignable*(owner: PSym, n: PNode): TAssignableResult =
         {tyVar, tyPtr, tyRef}:
       result = arLValue
     else:
-      result = isAssignable(owner, n.sons[0])
+      result = isAssignable(owner, n.sons[0], isUnsafeAddr)
     if result != arNone and sfDiscriminant in n.sons[1].sym.flags:
       result = arDiscriminant
   of nkBracketExpr:
@@ -208,23 +209,27 @@ proc isAssignable*(owner: PSym, n: PNode): TAssignableResult =
         {tyVar, tyPtr, tyRef}:
       result = arLValue
     else:
-      result = isAssignable(owner, n.sons[0])
+      result = isAssignable(owner, n.sons[0], isUnsafeAddr)
   of nkHiddenStdConv, nkHiddenSubConv, nkConv:
     # Object and tuple conversions are still addressable, so we skip them
     # XXX why is 'tyOpenArray' allowed here?
     if skipTypes(n.typ, abstractPtrs-{tyTypeDesc}).kind in
         {tyOpenArray, tyTuple, tyObject}:
-      result = isAssignable(owner, n.sons[1])
+      result = isAssignable(owner, n.sons[1], isUnsafeAddr)
     elif compareTypes(n.typ, n.sons[1].typ, dcEqIgnoreDistinct):
       # types that are equal modulo distinction preserve l-value:
-      result = isAssignable(owner, n.sons[1])
+      result = isAssignable(owner, n.sons[1], isUnsafeAddr)
   of nkHiddenDeref, nkDerefExpr, nkHiddenAddr:
     result = arLValue
   of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
-    result = isAssignable(owner, n.sons[0])
+    result = isAssignable(owner, n.sons[0], isUnsafeAddr)
   of nkCallKinds:
     # builtin slice keeps lvalue-ness:
-    if getMagic(n) == mSlice: result = isAssignable(owner, n.sons[1])
+    if getMagic(n) in {mArrGet, mSlice}:
+      result = isAssignable(owner, n.sons[1], isUnsafeAddr)
+  of nkStmtList, nkStmtListExpr:
+    if n.typ != nil:
+      result = isAssignable(owner, n.lastSon, isUnsafeAddr)
   else:
     discard
 

compiler/parser.nim

@@ -64,6 +64,7 @@ proc setBaseFlags*(n: PNode, base: TNumericalBase)
 proc parseSymbol*(p: var TParser, allowNil = false): PNode
 proc parseTry(p: var TParser; isExpr: bool): PNode
 proc parseCase(p: var TParser): PNode
+proc parseStmtPragma(p: var TParser): PNode
 # implementation
 
 proc getTok(p: var TParser) =
@@ -124,6 +125,9 @@ proc rawSkipComment(p: var TParser, node: PNode) =
 proc skipComment(p: var TParser, node: PNode) =
   if p.tok.indent < 0: rawSkipComment(p, node)
 
+proc flexComment(p: var TParser, node: PNode) =
+  if p.tok.indent < 0 or realInd(p): rawSkipComment(p, node)
+
 proc skipInd(p: var TParser) =
   if p.tok.indent >= 0:
     if not realInd(p): parMessage(p, errInvalidIndentation)
@@ -499,9 +503,12 @@ proc parsePar(p: var TParser): PNode =
   #| parKeyw = 'discard' | 'include' | 'if' | 'while' | 'case' | 'try'
   #|         | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let'
   #|         | 'when' | 'var' | 'mixin'
-  #| par = '(' optInd (&parKeyw complexOrSimpleStmt ^+ ';'
+  #| par = '(' optInd
-  #|                  | simpleExpr ('=' expr (';' complexOrSimpleStmt ^+ ';' )? )?
+  #|           ( &parKeyw complexOrSimpleStmt ^+ ';'
-  #|                             | (':' expr)? (',' (exprColonEqExpr comma?)*)?  )?
+  #|           | ';' complexOrSimpleStmt ^+ ';'
+  #|           | pragmaStmt
+  #|           | simpleExpr ( ('=' expr (';' complexOrSimpleStmt ^+ ';' )? )
+  #|                        | (':' expr (',' exprColonEqExpr     ^+ ',' )? ) ) )
   #|           optPar ')'
   #
   # unfortunately it's ambiguous: (expr: expr) vs (exprStmt); however a
@@ -521,6 +528,8 @@ proc parsePar(p: var TParser): PNode =
     getTok(p)
     optInd(p, result)
     semiStmtList(p, result)
+  elif p.tok.tokType == tkCurlyDotLe:
+    result.add(parseStmtPragma(p))
   elif p.tok.tokType != tkParRi:
     var a = simpleExpr(p)
     if p.tok.tokType == tkEquals:
@@ -881,12 +890,13 @@ proc parseTuple(p: var TParser, indentAllowed = false): PNode =
     skipComment(p, result)
     if realInd(p):
       withInd(p):
-        skipComment(p, result)
+        rawSkipComment(p, result)
         while true:
           case p.tok.tokType
           of tkSymbol, tkAccent:
             var a = parseIdentColonEquals(p, {})
-            skipComment(p, a)
+            if p.tok.indent < 0 or p.tok.indent >= p.currInd:
+              rawSkipComment(p, a)
             addSon(result, a)
           of tkEof: break
           else:
@@ -1602,6 +1612,7 @@ proc parseEnum(p: var TParser): PNode =
   getTok(p)
   addSon(result, ast.emptyNode)
   optInd(p, result)
+  flexComment(p, result)
   while true:
     var a = parseSymbol(p)
     if a.kind == nkEmpty: return
@@ -1615,12 +1626,14 @@ proc parseEnum(p: var TParser): PNode =
       a = newNodeP(nkEnumFieldDef, p)
       addSon(a, b)
       addSon(a, parseExpr(p))
-      skipComment(p, a)
+      if p.tok.indent < 0 or p.tok.indent >= p.currInd:
+        rawSkipComment(p, a)
     if p.tok.tokType == tkComma and p.tok.indent < 0:
       getTok(p)
       rawSkipComment(p, a)
     else:
-      skipComment(p, a)
+      if p.tok.indent < 0 or p.tok.indent >= p.currInd:
+        rawSkipComment(p, a)
     addSon(result, a)
     if p.tok.indent >= 0 and p.tok.indent <= p.currInd or
         p.tok.tokType == tkEof:
@@ -1641,7 +1654,7 @@ proc parseObjectWhen(p: var TParser): PNode =
     addSon(branch, parseExpr(p))
     colcom(p, branch)
     addSon(branch, parseObjectPart(p))
-    skipComment(p, branch)
+    flexComment(p, branch)
     addSon(result, branch)
     if p.tok.tokType != tkElif: break
   if p.tok.tokType == tkElse and sameInd(p):
@@ -1649,7 +1662,7 @@ proc parseObjectWhen(p: var TParser): PNode =
     eat(p, tkElse)
     colcom(p, branch)
     addSon(branch, parseObjectPart(p))
-    skipComment(p, branch)
+    flexComment(p, branch)
     addSon(result, branch)
 
 proc parseObjectCase(p: var TParser): PNode =
@@ -1669,7 +1682,7 @@ proc parseObjectCase(p: var TParser): PNode =
   addSon(a, ast.emptyNode)
   addSon(result, a)
   if p.tok.tokType == tkColon: getTok(p)
-  skipComment(p, result)
+  flexComment(p, result)
   var wasIndented = false
   let oldInd = p.currInd
   if realInd(p):
@@ -1718,7 +1731,8 @@ proc parseObjectPart(p: var TParser): PNode =
       result = parseObjectCase(p)
     of tkSymbol, tkAccent:
       result = parseIdentColonEquals(p, {withPragma})
-      skipComment(p, result)
+      if p.tok.indent < 0 or p.tok.indent >= p.currInd:
+        rawSkipComment(p, result)
     of tkNil, tkDiscard:
       result = newNodeP(nkNilLit, p)
       getTok(p)

compiler/passaux.nim

@@ -15,7 +15,7 @@ import
 proc verboseOpen(s: PSym): PPassContext =
   #MessageOut('compiling ' + s.name.s);
   result = nil                # we don't need a context
-  if gVerbosity > 0: rawMessage(hintProcessing, s.name.s)
+  rawMessage(hintProcessing, s.name.s)
 
 proc verboseProcess(context: PPassContext, n: PNode): PNode =
   result = n

compiler/passes.nim

@@ -170,10 +170,6 @@ proc processModule(module: PSym, stream: PLLStream, rd: PRodReader) =
     openPasses(a, module)
     if stream == nil:
       let filename = fileIdx.toFullPathConsiderDirty
-      if module.name.s == "-":
-        module.name.s = "stdinfile"
-        s = llStreamOpen(stdin)
-      else:
       s = llStreamOpen(filename, fmRead)
       if s == nil:
         rawMessage(errCannotOpenFile, filename)
@@ -194,8 +190,17 @@ proc processModule(module: PSym, stream: PLLStream, rd: PRodReader) =
       while true:
         var n = parseTopLevelStmt(p)
         if n.kind == nkEmpty: break
-        if not processTopLevelStmt(n, a): break
+        if sfNoForward in module.flags:
-
+          # read everything, no streaming possible
+          var sl = newNodeI(nkStmtList, n.info)
+          sl.add n
+          while true:
+            var n = parseTopLevelStmt(p)
+            if n.kind == nkEmpty: break
+            sl.add n
+          discard processTopLevelStmt(sl, a)
+          break
+        elif not processTopLevelStmt(n, a): break
       closeParsers(p)
       if s.kind != llsStdIn: break
     closePasses(a)

compiler/patterns.nim

@@ -130,7 +130,9 @@ proc matchNested(c: PPatternContext, p, n: PNode, rpn: bool): bool =
 
 proc matches(c: PPatternContext, p, n: PNode): bool =
   # hidden conversions (?)
-  if isPatternParam(c, p):
+  if nfNoRewrite in n.flags:
+    result = false
+  elif isPatternParam(c, p):
     result = bindOrCheck(c, p.sym, n)
   elif n.kind == nkSym and p.kind == nkIdent:
     result = p.ident.id == n.sym.name.id

compiler/platform.nim

@@ -158,8 +158,8 @@ type
   TSystemCPU* = enum # Also add CPU for in initialization section and
                      # alias conditionals to condsyms (end of module).
     cpuNone, cpuI386, cpuM68k, cpuAlpha, cpuPowerpc, cpuPowerpc64,
-    cpuSparc, cpuVm, cpuIa64, cpuAmd64, cpuMips, cpuArm,
+    cpuPowerpc64el, cpuSparc, cpuVm, cpuIa64, cpuAmd64, cpuMips, cpuMipsel,
-    cpuJS, cpuNimrodVM, cpuAVR
+    cpuArm, cpuArm64, cpuJS, cpuNimrodVM, cpuAVR, cpuMSP430
 
 type
   TEndian* = enum
@@ -175,15 +175,19 @@ const
     (name: "alpha", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
     (name: "powerpc", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32),
     (name: "powerpc64", intSize: 64, endian: bigEndian, floatSize: 64,bit: 64),
+    (name: "powerpc64el", intSize: 64, endian: littleEndian, floatSize: 64,bit: 64),
     (name: "sparc", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32),
     (name: "vm", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
     (name: "ia64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
     (name: "amd64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
     (name: "mips", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32),
+    (name: "mipsel", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
     (name: "arm", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
+    (name: "arm64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
     (name: "js", intSize: 32, endian: bigEndian,floatSize: 64,bit: 32),
     (name: "nimrodvm", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32),
-    (name: "avr", intSize: 16, endian: littleEndian, floatSize: 32, bit: 16)]
+    (name: "avr", intSize: 16, endian: littleEndian, floatSize: 32, bit: 16),
+    (name: "msp430", intSize: 16, endian: littleEndian, floatSize: 32, bit: 16)]
 
 var
   targetCPU*, hostCPU*: TSystemCPU

compiler/plugins/locals/locals.nim

@@ -9,7 +9,8 @@
 
 ## The builtin 'system.locals' implemented as a plugin.
 
-import plugins, ast, astalgo, magicsys, lookups, semdata, lowerings
+import compiler/plugins, compiler/ast, compiler/astalgo, compiler/magicsys,
+  compiler/lookups, compiler/semdata, compiler/lowerings
 
 proc semLocals(c: PContext, n: PNode): PNode =
   var counter = 0

compiler/pragmas.nim

@@ -25,19 +25,20 @@ const
     wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC,
     wAsmNoStackFrame, wError, wDiscardable, wNoInit, wDestructor, wCodegenDecl,
     wGensym, wInject, wRaises, wTags, wLocks, wDelegator, wGcSafe,
-    wOverride, wConstructor}
+    wOverride, wConstructor, wExportNims}
   converterPragmas* = procPragmas
-  methodPragmas* = procPragmas
+  methodPragmas* = procPragmas+{wBase}
   templatePragmas* = {wImmediate, wDeprecated, wError, wGensym, wInject, wDirty,
-    wDelegator}
+    wDelegator, wExportNims}
   macroPragmas* = {FirstCallConv..LastCallConv, wImmediate, wImportc, wExportc,
     wNodecl, wMagic, wNosideeffect, wCompilerproc, wDeprecated, wExtern,
-    wImportCpp, wImportObjC, wError, wDiscardable, wGensym, wInject, wDelegator}
+    wImportCpp, wImportObjC, wError, wDiscardable, wGensym, wInject, wDelegator,
+    wExportNims}
   iteratorPragmas* = {FirstCallConv..LastCallConv, wNosideeffect, wSideeffect,
     wImportc, wExportc, wNodecl, wMagic, wDeprecated, wBorrow, wExtern,
     wImportCpp, wImportObjC, wError, wDiscardable, wGensym, wInject, wRaises,
-    wTags, wLocks, wGcSafe}
+    wTags, wLocks, wGcSafe, wExportNims}
-  exprPragmas* = {wLine, wLocks}
+  exprPragmas* = {wLine, wLocks, wNoRewrite}
   stmtPragmas* = {wChecks, wObjChecks, wFieldChecks, wRangechecks,
     wBoundchecks, wOverflowchecks, wNilchecks, wAssertions, wWarnings, wHints,
     wLinedir, wStacktrace, wLinetrace, wOptimization, wHint, wWarning, wError,
@@ -54,15 +55,15 @@ const
     wPure, wHeader, wCompilerproc, wFinal, wSize, wExtern, wShallow,
     wImportCpp, wImportObjC, wError, wIncompleteStruct, wByCopy, wByRef,
     wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked,
-    wBorrow, wGcSafe}
+    wBorrow, wGcSafe, wExportNims}
   fieldPragmas* = {wImportc, wExportc, wDeprecated, wExtern,
-    wImportCpp, wImportObjC, wError, wGuard}
+    wImportCpp, wImportObjC, wError, wGuard, wBitsize}
   varPragmas* = {wImportc, wExportc, wVolatile, wRegister, wThreadVar, wNodecl,
     wMagic, wHeader, wDeprecated, wCompilerproc, wDynlib, wExtern,
     wImportCpp, wImportObjC, wError, wNoInit, wCompileTime, wGlobal,
-    wGensym, wInject, wCodegenDecl, wGuard, wGoto}
+    wGensym, wInject, wCodegenDecl, wGuard, wGoto, wExportNims}
   constPragmas* = {wImportc, wExportc, wHeader, wDeprecated, wMagic, wNodecl,
-    wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject}
+    wExtern, wImportCpp, wImportObjC, wError, wGensym, wInject, wExportNims}
   letPragmas* = varPragmas
   procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNosideeffect,
                       wThread, wRaises, wLocks, wTags, wGcSafe}
@@ -276,7 +277,8 @@ proc processNote(c: PContext, n: PNode) =
   if (n.kind == nkExprColonExpr) and (sonsLen(n) == 2) and
       (n.sons[0].kind == nkBracketExpr) and
       (n.sons[0].sons[1].kind == nkIdent) and
-      (n.sons[0].sons[0].kind == nkIdent) and (n.sons[1].kind == nkIdent):
+      (n.sons[0].sons[0].kind == nkIdent):
+      #and (n.sons[1].kind == nkIdent):
     var nk: TNoteKind
     case whichKeyword(n.sons[0].sons[0].ident)
     of wHint:
@@ -394,6 +396,8 @@ proc processCompile(c: PContext, n: PNode) =
   var found = findFile(s)
   if found == "": found = s
   var trunc = changeFileExt(found, "")
+  if not isAbsolute(found):
+    found = parentDir(n.info.toFullPath) / found
   extccomp.addExternalFileToCompile(found)
   extccomp.addFileToLink(completeCFilePath(trunc, false))
 
@@ -590,8 +594,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
                   validPragmas: TSpecialWords): bool =
   var it = n.sons[i]
   var key = if it.kind == nkExprColonExpr: it.sons[0] else: it
-  if key.kind == nkIdent:
+  if key.kind == nkBracketExpr:
-    var userPragma = strTableGet(c.userPragmas, key.ident)
+    processNote(c, it)
+    return
+  let ident = considerQuotedIdent(key)
+  var userPragma = strTableGet(c.userPragmas, ident)
   if userPragma != nil:
     inc c.instCounter
     if c.instCounter > 100:
@@ -602,7 +609,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
     n.sons[i] = userPragma.ast
     dec c.instCounter
   else:
-      var k = whichKeyword(key.ident)
+    var k = whichKeyword(ident)
     if k in validPragmas:
       case k
       of wExportc:
@@ -838,6 +845,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
         if sym == nil: pragmaLockStmt(c, it)
         elif sym.typ == nil: invalidPragma(it)
         else: sym.typ.lockLevel = pragmaLocks(c, it)
+      of wBitsize:
+        if sym == nil or sym.kind != skField or it.kind != nkExprColonExpr:
+          invalidPragma(it)
+        else:
+          sym.bitsize = expectIntLit(c, it)
       of wGuard:
         if sym == nil or sym.kind notin {skVar, skLet, skField}:
           invalidPragma(it)
@@ -848,6 +860,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
           invalidPragma(it)
         else:
           sym.flags.incl sfGoto
+      of wExportNims:
+        if sym == nil: invalidPragma(it)
+        else: magicsys.registerNimScriptSymbol(sym)
       of wInjectStmt:
         if it.kind != nkExprColonExpr:
           localError(it.info, errExprExpected)
@@ -859,9 +874,13 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
           c.module.flags.incl sfExperimental
         else:
           localError(it.info, "'experimental' pragma only valid as toplevel statement")
+      of wNoRewrite:
+        noVal(it)
+      of wBase:
+        noVal(it)
+        sym.flags.incl sfBase
       else: invalidPragma(it)
     else: invalidPragma(it)
-  else: processNote(c, it)
 
 proc implicitPragmas*(c: PContext, sym: PSym, n: PNode,
                       validPragmas: TSpecialWords) =

compiler/renderer.nim

@@ -31,6 +31,7 @@ type
     buf*: string
     pendingNL*: int        # negative if not active; else contains the
                            # indentation value
+    pendingWhitespace: int
     comStack*: seq[PNode]  # comment stack
     flags*: TRenderFlags
     checkAnon: bool        # we're in a context that can contain sfAnon
@@ -83,6 +84,7 @@ proc initSrcGen(g: var TSrcGen, renderFlags: TRenderFlags) =
   g.buf = ""
   g.flags = renderFlags
   g.pendingNL = -1
+  g.pendingWhitespace = -1
   g.checkAnon = false
 
 proc addTok(g: var TSrcGen, kind: TTokType, s: string) =
@@ -97,12 +99,21 @@ proc addPendingNL(g: var TSrcGen) =
     addTok(g, tkSpaces, "\n" & spaces(g.pendingNL))
     g.lineLen = g.pendingNL
     g.pendingNL = - 1
+    g.pendingWhitespace = -1
+  elif g.pendingWhitespace >= 0:
+    addTok(g, tkSpaces, spaces(g.pendingWhitespace))
+    g.pendingWhitespace = -1
 
 proc putNL(g: var TSrcGen, indent: int) =
   if g.pendingNL >= 0: addPendingNL(g)
   else: addTok(g, tkSpaces, "\n")
   g.pendingNL = indent
   g.lineLen = indent
+  g.pendingWhitespace = -1
+
+proc previousNL(g: TSrcGen): bool =
+  result = g.pendingNL >= 0 or (g.tokens.len > 0 and
+                                g.tokens[^1].kind == tkSpaces)
 
 proc putNL(g: var TSrcGen) =
   putNL(g, g.indent)
@@ -127,10 +138,13 @@ proc dedent(g: var TSrcGen) =
     dec(g.lineLen, IndentWidth)
 
 proc put(g: var TSrcGen, kind: TTokType, s: string) =
+  if kind != tkSpaces:
     addPendingNL(g)
     if len(s) > 0:
       addTok(g, kind, s)
       inc(g.lineLen, len(s))
+  else:
+    g.pendingWhitespace = s.len
 
 proc putLong(g: var TSrcGen, kind: TTokType, s: string, lineLen: int) =
   # use this for tokens over multiple lines.
@@ -767,6 +781,7 @@ proc gasm(g: var TSrcGen, n: PNode) =
   putWithSpace(g, tkAsm, "asm")
   gsub(g, n.sons[0])
   gcoms(g)
+  if n.sons.len > 1:
     gsub(g, n.sons[1])
 
 proc gident(g: var TSrcGen, n: PNode) =
@@ -1195,6 +1210,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     if renderNoPragmas notin g.flags:
       if g.inPragma <= 0:
         inc g.inPragma
+        #if not previousNL(g):
         put(g, tkSpaces, Space)
         put(g, tkCurlyDotLe, "{.")
         gcomma(g, n, emptyContext)

compiler/rodread.nim

@@ -19,8 +19,8 @@
 #    NIM:$fileversion\n
 #  - the module's id (even if the module changed, its ID will not!):
 #    ID:Ax3\n
-#  - CRC value of this module:
+#  - HASH value of this module:
-#    CRC:CRC-val\n
+#    HASH:HASH-val\n
 #  - a section containing the compiler options and defines this
 #    module has been compiled with:
 #    OPTIONS:options\n
@@ -33,7 +33,7 @@
 #    )
 #  - an include file dependency section:
 #    INCLUDES(
-#    <fileidx> <CRC of myfile.inc>\n # fileidx is the LINE in the file section!
+#    <fileidx> <Hash of myfile.inc>\n # fileidx is the LINE in the file section!
 #    )
 #  - a module dependency section:
 #    DEPS: <fileidx> <fileidx>\n
@@ -90,7 +90,7 @@
 
 import
   os, options, strutils, nversion, ast, astalgo, msgs, platform, condsyms,
-  ropes, idents, crc, idgen, types, rodutils, memfiles
+  ropes, idents, securehash, idgen, types, rodutils, memfiles, tables
 
 type
   TReasonForRecompile* = enum ## all the reasons that can trigger recompilation
@@ -98,7 +98,7 @@ type
     rrNone,                   # no need to recompile
     rrRodDoesNotExist,        # rod file does not exist
     rrRodInvalid,             # rod file is invalid
-    rrCrcChange,              # file has been edited since last recompilation
+    rrHashChange,             # file has been edited since last recompilation
     rrDefines,                # defines have changed
     rrOptions,                # options have changed
     rrInclDeps,               # an include has changed
@@ -136,7 +136,7 @@ type
     readerIndex: int
     line: int            # only used for debugging, but is always in the code
     moduleID: int
-    syms: TIdTable       # already processed symbols
+    syms: Table[int, PSym]       # already processed symbols
     memfile: MemFile     # unfortunately there is no point in time where we
                          # can close this! XXX
     methods*: TSymSeq
@@ -372,11 +372,11 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym =
   else:
     internalError(info, "decodeSym: no ident")
   #echo "decoding: {", ident.s
-  result = PSym(idTableGet(r.syms, id))
+  result = r.syms.getOrDefault(id)
   if result == nil:
     new(result)
     result.id = id
-    idTablePut(r.syms, result, result)
+    r.syms[result.id] = result
     if debugIds: registerID(result)
   elif result.id != id:
     internalError(info, "decodeSym: wrong id")
@@ -481,7 +481,7 @@ proc processInterf(r: PRodReader, module: PSym) =
     var s = newStub(r, w, key)
     s.owner = module
     strTableAdd(module.tab, s)
-    idTablePut(r.syms, s, s)
+    r.syms[s.id] = s
 
 proc processCompilerProcs(r: PRodReader, module: PSym) =
   if r.compilerProcsIdx == 0: internalError("processCompilerProcs")
@@ -491,11 +491,11 @@ proc processCompilerProcs(r: PRodReader, module: PSym) =
     inc(r.pos)
     var key = decodeVInt(r.s, r.pos)
     inc(r.pos)                # #10
-    var s = PSym(idTableGet(r.syms, key))
+    var s = r.syms.getOrDefault(key)
     if s == nil:
       s = newStub(r, w, key)
       s.owner = module
-      idTablePut(r.syms, s, s)
+      r.syms[s.id] = s
     strTableAdd(rodCompilerprocs, s)
 
 proc processIndex(r: PRodReader; idx: var TIndex; outf: File = nil) =
@@ -538,18 +538,20 @@ proc cmdChangeTriggersRecompilation(old, new: TCommands): bool =
   # else: trigger recompilation:
   result = true
 
-proc processRodFile(r: PRodReader, crc: TCrc32) = 
+proc processRodFile(r: PRodReader, hash: SecureHash) =
   var
     w: string
-    d, inclCrc: int
+    d: int
+  var inclHash: SecureHash
   while r.s[r.pos] != '\0':
     var section = rdWord(r)
     if r.reason != rrNone:
       break                   # no need to process this file further
     case section
-    of "CRC": 
+    of "HASH":
       inc(r.pos)              # skip ':'
-      if int(crc) != decodeVInt(r.s, r.pos): r.reason = rrCrcChange
+      if hash != parseSecureHash(decodeStr(r.s, r.pos)):
+        r.reason = rrHashChange
     of "ID":
       inc(r.pos)              # skip ':'
       r.moduleID = decodeVInt(r.s, r.pos)
@@ -596,9 +598,9 @@ proc processRodFile(r: PRodReader, crc: TCrc32) =
       while r.s[r.pos] != ')':
         w = r.files[decodeVInt(r.s, r.pos)].toFullPath
         inc(r.pos)            # skip ' '
-        inclCrc = decodeVInt(r.s, r.pos)
+        inclHash = parseSecureHash(decodeStr(r.s, r.pos))
         if r.reason == rrNone:
-          if not existsFile(w) or (inclCrc != int(crcFromFile(w))): 
+          if not existsFile(w) or (inclHash != secureHashFile(w)):
             r.reason = rrInclDeps
         if r.s[r.pos] == '\x0A':
           inc(r.pos)
@@ -649,7 +651,7 @@ proc startsWith(buf: cstring, token: string, pos = 0): bool =
   while s < token.len and buf[pos+s] == token[s]: inc s
   result = s == token.len
 
-proc newRodReader(modfilename: string, crc: TCrc32, 
+proc newRodReader(modfilename: string, hash: SecureHash,
                   readerIndex: int): PRodReader =
   new(result)
   try:
@@ -665,7 +667,7 @@ proc newRodReader(modfilename: string, crc: TCrc32,
   r.line = 1
   r.readerIndex = readerIndex
   r.filename = modfilename
-  initIdTable(r.syms)
+  r.syms = initTable[int, PSym]()
   # we terminate the file explicitly with ``\0``, so the cast to `cstring`
   # is safe:
   r.s = cast[cstring](r.memfile.mem)
@@ -701,8 +703,8 @@ type
     filename*: string
     reason*: TReasonForRecompile
     rd*: PRodReader
-    crc*: TCrc32
+    hash*: SecureHash
-    crcDone*: bool
+    hashDone*: bool
 
   TFileModuleMap = seq[TFileModuleRec]
 
@@ -735,7 +737,7 @@ proc getReader(moduleId: int): PRodReader =
   return nil
 
 proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym =
-  result = PSym(idTableGet(r.syms, id))
+  result = r.syms.getOrDefault(id)
   if result == nil:
     # load the symbol:
     var d = iiTableGet(r.index.tab, id)
@@ -794,14 +796,14 @@ proc loadMethods(r: PRodReader) =
     r.methods.add(rrGetSym(r, d, unknownLineInfo()))
     if r.s[r.pos] == ' ': inc(r.pos)
 
-proc getCRC*(fileIdx: int32): TCrc32 =
+proc getHash*(fileIdx: int32): SecureHash =
   internalAssert fileIdx >= 0 and fileIdx < gMods.len
 
-  if gMods[fileIdx].crcDone:
+  if gMods[fileIdx].hashDone:
-    return gMods[fileIdx].crc
+    return gMods[fileIdx].hash
 
-  result = crcFromFile(fileIdx.toFilename)
+  result = secureHashFile(fileIdx.toFullPath)
-  gMods[fileIdx].crc = result
+  gMods[fileIdx].hash = result
 
 template growCache*(cache, pos) =
   if cache.len <= pos: cache.setLen(pos+1)
@@ -813,16 +815,16 @@ proc checkDep(fileIdx: int32): TReasonForRecompile =
     # reason has already been computed for this module:
     return gMods[fileIdx].reason
   let filename = fileIdx.toFilename
-  var crc = getCRC(fileIdx)
+  var hash = getHash(fileIdx)
   gMods[fileIdx].reason = rrNone  # we need to set it here to avoid cycles
   result = rrNone
   var r: PRodReader = nil
   var rodfile = toGeneratedFile(filename.withPackageName, RodExt)
-  r = newRodReader(rodfile, crc, fileIdx)
+  r = newRodReader(rodfile, hash, fileIdx)
   if r == nil:
     result = (if existsFile(rodfile): rrRodInvalid else: rrRodDoesNotExist)
   else:
-    processRodFile(r, crc)
+    processRodFile(r, hash)
     result = r.reason
     if result == rrNone:
       # check modules it depends on
@@ -836,7 +838,7 @@ proc checkDep(fileIdx: int32): TReasonForRecompile =
         if res != rrNone:
           result = rrModDeps
           # we cannot break here, because of side-effects of `checkDep`
-  if result != rrNone and gVerbosity > 0:
+  if result != rrNone:
     rawMessage(hintProcessing, reasonToFrmt[result] % filename)
   if result != rrNone or optForceFullMake in gGlobalOptions:
     # recompilation is necessary:
@@ -857,7 +859,7 @@ proc handleSymbolFile(module: PSym): PRodReader =
   result = gMods[fileIdx].rd
   if result != nil:
     module.id = result.moduleID
-    idTablePut(result.syms, module, module)
+    result.syms[module.id] = module
     processInterf(result, module)
     processCompilerProcs(result, module)
     loadConverters(result)
@@ -896,6 +898,8 @@ proc getBody*(s: PSym): PNode =
   ## it may perform an expensive reload operation. Otherwise it's a simple
   ## accessor.
   assert s.kind in routineKinds
+  # prevent crashes due to incorrect macro transformations (bug #2377)
+  if s.ast.isNil or bodyPos >= s.ast.len: return ast.emptyNode
   result = s.ast.sons[bodyPos]
   if result == nil:
     assert s.offset != 0
@@ -1017,7 +1021,7 @@ proc writeType(f: File; t: PType) =
   f.write("]\n")
 
 proc viewFile(rodfile: string) =
-  var r = newRodReader(rodfile, 0, 0)
+  var r = newRodReader(rodfile, secureHash(""), 0)
   if r == nil:
     rawMessage(errGenerated, "cannot open file (or maybe wrong version):" &
        rodfile)
@@ -1027,30 +1031,30 @@ proc viewFile(rodfile: string) =
   while r.s[r.pos] != '\0':
     let section = rdWord(r)
     case section
-    of "CRC":
+    of "HASH":
       inc(r.pos)              # skip ':'
-      outf.writeln("CRC:", $decodeVInt(r.s, r.pos))
+      outf.writeLine("HASH:", $decodeVInt(r.s, r.pos))
     of "ID":
       inc(r.pos)              # skip ':'
       r.moduleID = decodeVInt(r.s, r.pos)
       setId(r.moduleID)
-      outf.writeln("ID:", $r.moduleID)
+      outf.writeLine("ID:", $r.moduleID)
     of "ORIGFILE":
       inc(r.pos)
       r.origFile = decodeStr(r.s, r.pos)
-      outf.writeln("ORIGFILE:", r.origFile)
+      outf.writeLine("ORIGFILE:", r.origFile)
     of "OPTIONS":
       inc(r.pos)              # skip ':'
       r.options = cast[TOptions](int32(decodeVInt(r.s, r.pos)))
-      outf.writeln("OPTIONS:", $r.options)
+      outf.writeLine("OPTIONS:", $r.options)
     of "GOPTIONS":
       inc(r.pos)              # skip ':'
       let dep = cast[TGlobalOptions](int32(decodeVInt(r.s, r.pos)))
-      outf.writeln("GOPTIONS:", $dep)
+      outf.writeLine("GOPTIONS:", $dep)
     of "CMD":
       inc(r.pos)              # skip ':'
       let dep = cast[TCommands](int32(decodeVInt(r.s, r.pos)))
-      outf.writeln("CMD:", $dep)
+      outf.writeLine("CMD:", $dep)
     of "DEFINES":
       inc(r.pos)              # skip ':'
       var d = 0
@@ -1072,7 +1076,7 @@ proc viewFile(rodfile: string) =
         r.files.add(finalPath.fileInfoIdx)
         inc(r.pos)            # skip #10
         inc(r.line)
-        outf.writeln finalPath
+        outf.writeLine finalPath
       if r.s[r.pos] == ')': inc(r.pos)
       outf.write(")\n")
     of "INCLUDES":
@@ -1082,11 +1086,11 @@ proc viewFile(rodfile: string) =
       while r.s[r.pos] != ')':
         let w = r.files[decodeVInt(r.s, r.pos)]
         inc(r.pos)            # skip ' '
-        let inclCrc = decodeVInt(r.s, r.pos)
+        let inclHash = decodeVInt(r.s, r.pos)
         if r.s[r.pos] == '\x0A':
           inc(r.pos)
           inc(r.line)
-        outf.write(w, " ", inclCrc, "\n")
+        outf.write(w, " ", inclHash, "\n")
       if r.s[r.pos] == ')': inc(r.pos)
       outf.write(")\n")
     of "DEPS":

compiler/rodwrite.nim

@@ -13,14 +13,15 @@
 
 import
   intsets, os, options, strutils, nversion, ast, astalgo, msgs, platform,
-  condsyms, ropes, idents, crc, rodread, passes, importer, idgen, rodutils
+  condsyms, ropes, idents, securehash, rodread, passes, importer, idgen,
+  rodutils
 
 # implementation
 
 type
   TRodWriter = object of TPassContext
     module: PSym
-    crc: TCrc32
+    hash: SecureHash
     options: TOptions
     defines: string
     inclDeps: string
@@ -38,7 +39,7 @@ type
 
   PRodWriter = ref TRodWriter
 
-proc newRodWriter(crc: TCrc32, module: PSym): PRodWriter
+proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter
 proc addModDep(w: PRodWriter, dep: string)
 proc addInclDep(w: PRodWriter, dep: string)
 proc addInterfaceSym(w: PRodWriter, s: PSym)
@@ -62,7 +63,7 @@ proc fileIdx(w: PRodWriter, filename: string): int =
 template filename*(w: PRodWriter): string =
   w.module.filename
 
-proc newRodWriter(crc: TCrc32, module: PSym): PRodWriter = 
+proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter =
   new(result)
   result.sstack = @[]
   result.tstack = @[]
@@ -70,7 +71,7 @@ proc newRodWriter(crc: TCrc32, module: PSym): PRodWriter =
   initIiTable(result.imports.tab)
   result.index.r = ""
   result.imports.r = ""
-  result.crc = crc
+  result.hash = hash
   result.module = module
   result.defines = getDefines()
   result.options = options.gOptions
@@ -96,7 +97,7 @@ proc addInclDep(w: PRodWriter, dep: string) =
   var resolved = dep.findModule(w.module.info.toFullPath)
   encodeVInt(fileIdx(w, dep), w.inclDeps)
   add(w.inclDeps, " ")
-  encodeVInt(crcFromFile(resolved), w.inclDeps)
+  encodeStr($secureHashFile(resolved), w.inclDeps)
   add(w.inclDeps, rodNL)
 
 proc pushType(w: PRodWriter, t: PType) =
@@ -335,8 +336,9 @@ proc symStack(w: PRodWriter): int =
       inc result
     elif iiTableGet(w.index.tab, s.id) == InvalidKey:
       var m = getModule(s)
-      if m == nil: internalError("symStack: module nil: " & s.name.s)
+      if m == nil and s.kind != skPackage:
-      if (m.id == w.module.id) or (sfFromGeneric in s.flags): 
+        internalError("symStack: module nil: " & s.name.s)
+      if s.kind == skPackage or m.id == w.module.id or sfFromGeneric in s.flags:
         # put definition in here
         var L = w.data.len
         addToIndex(w.index, s.id, L)
@@ -364,12 +366,12 @@ proc symStack(w: PRodWriter): int =
       elif iiTableGet(w.imports.tab, s.id) == InvalidKey:
         addToIndex(w.imports, s.id, m.id)
         when debugWrittenIds:
-          if not Contains(debugWritten, s.id):
+          if not contains(debugWritten, s.id):
             echo(w.filename)
             debug(s)
             debug(s.owner)
             debug(m)
-            InternalError("Symbol referred to but never written")
+            internalError("Symbol referred to but never written")
     inc(i)
   setLen(w.sstack, result)
 
@@ -439,9 +441,9 @@ proc writeRod(w: PRodWriter) =
   f.write(orig)
   f.write(rodNL)
 
-  var crc = "CRC:"
+  var hash = "HASH:"
-  encodeVInt(w.crc, crc)
+  encodeStr($w.hash, hash)
-  f.write(crc)
+  f.write(hash)
   f.write(rodNL)
 
   var options = "OPTIONS:"
@@ -574,7 +576,7 @@ proc process(c: PPassContext, n: PNode): PNode =
 
 proc myOpen(module: PSym): PPassContext =
   if module.id < 0: internalError("rodwrite: module ID not set")
-  var w = newRodWriter(module.fileIdx.getCRC, module)
+  var w = newRodWriter(module.fileIdx.getHash, module)
   rawAddInterfaceSym(w, module)
   result = w
 

compiler/scriptconfig.nim

@@ -0,0 +1,146 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Implements the new configuration system for Nim. Uses Nim as a scripting
+## language.
+
+import
+  ast, modules, passes, passaux, condsyms,
+  options, nimconf, lists, sem, semdata, llstream, vm, vmdef, commands, msgs,
+  os, times, osproc
+
+# we support 'cmpIgnoreStyle' natively for efficiency:
+from strutils import cmpIgnoreStyle
+
+proc listDirs(a: VmArgs, filter: set[PathComponent]) =
+  let dir = getString(a, 0)
+  var result: seq[string] = @[]
+  for kind, path in walkDir(dir):
+    if kind in filter: result.add path
+  setResult(a, result)
+
+proc setupVM*(module: PSym; scriptName: string): PEvalContext =
+  # For Nimble we need to export 'setupVM'.
+  result = newCtx(module)
+  result.mode = emRepl
+  registerAdditionalOps(result)
+
+  # captured vars:
+  var errorMsg: string
+  var vthisDir = scriptName.splitFile.dir
+
+  template cbconf(name, body) {.dirty.} =
+    result.registerCallback "stdlib.system." & astToStr(name),
+      proc (a: VmArgs) =
+        body
+
+  template cbos(name, body) {.dirty.} =
+    result.registerCallback "stdlib.system." & astToStr(name),
+      proc (a: VmArgs) =
+        try:
+          body
+        except OSError:
+          errorMsg = getCurrentExceptionMsg()
+
+  # Idea: Treat link to file as a file, but ignore link to directory to prevent
+  # endless recursions out of the box.
+  cbos listFiles:
+    listDirs(a, {pcFile, pcLinkToFile})
+  cbos listDirs:
+    listDirs(a, {pcDir})
+  cbos removeDir:
+    os.removeDir getString(a, 0)
+  cbos removeFile:
+    os.removeFile getString(a, 0)
+  cbos createDir:
+    os.createDir getString(a, 0)
+  cbos getOsError:
+    setResult(a, errorMsg)
+  cbos setCurrentDir:
+    os.setCurrentDir getString(a, 0)
+  cbos getCurrentDir:
+    setResult(a, os.getCurrentDir())
+  cbos moveFile:
+    os.moveFile(getString(a, 0), getString(a, 1))
+  cbos copyFile:
+    os.copyFile(getString(a, 0), getString(a, 1))
+  cbos getLastModificationTime:
+    setResult(a, toSeconds(getLastModificationTime(getString(a, 0))))
+
+  cbos rawExec:
+    setResult(a, osproc.execCmd getString(a, 0))
+
+  cbconf getEnv:
+    setResult(a, os.getEnv(a.getString 0))
+  cbconf existsEnv:
+    setResult(a, os.existsEnv(a.getString 0))
+  cbconf dirExists:
+    setResult(a, os.dirExists(a.getString 0))
+  cbconf fileExists:
+    setResult(a, os.fileExists(a.getString 0))
+
+  cbconf thisDir:
+    setResult(a, vthisDir)
+  cbconf put:
+    options.setConfigVar(getString(a, 0), getString(a, 1))
+  cbconf get:
+    setResult(a, options.getConfigVar(a.getString 0))
+  cbconf exists:
+    setResult(a, options.existsConfigVar(a.getString 0))
+  cbconf nimcacheDir:
+    setResult(a, options.getNimcacheDir())
+  cbconf paramStr:
+    setResult(a, os.paramStr(int a.getInt 0))
+  cbconf paramCount:
+    setResult(a, os.paramCount())
+  cbconf cmpIgnoreStyle:
+    setResult(a, strutils.cmpIgnoreStyle(a.getString 0, a.getString 1))
+  cbconf cmpIgnoreCase:
+    setResult(a, strutils.cmpIgnoreCase(a.getString 0, a.getString 1))
+  cbconf setCommand:
+    options.command = a.getString 0
+    let arg = a.getString 1
+    if arg.len > 0:
+      gProjectName = arg
+      try:
+        gProjectFull = canonicalizePath(gProjectPath / gProjectName)
+      except OSError:
+        gProjectFull = gProjectName
+  cbconf getCommand:
+    setResult(a, options.command)
+  cbconf switch:
+    processSwitch(a.getString 0, a.getString 1, passPP, unknownLineInfo())
+
+
+proc runNimScript*(scriptName: string) =
+  passes.gIncludeFile = includeModule
+  passes.gImportModule = importModule
+  initDefines()
+
+  defineSymbol("nimscript")
+  defineSymbol("nimconfig")
+  registerPass(semPass)
+  registerPass(evalPass)
+
+  appendStr(searchPaths, options.libpath)
+
+  var m = makeModule(scriptName)
+  incl(m.flags, sfMainModule)
+  vm.globalCtx = setupVM(m, scriptName)
+
+  compileSystemModule()
+  processModule(m, llStreamOpen(scriptName, fmRead), nil)
+
+  # ensure we load 'system.nim' again for the real non-config stuff!
+  resetAllModulesHard()
+  vm.globalCtx = nil
+  # do not remove the defined symbols
+  #initDefines()
+  undefSymbol("nimscript")
+  undefSymbol("nimconfig")

compiler/sem.nim

@@ -171,11 +171,15 @@ proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym =
   result = newSym(kind, considerQuotedIdent(n), getCurrOwner(), n.info)
 
 proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
+  proc `$`(kind: TSymKind): string = substr(system.`$`(kind), 2).toLower
+
   # like newSymS, but considers gensym'ed symbols
   if n.kind == nkSym:
     # and sfGenSym in n.sym.flags:
     result = n.sym
-    internalAssert result.kind == kind
+    if result.kind != kind:
+      localError(n.info, "cannot use symbol of kind '" &
+                 $result.kind & "' as a '" & $kind & "'")
     # when there is a nested proc inside a template, semtmpl
     # will assign a wrong owner during the first pass over the
     # template; we must fix it here: see #909
@@ -421,7 +425,11 @@ proc myOpenCached(module: PSym, rd: PRodReader): PPassContext =
   for m in items(rd.methods): methodDef(m, true)
 
 proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =
-  result = semStmt(c, n)
+  if sfNoForward in c.module.flags:
+    result = semAllTypeSections(c, n)
+  else:
+    result = n
+  result = semStmt(c, result)
   # BUGFIX: process newly generated generics here, not at the end!
   if c.lastGenericIdx < c.generics.len:
     var a = newNodeI(nkStmtList, n.info)

compiler/semcall.nim

@@ -95,7 +95,7 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) =
   # Gives a detailed error message; this is separated from semOverloadedCall,
   # as semOverlodedCall is already pretty slow (and we need this information
   # only in case of an error).
-  if c.inCompilesContext > 0:
+  if c.compilesContextId > 0 and optReportConceptFailures notin gGlobalOptions:
     # fail fast:
     globalError(n.info, errTypeMismatch, "")
   if errors.isNil or errors.len == 0:
@@ -133,6 +133,9 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) =
     add(candidates, "\n")
   if candidates != "":
     add(result, "\n" & msgKindToString(errButExpected) & "\n" & candidates)
+  if c.compilesContextId > 0 and optReportConceptFailures in gGlobalOptions:
+    globalError(n.info, errGenerated, result)
+  else:
     localError(n.info, errGenerated, result)
 
 proc gatherUsedSyms(c: PContext, usedSyms: var seq[PNode]) =
@@ -209,7 +212,10 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
       pickBest(callOp)
 
     if overloadsState == csEmpty and result.state == csEmpty:
-      localError(n.info, errUndeclaredIdentifier, considerQuotedIdent(f).s)
+      if nfDotField in n.flags and nfExplicitCall notin n.flags:
+        localError(n.info, errUndeclaredField, considerQuotedIdent(f).s)
+      else:
+        localError(n.info, errUndeclaredRoutine, considerQuotedIdent(f).s)
       return
     elif result.state != csMatch:
       if nfExprCall in n.flags:
@@ -232,7 +238,7 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
     internalAssert result.state == csMatch
     #writeMatches(result)
     #writeMatches(alt)
-    if c.inCompilesContext > 0:
+    if c.compilesContextId > 0:
       # quick error message for performance of 'compiles' built-in:
       globalError(n.info, errGenerated, "ambiguous call")
     elif gErrorCounter == 0:
@@ -302,8 +308,25 @@ proc semResolvedCall(c: PContext, n: PNode, x: TCandidate): PNode =
     if containsGenericType(result.typ) or x.fauxMatch == tyUnknown:
       result.typ = newTypeS(x.fauxMatch, c)
     return
-  if finalCallee.ast.sons[genericParamsPos].kind != nkEmpty:
+  let gp = finalCallee.ast.sons[genericParamsPos]
+  if gp.kind != nkEmpty:
+    if x.calleeSym.kind notin {skMacro, skTemplate}:
+      if x.calleeSym.magic in {mArrGet, mArrPut}:
+        finalCallee = x.calleeSym
+      else:
         finalCallee = generateInstance(c, x.calleeSym, x.bindings, n.info)
+    else:
+      # For macros and templates, the resolved generic params
+      # are added as normal params.
+      for s in instantiateGenericParamList(c, gp, x.bindings):
+        case s.kind
+          of skConst:
+            x.call.add s.ast
+          of skType:
+            x.call.add newSymNode(s, n.info)
+          else:
+            internalAssert false
+
   result = x.call
   instGenericConvertersSons(c, result, x)
   result.sons[0] = newSymNode(finalCallee, result.sons[0].info)

compiler/semdata.nim

@@ -45,7 +45,8 @@ type
   TExprFlag* = enum
     efLValue, efWantIterator, efInTypeof,
     efWantStmt, efAllowStmt, efDetermineType,
-    efAllowDestructor, efWantValue, efOperand, efNoSemCheck
+    efAllowDestructor, efWantValue, efOperand, efNoSemCheck,
+    efNoProcvarCheck
   TExprFlags* = set[TExprFlag]
 
   TTypeAttachedOp* = enum
@@ -70,7 +71,8 @@ type
     inTypeClass*: int          # > 0 if we are in a user-defined type class
     inGenericContext*: int     # > 0 if we are in a generic type
     inUnrolledContext*: int    # > 0 if we are unrolling a loop
-    inCompilesContext*: int    # > 0 if we are in a ``compiles`` magic
+    compilesContextId*: int    # > 0 if we are in a ``compiles`` magic
+    compilesContextIdGenerator*: int
     inGenericInst*: int        # > 0 if we are instantiating a generic
     converters*: TSymSeq       # sequence of converters
     patterns*: TSymSeq         # sequence of pattern matchers

compiler/semdestruct.nim

@@ -24,7 +24,6 @@ var
   destructorName = getIdent"destroy_"
   destructorParam = getIdent"this_"
   destructorPragma = newIdentNode(getIdent"destructor", unknownLineInfo())
-  rangeDestructorProc*: PSym
 
 proc instantiateDestructor(c: PContext, typ: PType): PType
 
@@ -141,9 +140,7 @@ proc instantiateDestructor(c: PContext, typ: PType): PType =
   case t.kind
   of tySequence, tyArray, tyArrayConstr, tyOpenArray, tyVarargs:
     if instantiateDestructor(c, t.sons[0]) != nil:
-      if rangeDestructorProc == nil:
+      t.destructor = getCompilerProc"nimDestroyRange"
-        rangeDestructorProc = searchInScopes(c, getIdent"nimDestroyRange")
-      t.destructor = rangeDestructorProc
       return t
     else:
       return nil
@@ -177,6 +174,15 @@ proc instantiateDestructor(c: PContext, typ: PType): PType =
   else:
     return nil
 
+proc createDestructorCall(c: PContext, s: PSym): PNode =
+  let varTyp = s.typ
+  if varTyp == nil or sfGlobal in s.flags: return
+  let destructableT = instantiateDestructor(c, varTyp)
+  if destructableT != nil:
+    let call = semStmt(c, newNode(nkCall, s.info, @[
+      useSym(destructableT.destructor), useSym(s)]))
+    result = newNode(nkDefer, s.info, @[call])
+
 proc insertDestructors(c: PContext,
                        varSection: PNode): tuple[outer, inner: PNode] =
   # Accepts a var or let section.

compiler/semexprs.nim

@@ -33,6 +33,7 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     if result.typ.kind == tyVar: result = newDeref(result)
   elif {efWantStmt, efAllowStmt} * flags != {}:
     result.typ = newTypeS(tyEmpty, c)
+    result.typ.flags.incl tfVoid
   else:
     localError(n.info, errExprXHasNoType,
                renderTree(result, {renderNoComments}))
@@ -51,7 +52,7 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     result.typ = errorType(c)
   else:
     # XXX tyGenericInst here?
-    semProcvarCheck(c, result)
+    if efNoProcvarCheck notin flags: semProcvarCheck(c, result)
     if result.typ.kind == tyVar: result = newDeref(result)
     semDestructorCheck(c, result, flags)
 
@@ -123,6 +124,9 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
         return newSymNode(u, n.info)
     result = newSymNode(s, n.info)
   of skVar, skLet, skResult, skForVar:
+    if s.magic == mNimvm:
+      localError(n.info, "illegal context for 'nimvm' magic")
+
     markUsed(n.info, s)
     styleCheckUse(n.info, s)
     # if a proc accesses a global variable, it is not side effect free:
@@ -288,8 +292,6 @@ proc semConv(c: PContext, n: PNode): PNode =
 
 proc semCast(c: PContext, n: PNode): PNode =
   ## Semantically analyze a casting ("cast[type](param)")
-  if optSafeCode in gGlobalOptions: localError(n.info, errCastNotInSafeMode)
-  #incl(c.p.owner.flags, sfSideEffect)
   checkSonsLen(n, 2)
   result = newNodeI(nkCast, n.info)
   result.typ = semTypeNode(c, n.sons[0], nil)
@@ -448,18 +450,18 @@ proc changeType(n: PNode, newType: PType, check: bool) =
     let tup = newType.skipTypes({tyGenericInst})
     if tup.kind != tyTuple:
       if tup.kind == tyObject: return
-      internalError(n.info, "changeType: no tuple type for constructor")
+      globalError(n.info, "no tuple type for constructor")
     elif sonsLen(n) > 0 and n.sons[0].kind == nkExprColonExpr:
       # named tuple?
       for i in countup(0, sonsLen(n) - 1):
         var m = n.sons[i].sons[0]
         if m.kind != nkSym:
-          internalError(m.info, "changeType(): invalid tuple constr")
+          globalError(m.info, "invalid tuple constructor")
           return
         if tup.n != nil:
           var f = getSymFromList(tup.n, m.sym.name)
           if f == nil:
-            internalError(m.info, "changeType(): invalid identifier")
+            globalError(m.info, "unknown identifier: " & m.sym.name.s)
             return
           changeType(n.sons[i].sons[1], f.typ, check)
         else:
@@ -474,7 +476,7 @@ proc changeType(n: PNode, newType: PType, check: bool) =
           addSon(a, m)
           changeType(m, tup.sons[i], check)
   of nkCharLit..nkUInt64Lit:
-    if check:
+    if check and n.kind != nkUInt64Lit:
       let value = n.intVal
       if value < firstOrd(newType) or value > lastOrd(newType):
         localError(n.info, errGenerated, "cannot convert " & $value &
@@ -596,8 +598,8 @@ proc skipObjConv(n: PNode): PNode =
   of nkObjUpConv, nkObjDownConv: result = n.sons[0]
   else: result = n
 
-proc isAssignable(c: PContext, n: PNode): TAssignableResult =
+proc isAssignable(c: PContext, n: PNode; isUnsafeAddr=false): TAssignableResult =
-  result = parampatterns.isAssignable(c.p.owner, n)
+  result = parampatterns.isAssignable(c.p.owner, n, isUnsafeAddr)
 
 proc newHiddenAddrTaken(c: PContext, n: PNode): PNode =
   if n.kind == nkHiddenDeref and not (gCmd == cmdCompileToCpp or
@@ -805,6 +807,9 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
       return semExpr(c, result, flags)
   else:
     n.sons[0] = semExpr(c, n.sons[0])
+    let t = n.sons[0].typ
+    if t != nil and t.kind == tyVar:
+      n.sons[0] = newDeref(n.sons[0])
   let nOrig = n.copyTree
   semOpAux(c, n)
   var t: PType = nil
@@ -814,7 +819,7 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
     # This is a proc variable, apply normal overload resolution
     let m = resolveIndirectCall(c, n, nOrig, t)
     if m.state != csMatch:
-      if c.inCompilesContext > 0:
+      if c.compilesContextId > 0:
         # speed up error generation:
         globalError(n.info, errTypeMismatch, "")
         return emptyNode
@@ -1152,7 +1157,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
     result.add(x[0])
     return
   checkMinSonsLen(n, 2)
-  n.sons[0] = semExprWithType(c, n.sons[0])
+  n.sons[0] = semExprWithType(c, n.sons[0], {efNoProcvarCheck})
   let arr = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyPtr, tyRef})
   case arr.kind
   of tyArray, tyOpenArray, tyVarargs, tyArrayConstr, tySequence, tyString,
@@ -1191,6 +1196,16 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode =
     else:
       localError(n.info, errIndexTypesDoNotMatch)
     result = n
+  else:
+    let s = if n.sons[0].kind == nkSym: n.sons[0].sym
+            elif n[0].kind in nkSymChoices: n.sons[0][0].sym
+            else: nil
+    if s != nil and s.kind in {skProc, skMethod, skConverter}+skIterators:
+      # type parameters: partial generic specialization
+      n.sons[0] = semSymGenericInstantiation(c, n.sons[0], s)
+      result = explicitGenericInstantiation(c, n, s)
+    elif s != nil and s.kind == skType:
+      result = symNodeFromType(c, semTypeNode(c, n, nil), n.info)
     else:
       c.p.bracketExpr = n.sons[0]
 
@@ -1246,7 +1261,7 @@ proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
 template resultTypeIsInferrable(typ: PType): expr =
   typ.isMetaType and typ.kind != tyTypeDesc
 
-proc semAsgn(c: PContext, n: PNode): PNode =
+proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
   checkSonsLen(n, 2)
   var a = n.sons[0]
   case a.kind
@@ -1269,18 +1284,29 @@ proc semAsgn(c: PContext, n: PNode): PNode =
     # --> `[]=`(a, i, x)
     let oldBracketExpr = c.p.bracketExpr
     a = semSubscript(c, a, {efLValue})
-    if a == nil:
+    if a == nil and mode != noOverloadedSubscript:
       result = buildOverloadedSubscripts(n.sons[0], getIdent"[]=")
       add(result, n[1])
       result = semExprNoType(c, result)
       c.p.bracketExpr = oldBracketExpr
       return result
+    elif a == nil:
+      localError(n.info, "could not resolve: " & $n[0])
+      return n
     c.p.bracketExpr = oldBracketExpr
   of nkCurlyExpr:
     # a{i} = x -->  `{}=`(a, i, x)
     result = buildOverloadedSubscripts(n.sons[0], getIdent"{}=")
     add(result, n[1])
     return semExprNoType(c, result)
+  of nkPar:
+    if a.len >= 2:
+      # unfortunately we need to rewrite ``(x, y) = foo()`` already here so
+      # that overloading of the assignment operator still works. Usually we
+      # prefer to do these rewritings in transf.nim:
+      return semStmt(c, lowerTupleUnpackingForAsgn(n, c.p.owner))
+    else:
+      a = semExprWithType(c, a, {efLValue})
   else:
     a = semExprWithType(c, a, {efLValue})
   n.sons[0] = a
@@ -1311,7 +1337,8 @@ proc semAsgn(c: PContext, n: PNode): PNode =
           typeMismatch(n, lhs.typ, rhs.typ)
 
     n.sons[1] = fitNode(c, le, rhs)
-    if tfHasAsgn in lhs.typ.flags and not lhsIsResult:
+    if tfHasAsgn in lhs.typ.flags and not lhsIsResult and
+        mode != noOverloadedAsgn:
       return overloadedAsgn(c, lhs, n.sons[1])
 
     fixAbstractType(c, n)
@@ -1369,7 +1396,7 @@ proc semProcBody(c: PContext, n: PNode): PNode =
      c.p.resultSym != nil and c.p.resultSym.typ.isMetaType:
     if isEmptyType(result.typ):
       # we inferred a 'void' return type:
-      c.p.resultSym.typ = nil
+      c.p.resultSym.typ = errorType(c)
       c.p.owner.typ.sons[0] = nil
     else:
       localError(c.p.resultSym.info, errCannotInferReturnType)
@@ -1610,7 +1637,9 @@ proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   # watch out, hacks ahead:
   let oldErrorCount = msgs.gErrorCounter
   let oldErrorMax = msgs.gErrorMax
-  inc c.inCompilesContext
+  let oldCompilesId = c.compilesContextId
+  inc c.compilesContextIdGenerator
+  c.compilesContextId = c.compilesContextIdGenerator
   # do not halt after first error:
   msgs.gErrorMax = high(int)
 
@@ -1628,12 +1657,15 @@ proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   let oldInGenericInst = c.inGenericInst
   let oldProcCon = c.p
   c.generics = @[]
+  var err: string
   try:
     result = semExpr(c, n, flags)
     if msgs.gErrorCounter != oldErrorCount: result = nil
   except ERecoverableError:
-    discard
+    if optReportConceptFailures in gGlobalOptions:
+      err = getCurrentExceptionMsg()
   # undo symbol table changes (as far as it's possible):
+  c.compilesContextId = oldCompilesId
   c.generics = oldGenerics
   c.inGenericContext = oldInGenericContext
   c.inUnrolledContext = oldInUnrolledContext
@@ -1642,10 +1674,11 @@ proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   msgs.setInfoContextLen(oldContextLen)
   setLen(gOwners, oldOwnerLen)
   c.currentScope = oldScope
-  dec c.inCompilesContext
   errorOutputs = oldErrorOutputs
   msgs.gErrorCounter = oldErrorCount
   msgs.gErrorMax = oldErrorMax
+  if optReportConceptFailures in gGlobalOptions and not err.isNil:
+    localError(n.info, err)
 
 proc semCompiles(c: PContext, n: PNode, flags: TExprFlags): PNode =
   # we replace this node by a 'true' or 'false' node:
@@ -1699,10 +1732,13 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
   case s.magic # magics that need special treatment
   of mAddr:
     checkSonsLen(n, 2)
-    result = semAddr(c, n.sons[1])
+    result = semAddr(c, n.sons[1], s.name.s == "unsafeAddr")
   of mTypeOf:
     checkSonsLen(n, 2)
     result = semTypeOf(c, n.sons[1])
+  #of mArrGet: result = semArrGet(c, n, flags)
+  #of mArrPut: result = semArrPut(c, n, flags)
+  #of mAsgn: result = semAsgnOpr(c, n)
   of mDefined: result = semDefined(c, setMs(n, s), false)
   of mDefinedInScope: result = semDefined(c, setMs(n, s), true)
   of mCompiles: result = semCompiles(c, setMs(n, s), flags)
@@ -1719,6 +1755,8 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     result = newStrNodeT(renderTree(n[1], {renderNoComments}), n)
     result.typ = getSysType(tyString)
   of mParallel:
+    if not experimentalMode(c):
+      localError(n.info, "use the {.experimental.} pragma to enable 'parallel'")
     result = setMs(n, s)
     var x = n.lastSon
     if x.kind == nkDo: x = x.sons[bodyPos]
@@ -1727,13 +1765,17 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     dec c.inParallelStmt
   of mSpawn:
     result = setMs(n, s)
-    result.sons[1] = semExpr(c, n.sons[1])
+    for i in 1 .. <n.len:
-    if not result[1].typ.isEmptyType:
+      result.sons[i] = semExpr(c, n.sons[i])
-      if spawnResult(result[1].typ, c.inParallelStmt > 0) == srFlowVar:
+    let typ = result[^1].typ
-        result.typ = createFlowVar(c, result[1].typ, n.info)
+    if not typ.isEmptyType:
+      if spawnResult(typ, c.inParallelStmt > 0) == srFlowVar:
+        result.typ = createFlowVar(c, typ, n.info)
       else:
-        result.typ = result[1].typ
+        result.typ = typ
-      result.add instantiateCreateFlowVarCall(c, result[1].typ, n.info).newSymNode
+      result.add instantiateCreateFlowVarCall(c, typ, n.info).newSymNode
+    else:
+      result.add emptyNode
   of mProcCall:
     result = setMs(n, s)
     result.sons[1] = semExpr(c, n.sons[1])
@@ -1749,11 +1791,30 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode =
     if semCheck: result = semStmt(c, e) # do not open a new scope!
     else: result = e
 
+  # Check if the node is "when nimvm"
+  # when nimvm:
+  #   ...
+  # else:
+  #   ...
+  var whenNimvm = false
+  if n.sons.len == 2 and n.sons[0].kind == nkElifBranch and
+      n.sons[1].kind == nkElse:
+    let exprNode = n.sons[0].sons[0]
+    if exprNode.kind == nkIdent:
+      whenNimvm = lookUp(c, exprNode).magic == mNimvm
+    elif exprNode.kind == nkSym:
+      whenNimvm = exprNode.sym.magic == mNimvm
+
   for i in countup(0, sonsLen(n) - 1):
     var it = n.sons[i]
     case it.kind
     of nkElifBranch, nkElifExpr:
       checkSonsLen(it, 2)
+      if whenNimvm:
+        if semCheck:
+          it.sons[1] = semStmt(c, it.sons[1])
+        result = n # when nimvm is not elimited until codegen
+      else:
         var e = semConstExpr(c, it.sons[0])
         if e.kind != nkIntLit:
           # can happen for cascading errors, assume false
@@ -1763,8 +1824,11 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode =
           setResult(it.sons[1])
     of nkElse, nkElseExpr:
       checkSonsLen(it, 1)
+      if result == nil or whenNimvm:
+        if semCheck:
+          it.sons[0] = semStmt(c, it.sons[0])
         if result == nil:
-        setResult(it.sons[0])
+          result = it.sons[0]
     else: illFormedAst(n)
   if result == nil:
     result = newNodeI(nkEmpty, n.info)
@@ -2026,6 +2090,19 @@ proc semExport(c: PContext, n: PNode): PNode =
     c.module.ast.add x
   result = n
 
+proc shouldBeBracketExpr(n: PNode): bool =
+  assert n.kind in nkCallKinds
+  let a = n.sons[0]
+  if a.kind in nkCallKinds:
+    let b = a[0]
+    if b.kind in nkSymChoices:
+      for i in 0..<b.len:
+        if b[i].sym.magic == mArrGet:
+          let be = newNodeI(nkBracketExpr, n.info)
+          for i in 1..<a.len: be.add(a[i])
+          n.sons[0] = be
+          return true
+
 proc setGenericParams(c: PContext, n: PNode) =
   for i in 1 .. <n.len:
     n[i].typ = semTypeNode(c, n[i], nil)
@@ -2133,7 +2210,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
       else:
         #liMessage(n.info, warnUser, renderTree(n));
         result = semIndirectOp(c, n, flags)
-    elif n[0].kind == nkBracketExpr and isSymChoice(n[0][0]):
+    elif (n[0].kind == nkBracketExpr or shouldBeBracketExpr(n)) and
+        isSymChoice(n[0][0]):
       # indirectOp can deal with explicit instantiations; the fixes
       # the 'newSeq[T](x)' bug
       setGenericParams(c, n.sons[0])
@@ -2147,18 +2225,13 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
       result = semWhen(c, n, true)
     else:
       result = semWhen(c, n, false)
+      if result == n:
+        # This is a "when nimvm" stmt.
+        result = semWhen(c, n, true)
+      else:
         result = semExpr(c, result, flags)
   of nkBracketExpr:
     checkMinSonsLen(n, 1)
-    var s = qualifiedLookUp(c, n.sons[0], {checkUndeclared})
-    if (s != nil and s.kind in {skProc, skMethod, skConverter}+skIterators) or
-        n[0].kind in nkSymChoices:
-      # type parameters: partial generic specialization
-      n.sons[0] = semSymGenericInstantiation(c, n.sons[0], s)
-      result = explicitGenericInstantiation(c, n, s)
-    elif s != nil and s.kind in {skType}:
-      result = symNodeFromType(c, semTypeNode(c, n, nil), n.info)
-    else:
     result = semArrayAccess(c, n, flags)
   of nkCurlyExpr:
     result = semExpr(c, buildOverloadedSubscripts(n, getIdent"{}"), flags)
@@ -2254,7 +2327,10 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   of nkStaticStmt:
     result = semStaticStmt(c, n)
   of nkDefer:
-    localError(n.info, errGenerated, "'defer' not allowed in this context")
+    n.sons[0] = semExpr(c, n.sons[0])
+    if not n.sons[0].typ.isEmptyType and not implicitlyDiscardable(n.sons[0]):
+      localError(n.info, errGenerated, "'defer' takes a 'void' expression")
+    #localError(n.info, errGenerated, "'defer' not allowed in this context")
   else:
     localError(n.info, errInvalidExpressionX,
                renderTree(n, {renderNoComments}))

compiler/semfold.nim

@@ -179,7 +179,7 @@ proc getIntervalType*(m: TMagic, n: PNode): PType =
       else:
         result = makeRangeF(a, abs(getFloat(a.n.sons[1])),
                                abs(getFloat(a.n.sons[0])))
-  of mAbsI, mAbsI64:
+  of mAbsI:
     let a = n.sons[1].typ
     if isIntRange(a):
       if a.n[0].intVal <= 0:
@@ -200,13 +200,13 @@ proc getIntervalType*(m: TMagic, n: PNode): PType =
     if isIntRange(a) and isIntLit(b):
       result = makeRange(a, pickMinInt(n.sons[1]) |-| pickMinInt(n.sons[2]),
                             pickMaxInt(n.sons[1]) |-| pickMaxInt(n.sons[2]))
-  of mAddI, mAddI64, mAddU:
+  of mAddI, mAddU:
     commutativeOp(`|+|`)
-  of mMulI, mMulI64, mMulU:
+  of mMulI, mMulU:
     commutativeOp(`|*|`)
-  of mSubI, mSubI64, mSubU:
+  of mSubI, mSubU:
     binaryOp(`|-|`)
-  of mBitandI, mBitandI64:
+  of mBitandI:
     # since uint64 is still not even valid for 'range' (since it's no ordinal
     # yet), we exclude it from the list (see bug #1638) for now:
     var a = n.sons[1]
@@ -225,7 +225,7 @@ proc getIntervalType*(m: TMagic, n: PNode): PType =
         result = makeRange(a.typ, 0, b.intVal-1)
       else:
         result = makeRange(a.typ, b.intVal+1, 0)
-  of mModI, mModI64:
+  of mModI:
     # so ... if you ever wondered about modulo's signedness; this defines it:
     let a = n.sons[1]
     let b = n.sons[2]
@@ -234,7 +234,7 @@ proc getIntervalType*(m: TMagic, n: PNode): PType =
         result = makeRange(a.typ, -(b.intVal-1), b.intVal-1)
       else:
         result = makeRange(a.typ, b.intVal+1, -(b.intVal+1))
-  of mDivI, mDivI64, mDivU:
+  of mDivI, mDivU:
     binaryOp(`|div|`)
   of mMinI:
     commutativeOp(min)
@@ -243,8 +243,8 @@ proc getIntervalType*(m: TMagic, n: PNode): PType =
   else: discard
 
 discard """
-  mShlI, mShlI64,
+  mShlI,
-  mShrI, mShrI64, mAddF64, mSubF64, mMulF64, mDivF64, mMaxF64, mMinF64
+  mShrI, mAddF64, mSubF64, mMulF64, mDivF64, mMaxF64, mMinF64
 """
 
 proc evalIs(n, a: PNode): PNode =
@@ -285,7 +285,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mUnaryMinusF64: result = newFloatNodeT(- getFloat(a), n)
   of mNot: result = newIntNodeT(1 - getInt(a), n)
   of mCard: result = newIntNodeT(nimsets.cardSet(a), n)
-  of mBitnotI, mBitnotI64: result = newIntNodeT(not getInt(a), n)
+  of mBitnotI: result = newIntNodeT(not getInt(a), n)
   of mLengthStr, mXLenStr:
     if a.kind == nkNilLit: result = newIntNodeT(0, n)
     else: result = newIntNodeT(len(getStr(a)), n)
@@ -298,7 +298,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
     result = newFloatNodeT(toFloat(int(getInt(a))), n)
   of mToInt, mToBiggestInt: result = newIntNodeT(system.toInt(getFloat(a)), n)
   of mAbsF64: result = newFloatNodeT(abs(getFloat(a)), n)
-  of mAbsI, mAbsI64:
+  of mAbsI:
     if getInt(a) >= 0: result = a
     else: result = newIntNodeT(- getInt(a), n)
   of mZe8ToI, mZe8ToI64, mZe16ToI, mZe16ToI64, mZe32ToI64, mZeIToI64:
@@ -307,19 +307,19 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mToU8: result = newIntNodeT(getInt(a) and 0x000000FF, n)
   of mToU16: result = newIntNodeT(getInt(a) and 0x0000FFFF, n)
   of mToU32: result = newIntNodeT(getInt(a) and 0x00000000FFFFFFFF'i64, n)
-  of mUnaryLt: result = newIntNodeT(getOrdValue(a) - 1, n)
+  of mUnaryLt: result = newIntNodeT(getOrdValue(a) |-| 1, n)
-  of mSucc: result = newIntNodeT(getOrdValue(a) + getInt(b), n)
+  of mSucc: result = newIntNodeT(getOrdValue(a) |+| getInt(b), n)
-  of mPred: result = newIntNodeT(getOrdValue(a) - getInt(b), n)
+  of mPred: result = newIntNodeT(getOrdValue(a) |-| getInt(b), n)
-  of mAddI, mAddI64: result = newIntNodeT(getInt(a) + getInt(b), n)
+  of mAddI: result = newIntNodeT(getInt(a) |+| getInt(b), n)
-  of mSubI, mSubI64: result = newIntNodeT(getInt(a) - getInt(b), n)
+  of mSubI: result = newIntNodeT(getInt(a) |-| getInt(b), n)
-  of mMulI, mMulI64: result = newIntNodeT(getInt(a) * getInt(b), n)
+  of mMulI: result = newIntNodeT(getInt(a) |*| getInt(b), n)
   of mMinI:
     if getInt(a) > getInt(b): result = newIntNodeT(getInt(b), n)
     else: result = newIntNodeT(getInt(a), n)
   of mMaxI:
     if getInt(a) > getInt(b): result = newIntNodeT(getInt(a), n)
     else: result = newIntNodeT(getInt(b), n)
-  of mShlI, mShlI64:
+  of mShlI:
     case skipTypes(n.typ, abstractRange).kind
     of tyInt8: result = newIntNodeT(int8(getInt(a)) shl int8(getInt(b)), n)
     of tyInt16: result = newIntNodeT(int16(getInt(a)) shl int16(getInt(b)), n)
@@ -327,7 +327,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
     of tyInt64, tyInt, tyUInt..tyUInt64:
       result = newIntNodeT(`shl`(getInt(a), getInt(b)), n)
     else: internalError(n.info, "constant folding for shl")
-  of mShrI, mShrI64:
+  of mShrI:
     case skipTypes(n.typ, abstractRange).kind
     of tyInt8: result = newIntNodeT(int8(getInt(a)) shr int8(getInt(b)), n)
     of tyInt16: result = newIntNodeT(int16(getInt(a)) shr int16(getInt(b)), n)
@@ -335,14 +335,14 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
     of tyInt64, tyInt, tyUInt..tyUInt64:
       result = newIntNodeT(`shr`(getInt(a), getInt(b)), n)
     else: internalError(n.info, "constant folding for shr")
-  of mDivI, mDivI64:
+  of mDivI:
     let y = getInt(b)
     if y != 0:
-      result = newIntNodeT(getInt(a) div y, n)
+      result = newIntNodeT(`|div|`(getInt(a), y), n)
-  of mModI, mModI64:
+  of mModI:
     let y = getInt(b)
     if y != 0:
-      result = newIntNodeT(getInt(a) mod y, n)
+      result = newIntNodeT(`|mod|`(getInt(a), y), n)
   of mAddF64: result = newFloatNodeT(getFloat(a) + getFloat(b), n)
   of mSubF64: result = newFloatNodeT(getFloat(a) - getFloat(b), n)
   of mMulF64: result = newFloatNodeT(getFloat(a) * getFloat(b), n)
@@ -359,11 +359,11 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
     if getFloat(a) > getFloat(b): result = newFloatNodeT(getFloat(b), n)
     else: result = newFloatNodeT(getFloat(a), n)
   of mIsNil: result = newIntNodeT(ord(a.kind == nkNilLit), n)
-  of mLtI, mLtI64, mLtB, mLtEnum, mLtCh:
+  of mLtI, mLtB, mLtEnum, mLtCh:
     result = newIntNodeT(ord(getOrdValue(a) < getOrdValue(b)), n)
-  of mLeI, mLeI64, mLeB, mLeEnum, mLeCh:
+  of mLeI, mLeB, mLeEnum, mLeCh:
     result = newIntNodeT(ord(getOrdValue(a) <= getOrdValue(b)), n)
-  of mEqI, mEqI64, mEqB, mEqEnum, mEqCh:
+  of mEqI, mEqB, mEqEnum, mEqCh:
     result = newIntNodeT(ord(getOrdValue(a) == getOrdValue(b)), n)
   of mLtF64: result = newIntNodeT(ord(getFloat(a) < getFloat(b)), n)
   of mLeF64: result = newIntNodeT(ord(getFloat(a) <= getFloat(b)), n)
@@ -375,9 +375,9 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
     result = newIntNodeT(ord(`<%`(getOrdValue(a), getOrdValue(b))), n)
   of mLeU, mLeU64:
     result = newIntNodeT(ord(`<=%`(getOrdValue(a), getOrdValue(b))), n)
-  of mBitandI, mBitandI64, mAnd: result = newIntNodeT(a.getInt and b.getInt, n)
+  of mBitandI, mAnd: result = newIntNodeT(a.getInt and b.getInt, n)
-  of mBitorI, mBitorI64, mOr: result = newIntNodeT(getInt(a) or getInt(b), n)
+  of mBitorI, mOr: result = newIntNodeT(getInt(a) or getInt(b), n)
-  of mBitxorI, mBitxorI64, mXor: result = newIntNodeT(a.getInt xor b.getInt, n)
+  of mBitxorI, mXor: result = newIntNodeT(a.getInt xor b.getInt, n)
   of mAddU: result = newIntNodeT(`+%`(getInt(a), getInt(b)), n)
   of mSubU: result = newIntNodeT(`-%`(getInt(a), getInt(b)), n)
   of mMulU: result = newIntNodeT(`*%`(getInt(a), getInt(b)), n)
@@ -430,14 +430,10 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mCompileOptionArg:
     result = newIntNodeT(ord(
       testCompileOptionArg(getStr(a), getStr(b), n.info)), n)
-  of mNewString, mNewStringOfCap,
+  of mEqProc:
-     mExit, mInc, ast.mDec, mEcho, mSwap, mAppendStrCh,
+    result = newIntNodeT(ord(
-     mAppendStrStr, mAppendSeqElem, mSetLengthStr, mSetLengthSeq,
+        exprStructuralEquivalent(a, b, strictSymEquality=true)), n)
-     mParseExprToAst, mParseStmtToAst, mExpandToAst, mTypeTrait, mDotDot,
+  else: discard
-     mNLen..mNError, mEqRef, mSlurp, mStaticExec, mNGenSym, mSpawn,
-     mParallel, mPlugin:
-    discard
-  else: internalError(a.info, "evalOp(" & $m & ')')
 
 proc getConstIfExpr(c: PSym, n: PNode): PNode =
   result = nil
@@ -540,7 +536,7 @@ proc foldConv*(n, a: PNode; check = false): PNode =
   of tyFloat..tyFloat64:
     case skipTypes(a.typ, abstractRange).kind
     of tyInt..tyInt64, tyEnum, tyBool, tyChar:
-      result = newFloatNodeT(toFloat(int(getOrdValue(a))), n)
+      result = newFloatNodeT(toBiggestFloat(getOrdValue(a)), n)
     else:
       result = a
       result.typ = n.typ

compiler/semgnrc.nim

@@ -30,6 +30,7 @@ type
   GenericCtx = object
     toMixin: IntSet
     cursorInBody: bool # only for nimsuggest
+    bracketExpr: PNode
 
 type
   TSemGenericFlag = enum
@@ -57,7 +58,7 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
   of skUnknown:
     # Introduced in this pass! Leave it as an identifier.
     result = n
-  of skProc, skMethod, skIterators, skConverter:
+  of skProc, skMethod, skIterators, skConverter, skModule:
     result = symChoice(c, n, s, scOpen)
   of skTemplate:
     if macroToExpand(s):
@@ -141,11 +142,13 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
       elif s.name.id in ctx.toMixin:
         result = newDot(result, symChoice(c, n, s, scForceOpen))
       else:
-        let sym = semGenericStmtSymbol(c, n, s, ctx)
+        let syms = semGenericStmtSymbol(c, n, s, ctx)
-        if sym.kind == nkSym:
+        if syms.kind == nkSym:
-          result = newDot(result, symChoice(c, n, s, scForceOpen))
+          let choice = symChoice(c, n, s, scForceOpen)
+          choice.kind = nkClosedSymChoice
+          result = newDot(result, choice)
         else:
-          result = newDot(result, sym)
+          result = newDot(result, syms)
 
 proc addTempDecl(c: PContext; n: PNode; kind: TSymKind) =
   let s = newSymS(skUnknown, getIdentNode(n), c)
@@ -223,8 +226,12 @@ proc semGenericStmt(c: PContext, n: PNode,
       of skUnknown, skParam:
         # Leave it as an identifier.
         discard
-      of skProc, skMethod, skIterators, skConverter:
+      of skProc, skMethod, skIterators, skConverter, skModule:
         result.sons[0] = symChoice(c, fn, s, scOption)
+        # do not check of 's.magic==mRoof' here because it might be some
+        # other '^' but after overload resolution the proper one:
+        if ctx.bracketExpr != nil and n.len == 2 and s.name.s == "^":
+          result.add ctx.bracketExpr
         first = 1
       of skGenericParam:
         result.sons[0] = newSymNodeTypeDesc(s, fn.info)
@@ -249,6 +256,40 @@ proc semGenericStmt(c: PContext, n: PNode,
     let flags = if mixinContext: flags+{withinMixin} else: flags
     for i in countup(first, sonsLen(result) - 1):
       result.sons[i] = semGenericStmt(c, result.sons[i], flags, ctx)
+  of nkCurlyExpr:
+    result = newNodeI(nkCall, n.info)
+    result.add newIdentNode(getIdent("{}"), n.info)
+    for i in 0 ..< n.len: result.add(n[i])
+    result = semGenericStmt(c, result, flags, ctx)
+  of nkBracketExpr:
+    result = newNodeI(nkCall, n.info)
+    result.add newIdentNode(getIdent("[]"), n.info)
+    for i in 0 ..< n.len: result.add(n[i])
+    withBracketExpr ctx, n.sons[0]:
+      result = semGenericStmt(c, result, flags, ctx)
+  of nkAsgn, nkFastAsgn:
+    checkSonsLen(n, 2)
+    let a = n.sons[0]
+    let b = n.sons[1]
+
+    let k = a.kind
+    case k
+    of nkCurlyExpr:
+      result = newNodeI(nkCall, n.info)
+      result.add newIdentNode(getIdent("{}="), n.info)
+      for i in 0 ..< a.len: result.add(a[i])
+      result.add(b)
+      result = semGenericStmt(c, result, flags, ctx)
+    of nkBracketExpr:
+      result = newNodeI(nkCall, n.info)
+      result.add newIdentNode(getIdent("[]="), n.info)
+      for i in 0 ..< a.len: result.add(a[i])
+      result.add(b)
+      withBracketExpr ctx, a.sons[0]:
+        result = semGenericStmt(c, result, flags, ctx)
+    else:
+      for i in countup(0, sonsLen(n) - 1):
+        result.sons[i] = semGenericStmt(c, n.sons[i], flags, ctx)
   of nkIfStmt:
     for i in countup(0, sonsLen(n)-1):
       n.sons[i] = semGenericStmtScope(c, n.sons[i], flags, ctx)
@@ -355,7 +396,7 @@ proc semGenericStmt(c: PContext, n: PNode,
         of nkEnumFieldDef: a = n.sons[i].sons[0]
         of nkIdent: a = n.sons[i]
         else: illFormedAst(n)
-        addDecl(c, newSymS(skUnknown, getIdentNode(a.sons[i]), c))
+        addDecl(c, newSymS(skUnknown, getIdentNode(a), c))
   of nkObjectTy, nkTupleTy, nkTupleClassTy:
     discard
   of nkFormalParams:

compiler/seminst.nim

@@ -10,14 +10,10 @@
 # This module implements the instantiation of generic procs.
 # included from sem.nim
 
-proc instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable,
+iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable): PSym =
-                                 entry: var TInstantiation) =
+  internalAssert n.kind == nkGenericParams
-  if n.kind != nkGenericParams:
-    internalError(n.info, "instantiateGenericParamList; no generic params")
-  newSeq(entry.concreteTypes, n.len)
   for i, a in n.pairs:
-    if a.kind != nkSym:
+    internalAssert a.kind == nkSym
-      internalError(a.info, "instantiateGenericParamList; no symbol")
     var q = a.sym
     if q.typ.kind notin {tyTypeDesc, tyGenericParam, tyStatic, tyIter}+tyTypeClasses:
       continue
@@ -42,8 +38,7 @@ proc instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable,
       #t = ReplaceTypeVarsT(cl, t)
     s.typ = t
     if t.kind == tyStatic: s.ast = t.n
-    addDecl(c, s)
+    yield s
-    entry.concreteTypes[i] = t
 
 proc sameInstantiation(a, b: TInstantiation): bool =
   if a.concreteTypes.len == b.concreteTypes.len:
@@ -52,10 +47,11 @@ proc sameInstantiation(a, b: TInstantiation): bool =
                           flags = {ExactTypeDescValues}): return
     result = true
 
-proc genericCacheGet(genericSym: PSym, entry: TInstantiation): PSym =
+proc genericCacheGet(genericSym: PSym, entry: TInstantiation;
+                     id: CompilesId): PSym =
   if genericSym.procInstCache != nil:
     for inst in genericSym.procInstCache:
-      if sameInstantiation(entry, inst[]):
+      if inst.compilesId == id and sameInstantiation(entry, inst[]):
         return inst.sym
 
 proc removeDefaultParamValues(n: PNode) =
@@ -169,7 +165,7 @@ proc instantiateProcType(c: PContext, pt: TIdTable,
   addDecl(c, prc)
 
   pushInfoContext(info)
-  var cl = initTypeVars(c, pt, info)
+  var cl = initTypeVars(c, pt, info, nil)
   var result = instCopyType(cl, prc.typ)
   let originalParams = result.n
   result.n = originalParams.shallowCopy
@@ -217,7 +213,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   ## The `pt` parameter is a type-unsafe mapping table used to link generic
   ## parameters to their concrete types within the generic instance.
   # no need to instantiate generic templates/macros:
-  if fn.kind in {skTemplate, skMacro}: return fn
+  internalAssert fn.kind notin {skMacro, skTemplate}
   # generates an instantiated proc
   if c.instCounter > 1000: internalError(fn.ast.info, "nesting too deep")
   inc(c.instCounter)
@@ -226,30 +222,49 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   # NOTE: for access of private fields within generics from a different module
   # we set the friend module:
   c.friendModules.add(getModule(fn))
-  #let oldScope = c.currentScope
+  let oldInTypeClass = c.inTypeClass
-  #c.currentScope = fn.scope
+  c.inTypeClass = 0
+  let oldScope = c.currentScope
+  while not isTopLevel(c): c.currentScope = c.currentScope.parent
   result = copySym(fn, false)
   incl(result.flags, sfFromGeneric)
   result.owner = fn
   result.ast = n
   pushOwner(result)
+
   openScope(c)
-  internalAssert n.sons[genericParamsPos].kind != nkEmpty
+  let gp = n.sons[genericParamsPos]
+  internalAssert gp.kind != nkEmpty
   n.sons[namePos] = newSymNode(result)
   pushInfoContext(info)
   var entry = TInstantiation.new
   entry.sym = result
-  instantiateGenericParamList(c, n.sons[genericParamsPos], pt, entry[])
+  # we need to compare both the generic types and the concrete types:
+  # generic[void](), generic[int]()
+  # see ttypeor.nim test.
+  var i = 0
+  newSeq(entry.concreteTypes, fn.typ.len+gp.len-1)
+  for s in instantiateGenericParamList(c, gp, pt):
+    addDecl(c, s)
+    entry.concreteTypes[i] = s.typ
+    inc i
   pushProcCon(c, result)
   instantiateProcType(c, pt, result, info)
+  for j in 1 .. result.typ.len-1:
+    entry.concreteTypes[i] = result.typ.sons[j]
+    inc i
+  if tfTriggersCompileTime in result.typ.flags:
+    incl(result.flags, sfCompileTime)
   n.sons[genericParamsPos] = ast.emptyNode
-  var oldPrc = genericCacheGet(fn, entry[])
+  var oldPrc = genericCacheGet(fn, entry[], c.compilesContextId)
   if oldPrc == nil:
     # we MUST not add potentially wrong instantiations to the caching mechanism.
     # This means recursive instantiations behave differently when in
     # a ``compiles`` context but this is the lesser evil. See
     # bug #1055 (tevilcompiles).
-    if c.inCompilesContext == 0: fn.procInstCache.safeAdd(entry)
+    #if c.compilesContextId == 0:
+    entry.compilesId = c.compilesContextId
+    fn.procInstCache.safeAdd(entry)
     c.generics.add(makeInstPair(fn, entry))
     if n.sons[pragmasPos].kind != nkEmpty:
       pragma(c, result, n.sons[pragmasPos], allRoutinePragmas)
@@ -264,7 +279,8 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   popInfoContext()
   closeScope(c)           # close scope for parameters
   popOwner()
-  #c.currentScope = oldScope
+  c.currentScope = oldScope
   discard c.friendModules.pop()
   dec(c.instCounter)
+  c.inTypeClass = oldInTypeClass
   if result.kind == skMethod: finishMethod(c, result)

compiler/semmacrosanity.nim

@@ -38,14 +38,16 @@ proc annotateType*(n: PNode, t: PType) =
   # Note: x can be unequal to t and we need to be careful to use 't'
   # to not to skip tyGenericInst
   case n.kind
-  of nkPar:
+  of nkObjConstr:
-    if x.kind == tyObject:
     n.typ = t
-      for i in 0 .. <n.len:
+    for i in 1 .. <n.len:
-        let field = x.n.ithField(i)
+      let field = x.n.ithField(i - 1)
       if field.isNil: globalError n.info, "invalid field at index " & $i
-        else: annotateType(n.sons[i], field.typ)
+      else:
-    elif x.kind == tyTuple:
+        internalAssert(n.sons[i].kind == nkExprColonExpr)
+        annotateType(n.sons[i].sons[1], field.typ)
+  of nkPar:
+    if x.kind == tyTuple:
       n.typ = t
       for i in 0 .. <n.len:
         if i >= x.len: globalError n.info, "invalid field at index " & $i
@@ -53,7 +55,7 @@ proc annotateType*(n: PNode, t: PType) =
     elif x.kind == tyProc and x.callConv == ccClosure:
       n.typ = t
     else:
-      globalError(n.info, "() must have an object or tuple type")
+      globalError(n.info, "() must have a tuple type")
   of nkBracket:
     if x.kind in {tyArrayConstr, tyArray, tySequence, tyOpenArray}:
       n.typ = t

compiler/semmagic.nim

@@ -10,10 +10,12 @@
 # This include file implements the semantic checking for magics.
 # included from sem.nim
 
-proc semAddr(c: PContext; n: PNode): PNode =
+proc semAddr(c: PContext; n: PNode; isUnsafeAddr=false): PNode =
   result = newNodeI(nkAddr, n.info)
   let x = semExprWithType(c, n)
-  if isAssignable(c, x) notin {arLValue, arLocalLValue}:
+  if x.kind == nkSym:
+    x.sym.flags.incl(sfAddrTaken)
+  if isAssignable(c, x, isUnsafeAddr) notin {arLValue, arLocalLValue}:
     localError(n.info, errExprHasNoAddress)
   result.add x
   result.typ = makePtrType(c, x.typ)
@@ -24,6 +26,41 @@ proc semTypeOf(c: PContext; n: PNode): PNode =
   result.add typExpr
   result.typ = makeTypeDesc(c, typExpr.typ.skipTypes({tyTypeDesc, tyIter}))
 
+type
+  SemAsgnMode = enum asgnNormal, noOverloadedSubscript, noOverloadedAsgn
+
+proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode
+proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode
+
+proc skipAddr(n: PNode): PNode {.inline.} =
+  (if n.kind == nkHiddenAddr: n.sons[0] else: n)
+
+proc semArrGet(c: PContext; n: PNode; flags: TExprFlags): PNode =
+  result = newNodeI(nkBracketExpr, n.info)
+  for i in 1..<n.len: result.add(n[i])
+  let oldBracketExpr = c.p.bracketExpr
+  result = semSubscript(c, result, flags)
+  c.p.bracketExpr = oldBracketExpr
+  if result.isNil:
+    localError(n.info, "could not resolve: " & $n)
+    result = n
+
+proc semArrPut(c: PContext; n: PNode; flags: TExprFlags): PNode =
+  # rewrite `[]=`(a, i, x)  back to ``a[i] = x``.
+  let b = newNodeI(nkBracketExpr, n.info)
+  b.add(n[1].skipAddr)
+  for i in 2..n.len-2: b.add(n[i])
+  result = newNodeI(nkAsgn, n.info, 2)
+  result.sons[0] = b
+  result.sons[1] = n.lastSon
+  result = semAsgn(c, result, noOverloadedSubscript)
+
+proc semAsgnOpr(c: PContext; n: PNode): PNode =
+  result = newNodeI(nkAsgn, n.info, 2)
+  result.sons[0] = n[1]
+  result.sons[1] = n[2]
+  result = semAsgn(c, result, noOverloadedAsgn)
+
 proc semIsPartOf(c: PContext, n: PNode, flags: TExprFlags): PNode =
   var r = isPartOf(n[1], n[2])
   result = newIntNodeT(ord(r), n)
@@ -119,10 +156,13 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
   case n[0].sym.magic
   of mAddr:
     checkSonsLen(n, 2)
-    result = semAddr(c, n.sons[1])
+    result = semAddr(c, n.sons[1], n[0].sym.name.s == "unsafeAddr")
   of mTypeOf:
     checkSonsLen(n, 2)
     result = semTypeOf(c, n.sons[1])
+  of mArrGet: result = semArrGet(c, n, flags)
+  of mArrPut: result = semArrPut(c, n, flags)
+  of mAsgn: result = semAsgnOpr(c, n)
   of mIsPartOf: result = semIsPartOf(c, n, flags)
   of mTypeTrait: result = semTypeTraits(c, n)
   of mAstToStr:
@@ -143,25 +183,28 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
       if isNegative(n.sons[1]) or (n.len > 2 and isNegative(n.sons[2])):
         localError(n.info, "use '^' instead of '-'; negative indexing is obsolete")
   of mRoof:
-    # error correction:
+    let bracketExpr = if n.len == 3: n.sons[2] else: c.p.bracketExpr
-    result = n.sons[1]
+    if bracketExpr.isNil:
-    if c.p.bracketExpr.isNil:
       localError(n.info, "no surrounding array access context for '^'")
-    elif c.p.bracketExpr.checkForSideEffects != seNoSideEffect:
+      result = n.sons[1]
+    elif bracketExpr.checkForSideEffects != seNoSideEffect:
       localError(n.info, "invalid context for '^' as '$#' has side effects" %
-        renderTree(c.p.bracketExpr))
+        renderTree(bracketExpr))
-    elif c.p.bracketExpr.typ.isStrangeArray:
+      result = n.sons[1]
+    elif bracketExpr.typ.isStrangeArray:
       localError(n.info, "invalid context for '^' as len!=high+1 for '$#'" %
-        renderTree(c.p.bracketExpr))
+        renderTree(bracketExpr))
+      result = n.sons[1]
     else:
       # ^x  is rewritten to: len(a)-x
       let lenExpr = newNodeI(nkCall, n.info)
       lenExpr.add newIdentNode(getIdent"len", n.info)
-      lenExpr.add c.p.bracketExpr
+      lenExpr.add bracketExpr
       let lenExprB = semExprWithType(c, lenExpr)
       if lenExprB.typ.isNil or not isOrdinalType(lenExprB.typ):
         localError(n.info, "'$#' has to be of an ordinal type for '^'" %
           renderTree(lenExpr))
+        result = n.sons[1]
       else:
         result = newNodeIT(nkCall, n.info, getSysType(tyInt))
         result.add newSymNode(createMagic("-", mSubI), n.info)

compiler/semparallel.nim

@@ -128,10 +128,10 @@ template `?`(x): expr = x.renderTree
 proc checkLe(c: AnalysisCtx; a, b: PNode) =
   case proveLe(c.guards, a, b)
   of impUnknown:
-    localError(a.info, "cannot prove: " & ?a & " <= " & ?b)
+    localError(a.info, "cannot prove: " & ?a & " <= " & ?b & " (bounds check)")
   of impYes: discard
   of impNo:
-    localError(a.info, "can prove: " & ?a & " > " & ?b)
+    localError(a.info, "can prove: " & ?a & " > " & ?b & " (bounds check)")
 
 proc checkBounds(c: AnalysisCtx; arr, idx: PNode) =
   checkLe(c, arr.lowBound, idx)
@@ -156,19 +156,23 @@ proc addSlice(c: var AnalysisCtx; n: PNode; x, le, ri: PNode) =
 
 proc overlap(m: TModel; x,y,c,d: PNode) =
   #  X..Y and C..D overlap iff (X <= D and C <= Y)
+  case proveLe(m, c, y)
+  of impUnknown:
+    case proveLe(m, x, d)
+    of impNo: discard
+    of impUnknown, impYes:
+      localError(x.info,
+        "cannot prove: $# > $#; required for ($#)..($#) disjoint from ($#)..($#)" %
+            [?c, ?y, ?x, ?y, ?c, ?d])
+  of impYes:
     case proveLe(m, x, d)
     of impUnknown:
       localError(x.info,
         "cannot prove: $# > $#; required for ($#)..($#) disjoint from ($#)..($#)" %
           [?x, ?d, ?x, ?y, ?c, ?d])
     of impYes:
-    case proveLe(m, c, y)
+      localError(x.info, "($#)..($#) not disjoint from ($#)..($#)" %
-    of impUnknown:
-      localError(x.info,
-        "cannot prove: $# > $#; required for ($#)..($#) disjoint from ($#)..($#)" %
                 [?c, ?y, ?x, ?y, ?c, ?d])
-    of impYes:
-      localError(x.info, "($#)..($#) not disjoint from ($#)..($#)" % [?x, ?y, ?c, ?d])
     of impNo: discard
   of impNo: discard
 
@@ -278,10 +282,12 @@ proc analyseCall(c: var AnalysisCtx; n: PNode; op: PSym) =
         slot.stride = min(slot.stride, incr)
     analyseSons(c, n)
   elif op.name.s == "[]" and op.fromSystem:
-    c.addSlice(n, n[1], n[2][1], n[2][2])
+    let slice = n[2].skipStmtList
+    c.addSlice(n, n[1], slice[1], slice[2])
     analyseSons(c, n)
   elif op.name.s == "[]=" and op.fromSystem:
-    c.addSlice(n, n[1], n[2][1], n[2][2])
+    let slice = n[2].skipStmtList
+    c.addSlice(n, n[1], slice[1], slice[2])
     analyseSons(c, n)
   else:
     analyseSons(c, n)
@@ -363,7 +369,7 @@ proc analyse(c: var AnalysisCtx; n: PNode) =
             else: internalError(it.info, "slot already has a lower bound")
         if not isSpawned: analyse(c, value)
   of nkCaseStmt: analyseCase(c, n)
-  of nkIfStmt, nkIfExpr: analyseIf(c, n)
+  of nkWhen, nkIfStmt, nkIfExpr: analyseIf(c, n)
   of nkWhileStmt:
     analyse(c, n.sons[0])
     # 'while true' loop?
@@ -395,8 +401,9 @@ proc transformSlices(n: PNode): PNode =
       result = copyNode(n)
       result.add opSlice.newSymNode
       result.add n[1]
-      result.add n[2][1]
+      let slice = n[2].skipStmtList
-      result.add n[2][2]
+      result.add slice[1]
+      result.add slice[2]
       return result
   if n.safeLen > 0:
     result = shallowCopy(n)

compiler/sempass2.nim

@@ -9,7 +9,7 @@
 
 import
   intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
-  wordrecg, strutils, options, guards
+  wordrecg, strutils, options, guards, writetracking
 
 # Second semantic checking pass over the AST. Necessary because the old
 # way had some inherent problems. Performs:
@@ -17,7 +17,7 @@ import
 # * effect+exception tracking
 # * "usage before definition" checking
 # * checks for invalid usages of compiletime magics (not implemented)
-# * checks for invalid usages of PNimNode (not implemented)
+# * checks for invalid usages of NimNode (not implemented)
 # * later: will do an escape analysis for closures at least
 
 # Predefined effects:
@@ -29,21 +29,6 @@ import
 #   --> a TR macro can annotate the proc with user defined annotations
 #   --> the effect system can access these
 
-# Load&Store analysis is performed on *paths*. A path is an access like
-# obj.x.y[i].z; splitting paths up causes some problems:
-#
-# var x = obj.x
-# var z = x.y[i].z
-#
-# Alias analysis is affected by this too! A good solution is *type splitting*:
-# T becomes T1 and T2 if it's known that T1 and T2 can't alias.
-#
-# An aliasing problem and a race condition are effectively the same problem.
-# Type based alias analysis is nice but not sufficient; especially splitting
-# an array and filling it in parallel should be supported but is not easily
-# done: It essentially requires a built-in 'indexSplit' operation and dependent
-# typing.
-
 # ------------------------ exception and tag tracking -------------------------
 
 discard """
@@ -207,9 +192,9 @@ proc markGcUnsafe(a: PEffects; reason: PNode) =
       a.owner.gcUnsafetyReason = newSym(skUnknown, getIdent("<unknown>"),
                                         a.owner, reason.info)
 
-proc listGcUnsafety(s: PSym; onlyWarning: bool) =
+proc listGcUnsafety(s: PSym; onlyWarning: bool; cycleCheck: var IntSet) =
   let u = s.gcUnsafetyReason
-  if u != nil:
+  if u != nil and not cycleCheck.containsOrIncl(u.id):
     let msgKind = if onlyWarning: warnGcUnsafe2 else: errGenerated
     if u.kind in {skLet, skVar}:
       message(s.info, msgKind,
@@ -218,7 +203,7 @@ proc listGcUnsafety(s: PSym; onlyWarning: bool) =
     elif u.kind in routineKinds:
       # recursive call *always* produces only a warning so the full error
       # message is printed:
-      listGcUnsafety(u, true)
+      listGcUnsafety(u, true, cycleCheck)
       message(s.info, msgKind,
         "'$#' is not GC-safe as it calls '$#'" %
         [s.name.s, u.name.s])
@@ -227,6 +212,10 @@ proc listGcUnsafety(s: PSym; onlyWarning: bool) =
       message(u.info, msgKind,
         "'$#' is not GC-safe as it performs an indirect call here" % s.name.s)
 
+proc listGcUnsafety(s: PSym; onlyWarning: bool) =
+  var cycleCheck = initIntSet()
+  listGcUnsafety(s, onlyWarning, cycleCheck)
+
 proc useVar(a: PEffects, n: PNode) =
   let s = n.sym
   if isLocalVar(a, s):
@@ -237,9 +226,10 @@ proc useVar(a: PEffects, n: PNode) =
         message(n.info, warnUninit, s.name.s)
       # prevent superfluous warnings about the same variable:
       a.init.add s.id
-  if {sfGlobal, sfThread} * s.flags == {sfGlobal} and s.kind in {skVar, skLet}:
+  if {sfGlobal, sfThread} * s.flags != {} and s.kind in {skVar, skLet}:
     if s.guard != nil: guardGlobal(a, n, s.guard)
-    if (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem):
+    if {sfGlobal, sfThread} * s.flags == {sfGlobal} and
+        (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem):
       #if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n)
       markGcUnsafe(a, s)
 
@@ -433,17 +423,41 @@ proc documentEffect(n, x: PNode, effectType: TSpecialWord, idx: int): PNode =
     result = newNode(nkExprColonExpr, n.info, @[
       newIdentNode(getIdent(specialWords[effectType]), n.info), effects])
 
+proc documentWriteEffect(n: PNode; flag: TSymFlag; pragmaName: string): PNode =
+  let s = n.sons[namePos].sym
+  let params = s.typ.n
+
+  var effects = newNodeI(nkBracket, n.info)
+  for i in 1 ..< params.len:
+    if params[i].kind == nkSym and flag in params[i].sym.flags:
+      effects.add params[i]
+
+  if effects.len > 0:
+    result = newNode(nkExprColonExpr, n.info, @[
+      newIdentNode(getIdent(pragmaName), n.info), effects])
+
+proc documentNewEffect(n: PNode): PNode =
+  let s = n.sons[namePos].sym
+  if tfReturnsNew in s.typ.flags:
+    result = newIdentNode(getIdent("new"), n.info)
+
 proc documentRaises*(n: PNode) =
   if n.sons[namePos].kind != nkSym: return
   let pragmas = n.sons[pragmasPos]
   let p1 = documentEffect(n, pragmas, wRaises, exceptionEffects)
   let p2 = documentEffect(n, pragmas, wTags, tagEffects)
+  let p3 = documentWriteEffect(n, sfWrittenTo, "writes")
+  let p4 = documentNewEffect(n)
+  let p5 = documentWriteEffect(n, sfEscapes, "escapes")
 
-  if p1 != nil or p2 != nil:
+  if p1 != nil or p2 != nil or p3 != nil or p4 != nil or p5 != nil:
     if pragmas.kind == nkEmpty:
       n.sons[pragmasPos] = newNodeI(nkPragma, n.info)
     if p1 != nil: n.sons[pragmasPos].add p1
     if p2 != nil: n.sons[pragmasPos].add p2
+    if p3 != nil: n.sons[pragmasPos].add p3
+    if p4 != nil: n.sons[pragmasPos].add p4
+    if p5 != nil: n.sons[pragmasPos].add p5
 
 template notGcSafe(t): expr = {tfGcSafe, tfNoSideEffect} * t.flags == {}
 
@@ -721,7 +735,7 @@ proc track(tracked: PEffects, n: PNode) =
       # since 'var (a, b): T = ()' is not even allowed, there is always type
       # inference for (a, b) and thus no nil checking is necessary.
   of nkCaseStmt: trackCase(tracked, n)
-  of nkIfStmt, nkIfExpr: trackIf(tracked, n)
+  of nkWhen, nkIfStmt, nkIfExpr: trackIf(tracked, n)
   of nkBlockStmt, nkBlockExpr: trackBlock(tracked, n.sons[1])
   of nkWhileStmt:
     track(tracked, n.sons[0])
@@ -748,7 +762,7 @@ proc track(tracked: PEffects, n: PNode) =
     for i in 1 .. <len(n):
       let x = n.sons[i]
       track(tracked, x)
-      if sfDiscriminant in x.sons[0].sym.flags:
+      if x.sons[0].kind == nkSym and sfDiscriminant in x.sons[0].sym.flags:
         addDiscriminantFact(tracked.guards, x)
     setLen(tracked.guards, oldFacts)
   of nkPragmaBlock:
@@ -841,6 +855,8 @@ proc initEffects(effects: PNode; s: PSym; t: var TEffects) =
   newSeq(effects.sons, effectListLen)
   effects.sons[exceptionEffects] = newNodeI(nkArgList, s.info)
   effects.sons[tagEffects] = newNodeI(nkArgList, s.info)
+  effects.sons[usesEffects] = ast.emptyNode
+  effects.sons[writeEffects] = ast.emptyNode
 
   t.exc = effects.sons[exceptionEffects]
   t.tags = effects.sons[tagEffects]
@@ -895,6 +911,7 @@ proc trackProc*(s: PSym, body: PNode) =
     message(s.info, warnLockLevel,
       "declared lock level is $1, but real lock level is $2" %
         [$s.typ.lockLevel, $t.maxLockLevel])
+  when useWriteTracking: trackWrites(s, body)
 
 proc trackTopLevelStmt*(module: PSym; n: PNode) =
   if n.kind in {nkPragma, nkMacroDef, nkTemplateDef, nkProcDef,

compiler/semstmts.nim

@@ -369,6 +369,15 @@ proc addToVarSection(c: PContext; result: var PNode; orig, identDefs: PNode) =
   else:
     result.add identDefs
 
+proc addDefer(c: PContext; result: var PNode; s: PSym) =
+  let deferDestructorCall = createDestructorCall(c, s)
+  if deferDestructorCall != nil:
+    if result.kind != nkStmtList:
+      let oldResult = result
+      result = newNodeI(nkStmtList, result.info)
+      result.add oldResult
+    result.add deferDestructorCall
+
 proc isDiscardUnderscore(v: PSym): bool =
   if v.name.s == "_":
     v.flags.incl(sfGenSym)
@@ -469,6 +478,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
         if def.kind == nkPar: v.ast = def[j]
         v.typ = tup.sons[j]
         b.sons[j] = newSymNode(v)
+      addDefer(c, result, v)
       checkNilable(v)
       if sfCompileTime in v.flags: hasCompileTime = true
   if hasCompileTime: vm.setupCompileTimeVar(c.module, result)
@@ -744,10 +754,59 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
       if s.typ.kind in {tyObject, tyTuple} and not s.typ.n.isNil:
         checkForMetaFields(s.typ.n)
 
+proc semAllTypeSections(c: PContext; n: PNode): PNode =
+  proc gatherStmts(c: PContext; n: PNode; result: PNode) {.nimcall.} =
+    case n.kind
+    of nkIncludeStmt:
+      for i in 0..<n.len:
+        var f = checkModuleName(n.sons[i])
+        if f != InvalidFileIDX:
+          if containsOrIncl(c.includedFiles, f):
+            localError(n.info, errRecursiveDependencyX, f.toFilename)
+          else:
+            let code = gIncludeFile(c.module, f)
+            gatherStmts c, code, result
+            excl(c.includedFiles, f)
+    of nkStmtList:
+      for i in 0 ..< n.len:
+        gatherStmts(c, n.sons[i], result)
+    of nkTypeSection:
+      incl n.flags, nfSem
+      typeSectionLeftSidePass(c, n)
+      result.add n
+    else:
+      result.add n
+
+  result = newNodeI(nkStmtList, n.info)
+  gatherStmts(c, n, result)
+
+  template rec(name) =
+    for i in 0 ..< result.len:
+      if result[i].kind == nkTypeSection:
+        name(c, result[i])
+
+  rec typeSectionRightSidePass
+  rec typeSectionFinalPass
+  when false:
+    # too beautiful to delete:
+    template rec(name; setbit=false) =
+      proc `name rec`(c: PContext; n: PNode) {.nimcall.} =
+        if n.kind == nkTypeSection:
+          when setbit: incl n.flags, nfSem
+          name(c, n)
+        elif n.kind == nkStmtList:
+          for i in 0 ..< n.len:
+            `name rec`(c, n.sons[i])
+      `name rec`(c, n)
+    rec typeSectionLeftSidePass, true
+    rec typeSectionRightSidePass
+    rec typeSectionFinalPass
+
 proc semTypeSection(c: PContext, n: PNode): PNode =
   ## Processes a type section. This must be done in separate passes, in order
   ## to allow the type definitions in the section to reference each other
   ## without regard for the order of their definitions.
+  if sfNoForward notin c.module.flags or nfSem notin n.flags:
     typeSectionLeftSidePass(c, n)
     typeSectionRightSidePass(c, n)
     typeSectionFinalPass(c, n)
@@ -898,27 +957,35 @@ proc semDo(c: PContext, n: PNode, flags: TExprFlags): PNode =
 proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode =
   var n = n
 
-  n = replaceTypesInBody(c, pt, n)
+  let original = n.sons[namePos].sym
+  let s = copySym(original, false)
+  incl(s.flags, sfFromGeneric)
+
+  n = replaceTypesInBody(c, pt, n, original)
   result = n
-
+  s.ast = result
+  n.sons[namePos].sym = s
   n.sons[genericParamsPos] = emptyNode
-  n.sons[paramsPos] = n.typ.n
+  let params = n.typ.n
-
+  n.sons[paramsPos] = params
+  s.typ = n.typ
+  for i in 1..<params.len:
+    if params[i].typ.kind in {tyTypeDesc, tyGenericParam,
+                              tyFromExpr, tyFieldAccessor}+tyTypeClasses:
+      localError(params[i].info, "cannot infer type of parameter: " &
+                 params[i].sym.name.s)
   openScope(c)
-  var s = n.sons[namePos].sym
   pushOwner(s)
-  addParams(c, n.typ.n, skProc)
+  addParams(c, params, skProc)
   pushProcCon(c, s)
   addResult(c, n.typ.sons[0], n.info, skProc)
   addResultNode(c, n)
   let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
-  n.sons[bodyPos] = transformBody(c.module, semBody, n.sons[namePos].sym)
+  n.sons[bodyPos] = transformBody(c.module, semBody, s)
   popProcCon(c)
   popOwner()
   closeScope(c)
 
-  s.ast = result
-
   # alternative variant (not quite working):
   # var prc = arg[0].sym
   # let inferred = c.semGenerateInstance(c, prc, m.bindings, arg.info)
@@ -974,6 +1041,7 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
                  "signature for 'deepCopy' must be proc[T: ptr|ref](x: T): T")
     incl(s.flags, sfUsed)
   of "=":
+    if s.magic == mAsgn: return
     incl(s.flags, sfUsed)
     let t = s.typ
     if t.len == 3 and t.sons[0] == nil and t.sons[1].kind == tyVar:
@@ -1033,7 +1101,8 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     n.sons[namePos] = newSymNode(s)
     s.ast = n
     #s.scope = c.currentScope
-
+    when false:
+      # disable for now
       if sfNoForward in c.module.flags and
          sfSystemModule notin c.module.flags:
         addInterfaceOverloadableSymAt(c, c.currentScope, s)
@@ -1071,6 +1140,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
         # semParamList(c, n.sons[ParamsPos], nil, s)
   else:
     s.typ = newProcType(c, n.info)
+  if tfTriggersCompileTime in s.typ.flags: incl(s.flags, sfCompileTime)
   if n.sons[patternPos].kind != nkEmpty:
     n.sons[patternPos] = semPattern(c, n.sons[patternPos])
   if s.kind in skIterators:
@@ -1160,6 +1230,9 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
   if n.sons[patternPos].kind != nkEmpty:
     c.patterns.add(s)
   if isAnon: result.typ = s.typ
+  if isTopLevel(c) and s.kind != skClosureIterator and
+      s.typ.callConv == ccClosure:
+    message(s.info, warnDeprecated, "top level '.closure' calling convention")
 
 proc determineType(c: PContext, s: PSym) =
   if s.typ != nil: return
@@ -1268,6 +1341,8 @@ proc semPragmaBlock(c: PContext, n: PNode): PNode =
     of wLocks:
       result = n
       result.typ = n.sons[1].typ
+    of wNoRewrite:
+      incl(result.flags, nfNoRewrite)
     else: discard
 
 proc semStaticStmt(c: PContext, n: PNode): PNode =
@@ -1319,7 +1394,7 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
   for i in countup(0, length - 1):
     let k = n.sons[i].kind
     case k
-    of nkFinally, nkExceptBranch, nkDefer:
+    of nkFinally, nkExceptBranch:
       # stand-alone finally and except blocks are
       # transformed into regular try blocks:
       #
@@ -1360,7 +1435,7 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
             localError(result.info, "type class predicate failed")
         of tyUnknown: continue
         else: discard
-      if n.sons[i].typ == enforceVoidContext or usesResult(n.sons[i]):
+      if n.sons[i].typ == enforceVoidContext: #or usesResult(n.sons[i]):
         voidContext = true
         n.typ = enforceVoidContext
       if i == last and (length == 1 or efWantValue in flags):
@@ -1372,21 +1447,13 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
         n.typ = n.sons[i].typ
         if not isEmptyType(n.typ): n.kind = nkStmtListExpr
       case n.sons[i].kind
-      of nkVarSection, nkLetSection:
-        let (outer, inner) = insertDestructors(c, n.sons[i])
-        if outer != nil:
-          n.sons[i] = outer
-          var rest = newNode(nkStmtList, n.info, n.sons[i+1 .. length-1])
-          inner.addSon(semStmtList(c, rest, flags))
-          n.sons.setLen(i+1)
-          return
       of LastBlockStmts:
         for j in countup(i + 1, length - 1):
           case n.sons[j].kind
           of nkPragma, nkCommentStmt, nkNilLit, nkEmpty: discard
           else: localError(n.sons[j].info, errStmtInvalidAfterReturn)
       else: discard
-  if result.len == 1:
+  if result.len == 1 and result.sons[0].kind != nkDefer:
     result = result.sons[0]
   when defined(nimfix):
     if result.kind == nkCommentStmt and not result.comment.isNil and

compiler/semtempl.nim

@@ -110,6 +110,13 @@ type
     toBind, toMixin, toInject: IntSet
     owner: PSym
     cursorInBody: bool # only for nimsuggest
+    bracketExpr: PNode
+
+template withBracketExpr(ctx, x, body: untyped) =
+  let old = ctx.bracketExpr
+  ctx.bracketExpr = x
+  body
+  ctx.bracketExpr = old
 
 proc getIdentNode(c: var TemplCtx, n: PNode): PNode =
   case n.kind
@@ -184,6 +191,21 @@ proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) =
   else:
     let ident = getIdentNode(c, n)
     if not isTemplParam(c, ident):
+      # fix #2670, consider:
+      #
+      # when b:
+      #    var a = "hi"
+      # else:
+      #    var a = 5
+      # echo a
+      #
+      # We need to ensure that both 'a' produce the same gensym'ed symbol.
+      # So we need only check the *current* scope.
+      let s = localSearchInScope(c.c, considerQuotedIdent ident)
+      if s != nil and s.owner == c.owner and sfGenSym in s.flags:
+        styleCheckUse(n.info, s)
+        replaceIdentBySym(n, newSymNode(s, n.info))
+      else:
         let local = newGenSym(k, ident, c)
         addPrelimDecl(c.c, local)
         styleCheckDef(n.info, local)
@@ -260,6 +282,53 @@ proc semTemplSomeDecl(c: var TemplCtx, n: PNode, symKind: TSymKind; start=0) =
       addLocalDecl(c, a.sons[j], symKind)
 
 proc semPattern(c: PContext, n: PNode): PNode
+
+proc semTemplBodySons(c: var TemplCtx, n: PNode): PNode =
+  result = n
+  for i in 0.. < n.len:
+    result.sons[i] = semTemplBody(c, n.sons[i])
+
+proc wrapInBind(c: var TemplCtx; n: PNode; opr: string): PNode =
+  let ident = getIdent(opr)
+  if ident.id in c.toInject: return n
+
+  let s = searchInScopes(c.c, ident)
+  if s != nil:
+    var callee: PNode
+    if contains(c.toBind, s.id):
+      callee = symChoice(c.c, n, s, scClosed)
+    elif contains(c.toMixin, s.name.id):
+      callee = symChoice(c.c, n, s, scForceOpen)
+    elif s.owner == c.owner and sfGenSym in s.flags:
+      # template tmp[T](x: var seq[T]) =
+      # var yz: T
+      incl(s.flags, sfUsed)
+      callee = newSymNode(s, n.info)
+      styleCheckUse(n.info, s)
+    else:
+      callee = semTemplSymbol(c.c, n, s)
+
+    let call = newNodeI(nkCall, n.info)
+    call.add(callee)
+    for i in 0 .. n.len-1: call.add(n[i])
+    result = newNodeI(nkBind, n.info, 2)
+    result.sons[0] = n
+    result.sons[1] = call
+  else:
+    result = n
+
+proc oprIsRoof(n: PNode): bool =
+  const roof = "^"
+  case n.kind
+  of nkIdent: result = n.ident.s == roof
+  of nkSym: result = n.sym.name.s == roof
+  of nkAccQuoted:
+    if n.len == 1:
+      result = oprIsRoof(n.sons[0])
+  of nkOpenSymChoice, nkClosedSymChoice:
+    result = oprIsRoof(n.sons[0])
+  else: discard
+
 proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
   result = n
   semIdeForTemplateOrGenericCheck(n, c.cursorInBody)
@@ -402,10 +471,48 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
     result.sons[1] = semTemplBody(c, n.sons[1])
   of nkPragma:
     result = onlyReplaceParams(c, n)
+  of nkBracketExpr:
+    result = newNodeI(nkCall, n.info)
+    result.add newIdentNode(getIdent("[]"), n.info)
+    for i in 0 ..< n.len: result.add(n[i])
+    let n0 = semTemplBody(c, n.sons[0])
+    withBracketExpr c, n0:
+      result = semTemplBodySons(c, result)
+  of nkCurlyExpr:
+    result = newNodeI(nkCall, n.info)
+    result.add newIdentNode(getIdent("{}"), n.info)
+    for i in 0 ..< n.len: result.add(n[i])
+    result = semTemplBodySons(c, result)
+  of nkAsgn, nkFastAsgn:
+    checkSonsLen(n, 2)
+    let a = n.sons[0]
+    let b = n.sons[1]
+
+    let k = a.kind
+    case k
+    of nkBracketExpr:
+      result = newNodeI(nkCall, n.info)
+      result.add newIdentNode(getIdent("[]="), n.info)
+      for i in 0 ..< a.len: result.add(a[i])
+      result.add(b)
+      let a0 = semTemplBody(c, a.sons[0])
+      withBracketExpr c, a0:
+        result = semTemplBodySons(c, result)
+    of nkCurlyExpr:
+      result = newNodeI(nkCall, n.info)
+      result.add newIdentNode(getIdent("{}="), n.info)
+      for i in 0 ..< a.len: result.add(a[i])
+      result.add(b)
+      result = semTemplBodySons(c, result)
     else:
+      result = semTemplBodySons(c, n)
+  of nkCallKinds-{nkPostfix}:
+    result = semTemplBodySons(c, n)
+    if c.bracketExpr != nil and n.len == 2 and oprIsRoof(n.sons[0]):
+      result.add c.bracketExpr
+  of nkDotExpr, nkAccQuoted:
     # dotExpr is ambiguous: note that we explicitly allow 'x.TemplateParam',
     # so we use the generic code for nkDotExpr too
-    if n.kind == nkDotExpr or n.kind == nkAccQuoted:
     let s = qualifiedLookUp(c.c, n, {})
     if s != nil:
       # do not symchoice a quoted template parameter (bug #2390):
@@ -420,9 +527,9 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
         return symChoice(c.c, n, s, scForceOpen)
       else:
         return symChoice(c.c, n, s, scOpen)
-    result = n
+    result = semTemplBodySons(c, n)
-    for i in countup(0, sonsLen(n) - 1):
+  else:
-      result.sons[i] = semTemplBody(c, n.sons[i])
+    result = semTemplBodySons(c, n)
 
 proc semTemplBodyDirty(c: var TemplCtx, n: PNode): PNode =
   result = n
@@ -452,30 +559,9 @@ proc semTemplBodyDirty(c: var TemplCtx, n: PNode): PNode =
     for i in countup(0, sonsLen(n) - 1):
       result.sons[i] = semTemplBodyDirty(c, n.sons[i])
 
-proc transformToExpr(n: PNode): PNode =
-  var realStmt: int
-  result = n
-  case n.kind
-  of nkStmtList:
-    realStmt = - 1
-    for i in countup(0, sonsLen(n) - 1):
-      case n.sons[i].kind
-      of nkCommentStmt, nkEmpty, nkNilLit:
-        discard
-      else:
-        if realStmt == - 1: realStmt = i
-        else: realStmt = - 2
-    if realStmt >= 0: result = transformToExpr(n.sons[realStmt])
-    else: n.kind = nkStmtListExpr
-  of nkBlockStmt:
-    n.kind = nkBlockExpr
-    #nkIfStmt: n.kind = nkIfExpr // this is not correct!
-  else:
-    discard
-
 proc semTemplateDef(c: PContext, n: PNode): PNode =
   var s: PSym
-  if c.p.owner.kind == skModule:
+  if isTopLevel(c):
     s = semIdentVis(c, skTemplate, n.sons[0], {sfExported})
     incl(s.flags, sfGlobal)
   else:
@@ -530,8 +616,6 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
     n.sons[bodyPos] = semTemplBodyDirty(ctx, n.sons[bodyPos])
   else:
     n.sons[bodyPos] = semTemplBody(ctx, n.sons[bodyPos])
-  if s.typ.sons[0].kind notin {tyStmt, tyTypeDesc}:
-    n.sons[bodyPos] = transformToExpr(n.sons[bodyPos])
   # only parameters are resolved, no type checking is performed
   semIdeForTemplateOrGeneric(c, n.sons[bodyPos], ctx.cursorInBody)
   closeScope(c)
@@ -585,6 +669,11 @@ proc semPatternBody(c: var TemplCtx, n: PNode): PNode =
       localError(n.info, errInvalidExpression)
       result = n
 
+  proc stupidStmtListExpr(n: PNode): bool =
+    for i in 0 .. n.len-2:
+      if n[i].kind notin {nkEmpty, nkCommentStmt}: return false
+    result = true
+
   result = n
   case n.kind
   of nkIdent:
@@ -610,6 +699,12 @@ proc semPatternBody(c: var TemplCtx, n: PNode): PNode =
         localError(n.info, errInvalidExpression)
     else:
       localError(n.info, errInvalidExpression)
+  of nkStmtList, nkStmtListExpr:
+    if stupidStmtListExpr(n):
+      result = semPatternBody(c, n.lastSon)
+    else:
+      for i in countup(0, sonsLen(n) - 1):
+        result.sons[i] = semPatternBody(c, n.sons[i])
   of nkCallKinds:
     let s = qualifiedLookUp(c.c, n.sons[0], {})
     if s != nil:

compiler/semtypes.nim

@@ -23,6 +23,9 @@ proc newConstraint(c: PContext, k: TTypeKind): PType =
 
 proc semEnum(c: PContext, n: PNode, prev: PType): PType =
   if n.sonsLen == 0: return newConstraint(c, tyEnum)
+  elif n.sonsLen == 1:
+    # don't create an empty tyEnum; fixes #3052
+    return errorType(c)
   var
     counter, x: BiggestInt
     e: PSym
@@ -130,6 +133,7 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType =
     let isCall = ord(n.kind in nkCallKinds)
     let n = if n[0].kind == nkBracket: n[0] else: n
     checkMinSonsLen(n, 1)
+    var base = semTypeNode(c, n.lastSon, nil)
     result = newOrPrevType(kind, prev, c)
     # check every except the last is an object:
     for i in isCall .. n.len-2:
@@ -137,7 +141,6 @@ proc semAnyRef(c: PContext; n: PNode; kind: TTypeKind; prev: PType): PType =
       if region.skipTypes({tyGenericInst}).kind notin {tyError, tyObject}:
         message n[i].info, errGenerated, "region needs to be an object type"
       addSonSkipIntLit(result, region)
-    var base = semTypeNode(c, n.lastSon, nil)
     addSonSkipIntLit(result, base)
 
 proc semVarType(c: PContext, n: PNode, prev: PType): PType =
@@ -251,19 +254,21 @@ proc semArrayIndex(c: PContext, n: PNode): PType =
 
 proc semArray(c: PContext, n: PNode, prev: PType): PType =
   var base: PType
-  result = newOrPrevType(tyArray, prev, c)
   if sonsLen(n) == 3:
     # 3 = length(array indx base)
-    var indx = semArrayIndex(c, n[1])
+    let indx = semArrayIndex(c, n[1])
-    addSonSkipIntLit(result, indx)
+    var indxB = indx
-    if indx.kind == tyGenericInst: indx = lastSon(indx)
+    if indxB.kind == tyGenericInst: indxB = lastSon(indxB)
-    if indx.kind notin {tyGenericParam, tyStatic, tyFromExpr}:
+    if indxB.kind notin {tyGenericParam, tyStatic, tyFromExpr}:
-      if not isOrdinalType(indx):
+      if not isOrdinalType(indxB):
         localError(n.sons[1].info, errOrdinalTypeExpected)
-      elif enumHasHoles(indx):
+      elif enumHasHoles(indxB):
         localError(n.sons[1].info, errEnumXHasHoles,
-                   typeToString(indx.skipTypes({tyRange})))
+                   typeToString(indxB.skipTypes({tyRange})))
     base = semTypeNode(c, n.sons[2], nil)
+    # ensure we only construct a tyArray when there was no error (bug #3048):
+    result = newOrPrevType(tyArray, prev, c)
+    addSonSkipIntLit(result, indx)
     addSonSkipIntLit(result, base)
   else:
     localError(n.info, errArrayExpectsTwoTypeParams)
@@ -503,8 +508,9 @@ proc semRecordCase(c: PContext, n: PNode, check: var IntSet, pos: var int,
   var typ = skipTypes(a.sons[0].typ, abstractVar-{tyTypeDesc})
   if not isOrdinalType(typ):
     localError(n.info, errSelectorMustBeOrdinal)
-  elif firstOrd(typ) < 0:
+  elif firstOrd(typ) != 0:
-    localError(n.info, errOrdXMustNotBeNegative, a.sons[0].sym.name.s)
+    localError(n.info, errGenerated, "low(" & $a.sons[0].sym.name.s &
+                                     ") must be 0 for discriminant")
   elif lengthOrd(typ) > 0x00007FFF:
     localError(n.info, errLenXinvalid, a.sons[0].sym.name.s)
   var chckCovered = true
@@ -629,7 +635,7 @@ proc skipGenericInvocation(t: PType): PType {.inline.} =
   result = t
   if result.kind == tyGenericInvocation:
     result = result.sons[0]
-  while result.kind in {tyGenericInst, tyGenericBody}:
+  while result.kind in {tyGenericInst, tyGenericBody, tyRef, tyPtr}:
     result = lastSon(result)
 
 proc addInheritedFields(c: PContext, check: var IntSet, pos: var int,
@@ -651,7 +657,7 @@ proc semObjectNode(c: PContext, n: PNode, prev: PType): PType =
     if base.isNil:
       localError(n.info, errIllegalRecursionInTypeX, "object")
     else:
-      var concreteBase = skipGenericInvocation(base).skipTypes(skipPtrs)
+      var concreteBase = skipGenericInvocation(base)
       if concreteBase.kind == tyObject and tfFinal notin concreteBase.flags:
         addInheritedFields(c, check, pos, concreteBase)
       else:
@@ -712,12 +718,12 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
   if paramType == nil: return # (e.g. proc return type)
 
   proc addImplicitGenericImpl(typeClass: PType, typId: PIdent): PType =
-    let finalTypId = if typId != nil: typId
-                     else: getIdent(paramName & ":type")
     if genericParams == nil:
       # This happens with anonymous proc types appearing in signatures
       # XXX: we need to lift these earlier
       return
+    let finalTypId = if typId != nil: typId
+                     else: getIdent(paramName & ":type")
     # is this a bindOnce type class already present in the param list?
     for i in countup(0, genericParams.len - 1):
       if genericParams.sons[i].sym.name.id == finalTypId.id:
@@ -751,7 +757,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
 
   case paramType.kind:
   of tyAnything:
-    result = addImplicitGeneric(newTypeS(tyGenericParam, c))
+    result = addImplicitGenericImpl(newTypeS(tyGenericParam, c), nil)
 
   of tyStatic:
     # proc(a: expr{string}, b: expr{nkLambda})
@@ -862,6 +868,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
   of tyExpr:
     if procKind notin {skMacro, skTemplate}:
       result = addImplicitGeneric(newTypeS(tyAnything, c))
+      #result = addImplicitGenericImpl(newTypeS(tyGenericParam, c), nil)
 
   of tyGenericParam:
     markUsed(info, paramType.sym)
@@ -964,17 +971,25 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
   elif kind == skIterator:
     # XXX This is special magic we should likely get rid of
     r = newTypeS(tyExpr, c)
+    message(n.info, warnDeprecated, "implicit return type for 'iterator'")
 
   if r != nil:
     # turn explicit 'void' return type into 'nil' because the rest of the
     # compiler only checks for 'nil':
     if skipTypes(r, {tyGenericInst}).kind != tyEmpty:
       # 'auto' as a return type does not imply a generic:
-      if r.kind != tyExpr:
+      if r.kind == tyAnything:
+        # 'p(): auto' and 'p(): expr' are equivalent, but the rest of the
+        # compiler is hardly aware of 'auto':
+        r = newTypeS(tyExpr, c)
+      elif r.kind != tyExpr:
         if r.sym == nil or sfAnon notin r.sym.flags:
           let lifted = liftParamType(c, kind, genericParams, r, "result",
                                      n.sons[0].info)
-          if lifted != nil: r = lifted
+          if lifted != nil:
+            r = lifted
+            #if r.kind != tyGenericParam:
+            #echo "came here for ", typeToString(r)
             r.flags.incl tfRetType
         r = skipIntLit(r)
         if kind == skIterator:
@@ -1049,6 +1064,7 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType =
     return newOrPrevType(tyError, prev, c)
   else:
     var m = newCandidate(c, t)
+    m.isNoCall = true
     matches(c, n, copyTree(n), m)
 
     if m.state != csMatch and not m.typedescMatched:
@@ -1085,6 +1101,7 @@ proc semTypeExpr(c: PContext, n: PNode): PType =
     result = n.typ.base
   else:
     localError(n.info, errTypeExpected, n.renderTree)
+    result = errorType(c)
 
 proc freshType(res, prev: PType): PType {.inline.} =
   if prev.isNil:
@@ -1140,7 +1157,17 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     else:
       result = semAnonTuple(c, n, prev)
   of nkCallKinds:
-    if isRange(n):
+    let x = n[0]
+    let ident = case x.kind
+                of nkIdent: x.ident
+                of nkSym: x.sym.name
+                of nkClosedSymChoice, nkOpenSymChoice: x[0].sym.name
+                else: nil
+    if ident != nil and ident.s == "[]":
+      let b = newNodeI(nkBracketExpr, n.info)
+      for i in 1..<n.len: b.add(n[i])
+      result = semTypeNode(c, b, prev)
+    elif ident != nil and ident.id == ord(wDotDot):
       result = semRangeAux(c, n, prev)
     elif n[0].kind notin nkIdentKinds:
       result = semTypeExpr(c, n)
@@ -1326,12 +1353,23 @@ proc processMagicType(c: PContext, m: PSym) =
     rawAddSon(m.typ, newTypeS(tyEmpty, c))
   of mIntSetBaseType: setMagicType(m, tyRange, intSize)
   of mNil: setMagicType(m, tyNil, ptrSize)
-  of mExpr: setMagicType(m, tyExpr, 0)
+  of mExpr:
-  of mStmt: setMagicType(m, tyStmt, 0)
+    if m.name.s == "auto":
+      setMagicType(m, tyAnything, 0)
+    else:
+      setMagicType(m, tyExpr, 0)
+      if m.name.s == "expr": m.typ.flags.incl tfOldSchoolExprStmt
+  of mStmt:
+    setMagicType(m, tyStmt, 0)
+    if m.name.s == "stmt": m.typ.flags.incl tfOldSchoolExprStmt
   of mTypeDesc:
     setMagicType(m, tyTypeDesc, 0)
     rawAddSon(m.typ, newTypeS(tyNone, c))
-  of mVoidType: setMagicType(m, tyEmpty, 0)
+  of mVoidType:
+    setMagicType(m, tyEmpty, 0)
+    # for historical reasons we conflate 'void' with 'empty' so that '@[]'
+    # has the type 'seq[void]'.
+    m.typ.flags.incl tfVoid
   of mArray:
     setMagicType(m, tyArray, 0)
   of mOpenArray:
@@ -1348,7 +1386,8 @@ proc processMagicType(c: PContext, m: PSym) =
   of mOrdinal:
     setMagicType(m, tyOrdinal, 0)
     rawAddSon(m.typ, newTypeS(tyNone, c))
-  of mPNimrodNode: discard
+  of mPNimrodNode:
+    incl m.typ.flags, tfTriggersCompileTime
   of mShared:
     setMagicType(m, tyObject, 0)
     m.typ.n = newNodeI(nkRecList, m.info)

compiler/semtypinst.nim

@@ -75,8 +75,12 @@ proc searchInstTypes*(key: PType): PType =
 proc cacheTypeInst*(inst: PType) =
   # XXX: add to module's generics
   #      update the refcount
-  let genericTyp = inst.sons[0]
+  let gt = inst.sons[0]
-  genericTyp.sym.typeInstCache.safeAdd(inst)
+  let t = if gt.kind == tyGenericBody: gt.lastSon else: gt
+  if t.kind in {tyStatic, tyGenericParam, tyIter} + tyTypeClasses:
+    return
+  gt.sym.typeInstCache.safeAdd(inst)
+
 
 type
   TReplTypeVars* {.final.} = object
@@ -90,6 +94,8 @@ type
     allowMetaTypes*: bool     # allow types such as seq[Number]
                               # i.e. the result contains unresolved generics
     skipTypedesc*: bool       # wether we should skip typeDescs
+    owner*: PSym              # where this instantiation comes from
+    recursionLimit: int
 
 proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType
 proc replaceTypeVarsS(cl: var TReplTypeVars, s: PSym): PSym
@@ -207,6 +213,9 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode): PNode =
 
 proc replaceTypeVarsS(cl: var TReplTypeVars, s: PSym): PSym =
   if s == nil: return nil
+  # symbol is not our business:
+  if cl.owner != nil and s.owner != cl.owner:
+    return s
   result = PSym(idTableGet(cl.symMap, s))
   if result == nil:
     result = copySym(s, false)
@@ -365,6 +374,19 @@ proc propagateFieldFlags(t: PType, n: PNode) =
   else: discard
 
 proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
+  template bailout =
+    if cl.recursionLimit > 100:
+      # bail out, see bug #2509. But note this caching is in general wrong,
+      # look at this example where TwoVectors should not share the generic
+      # instantiations (bug #3112):
+
+      # type
+      #   Vector[N: static[int]] = array[N, float64]
+      #   TwoVectors[Na, Nb: static[int]] = (Vector[Na], Vector[Nb])
+      result = PType(idTableGet(cl.localCache, t))
+      if result != nil: return result
+    inc cl.recursionLimit
+
   result = t
   if t == nil: return
 
@@ -420,8 +442,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
     result = t
 
   of tyGenericInst:
-    result = PType(idTableGet(cl.localCache, t))
+    bailout()
-    if result != nil: return result
     result = instCopyType(cl, t)
     idTablePut(cl.localCache, t, result)
     for i in 1 .. <result.sonsLen:
@@ -431,8 +452,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
   else:
     if containsGenericType(t):
       #if not cl.allowMetaTypes:
-      result = PType(idTableGet(cl.localCache, t))
+      bailout()
-      if result != nil: return result
       result = instCopyType(cl, t)
       result.size = -1 # needs to be recomputed
       #if not cl.allowMetaTypes:
@@ -440,8 +460,14 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
 
       for i in countup(0, sonsLen(result) - 1):
         if result.sons[i] != nil:
-          result.sons[i] = replaceTypeVarsT(cl, result.sons[i])
+          var r = replaceTypeVarsT(cl, result.sons[i])
-          propagateToOwner(result, result.sons[i])
+          if result.kind == tyObject:
+            # carefully coded to not skip the precious tyGenericInst:
+            let r2 = r.skipTypes({tyGenericInst})
+            if r2.kind in {tyPtr, tyRef}:
+              r = skipTypes(r2, {tyPtr, tyRef})
+          result.sons[i] = r
+          propagateToOwner(result, r)
 
       result.n = replaceTypeVarsN(cl, result.n)
 
@@ -459,22 +485,33 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
 
       else: discard
 
-proc initTypeVars*(p: PContext, pt: TIdTable, info: TLineInfo): TReplTypeVars =
+proc initTypeVars*(p: PContext, pt: TIdTable, info: TLineInfo;
+                   owner: PSym): TReplTypeVars =
   initIdTable(result.symMap)
   copyIdTable(result.typeMap, pt)
   initIdTable(result.localCache)
   result.info = info
   result.c = p
+  result.owner = owner
 
-proc replaceTypesInBody*(p: PContext, pt: TIdTable, n: PNode): PNode =
+proc replaceTypesInBody*(p: PContext, pt: TIdTable, n: PNode;
-  var cl = initTypeVars(p, pt, n.info)
+                         owner: PSym): PNode =
+  var cl = initTypeVars(p, pt, n.info, owner)
+  pushInfoContext(n.info)
+  result = replaceTypeVarsN(cl, n)
+  popInfoContext()
+
+proc replaceTypesForLambda*(p: PContext, pt: TIdTable, n: PNode;
+                            original, new: PSym): PNode =
+  var cl = initTypeVars(p, pt, n.info, original)
+  idTablePut(cl.symMap, original, new)
   pushInfoContext(n.info)
   result = replaceTypeVarsN(cl, n)
   popInfoContext()
 
 proc generateTypeInstance*(p: PContext, pt: TIdTable, info: TLineInfo,
                            t: PType): PType =
-  var cl = initTypeVars(p, pt, info)
+  var cl = initTypeVars(p, pt, info, nil)
   pushInfoContext(info)
   result = replaceTypeVarsT(cl, t)
   popInfoContext()

compiler/service.nim

@@ -18,7 +18,7 @@ when useCaas:
 
 # We cache modules and the dependency graph. However, we don't check for
 # file changes but expect the client to tell us about them, otherwise the
-# repeated CRC calculations may turn out to be too slow.
+# repeated hash calculations may turn out to be too slow.
 
 var
   curCaasCmd* = ""

compiler/sigmatch.nim

@@ -47,6 +47,7 @@ type
     coerceDistincts*: bool   # this is an explicit coercion that can strip away
                              # a distrinct type
     typedescMatched*: bool
+    isNoCall*: bool          # misused for generic type instantiations C[T]
     inheritancePenalty: int  # to prefer closest father object type
     errors*: CandidateErrors # additional clarifications to be displayed to the
                              # user if overload resolution fails
@@ -57,6 +58,10 @@ type
     isSubtype,
     isSubrange,              # subrange of the wanted type; no type conversion
                              # but apart from that counts as ``isSubtype``
+    isBothMetaConvertible    # generic proc parameter was matched against
+                             # generic type, e.g., map(mySeq, x=>x+1),
+                             # maybe recoverable by rerun if the parameter is
+                             # the proc's return value
     isInferred,              # generic proc was matched against a concrete type
     isInferredConvertible,   # same as above, but requiring proc CC conversion
     isGeneric,
@@ -158,11 +163,15 @@ proc sumGeneric(t: PType): int =
       t = t.sons[0]
       inc result
       inc isvar
+    of tyTypeDesc:
+      t = t.lastSon
+      if t.kind == tyEmpty: break
+      inc result
     of tyGenericInvocation, tyTuple:
       result += ord(t.kind == tyGenericInvocation)
       for i in 0 .. <t.len: result += t.sons[i].sumGeneric
       break
-    of tyGenericParam, tyExpr, tyStatic, tyStmt, tyTypeDesc: break
+    of tyGenericParam, tyExpr, tyStatic, tyStmt: break
     of tyBool, tyChar, tyEnum, tyObject, tyProc, tyPointer,
         tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128,
         tyUInt..tyUInt64:
@@ -218,12 +227,12 @@ proc cmpCandidates*(a, b: TCandidate): int =
   result = complexDisambiguation(a.callee, b.callee)
 
 proc writeMatches*(c: TCandidate) =
-  writeln(stdout, "exact matches: " & $c.exactMatches)
+  writeLine(stdout, "exact matches: " & $c.exactMatches)
-  writeln(stdout, "generic matches: " & $c.genericMatches)
+  writeLine(stdout, "generic matches: " & $c.genericMatches)
-  writeln(stdout, "subtype matches: " & $c.subtypeMatches)
+  writeLine(stdout, "subtype matches: " & $c.subtypeMatches)
-  writeln(stdout, "intconv matches: " & $c.intConvMatches)
+  writeLine(stdout, "intconv matches: " & $c.intConvMatches)
-  writeln(stdout, "conv matches: " & $c.convMatches)
+  writeLine(stdout, "conv matches: " & $c.convMatches)
-  writeln(stdout, "inheritance: " & $c.inheritancePenalty)
+  writeLine(stdout, "inheritance: " & $c.inheritancePenalty)
 
 proc argTypeToString(arg: PNode; prefer: TPreferedDesc): string =
   if arg.kind in nkSymChoices:
@@ -264,6 +273,9 @@ proc concreteType(c: TCandidate, t: PType): PType =
     addSonSkipIntLit(result, t.sons[1]) # XXX: semantic checking for the type?
   of tyNil:
     result = nil              # what should it be?
+  of tyTypeDesc:
+    if c.isNoCall: result = t
+    else: result = nil
   of tySequence, tySet:
     if t.sons[0].kind == tyEmpty: result = nil
     else: result = t
@@ -381,8 +393,29 @@ proc inconsistentVarTypes(f, a: PType): bool {.inline.} =
   result = f.kind != a.kind and (f.kind == tyVar or a.kind == tyVar)
 
 proc procParamTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
-  var f = f
+  ## For example we have:
+  ## .. code-block:: nim
+  ##   proc myMap[T,S](sIn: seq[T], f: proc(x: T): S): seq[S] = ...
+  ##   proc innerProc[Q,W](q: Q): W = ...
+  ## And we want to match: myMap(@[1,2,3], innerProc)
+  ## This proc (procParamTypeRel) will do the following steps in
+  ## three different calls:
+  ## - matches f=T to a=Q. Since f is metatype, we resolve it
+  ##    to int (which is already known at this point). So in this case
+  ##    Q=int mapping will be saved to c.bindings.
+  ## - matches f=S to a=W. Both of these metatypes are unknown, so we
+  ##    return with isBothMetaConvertible to ask for rerun.
+  ## - matches f=S to a=W. At this point the return type of innerProc
+  ##    is known (we get it from c.bindings). We can use that value
+  ##    to match with f, and save back to c.bindings.
+  var
+    f = f
+    a = a
 
+  if a.isMetaType:
+    let aResolved = PType(idTableGet(c.bindings, a))
+    if aResolved != nil:
+      a = aResolved
   if a.isMetaType:
     if f.isMetaType:
       # We are matching a generic proc (as proc param)
@@ -393,12 +426,15 @@ proc procParamTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
       f = generateTypeInstance(c.c, c.bindings, c.call.info, f)
       if f == nil or f.isMetaType:
         # no luck resolving the type, so the inference fails
-        return isNone
+        return isBothMetaConvertible
+    # Note that this typeRel call will save a's resolved type into c.bindings
     let reverseRel = typeRel(c, a, f)
     if reverseRel >= isGeneric:
       result = isInferred
       #inc c.genericMatches
   else:
+    # Note that this typeRel call will save f's resolved type into c.bindings
+    # if f is metatype.
     result = typeRel(c, f, a)
 
   if result <= isSubtype or inconsistentVarTypes(f, a):
@@ -442,8 +478,9 @@ proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
     elif f.callConv != a.callConv:
       # valid to pass a 'nimcall' thingie to 'closure':
       if f.callConv == ccClosure and a.callConv == ccDefault:
-        result = if result != isInferred: isConvertible
+        result = if result == isInferred: isInferredConvertible
-                 else: isInferredConvertible
+                 elif result == isBothMetaConvertible: isBothMetaConvertible
+                 else: isConvertible
       else:
         return isNone
     when useEffectSystem:
@@ -474,6 +511,9 @@ proc typeRangeRel(f, a: PType): TTypeRelation {.noinline.} =
 proc matchUserTypeClass*(c: PContext, m: var TCandidate,
                          ff, a: PType): TTypeRelation =
   var body = ff.skipTypes({tyUserTypeClassInst})
+  if c.inTypeClass > 20:
+    localError(body.n[3].info, $body.n[3] & " too nested for type matching")
+    return isNone
 
   openScope(c)
   inc c.inTypeClass
@@ -553,7 +593,7 @@ proc tryResolvingStaticExpr(c: var TCandidate, n: PNode): PNode =
   # Here, N-1 will be initially nkStaticExpr that can be evaluated only after
   # N is bound to a concrete value during the matching of the first param.
   # This proc is used to evaluate such static expressions.
-  let instantiated = replaceTypesInBody(c.c, c.bindings, n)
+  let instantiated = replaceTypesInBody(c.c, c.bindings, n, nil)
   result = c.c.semExpr(c.c, instantiated)
 
 proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
@@ -644,6 +684,13 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
   of tyAnything:
     return if f.kind == tyAnything: isGeneric
            else: isNone
+
+  of tyUserTypeClass, tyUserTypeClassInst:
+    # consider this: 'var g: Node' *within* a concept where 'Node'
+    # is a concept too (tgraph)
+    let x = typeRel(c, a, f, false)
+    if x >= isGeneric:
+      return isGeneric
   else: discard
 
   case f.kind
@@ -727,8 +774,12 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
         result = isNone
     else: discard
   of tyOpenArray, tyVarargs:
-    # varargs[expr] is special
+    # varargs[expr] is special too but handled earlier. So we only need to
-    if f.kind == tyVarargs and f.sons[0].kind == tyExpr: return
+    # handle varargs[stmt] which is the same as varargs[typed]:
+    if f.kind == tyVarargs:
+      if tfOldSchoolExprStmt in f.sons[0].flags:
+        if f.sons[0].kind == tyExpr: return
+      elif f.sons[0].kind == tyStmt: return
     case a.kind
     of tyOpenArray, tyVarargs:
       result = typeRel(c, base(f), base(a))
@@ -748,6 +799,13 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
         result = isConvertible
       elif typeRel(c, base(f), a.sons[0]) >= isGeneric:
         result = isConvertible
+    of tyString:
+      if f.kind == tyOpenArray:
+        if f.sons[0].kind == tyChar:
+          result = isConvertible
+        elif f.sons[0].kind == tyGenericParam and a.len > 0 and
+            typeRel(c, base(f), base(a)) >= isGeneric:
+          result = isConvertible
     else: discard
   of tySequence:
     case a.kind
@@ -1091,6 +1149,8 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation =
       result = isNone
 
   of tyStmt:
+    if aOrig != nil and tfOldSchoolExprStmt notin f.flags:
+      put(c.bindings, f, aOrig)
     result = isGeneric
 
   of tyProxy:
@@ -1204,7 +1264,7 @@ proc incMatches(m: var TCandidate; r: TTypeRelation; convMatch = 1) =
   case r
   of isConvertible, isIntConv: inc(m.convMatches, convMatch)
   of isSubtype, isSubrange: inc(m.subtypeMatches)
-  of isGeneric, isInferred: inc(m.genericMatches)
+  of isGeneric, isInferred, isBothMetaConvertible: inc(m.genericMatches)
   of isFromIntLit: inc(m.intConvMatches, 256)
   of isInferredConvertible:
     inc(m.convMatches)
@@ -1270,6 +1330,29 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType,
     put(m.bindings, f, inlined)
     return argSemantized
 
+  # If r == isBothMetaConvertible then we rerun typeRel.
+  # bothMetaCounter is for safety to avoid any infinite loop,
+  #  I don't have any example when it is needed.
+  # lastBindingsLenth is used to check whether m.bindings remains the same,
+  #  because in that case there is no point in continuing.
+  var bothMetaCounter = 0
+  var lastBindingsLength = -1
+  while r == isBothMetaConvertible and
+      lastBindingsLength != m.bindings.counter and
+      bothMetaCounter < 100:
+    lastBindingsLength = m.bindings.counter
+    inc(bothMetaCounter)
+    if arg.kind in {nkProcDef, nkIteratorDef} + nkLambdaKinds:
+      result = c.semInferredLambda(c, m.bindings, arg)
+    elif arg.kind != nkSym:
+      return nil
+    else:
+      let inferred = c.semGenerateInstance(c, arg.sym, m.bindings, arg.info)
+      result = newSymNode(inferred, arg.info)
+    inc(m.convMatches)
+    arg = result
+    r = typeRel(m, f, arg.typ)
+
   case r
   of isConvertible:
     inc(m.convMatches)
@@ -1281,6 +1364,9 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType,
     result = implicitConv(nkHiddenStdConv, f, arg, m, c)
   of isSubtype:
     inc(m.subtypeMatches)
+    if f.kind == tyTypeDesc:
+      result = arg
+    else:
       result = implicitConv(nkHiddenSubConv, f, arg, m, c)
   of isSubrange:
     inc(m.subtypeMatches)
@@ -1291,6 +1377,8 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType,
   of isInferred, isInferredConvertible:
     if arg.kind in {nkProcDef, nkIteratorDef} + nkLambdaKinds:
       result = c.semInferredLambda(c, m.bindings, arg)
+    elif arg.kind != nkSym:
+      return nil
     else:
       let inferred = c.semGenerateInstance(c, arg.sym, m.bindings, arg.info)
       result = newSymNode(inferred, arg.info)
@@ -1310,6 +1398,9 @@ proc paramTypesMatchAux(m: var TCandidate, f, argType: PType,
       result.typ = getInstantiatedType(c, arg, m, f)
     else:
       result = arg
+  of isBothMetaConvertible:
+    # This is the result for the 101th time.
+    result = nil
   of isFromIntLit:
     # too lazy to introduce another ``*matches`` field, so we conflate
     # ``isIntConv`` and ``isIntLit`` here:
@@ -1454,6 +1545,10 @@ proc incrIndexType(t: PType) =
   assert t.kind == tyArrayConstr
   inc t.sons[0].n.sons[1].intVal
 
+template isVarargsUntyped(x): expr =
+  x.kind == tyVarargs and x.sons[0].kind == tyExpr and
+    tfOldSchoolExprStmt notin x.sons[0].flags
+
 proc matchesAux(c: PContext, n, nOrig: PNode,
                 m: var TCandidate, marker: var IntSet) =
   template checkConstraint(n: expr) {.immediate, dirty.} =
@@ -1482,10 +1577,17 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
   var formalLen = m.callee.n.len
   addSon(m.call, copyTree(n.sons[0]))
   var container: PNode = nil # constructed container
-  var formal: PSym = nil
+  var formal: PSym = if formalLen > 1: m.callee.n.sons[1].sym else: nil
 
   while a < n.len:
-    if n.sons[a].kind == nkExprEqExpr:
+    if a >= formalLen-1 and formal != nil and formal.typ.isVarargsUntyped:
+      if container.isNil:
+        container = newNodeIT(nkBracket, n.sons[a].info, arrayConstr(c, n.info))
+        setSon(m.call, formal.position + 1, container)
+      else:
+        incrIndexType(container.typ)
+      addSon(container, n.sons[a])
+    elif n.sons[a].kind == nkExprEqExpr:
       # named param
       # check if m.callee has such a param:
       prepareNamedParam(n.sons[a])
@@ -1507,7 +1609,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
       n.sons[a].sons[1] = prepareOperand(c, formal.typ, n.sons[a].sons[1])
       n.sons[a].typ = n.sons[a].sons[1].typ
       var arg = paramTypesMatch(m, formal.typ, n.sons[a].typ,
-                                n.sons[a].sons[1], nOrig.sons[a].sons[1])
+                                n.sons[a].sons[1], n.sons[a].sons[1])
       if arg == nil:
         m.state = csNoMatch
         return
@@ -1536,7 +1638,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode,
             addSon(m.call, copyTree(n.sons[a]))
         elif formal != nil and formal.typ.kind == tyVarargs:
           # beware of the side-effects in 'prepareOperand'! So only do it for
-          # varags matching. See tests/metatype/tstatic_overloading.
+          # varargs matching. See tests/metatype/tstatic_overloading.
           m.baseTypeMatch = false
           n.sons[a] = prepareOperand(c, formal.typ, n.sons[a])
           var arg = paramTypesMatch(m, formal.typ, n.sons[a].typ,
@@ -1599,6 +1701,10 @@ proc partialMatch*(c: PContext, n, nOrig: PNode, m: var TCandidate) =
   matchesAux(c, n, nOrig, m, marker)
 
 proc matches*(c: PContext, n, nOrig: PNode, m: var TCandidate) =
+  if m.calleeSym != nil and m.calleeSym.magic in {mArrGet, mArrPut}:
+    m.state = csMatch
+    m.call = n
+    return
   var marker = initIntSet()
   matchesAux(c, n, nOrig, m, marker)
   if m.state == csNoMatch: return

compiler/suggest.nim

@@ -15,57 +15,79 @@ import algorithm, sequtils
 
 const
   sep = '\t'
-  sectionSuggest = "sug"
+
-  sectionDef = "def"
+type
-  sectionContext = "con"
+  Suggest* = object
-  sectionUsage = "use"
+    section*: IdeCmd
+    qualifiedPath*: seq[string]
+    filePath*: string
+    line*: int                   # Starts at 1
+    column*: int                 # Starts at 0
+    doc*: string           # Not escaped (yet)
+    symkind*: TSymKind
+    forth*: string               # XXX TODO object on symkind
+
+var
+  suggestionResultHook*: proc (result: Suggest) {.closure.}
 
 #template sectionSuggest(): expr = "##begin\n" & getStackTrace() & "##end\n"
 
 template origModuleName(m: PSym): string = m.name.s
 
-proc symToStr(s: PSym, isLocal: bool, section: string, li: TLineInfo): string = 
+proc symToSuggest(s: PSym, isLocal: bool, section: string, li: TLineInfo): Suggest =
-  result = section
+  result.section = parseIdeCmd(section)
-  result.add(sep)
   if optIdeTerse in gGlobalOptions:
-    if s.kind in routineKinds:
+    result.symkind = s.kind
-      result.add renderTree(s.ast, {renderNoBody, renderNoComments,
+    result.filePath = toFullPath(li)
-                                    renderDocComments, renderNoPragmas})
+    result.line = toLinenumber(li)
+    result.column = toColumn(li)
   else:
-      result.add s.name.s
+    result.symkind = s.kind
-    result.add(sep)
+    result.qualifiedPath = @[]
-    result.add(toFullPath(li))
-    result.add(sep)
-    result.add($toLinenumber(li))
-    result.add(sep)
-    result.add($toColumn(li))
-  else:
-    result.add($s.kind)
-    result.add(sep)
     if not isLocal and s.kind != skModule:
       let ow = s.owner
       if ow.kind != skModule and ow.owner != nil:
         let ow2 = ow.owner
-        result.add(ow2.origModuleName)
+        result.qualifiedPath.add(ow2.origModuleName)
-        result.add('.')
+      result.qualifiedPath.add(ow.origModuleName)
-      result.add(ow.origModuleName)
+    result.qualifiedPath.add(s.name.s)
-      result.add('.')
+
-    result.add(s.name.s)
-    result.add(sep)
     if s.typ != nil:
-      result.add(typeToString(s.typ))
+      result.forth = typeToString(s.typ)
+    else:
+      result.forth = ""
+    result.filePath = toFullPath(li)
+    result.line = toLinenumber(li)
+    result.column = toColumn(li)
+    when not defined(noDocgen):
+      result.doc = s.extractDocComment
+
+proc `$`(suggest: Suggest): string =
+  result = $suggest.section
   result.add(sep)
-    result.add(toFullPath(li))
+  result.add($suggest.symkind)
   result.add(sep)
-    result.add($toLinenumber(li))
+  result.add(suggest.qualifiedPath.join("."))
   result.add(sep)
-    result.add($toColumn(li))
+  result.add(suggest.forth)
+  result.add(sep)
+  result.add(suggest.filePath)
+  result.add(sep)
+  result.add($suggest.line)
+  result.add(sep)
+  result.add($suggest.column)
   result.add(sep)
   when not defined(noDocgen):
-      result.add(s.extractDocComment.escape)
+    result.add(suggest.doc.escape)
 
-proc symToStr(s: PSym, isLocal: bool, section: string): string = 
+proc symToSuggest(s: PSym, isLocal: bool, section: string): Suggest =
-  result = symToStr(s, isLocal, section, s.info)
+  result = symToSuggest(s, isLocal, section, s.info)
+
+proc suggestResult(s: Suggest) =
+  if not isNil(suggestionResultHook):
+    suggestionResultHook(s)
+  else:
+    suggestWriteln($(s))
 
 proc filterSym(s: PSym): bool {.inline.} =
   result = s.kind != skModule
@@ -84,7 +106,7 @@ proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} =
 
 proc suggestField(c: PContext, s: PSym, outputs: var int) =
   if filterSym(s) and fieldVisible(c, s):
-    suggestWriteln(symToStr(s, isLocal=true, sectionSuggest))
+    suggestResult(symToSuggest(s, isLocal=true, $ideSug))
     inc outputs
 
 template wholeSymTab(cond, section: expr) {.immediate.} =
@@ -97,7 +119,7 @@ template wholeSymTab(cond, section: expr) {.immediate.} =
     for item in entries:
       let it {.inject.} = item
       if cond:
-        suggestWriteln(symToStr(it, isLocal = isLocal, section))
+        suggestResult(symToSuggest(it, isLocal = isLocal, section))
         inc outputs
 
 proc suggestSymList(c: PContext, list: PNode, outputs: var int) =
@@ -140,7 +162,7 @@ proc argsFit(c: PContext, candidate: PSym, n, nOrig: PNode): bool =
 
 proc suggestCall(c: PContext, n, nOrig: PNode, outputs: var int) =
   wholeSymTab(filterSym(it) and nameFits(c, it, n) and argsFit(c, it, n, nOrig),
-              sectionContext)
+              $ideCon)
 
 proc typeFits(c: PContext, s: PSym, firstArg: PType): bool {.inline.} =
   if s.typ != nil and sonsLen(s.typ) > 1 and s.typ.sons[1] != nil:
@@ -157,7 +179,7 @@ proc typeFits(c: PContext, s: PSym, firstArg: PType): bool {.inline.} =
 
 proc suggestOperations(c: PContext, n: PNode, typ: PType, outputs: var int) =
   assert typ != nil
-  wholeSymTab(filterSymNoOpr(it) and typeFits(c, it, typ), sectionSuggest)
+  wholeSymTab(filterSymNoOpr(it) and typeFits(c, it, typ), $ideSug)
 
 proc suggestEverything(c: PContext, n: PNode, outputs: var int) =
   # do not produce too many symbols:
@@ -166,7 +188,7 @@ proc suggestEverything(c: PContext, n: PNode, outputs: var int) =
     if scope == c.topLevelScope: isLocal = false
     for it in items(scope.symbols):
       if filterSym(it):
-        suggestWriteln(symToStr(it, isLocal = isLocal, sectionSuggest))
+        suggestResult(symToSuggest(it, isLocal = isLocal, $ideSug))
         inc outputs
     if scope == c.topLevelScope: break
 
@@ -181,12 +203,12 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) =
         # all symbols accessible, because we are in the current module:
         for it in items(c.topLevelScope.symbols):
           if filterSym(it):
-            suggestWriteln(symToStr(it, isLocal=false, sectionSuggest))
+            suggestResult(symToSuggest(it, isLocal=false, $ideSug))
             inc outputs
       else:
         for it in items(n.sym.tab):
           if filterSym(it):
-            suggestWriteln(symToStr(it, isLocal=false, sectionSuggest))
+            suggestResult(symToSuggest(it, isLocal=false, $ideSug))
             inc outputs
     else:
       # fallback:
@@ -263,16 +285,16 @@ var
 proc findUsages(info: TLineInfo; s: PSym) =
   if usageSym == nil and isTracked(info, s.name.s.len):
     usageSym = s
-    suggestWriteln(symToStr(s, isLocal=false, sectionUsage))
+    suggestResult(symToSuggest(s, isLocal=false, $ideUse))
   elif s == usageSym:
     if lastLineInfo != info:
-      suggestWriteln(symToStr(s, isLocal=false, sectionUsage, info))
+      suggestResult(symToSuggest(s, isLocal=false, $ideUse, info))
     lastLineInfo = info
 
 proc findDefinition(info: TLineInfo; s: PSym) =
   if s.isNil: return
   if isTracked(info, s.name.s.len):
-    suggestWriteln(symToStr(s, isLocal=false, sectionDef))
+    suggestResult(symToSuggest(s, isLocal=false, $ideDef))
     suggestQuit()
 
 proc ensureIdx[T](x: var T, y: int) =
@@ -287,6 +309,10 @@ proc suggestSym*(info: TLineInfo; s: PSym) {.inline.} =
     findUsages(info, s)
   elif gIdeCmd == ideDef:
     findDefinition(info, s)
+  elif gIdeCmd == ideDus and s != nil:
+    if isTracked(info, s.name.s.len):
+      suggestResult(symToSuggest(s, isLocal=false, $ideDef))
+    findUsages(info, s)
 
 proc markUsed(info: TLineInfo; s: PSym) =
   incl(s.flags, sfUsed)
@@ -313,8 +339,8 @@ proc suggestExpr*(c: PContext, node: PNode) =
     if cp == cpNone: return
   var outputs = 0
   # This keeps semExpr() from coming here recursively:
-  if c.inCompilesContext > 0: return
+  if c.compilesContextId > 0: return
-  inc(c.inCompilesContext)
+  inc(c.compilesContextId)
 
   if gIdeCmd == ideSug:
     var n = if nfIsCursor in node.flags: node else: findClosestDot(node)
@@ -343,8 +369,8 @@ proc suggestExpr*(c: PContext, node: PNode) =
         addSon(a, x)
       suggestCall(c, a, n, outputs)
 
-  dec(c.inCompilesContext)
+  dec(c.compilesContextId)
-  if outputs > 0 and gIdeCmd != ideUse: suggestQuit()
+  if outputs > 0 and gIdeCmd notin {ideUse, ideDus}: suggestQuit()
 
 proc suggestStmt*(c: PContext, n: PNode) =
   suggestExpr(c, n)

compiler/syntaxes.nim

@@ -92,10 +92,15 @@ proc parsePipe(filename: string, inputStream: PLLStream): PNode =
     var line = newStringOfCap(80)
     discard llStreamReadLine(s, line)
     var i = utf8Bom(line)
+    var linenumber = 1
     if containsShebang(line, i):
       discard llStreamReadLine(s, line)
       i = 0
-    if line[i] == '#' and line[i+1] == '!':
+      inc linenumber
+    if line[i] == '#' and line[i+1] in {'?', '!'}:
+      if line[i+1] == '!':
+        message(newLineInfo(filename, linenumber, 1),
+                warnDeprecated, "use '#?' instead; '#!'")
       inc(i, 2)
       while line[i] in Whitespace: inc(i)
       var q: TParser
@@ -139,7 +144,7 @@ proc applyFilter(p: var TParsers, n: PNode, filename: string,
   of filtReplace:
     result = filterReplace(stdin, filename, n)
   if f != filtNone:
-    if gVerbosity >= 2: 
+    if hintCodeBegin in gNotes:
       rawMessage(hintCodeBegin, [])
       msgWriteln(result.s)
       rawMessage(hintCodeEnd, [])

compiler/transf.nim

@@ -16,6 +16,7 @@
 # * converts "continue" to "break"; disambiguates "break"
 # * introduces method dispatchers
 # * performs lambda lifting for closure support
+# * transforms 'defer' into a 'try finally' statement
 
 import
   intsets, strutils, lists, options, ast, astalgo, trees, treetab, msgs, os,
@@ -44,6 +45,7 @@ type
     inlining: int            # > 0 if we are in inlining context (copy vars)
     nestedProcs: int         # > 0 if we are in a nested proc
     contSyms, breakSyms: seq[PSym]  # to transform 'continue' and 'break'
+    deferDetected: bool
   PTransf = ref TTransfContext
 
 proc newTransNode(a: PNode): PTransNode {.inline.} =
@@ -113,7 +115,7 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
   #  return liftIterSym(n)
   var b: PNode
   var tc = c.transCon
-  if sfBorrow in n.sym.flags:
+  if sfBorrow in n.sym.flags and n.sym.kind in routineKinds:
     # simply exchange the symbol:
     b = n.sym.getBody
     if b.kind != nkSym: internalError(n.info, "wrong AST for borrowed symbol")
@@ -137,7 +139,7 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode =
     if it.kind == nkCommentStmt:
       result[i] = PTransNode(it)
     elif it.kind == nkIdentDefs:
-      if it.sons[0].kind != nkSym: internalError(it.info, "transformVarSection")
+      if it.sons[0].kind == nkSym:
         internalAssert(it.len == 3)
         var newVar = copySym(it.sons[0].sym)
         incl(newVar.flags, sfFromGeneric)
@@ -154,6 +156,9 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode =
         defs[2] = transform(c, it.sons[2])
         newVar.ast = defs[2].PNode
         result[i] = defs
+      else:
+        # has been transformed into 'param.x' for closure iterators, so keep it:
+        result[i] = PTransNode(it)
     else:
       if it.kind != nkVarTuple:
         internalError(it.info, "transformVarSection: not nkVarTuple")
@@ -299,6 +304,7 @@ proc transformYield(c: PTransf, n: PNode): PTransNode =
   var e = n.sons[0]
   # c.transCon.forStmt.len == 3 means that there is one for loop variable
   # and thus no tuple unpacking:
+  if e.typ.isNil: return result # can happen in nimsuggest for unknown reasons
   if skipTypes(e.typ, {tyGenericInst}).kind == tyTuple and
       c.transCon.forStmt.len != 3:
     e = skipConv(e)
@@ -465,10 +471,13 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
   var call = n.sons[length - 2]
 
   let labl = newLabel(c, n)
-  c.breakSyms.add(labl)
   result = newTransNode(nkBlockStmt, n.info, 2)
   result[0] = newSymNode(labl).PTransNode
-
+  if call.typ.isNil:
+    # see bug #3051
+    result[1] = newNode(nkEmpty).PTransNode
+    return result
+  c.breakSyms.add(labl)
   if call.typ.kind != tyIter and
     (call.kind notin nkCallKinds or call.sons[0].kind != nkSym or
       call.sons[0].sym.kind != skIterator):
@@ -479,10 +488,10 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
 
   #echo "transforming: ", renderTree(n)
   var stmtList = newTransNode(nkStmtList, n.info, 0)
+  result[1] = stmtList
 
   var loopBody = transformLoopBody(c, n.sons[length-1])
 
-  result[1] = stmtList
   discard c.breakSyms.pop
 
   var v = newNodeI(nkVarSection, n.info)
@@ -677,6 +686,14 @@ proc commonOptimizations*(c: PSym, n: PNode): PNode =
       result = n
 
 proc transform(c: PTransf, n: PNode): PTransNode =
+  when false:
+    var oldDeferAnchor: PNode
+    if n.kind in {nkElifBranch, nkOfBranch, nkExceptBranch, nkElifExpr,
+                  nkElseExpr, nkElse, nkForStmt, nkWhileStmt, nkFinally,
+                  nkBlockStmt, nkBlockExpr}:
+      oldDeferAnchor = c.deferAnchor
+      c.deferAnchor = n
+
   case n.kind
   of nkSym:
     result = transformSym(c, n)
@@ -709,13 +726,36 @@ proc transform(c: PTransf, n: PNode): PTransNode =
     result = transformFor(c, n)
   of nkParForStmt:
     result = transformSons(c, n)
-  of nkCaseStmt: result = transformCase(c, n)
+  of nkCaseStmt:
+    result = transformCase(c, n)
+  of nkWhileStmt: result = transformWhile(c, n)
+  of nkBlockStmt, nkBlockExpr:
+    result = transformBlock(c, n)
+  of nkDefer:
+    c.deferDetected = true
+    result = transformSons(c, n)
+    when false:
+      let deferPart = newNodeI(nkFinally, n.info)
+      deferPart.add n.sons[0]
+      let tryStmt = newNodeI(nkTryStmt, n.info)
+      if c.deferAnchor.isNil:
+        tryStmt.add c.root
+        c.root = tryStmt
+        result = PTransNode(tryStmt)
+      else:
+        # modify the corresponding *action*, don't rely on nkStmtList:
+        let L = c.deferAnchor.len-1
+        tryStmt.add c.deferAnchor.sons[L]
+        c.deferAnchor.sons[L] = tryStmt
+        result = newTransNode(nkCommentStmt, n.info, 0)
+      tryStmt.addSon(deferPart)
+      # disable the original 'defer' statement:
+      n.kind = nkCommentStmt
   of nkContinueStmt:
     result = PTransNode(newNodeI(nkBreakStmt, n.info))
     var labl = c.contSyms[c.contSyms.high]
     add(result, PTransNode(newSymNode(labl)))
   of nkBreakStmt: result = transformBreak(c, n)
-  of nkWhileStmt: result = transformWhile(c, n)
   of nkCallKinds:
     result = transformCall(c, n)
   of nkAddr, nkHiddenAddr:
@@ -751,8 +791,6 @@ proc transform(c: PTransf, n: PNode): PTransNode =
       result = transformYield(c, n)
     else:
       result = transformSons(c, n)
-  of nkBlockStmt, nkBlockExpr:
-    result = transformBlock(c, n)
   of nkIdentDefs, nkConstDef:
     result = transformSons(c, n)
     # XXX comment handling really sucks:
@@ -761,6 +799,8 @@ proc transform(c: PTransf, n: PNode): PTransNode =
   of nkClosure: return PTransNode(n)
   else:
     result = transformSons(c, n)
+  when false:
+    if oldDeferAnchor != nil: c.deferAnchor = oldDeferAnchor
   var cnst = getConstExpr(c.module, PNode(result))
   # we inline constants if they are not complex constants:
   if cnst != nil and not dontInlineConstant(n, cnst):
@@ -782,12 +822,54 @@ proc openTransf(module: PSym, filename: string): PTransf =
   result.breakSyms = @[]
   result.module = module
 
+proc flattenStmts(n: PNode) =
+  var goOn = true
+  while goOn:
+    goOn = false
+    var i = 0
+    while i < n.len:
+      let it = n[i]
+      if it.kind in {nkStmtList, nkStmtListExpr}:
+        n.sons[i..i] = it.sons[0..<it.len]
+        goOn = true
+      inc i
+
+proc liftDeferAux(n: PNode) =
+  if n.kind in {nkStmtList, nkStmtListExpr}:
+    flattenStmts(n)
+    var goOn = true
+    while goOn:
+      goOn = false
+      let last = n.len-1
+      for i in 0..last:
+        if n.sons[i].kind == nkDefer:
+          let deferPart = newNodeI(nkFinally, n.sons[i].info)
+          deferPart.add n.sons[i].sons[0]
+          var tryStmt = newNodeI(nkTryStmt, n.sons[i].info)
+          var body = newNodeI(n.kind, n.sons[i].info)
+          if i < last:
+            body.sons = n.sons[(i+1)..last]
+          tryStmt.addSon(body)
+          tryStmt.addSon(deferPart)
+          n.sons[i] = tryStmt
+          n.sons.setLen(i+1)
+          n.typ = n.sons[i].typ
+          goOn = true
+          break
+  for i in 0..n.safeLen-1:
+    liftDeferAux(n.sons[i])
+
+template liftDefer(c, root) =
+  if c.deferDetected:
+    liftDeferAux(root)
+
 proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
   if nfTransf in n.flags or prc.kind in {skTemplate}:
     result = n
   else:
     var c = openTransf(module, "")
     result = processTransf(c, n, prc)
+    liftDefer(c, result)
     result = liftLambdas(prc, result)
     #if prc.kind == skClosureIterator:
     #  result = lambdalifting.liftIterator(prc, result)
@@ -802,6 +884,7 @@ proc transformStmt*(module: PSym, n: PNode): PNode =
   else:
     var c = openTransf(module, "")
     result = processTransf(c, n, module)
+    liftDefer(c, result)
     result = liftLambdasForTopLevel(module, result)
     incl(result.flags, nfTransf)
     when useEffectSystem: trackTopLevelStmt(module, result)
@@ -812,4 +895,5 @@ proc transformExpr*(module: PSym, n: PNode): PNode =
   else:
     var c = openTransf(module, "")
     result = processTransf(c, n, module)
+    liftDefer(c, result)
     incl(result.flags, nfTransf)

compiler/trees.nim

@@ -36,13 +36,16 @@ proc cyclicTree*(n: PNode): bool =
   var s = newNodeI(nkEmpty, n.info)
   result = cyclicTreeAux(n, s)
 
-proc exprStructuralEquivalent*(a, b: PNode): bool =
+proc exprStructuralEquivalent*(a, b: PNode; strictSymEquality=false): bool =
   result = false
   if a == b:
     result = true
   elif (a != nil) and (b != nil) and (a.kind == b.kind):
     case a.kind
     of nkSym:
+      if strictSymEquality:
+        result = a.sym == b.sym
+      else:
         # don't go nuts here: same symbol as string is enough:
         result = a.sym.name.id == b.sym.name.id
     of nkIdent: result = a.ident.id == b.ident.id
@@ -53,7 +56,8 @@ proc exprStructuralEquivalent*(a, b: PNode): bool =
     else:
       if sonsLen(a) == sonsLen(b):
         for i in countup(0, sonsLen(a) - 1):
-          if not exprStructuralEquivalent(a.sons[i], b.sons[i]): return
+          if not exprStructuralEquivalent(a.sons[i], b.sons[i],
+                                          strictSymEquality): return
         result = true
 
 proc sameTree*(a, b: PNode): bool =

compiler/treetab.nim

@@ -12,7 +12,7 @@
 import
   hashes, ast, astalgo, types
 
-proc hashTree(n: PNode): THash = 
+proc hashTree(n: PNode): Hash =
   if n == nil: return
   result = ord(n.kind)
   case n.kind
@@ -53,8 +53,8 @@ proc treesEquivalent(a, b: PNode): bool =
         result = true
     if result: result = sameTypeOrNil(a.typ, b.typ)
 
-proc nodeTableRawGet(t: TNodeTable, k: THash, key: PNode): int = 
+proc nodeTableRawGet(t: TNodeTable, k: Hash, key: PNode): int =
-  var h: THash = k and high(t.data)
+  var h: Hash = k and high(t.data)
   while t.data[h].key != nil:
     if (t.data[h].h == k) and treesEquivalent(t.data[h].key, key):
       return h
@@ -66,9 +66,9 @@ proc nodeTableGet*(t: TNodeTable, key: PNode): int =
   if index >= 0: result = t.data[index].val
   else: result = low(int)
 
-proc nodeTableRawInsert(data: var TNodePairSeq, k: THash, key: PNode, 
+proc nodeTableRawInsert(data: var TNodePairSeq, k: Hash, key: PNode,
                         val: int) =
-  var h: THash = k and high(data)
+  var h: Hash = k and high(data)
   while data[h].key != nil: h = nextTry(h, high(data))
   assert(data[h].key == nil)
   data[h].h = k
@@ -77,7 +77,7 @@ proc nodeTableRawInsert(data: var TNodePairSeq, k: THash, key: PNode,
 
 proc nodeTablePut*(t: var TNodeTable, key: PNode, val: int) =
   var n: TNodePairSeq
-  var k: THash = hashTree(key)
+  var k: Hash = hashTree(key)
   var index = nodeTableRawGet(t, k, key)
   if index >= 0:
     assert(t.data[index].key != nil)
@@ -94,7 +94,7 @@ proc nodeTablePut*(t: var TNodeTable, key: PNode, val: int) =
 
 proc nodeTableTestOrSet*(t: var TNodeTable, key: PNode, val: int): int =
   var n: TNodePairSeq
-  var k: THash = hashTree(key)
+  var k: Hash = hashTree(key)
   var index = nodeTableRawGet(t, k, key)
   if index >= 0:
     assert(t.data[index].key != nil)

compiler/types.nim

@@ -395,7 +395,7 @@ proc rangeToStr(n: PNode): string =
 
 const
   typeToStr: array[TTypeKind, string] = ["None", "bool", "Char", "empty",
-    "Array Constructor [$1]", "nil", "expr", "stmt", "typeDesc",
+    "Array Constructor [$1]", "nil", "untyped", "typed", "typeDesc",
     "GenericInvocation", "GenericBody", "GenericInst", "GenericParam",
     "distinct $1", "enum", "ordinal[$1]", "array[$1, $2]", "object", "tuple",
     "set[$1]", "range[$1]", "ptr ", "ref ", "var ", "seq[$1]", "proc",
@@ -411,6 +411,10 @@ const
 
 const preferToResolveSymbols = {preferName, preferModuleInfo, preferGenericArg}
 
+proc addTypeFlags(name: var string, typ: PType) {.inline.} =
+  if tfShared in typ.flags: name = "shared " & name
+  if tfNotNil in typ.flags: name.add(" not nil")
+
 proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
   var t = typ
   result = ""
@@ -418,11 +422,13 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
   if prefer in preferToResolveSymbols and t.sym != nil and
        sfAnon notin t.sym.flags:
     if t.kind == tyInt and isIntLit(t):
-      return t.sym.name.s & " literal(" & $t.n.intVal & ")"
+      result = t.sym.name.s & " literal(" & $t.n.intVal & ")"
-    if prefer == preferName or t.sym.owner.isNil:
+    elif prefer == preferName or t.sym.owner.isNil:
-      return t.sym.name.s
+      result = t.sym.name.s
     else:
-      return t.sym.owner.name.s & '.' & t.sym.name.s
+      result = t.sym.owner.name.s & '.' & t.sym.name.s
+    result.addTypeFlags(t)
+    return
   case t.kind
   of tyInt:
     if not isIntLit(t) or prefer == preferExported:
@@ -481,7 +487,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
     result = "not " & typeToString(t.sons[0])
   of tyExpr:
     internalAssert t.len == 0
-    result = "expr"
+    result = "untyped"
   of tyFromExpr, tyFieldAccessor:
     result = renderTree(t.n)
   of tyArray:
@@ -563,8 +569,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string =
     result = typeToStr[t.kind] % typeToString(t.sons[0])
   else:
     result = typeToStr[t.kind]
-  if tfShared in t.flags: result = "shared " & result
+  result.addTypeFlags(t)
-  if tfNotNil in t.flags: result.add(" not nil")
 
 proc resultType(t: PType): PType =
   assert(t.kind == tyProc)
@@ -986,7 +991,9 @@ proc compareTypes*(x, y: PType,
   var c = initSameTypeClosure()
   c.cmp = cmp
   c.flags = flags
-  result = sameTypeAux(x, y, c)
+  if x == y: result = true
+  elif x.isNil or y.isNil: result = false
+  else: result = sameTypeAux(x, y, c)
 
 proc inheritanceDiff*(a, b: PType): int =
   # | returns: 0 iff `a` == `b`
@@ -1260,6 +1267,9 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt =
       else: result = 8
     a = result
   of tySet:
+    if typ.sons[0].kind == tyGenericParam:
+      result = szUnknownSize
+    else:
       length = lengthOrd(typ.sons[0])
       if length <= 8: result = 1
       elif length <= 16: result = 2

compiler/vm.nim

@@ -83,7 +83,7 @@ proc stackTrace(c: PCtx, tos: PStackFrame, pc: int,
 
 proc bailOut(c: PCtx; tos: PStackFrame) =
   stackTrace(c, tos, c.exceptionInstr, errUnhandledExceptionX,
-             c.currentExceptionA.sons[2].strVal)
+             c.currentExceptionA.sons[3].skipColon.strVal)
 
 when not defined(nimComputedGoto):
   {.pragma: computedGoto.}
@@ -120,10 +120,10 @@ template decodeBx(k: expr) {.immediate, dirty.} =
 template move(a, b: expr) {.immediate, dirty.} = system.shallowCopy(a, b)
 # XXX fix minor 'shallowCopy' overloading bug in compiler
 
-proc createStrKeepNode(x: var TFullReg) =
+proc createStrKeepNode(x: var TFullReg; keepNode=true) =
   if x.node.isNil:
     x.node = newNode(nkStrLit)
-  elif x.node.kind == nkNilLit:
+  elif x.node.kind == nkNilLit and keepNode:
     when defined(useNodeIds):
       let id = x.node.id
     system.reset(x.node[])
@@ -311,7 +311,7 @@ proc opConv*(dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): bool =
           if f.position == x:
             dest.node.strVal = if f.ast.isNil: f.name.s else: f.ast.strVal
             return
-        internalError("opConv for enum")
+        dest.node.strVal = styp.sym.name.s & " " & $x
     of tyInt..tyInt64:
       dest.node.strVal = $src.intVal
     of tyUInt..tyUInt64:
@@ -320,7 +320,18 @@ proc opConv*(dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): bool =
       dest.node.strVal = if src.intVal == 0: "false" else: "true"
     of tyFloat..tyFloat128:
       dest.node.strVal = $src.floatVal
-    of tyString, tyCString:
+    of tyString:
+      dest.node.strVal = src.node.strVal
+    of tyCString:
+      if src.node.kind == nkBracket:
+        # Array of chars
+        var strVal = ""
+        for son in src.node.sons:
+          let c = char(son.intVal)
+          if c == '\0': break
+          strVal.add(c)
+        dest.node.strVal = strVal
+      else:
         dest.node.strVal = src.node.strVal
     of tyChar:
       dest.node.strVal = $chr(src.intVal)
@@ -351,7 +362,7 @@ proc opConv*(dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): bool =
         myreset(dest); dest.kind = rkFloat
       case skipTypes(srctyp, abstractRange).kind
       of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyBool, tyChar:
-        dest.floatVal = toFloat(src.intVal.int)
+        dest.floatVal = toBiggestFloat(src.intVal)
       else:
         dest.floatVal = src.floatVal
     else:
@@ -383,8 +394,9 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     let instr = c.code[pc]
     let ra = instr.regA
     #if c.traceActive:
-    #  echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra
+    #echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra, " rb ", instr.regB, " rc ", instr.regC
     #  message(c.debug[pc], warnUser, "Trace")
+
     case instr.opcode
     of opcEof: return regs[ra]
     of opcRet:
@@ -407,8 +419,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       decodeB(rkInt)
       regs[ra].intVal = regs[rb].intVal
     of opcAsgnStr:
-      decodeB(rkNode)
+      decodeBC(rkNode)
-      createStrKeepNode regs[ra]
+      createStrKeepNode regs[ra], rc != 0
       regs[ra].node.strVal = regs[rb].node.strVal
     of opcAsgnFloat:
       decodeB(rkFloat)
@@ -431,7 +443,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         assert regs[rb].kind == rkNode
         let nb = regs[rb].node
         case nb.kind
-        of nkCharLit..nkInt64Lit:
+        of nkCharLit..nkUInt64Lit:
           ensureKind(rkInt)
           regs[ra].intVal = nb.intVal
         of nkFloatLit..nkFloat64Lit:
@@ -474,14 +486,19 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       decodeBC(rkNode)
       let src = regs[rb].node
       if src.kind notin {nkEmpty..nkNilLit}:
-        let n = src.sons[rc].skipColon
+        let n = src.sons[rc + ord(src.kind == nkObjConstr)].skipColon
         regs[ra].node = n
       else:
         stackTrace(c, tos, pc, errNilAccess)
     of opcWrObj:
       # a.b = c
       decodeBC(rkNode)
-      putIntoNode(regs[ra].node.sons[rb], regs[rc])
+      let shiftedRb = rb + ord(regs[ra].node.kind == nkObjConstr)
+      let dest = regs[ra].node
+      if dest.sons[shiftedRb].kind == nkExprColonExpr:
+        putIntoNode(dest.sons[shiftedRb].sons[1], regs[rc])
+      else:
+        putIntoNode(dest.sons[shiftedRb], regs[rc])
     of opcWrStrIdx:
       decodeBC(rkNode)
       let idx = regs[rb].intVal.int
@@ -494,7 +511,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       regs[ra].regAddr = addr(regs[rb])
     of opcAddrNode:
       decodeB(rkNodeAddr)
+      if regs[rb].kind == rkNode:
         regs[ra].nodeAddr = addr(regs[rb].node)
+      else:
+        stackTrace(c, tos, pc, errGenerated, "limited VM support for 'addr'")
     of opcLdDeref:
       # a = b[]
       let ra = instr.regA
@@ -509,8 +529,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       of rkNode:
         if regs[rb].node.kind == nkNilLit:
           stackTrace(c, tos, pc, errNilAccess)
-        assert regs[rb].node.kind == nkRefTy
+        if regs[rb].node.kind == nkRefTy:
           regs[ra].node = regs[rb].node.sons[0]
+        else:
+          stackTrace(c, tos, pc, errGenerated, "limited VM support for 'ref'")
       else:
         stackTrace(c, tos, pc, errNilAccess)
     of opcWrDeref:
@@ -682,11 +704,19 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcLtu:
       decodeBC(rkInt)
       regs[ra].intVal = ord(regs[rb].intVal <% regs[rc].intVal)
-    of opcEqRef, opcEqNimrodNode:
+    of opcEqRef:
       decodeBC(rkInt)
       regs[ra].intVal = ord((regs[rb].node.kind == nkNilLit and
                              regs[rc].node.kind == nkNilLit) or
                              regs[rb].node == regs[rc].node)
+    of opcEqNimrodNode:
+      decodeBC(rkInt)
+      regs[ra].intVal =
+        ord(exprStructuralEquivalent(regs[rb].node, regs[rc].node,
+                                     strictSymEquality=true))
+    of opcSameNodeType:
+      decodeBC(rkInt)
+      regs[ra].intVal = ord(regs[rb].node.typ.sameTypeOrNil regs[rc].node.typ)
     of opcXor:
       decodeBC(rkInt)
       regs[ra].intVal = ord(regs[rb].intVal != regs[rc].intVal)
@@ -770,16 +800,24 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         regs[ra].node.add(copyTree(regs[rb].regToNode))
       else:
         stackTrace(c, tos, pc, errNilAccess)
+    of opcGetImpl:
+      decodeB(rkNode)
+      let a = regs[rb].node
+      if a.kind == nkSym:
+        regs[ra].node = if a.sym.ast.isNil: newNode(nkNilLit)
+                        else: copyTree(a.sym.ast)
+      else:
+        stackTrace(c, tos, pc, errFieldXNotFound, "symbol")
     of opcEcho:
       let rb = instr.regB
       if rb == 1:
-        msgWriteln(regs[ra].node.strVal)
+        msgWriteln(regs[ra].node.strVal, {msgStdout})
       else:
         var outp = ""
         for i in ra..ra+rb-1:
           #if regs[i].kind != rkNode: debug regs[i]
           outp.add(regs[i].node.strVal)
-        msgWriteln(outp)
+        msgWriteln(outp, {msgStdout})
     of opcContainsSet:
       decodeBC(rkInt)
       regs[ra].intVal = ord(inSet(regs[rb].node, regs[rc].regToNode))
@@ -1047,18 +1085,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
           # set to default value:
           for i in oldLen .. <newLen:
             regs[ra].node.sons[i] = newNodeI(nkEmpty, c.debug[pc])
-    of opcSwap:
-      let rb = instr.regB
-      if regs[ra].kind == regs[rb].kind:
-        case regs[ra].kind
-        of rkNone: discard
-        of rkInt: swap regs[ra].intVal, regs[rb].intVal
-        of rkFloat: swap regs[ra].floatVal, regs[rb].floatVal
-        of rkNode: swap regs[ra].node, regs[rb].node
-        of rkRegisterAddr: swap regs[ra].regAddr, regs[rb].regAddr
-        of rkNodeAddr: swap regs[ra].nodeAddr, regs[rb].nodeAddr
-      else:
-        internalError(c.debug[pc], "cannot swap operands")
     of opcReset:
       internalError(c.debug[pc], "too implement")
     of opcNarrowS:
@@ -1121,7 +1147,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       decodeB(rkInt)
       let a = regs[rb].node
       case a.kind
-      of nkCharLit..nkInt64Lit: regs[ra].intVal = a.intVal
+      of nkCharLit..nkUInt64Lit: regs[ra].intVal = a.intVal
       else: stackTrace(c, tos, pc, errFieldXNotFound, "intVal")
     of opcNFloatVal:
       decodeB(rkFloat)
@@ -1172,9 +1198,12 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
                                      c.module)
     of opcGorge:
       decodeBC(rkNode)
+      inc pc
+      let rd = c.code[pc].regA
+
       createStr regs[ra]
       regs[ra].node.strVal = opGorge(regs[rb].node.strVal,
-                                     regs[rc].node.strVal)
+                                     regs[rc].node.strVal, regs[rd].node.strVal)
     of opcNError:
       stackTrace(c, tos, pc, errUser, regs[ra].node.strVal)
     of opcNWarning:
@@ -1276,7 +1305,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcNSetIntVal:
       decodeB(rkNode)
       var dest = regs[ra].node
-      if dest.kind in {nkCharLit..nkInt64Lit} and
+      if dest.kind in {nkCharLit..nkUInt64Lit} and
          regs[rb].kind in {rkInt}:
         dest.intVal = regs[rb].intVal
       else:
@@ -1385,6 +1414,31 @@ proc execute(c: PCtx, start: int): PNode =
   newSeq(tos.slots, c.prc.maxSlots)
   result = rawExecute(c, start, tos).regToNode
 
+proc execProc*(c: PCtx; sym: PSym; args: openArray[PNode]): PNode =
+  if sym.kind in routineKinds:
+    if sym.typ.len-1 != args.len:
+      localError(sym.info,
+        "NimScript: expected $# arguments, but got $#" % [
+        $(sym.typ.len-1), $args.len])
+    else:
+      let start = genProc(c, sym)
+
+      var tos = PStackFrame(prc: sym, comesFrom: 0, next: nil)
+      let maxSlots = sym.offset
+      newSeq(tos.slots, maxSlots)
+
+      # setup parameters:
+      if not isEmptyType(sym.typ.sons[0]) or sym.kind == skMacro:
+        putIntoReg(tos.slots[0], getNullValue(sym.typ.sons[0], sym.info))
+      # XXX We could perform some type checking here.
+      for i in 1.. <sym.typ.len:
+        putIntoReg(tos.slots[i], args[i-1])
+
+      result = rawExecute(c, start, tos).regToNode
+  else:
+    localError(sym.info,
+      "NimScript: attempt to call non-routine: " & sym.name.s)
+
 proc evalStmt*(c: PCtx, n: PNode) =
   let n = transformExpr(c.module, n)
   let start = genStmt(c, n)
@@ -1399,13 +1453,17 @@ proc evalExpr*(c: PCtx, n: PNode): PNode =
   assert c.code[start].opcode != opcEof
   result = execute(c, start)
 
+proc getGlobalValue*(c: PCtx; s: PSym): PNode =
+  internalAssert s.kind in {skLet, skVar} and sfGlobal in s.flags
+  result = c.globals.sons[s.position-1]
+
 include vmops
 
 # for now we share the 'globals' environment. XXX Coming soon: An API for
 # storing&loading the 'globals' environment to get what a component system
 # requires.
 var
-  globalCtx: PCtx
+  globalCtx*: PCtx
 
 proc setupGlobalCtx(module: PSym) =
   if globalCtx.isNil:
@@ -1442,6 +1500,8 @@ proc evalConstExprAux(module, prc: PSym, n: PNode, mode: TEvalMode): PNode =
   let n = transformExpr(module, n)
   setupGlobalCtx(module)
   var c = globalCtx
+  let oldMode = c.mode
+  defer: c.mode = oldMode
   c.mode = mode
   let start = genExpr(c, n, requiresValue = mode!=emStaticStmt)
   if c.code[start].opcode == opcEof: return emptyNode
@@ -1465,12 +1525,20 @@ proc evalStaticStmt*(module: PSym, e: PNode, prc: PSym) =
 proc setupCompileTimeVar*(module: PSym, n: PNode) =
   discard evalConstExprAux(module, nil, n, emStaticStmt)
 
-proc setupMacroParam(x: PNode): PNode =
+proc setupMacroParam(x: PNode, typ: PType): TFullReg =
-  result = x
+  case typ.kind
-  if result.kind in {nkHiddenSubConv, nkHiddenStdConv}: result = result.sons[1]
+  of tyStatic:
-  result = canonValue(result)
+    putIntoReg(result, x)
-  result.flags.incl nfIsRef
+  of tyTypeDesc:
-  result.typ = x.typ
+    putIntoReg(result, x)
+  else:
+    result.kind = rkNode
+    var n = x
+    if n.kind in {nkHiddenSubConv, nkHiddenStdConv}: n = n.sons[1]
+    n = n.canonValue
+    n.flags.incl nfIsRef
+    n.typ = x.typ
+    result.node = n
 
 var evalMacroCounter: int
 
@@ -1506,10 +1574,17 @@ proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode =
   # return value:
   tos.slots[0].kind = rkNode
   tos.slots[0].node = newNodeIT(nkEmpty, n.info, sym.typ.sons[0])
+
   # setup parameters:
-  for i in 1 .. < min(tos.slots.len, L):
+  for i in 1.. <sym.typ.len:
-    tos.slots[i].kind = rkNode
+    tos.slots[i] = setupMacroParam(n.sons[i], sym.typ.sons[i])
-    tos.slots[i].node = setupMacroParam(n.sons[i])
+
+  if sfImmediate notin sym.flags:
+    let gp = sym.ast[genericParamsPos]
+    for i in 0 .. <gp.len:
+      let idx = sym.typ.len + i
+      tos.slots[idx] = setupMacroParam(n.sons[idx], gp[i].sym.typ)
+
   # temporary storage:
   #for i in L .. <maxSlots: tos.slots[i] = newNode(nkEmpty)
   result = rawExecute(c, start, tos).regToNode

compiler/vmdef.nim

@@ -60,12 +60,13 @@ type
     opcAddFloat, opcSubFloat, opcMulFloat, opcDivFloat, opcShrInt, opcShlInt,
     opcBitandInt, opcBitorInt, opcBitxorInt, opcAddu, opcSubu, opcMulu,
     opcDivu, opcModu, opcEqInt, opcLeInt, opcLtInt, opcEqFloat,
-    opcLeFloat, opcLtFloat, opcLeu, opcLtu, opcEqRef, opcEqNimrodNode, opcXor,
+    opcLeFloat, opcLtFloat, opcLeu, opcLtu,
-    opcNot, opcUnaryMinusInt, opcUnaryMinusFloat, opcBitnotInt,
+    opcEqRef, opcEqNimrodNode, opcSameNodeType,
+    opcXor, opcNot, opcUnaryMinusInt, opcUnaryMinusFloat, opcBitnotInt,
     opcEqStr, opcLeStr, opcLtStr, opcEqSet, opcLeSet, opcLtSet,
     opcMulSet, opcPlusSet, opcMinusSet, opcSymdiffSet, opcConcatStr,
     opcContainsSet, opcRepr, opcSetLenStr, opcSetLenSeq,
-    opcSwap, opcIsNil, opcOf, opcIs,
+    opcIsNil, opcOf, opcIs,
     opcSubStr, opcParseFloat, opcConv, opcCast,
     opcQuit, opcReset,
     opcNarrowS, opcNarrowU,
@@ -101,6 +102,7 @@ type
     opcEqIdent,
     opcStrToIdent,
     opcIdentToStr,
+    opcGetImpl,
 
     opcEcho,
     opcIndCall, # dest = call regStart, n; where regStart = fn, arg1, ...

compiler/vmdeps.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-import ast, types, msgs, osproc, streams, options, idents
+import ast, types, msgs, osproc, streams, options, idents, securehash
 
 proc readOutput(p: Process): string =
   result = ""
@@ -15,12 +15,33 @@ proc readOutput(p: Process): string =
   while not output.atEnd:
     result.add(output.readLine)
     result.add("\n")
+  if result.len > 0:
     result.setLen(result.len - "\n".len)
   discard p.waitForExit
 
-proc opGorge*(cmd, input: string): string =
+proc opGorge*(cmd, input, cache: string): string =
+  if cache.len > 0:# and optForceFullMake notin gGlobalOptions:
+    let h = secureHash(cmd & "\t" & input & "\t" & cache)
+    let filename = options.toGeneratedFile("gorge_" & $h, "txt")
+    var f: File
+    if open(f, filename):
+      result = f.readAll
+      f.close
+      return
+    var readSuccessful = false
     try:
-    var p = startProcess(cmd, options={poEvalCommand})
+      var p = startProcess(cmd, options={poEvalCommand, poStderrToStdout})
+      if input.len != 0:
+        p.inputStream.write(input)
+        p.inputStream.close()
+      result = p.readOutput
+      readSuccessful = true
+      writeFile(filename, result)
+    except IOError, OSError:
+      if not readSuccessful: result = ""
+  else:
+    try:
+      var p = startProcess(cmd, options={poEvalCommand, poStderrToStdout})
       if input.len != 0:
         p.inputStream.write(input)
         p.inputStream.close()

compiler/vmgen.nim

@@ -37,7 +37,7 @@ when hasFFI:
   import evalffi
 
 type
-  TGenFlag = enum gfNone, gfAddrOf
+  TGenFlag = enum gfAddrOf, gfFieldAccess
   TGenFlags = set[TGenFlag]
 
 proc debugInfo(info: TLineInfo): string =
@@ -74,8 +74,9 @@ proc codeListing(c: PCtx, result: var string, start=0; last = -1) =
       result.addf("\t$#\tr$#, L$#", ($opc).substr(3), x.regA,
                   i+x.regBx-wordExcess)
     elif opc in {opcLdConst, opcAsgnConst}:
-      result.addf("\t$#\tr$#, $#", ($opc).substr(3), x.regA,
+      let idx = x.regBx-wordExcess
-        c.constants[x.regBx-wordExcess].renderTree)
+      result.addf("\t$#\tr$#, $# ($#)", ($opc).substr(3), x.regA,
+        c.constants[idx].renderTree, $idx)
     elif opc in {opcMarshalLoad, opcMarshalStore}:
       let y = c.code[i+1]
       result.addf("\t$#\tr$#, r$#, $#", ($opc).substr(3), x.regA, x.regB,
@@ -169,8 +170,12 @@ proc getSlotKind(t: PType): TSlotKind =
 const
   HighRegisterPressure = 40
 
-proc getTemp(c: PCtx; typ: PType): TRegister =
+proc bestEffort(c: PCtx): TLineInfo =
-  let c = c.prc
+  (if c.prc == nil: c.module.info else: c.prc.sym.info)
+
+proc getTemp(cc: PCtx; tt: PType): TRegister =
+  let typ = tt.skipTypesOrNil({tyStatic})
+  let c = cc.prc
   # we prefer the same slot kind here for efficiency. Unfortunately for
   # discardable return types we may not know the desired type. This can happen
   # for e.g. mNAdd[Multiple]:
@@ -181,13 +186,13 @@ proc getTemp(c: PCtx; typ: PType): TRegister =
       return TRegister(i)
 
   # if register pressure is high, we re-use more aggressively:
-  if c.maxSlots >= HighRegisterPressure:
+  if c.maxSlots >= HighRegisterPressure and false:
     for i in 0 .. c.maxSlots-1:
       if not c.slots[i].inUse:
         c.slots[i] = (inUse: true, kind: k)
         return TRegister(i)
   if c.maxSlots >= high(TRegister):
-    internalError("cannot generate code; too many registers required")
+    globalError(cc.bestEffort, "VM problem: too many registers required")
   result = TRegister(c.maxSlots)
   c.slots[c.maxSlots] = (inUse: true, kind: k)
   inc c.maxSlots
@@ -196,9 +201,9 @@ proc freeTemp(c: PCtx; r: TRegister) =
   let c = c.prc
   if c.slots[r].kind in {slotSomeTemp..slotTempComplex}: c.slots[r].inUse = false
 
-proc getTempRange(c: PCtx; n: int; kind: TSlotKind): TRegister =
+proc getTempRange(cc: PCtx; n: int; kind: TSlotKind): TRegister =
   # if register pressure is high, we re-use more aggressively:
-  let c = c.prc
+  let c = cc.prc
   if c.maxSlots >= HighRegisterPressure or c.maxSlots+n >= high(TRegister):
     for i in 0 .. c.maxSlots-n:
       if not c.slots[i].inUse:
@@ -209,7 +214,7 @@ proc getTempRange(c: PCtx; n: int; kind: TSlotKind): TRegister =
           for k in result .. result+n-1: c.slots[k] = (inUse: true, kind: kind)
           return
   if c.maxSlots+n >= high(TRegister):
-    internalError("cannot generate code; too many registers required")
+    globalError(cc.bestEffort, "VM problem: too many registers required")
   result = TRegister(c.maxSlots)
   inc c.maxSlots, n
   for k in result .. result+n-1: c.slots[k] = (inUse: true, kind: kind)
@@ -305,7 +310,7 @@ proc genBreak(c: PCtx; n: PNode) =
       if c.prc.blocks[i].label == n.sons[0].sym:
         c.prc.blocks[i].fixups.add L1
         return
-    internalError(n.info, "cannot find 'break' target")
+    globalError(n.info, errGenerated, "VM problem: cannot find 'break' target")
   else:
     c.prc.blocks[c.prc.blocks.high].fixups.add L1
 
@@ -393,7 +398,7 @@ proc genLiteral(c: PCtx; n: PNode): int =
 proc unused(n: PNode; x: TDest) {.inline.} =
   if x >= 0:
     #debug(n)
-    internalError(n.info, "not unused")
+    globalError(n.info, "not unused")
 
 proc genCase(c: PCtx; n: PNode; dest: var TDest) =
   #  if (!expr1) goto L1;
@@ -509,10 +514,10 @@ proc needsAsgnPatch(n: PNode): bool =
 
 proc genField(n: PNode): TRegister =
   if n.kind != nkSym or n.sym.kind != skField:
-    internalError(n.info, "no field symbol")
+    globalError(n.info, "no field symbol")
   let s = n.sym
   if s.position > high(result):
-      internalError(n.info,
+    globalError(n.info,
         "too large offset! cannot generate code for: " & s.name.s)
   result = s.position
 
@@ -530,7 +535,7 @@ proc genIndex(c: PCtx; n: PNode; arr: PType): TRegister =
 proc genAsgnPatch(c: PCtx; le: PNode, value: TRegister) =
   case le.kind
   of nkBracketExpr:
-    let dest = c.genx(le.sons[0], {gfAddrOf})
+    let dest = c.genx(le.sons[0], {gfAddrOf, gfFieldAccess})
     let idx = c.genIndex(le.sons[1], le.sons[0].typ)
     c.gABC(le, opcWrArr, dest, idx, value)
     c.freeTemp(dest)
@@ -538,7 +543,7 @@ proc genAsgnPatch(c: PCtx; le: PNode, value: TRegister) =
   of nkDotExpr, nkCheckedFieldExpr:
     # XXX field checks here
     let left = if le.kind == nkDotExpr: le else: le.sons[0]
-    let dest = c.genx(left.sons[0], {gfAddrOf})
+    let dest = c.genx(left.sons[0], {gfAddrOf, gfFieldAccess})
     let idx = genField(left.sons[1])
     c.gABC(left, opcWrObj, dest, idx, value)
     c.freeTemp(dest)
@@ -596,6 +601,18 @@ proc genBinaryABC(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
   c.freeTemp(tmp)
   c.freeTemp(tmp2)
 
+proc genBinaryABCD(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
+  let
+    tmp = c.genx(n.sons[1])
+    tmp2 = c.genx(n.sons[2])
+    tmp3 = c.genx(n.sons[3])
+  if dest < 0: dest = c.getTemp(n.typ)
+  c.gABC(n, opc, dest, tmp, tmp2)
+  c.gABC(n, opc, tmp3)
+  c.freeTemp(tmp)
+  c.freeTemp(tmp2)
+  c.freeTemp(tmp3)
+
 proc genNarrow(c: PCtx; n: PNode; dest: TDest) =
   let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
   # uint is uint64 in the VM, we we only need to mask the result for
@@ -691,7 +708,7 @@ proc genConv(c: PCtx; n, arg: PNode; dest: var TDest; opc=opcConv) =
   if dest < 0: dest = c.getTemp(n.typ)
   c.gABC(n, opc, dest, tmp)
   c.gABx(n, opc, 0, genType(c, n.typ))
-  c.gABx(n, opc, 0, genType(c, arg.typ))
+  c.gABx(n, opc, 0, genType(c, arg.typ.skipTypes({tyStatic})))
   c.freeTemp(tmp)
 
 proc genCard(c: PCtx; n: PNode; dest: var TDest) =
@@ -710,9 +727,9 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     if dest < 0: dest = c.getTemp(n.typ)
     c.gABI(n, opcSubImmInt, dest, tmp, 1)
     c.freeTemp(tmp)
-  of mPred, mSubI, mSubI64:
+  of mPred, mSubI:
     c.genAddSubInt(n, dest, opcSubInt)
-  of mSucc, mAddI, mAddI64:
+  of mSucc, mAddI:
     c.genAddSubInt(n, dest, opcAddInt)
   of mInc, mDec:
     unused(n, dest)
@@ -759,28 +776,28 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     c.freeTemp(d)
     c.freeTemp(tmp)
   of mCard: genCard(c, n, dest)
-  of mMulI, mMulI64: genBinaryABCnarrow(c, n, dest, opcMulInt)
+  of mMulI: genBinaryABCnarrow(c, n, dest, opcMulInt)
-  of mDivI, mDivI64: genBinaryABCnarrow(c, n, dest, opcDivInt)
+  of mDivI: genBinaryABCnarrow(c, n, dest, opcDivInt)
-  of mModI, mModI64: genBinaryABCnarrow(c, n, dest, opcModInt)
+  of mModI: genBinaryABCnarrow(c, n, dest, opcModInt)
   of mAddF64: genBinaryABC(c, n, dest, opcAddFloat)
   of mSubF64: genBinaryABC(c, n, dest, opcSubFloat)
   of mMulF64: genBinaryABC(c, n, dest, opcMulFloat)
   of mDivF64: genBinaryABC(c, n, dest, opcDivFloat)
-  of mShrI, mShrI64: genBinaryABCnarrowU(c, n, dest, opcShrInt)
+  of mShrI: genBinaryABCnarrowU(c, n, dest, opcShrInt)
-  of mShlI, mShlI64: genBinaryABCnarrowU(c, n, dest, opcShlInt)
+  of mShlI: genBinaryABCnarrowU(c, n, dest, opcShlInt)
-  of mBitandI, mBitandI64: genBinaryABCnarrowU(c, n, dest, opcBitandInt)
+  of mBitandI: genBinaryABCnarrowU(c, n, dest, opcBitandInt)
-  of mBitorI, mBitorI64: genBinaryABCnarrowU(c, n, dest, opcBitorInt)
+  of mBitorI: genBinaryABCnarrowU(c, n, dest, opcBitorInt)
-  of mBitxorI, mBitxorI64: genBinaryABCnarrowU(c, n, dest, opcBitxorInt)
+  of mBitxorI: genBinaryABCnarrowU(c, n, dest, opcBitxorInt)
   of mAddU: genBinaryABCnarrowU(c, n, dest, opcAddu)
   of mSubU: genBinaryABCnarrowU(c, n, dest, opcSubu)
   of mMulU: genBinaryABCnarrowU(c, n, dest, opcMulu)
   of mDivU: genBinaryABCnarrowU(c, n, dest, opcDivu)
   of mModU: genBinaryABCnarrowU(c, n, dest, opcModu)
-  of mEqI, mEqI64, mEqB, mEqEnum, mEqCh:
+  of mEqI, mEqB, mEqEnum, mEqCh:
     genBinaryABC(c, n, dest, opcEqInt)
-  of mLeI, mLeI64, mLeEnum, mLeCh, mLeB:
+  of mLeI, mLeEnum, mLeCh, mLeB:
     genBinaryABC(c, n, dest, opcLeInt)
-  of mLtI, mLtI64, mLtEnum, mLtCh, mLtB:
+  of mLtI, mLtEnum, mLtCh, mLtB:
     genBinaryABC(c, n, dest, opcLtInt)
   of mEqF64: genBinaryABC(c, n, dest, opcEqFloat)
   of mLeF64: genBinaryABC(c, n, dest, opcLeFloat)
@@ -796,7 +813,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     genNarrow(c, n, dest)
   of mUnaryMinusF64: genUnaryABC(c, n, dest, opcUnaryMinusFloat)
   of mUnaryPlusI, mUnaryPlusF64: gen(c, n.sons[1], dest)
-  of mBitnotI, mBitnotI64:
+  of mBitnotI:
     genUnaryABC(c, n, dest, opcBitnotInt)
     genNarrowU(c, n, dest)
   of mZe8ToI, mZe8ToI64, mZe16ToI, mZe16ToI64, mZe32ToI64, mZeIToI64,
@@ -831,12 +848,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     c.freeTemp(tmp)
   of mSwap:
     unused(n, dest)
-    var
+    c.gen(lowerSwap(n, if c.prc == nil: c.module else: c.prc.sym))
-      d1 = c.genx(n.sons[1])
-      d2 = c.genx(n.sons[2])
-    c.gABC(n, opcSwap, d1, d2)
-    c.genAsgnPatch(n.sons[1], d1)
-    c.genAsgnPatch(n.sons[2], d2)
   of mIsNil: genUnaryABC(c, n, dest, opcIsNil)
   of mCopyStr:
     if dest < 0: dest = c.getTemp(n.typ)
@@ -934,8 +946,9 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     c.gABC(n, opcTypeTrait, dest, tmp)
     c.freeTemp(tmp)
   of mSlurp: genUnaryABC(c, n, dest, opcSlurp)
-  of mStaticExec: genBinaryABC(c, n, dest, opcGorge)
+  of mStaticExec: genBinaryABCD(c, n, dest, opcGorge)
   of mNLen: genUnaryABI(c, n, dest, opcLenSeq)
+  of mGetImpl: genUnaryABC(c, n, dest, opcGetImpl)
   of mNChild: genBinaryABC(c, n, dest, opcNChild)
   of mNSetChild, mNDel:
     unused(n, dest)
@@ -988,11 +1001,12 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
       if dest < 0: dest = c.getTemp(n.typ)
       c.gABx(n, opcNBindSym, dest, idx)
     else:
-      internalError(n.info, "invalid bindSym usage")
+      localError(n.info, "invalid bindSym usage")
   of mStrToIdent: genUnaryABC(c, n, dest, opcStrToIdent)
   of mIdentToStr: genUnaryABC(c, n, dest, opcIdentToStr)
   of mEqIdent: genBinaryABC(c, n, dest, opcEqIdent)
   of mEqNimrodNode: genBinaryABC(c, n, dest, opcEqNimrodNode)
+  of mSameNodeType: genBinaryABC(c, n, dest, opcSameNodeType)
   of mNLineInfo: genUnaryABC(c, n, dest, opcNLineInfo)
   of mNHint:
     unused(n, dest)
@@ -1013,7 +1027,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     c.gABC(n, opcCallSite, dest)
   of mNGenSym: genBinaryABC(c, n, dest, opcGenSym)
   of mMinI, mMaxI, mAbsF64, mMinF64, mMaxF64, mAbsI,
-     mAbsI64, mDotDot:
+     mDotDot:
     c.genCall(n, dest)
   of mExpandToAst:
     if n.len != 2:
@@ -1030,7 +1044,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
       globalError(n.info, "expandToAst requires a call expression")
   else:
     # mGCref, mGCunref,
-    internalError(n.info, "cannot generate code for: " & $m)
+    globalError(n.info, "cannot generate code for: " & $m)
 
 proc genMarshalLoad(c: PCtx, n: PNode, dest: var TDest) =
   ## Signature: proc to*[T](data: string): T
@@ -1062,6 +1076,7 @@ const
     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64}
 
 proc fitsRegister*(t: PType): bool =
+  assert t != nil
   t.skipTypes(abstractInst-{tyTypeDesc}).kind in {
     tyRange, tyEnum, tyBool, tyInt..tyUInt64, tyChar}
 
@@ -1076,10 +1091,32 @@ proc requiresCopy(n: PNode): bool =
 proc unneededIndirection(n: PNode): bool =
   n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind == tyRef
 
+proc canElimAddr(n: PNode): PNode =
+  case n.sons[0].kind
+  of nkObjUpConv, nkObjDownConv, nkChckRange, nkChckRangeF, nkChckRange64:
+    var m = n.sons[0].sons[0]
+    if m.kind in {nkDerefExpr, nkHiddenDeref}:
+      # addr ( nkConv ( deref ( x ) ) ) --> nkConv(x)
+      result = copyNode(n.sons[0])
+      result.add m.sons[0]
+  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+    var m = n.sons[0].sons[1]
+    if m.kind in {nkDerefExpr, nkHiddenDeref}:
+      # addr ( nkConv ( deref ( x ) ) ) --> nkConv(x)
+      result = copyNode(n.sons[0])
+      result.add m.sons[0]
+  else:
+    if n.sons[0].kind in {nkDerefExpr, nkHiddenDeref}:
+      # addr ( deref ( x )) --> x
+      result = n.sons[0].sons[0]
+
 proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
                   flags: TGenFlags) =
   # a nop for certain types
   let isAddr = opc in {opcAddrNode, opcAddrReg}
+  if isAddr and (let m = canElimAddr(n); m != nil):
+    gen(c, m, dest, flags)
+    return
   let newflags = if isAddr: flags+{gfAddrOf} else: flags
   # consider:
   # proc foo(f: var ref int) =
@@ -1092,6 +1129,8 @@ proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
   # nkAddr we must not use 'unneededIndirection', but for deref we use it.
   if not isAddr and unneededIndirection(n.sons[0]):
     gen(c, n.sons[0], dest, newflags)
+    if gfAddrOf notin flags and fitsRegister(n.typ):
+      c.gABC(n, opcNodeToReg, dest, dest)
   elif isAddr and isGlobal(n.sons[0]):
     gen(c, n.sons[0], dest, flags+{gfAddrOf})
   else:
@@ -1099,6 +1138,7 @@ proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
     if dest < 0: dest = c.getTemp(n.typ)
     if not isAddr:
       gABC(c, n, opc, dest, tmp)
+      assert n.typ != nil
       if gfAddrOf notin flags and fitsRegister(n.typ):
         c.gABC(n, opcNodeToReg, dest, dest)
     elif c.prc.slots[tmp].kind >= slotTempUnknown:
@@ -1132,7 +1172,7 @@ proc whichAsgnOpc(n: PNode; opc: TOpcode): TOpcode = opc
 proc genAsgn(c: PCtx; dest: TDest; ri: PNode; requiresCopy: bool) =
   let tmp = c.genx(ri)
   assert dest >= 0
-  gABC(c, ri, whichAsgnOpc(ri), dest, tmp)
+  gABC(c, ri, whichAsgnOpc(ri), dest, tmp, 1-ord(requiresCopy))
   c.freeTemp(tmp)
 
 proc setSlot(c: PCtx; v: PSym) =
@@ -1140,7 +1180,7 @@ proc setSlot(c: PCtx; v: PSym) =
   if v.position == 0:
     if c.prc.maxSlots == 0: c.prc.maxSlots = 1
     if c.prc.maxSlots >= high(TRegister):
-      internalError(v.info, "cannot generate code; too many registers required")
+      globalError(v.info, "cannot generate code; too many registers required")
     v.position = c.prc.maxSlots
     c.prc.slots[v.position] = (inUse: true,
         kind: if v.kind == skLet: slotFixedLet else: slotFixedVar)
@@ -1166,7 +1206,10 @@ proc checkCanEval(c: PCtx; n: PNode) =
   let s = n.sym
   if {sfCompileTime, sfGlobal} <= s.flags: return
   if s.kind in {skVar, skTemp, skLet, skParam, skResult} and
-      not s.isOwnedBy(c.prc.sym) and s.owner != c.module:
+      not s.isOwnedBy(c.prc.sym) and s.owner != c.module and c.mode != emRepl:
+    cannotEval(n)
+  elif s.kind in {skProc, skConverter, skMethod,
+                  skIterator, skClosureIterator} and sfForward in s.flags:
     cannotEval(n)
 
 proc isTemp(c: PCtx; dest: TDest): bool =
@@ -1183,9 +1226,10 @@ proc preventFalseAlias(c: PCtx; n: PNode; opc: TOpcode;
   # opcLdObj et al really means "load address". We sometimes have to create a
   # copy in order to not introduce false aliasing:
   # mylocal = a.b  # needs a copy of the data!
+  assert n.typ != nil
   if needsAdditionalCopy(n):
     var cc = c.getTemp(n.typ)
-    c.gABC(n, whichAsgnOpc(n), cc, value)
+    c.gABC(n, whichAsgnOpc(n), cc, value, 0)
     c.gABC(n, opc, dest, idx, cc)
     c.freeTemp(cc)
   else:
@@ -1194,7 +1238,7 @@ proc preventFalseAlias(c: PCtx; n: PNode; opc: TOpcode;
 proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) =
   case le.kind
   of nkBracketExpr:
-    let dest = c.genx(le.sons[0], {gfAddrOf})
+    let dest = c.genx(le.sons[0], {gfAddrOf, gfFieldAccess})
     let idx = c.genIndex(le.sons[1], le.sons[0].typ)
     let tmp = c.genx(ri)
     if le.sons[0].typ.skipTypes(abstractVarRange-{tyTypeDesc}).kind in {
@@ -1206,7 +1250,7 @@ proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) =
   of nkDotExpr, nkCheckedFieldExpr:
     # XXX field checks here
     let left = if le.kind == nkDotExpr: le else: le.sons[0]
-    let dest = c.genx(left.sons[0], {gfAddrOf})
+    let dest = c.genx(left.sons[0], {gfAddrOf, gfFieldAccess})
     let idx = genField(left.sons[1])
     let tmp = c.genx(ri)
     c.preventFalseAlias(left, opcWrObj, dest, idx, tmp)
@@ -1230,10 +1274,11 @@ proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) =
       internalAssert s.position > 0 or (s.position == 0 and
                                         s.kind in {skParam,skResult})
       var dest: TRegister = s.position + ord(s.kind == skParam)
+      assert le.typ != nil
       if needsAdditionalCopy(le) and s.kind in {skResult, skVar, skParam}:
         var cc = c.getTemp(le.typ)
         gen(c, ri, cc)
-        c.gABC(le, whichAsgnOpc(le), dest, cc)
+        c.gABC(le, whichAsgnOpc(le), dest, cc, 1)
         c.freeTemp(cc)
       else:
         gen(c, ri, dest)
@@ -1292,12 +1337,13 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
       if sfImportc in s.flags: c.importcSym(n.info, s)
       else: genGlobalInit(c, n, s)
     if dest < 0: dest = c.getTemp(n.typ)
+    assert s.typ != nil
     if gfAddrOf notin flags and fitsRegister(s.typ):
       var cc = c.getTemp(n.typ)
       c.gABx(n, opcLdGlobal, cc, s.position)
       c.gABC(n, opcNodeToReg, dest, cc)
       c.freeTemp(cc)
-    elif gfAddrOf in flags:
+    elif {gfAddrOf, gfFieldAccess} * flags == {gfAddrOf}:
       c.gABx(n, opcLdGlobalAddr, dest, s.position)
     else:
       c.gABx(n, opcLdGlobal, dest, s.position)
@@ -1353,9 +1399,11 @@ proc genCheckedObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
   genObjAccess(c, n.sons[0], dest, flags)
 
 proc genArrAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
-  if n.sons[0].typ.skipTypes(abstractVarRange-{tyTypeDesc}).kind in {
+  let arrayType = n.sons[0].typ.skipTypes(abstractVarRange-{tyTypeDesc}).kind
-      tyString, tyCString}:
+  if arrayType in {tyString, tyCString}:
     genArrAccess2(c, n, dest, opcLdStrIdx, {})
+  elif arrayType == tyTypeDesc:
+    c.genTypeLit(n.typ, dest)
   else:
     genArrAccess2(c, n, dest, opcLdArr, flags)
 
@@ -1368,8 +1416,11 @@ proc getNullValueAux(obj: PNode, result: PNode) =
     for i in countup(1, sonsLen(obj) - 1):
       getNullValueAux(lastSon(obj.sons[i]), result)
   of nkSym:
-    addSon(result, getNullValue(obj.sym.typ, result.info))
+    let field = newNodeI(nkExprColonExpr, result.info)
-  else: internalError(result.info, "getNullValueAux")
+    field.add(obj)
+    field.add(getNullValue(obj.sym.typ, result.info))
+    addSon(result, field)
+  else: globalError(result.info, "cannot create null element for: " & $obj)
 
 proc getNullValue(typ: PType, info: TLineInfo): PNode =
   var t = skipTypes(typ, abstractRange-{tyTypeDesc})
@@ -1394,7 +1445,8 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode =
       result.add(newNodeIT(nkNilLit, info, t))
       result.add(newNodeIT(nkNilLit, info, t))
   of tyObject:
-    result = newNodeIT(nkPar, info, t)
+    result = newNodeIT(nkObjConstr, info, t)
+    result.add(newNodeIT(nkEmpty, info, t))
     getNullValueAux(t.n, result)
     # initialize inherited fields:
     var base = t.sons[0]
@@ -1411,9 +1463,11 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode =
       addSon(result, getNullValue(t.sons[i], info))
   of tySet:
     result = newNodeIT(nkCurly, info, t)
-  else: internalError(info, "getNullValue: " & $t.kind)
+  else:
+    globalError(info, "cannot create null element for: " & $t.kind)
 
 proc ldNullOpcode(t: PType): TOpcode =
+  assert t != nil
   if fitsRegister(t): opcLdNullReg else: opcLdNull
 
 proc genVarSection(c: PCtx; n: PNode) =
@@ -1441,7 +1495,7 @@ proc genVarSection(c: PCtx; n: PNode) =
         if a.sons[2].kind != nkEmpty:
           let tmp = c.genx(a.sons[0], {gfAddrOf})
           let val = c.genx(a.sons[2])
-          c.preventFalseAlias(a, opcWrDeref, tmp, 0, val)
+          c.preventFalseAlias(a.sons[2], opcWrDeref, tmp, 0, val)
           c.freeTemp(val)
           c.freeTemp(tmp)
       else:
@@ -1449,13 +1503,15 @@ proc genVarSection(c: PCtx; n: PNode) =
         if a.sons[2].kind == nkEmpty:
           c.gABx(a, ldNullOpcode(s.typ), s.position, c.genType(s.typ))
         else:
+          assert s.typ != nil
           if not fitsRegister(s.typ):
             c.gABx(a, ldNullOpcode(s.typ), s.position, c.genType(s.typ))
           let le = a.sons[0]
+          assert le.typ != nil
           if not fitsRegister(le.typ) and s.kind in {skResult, skVar, skParam}:
             var cc = c.getTemp(le.typ)
             gen(c, a.sons[2], cc)
-            c.gABC(le, whichAsgnOpc(le), s.position.TRegister, cc)
+            c.gABC(le, whichAsgnOpc(le), s.position.TRegister, cc, 1)
             c.freeTemp(cc)
           else:
             gen(c, a.sons[2], s.position.TRegister)
@@ -1522,7 +1578,7 @@ proc genObjConstr(c: PCtx, n: PNode, dest: var TDest) =
                           dest, idx, tmp)
       c.freeTemp(tmp)
     else:
-      internalError(n.info, "invalid object constructor")
+      globalError(n.info, "invalid object constructor")
 
 proc genTupleConstr(c: PCtx, n: PNode, dest: var TDest) =
   if dest < 0: dest = c.getTemp(n.typ)
@@ -1596,8 +1652,13 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
         c.gABx(n, opcLdConst, dest, lit)
     of skType:
       genTypeLit(c, s.typ, dest)
+    of skGenericParam:
+      if c.prc.sym.kind == skMacro:
+        genRdVar(c, n, dest, flags)
       else:
         internalError(n.info, "cannot generate code for: " & s.name.s)
+    else:
+      globalError(n.info, errGenerated, "cannot generate code for: " & s.name.s)
   of nkCallKinds:
     if n.sons[0].kind == nkSym:
       let s = n.sons[0].sym
@@ -1631,7 +1692,10 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
   of nkBracketExpr: genArrAccess(c, n, dest, flags)
   of nkDerefExpr, nkHiddenDeref: genAddrDeref(c, n, dest, opcLdDeref, flags)
   of nkAddr, nkHiddenAddr: genAddrDeref(c, n, dest, opcAddrNode, flags)
-  of nkWhenStmt, nkIfStmt, nkIfExpr: genIf(c, n, dest)
+  of nkIfStmt, nkIfExpr: genIf(c, n, dest)
+  of nkWhenStmt:
+      # This is "when nimvm" node. Chose the first branch.
+      gen(c, n.sons[0].sons[1], dest)
   of nkCaseStmt: genCase(c, n, dest)
   of nkWhileStmt:
     unused(n, dest)
@@ -1682,7 +1746,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     c.freeTemp(tmp1)
     c.freeTemp(tmp2)
     if dest >= 0:
-      gABC(c, n, whichAsgnOpc(n), dest, tmp0)
+      gABC(c, n, whichAsgnOpc(n), dest, tmp0, 1)
       c.freeTemp(tmp0)
     else:
       dest = tmp0
@@ -1701,7 +1765,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     else:
       globalError(n.info, errGenerated, "VM is not allowed to 'cast'")
   else:
-    internalError n.info, "cannot generate VM code for " & n.renderTree
+    globalError(n.info, errGenerated, "cannot generate VM code for " & $n)
 
 proc removeLastEof(c: PCtx) =
   let last = c.code.len-1
@@ -1717,7 +1781,8 @@ proc genStmt*(c: PCtx; n: PNode): int =
   var d: TDest = -1
   c.gen(n, d)
   c.gABC(n, opcEof)
-  if d >= 0: internalError(n.info, "some destination set")
+  if d >= 0:
+    globalError(n.info, errGenerated, "VM problem: dest register is set")
 
 proc genExpr*(c: PCtx; n: PNode, requiresValue = true): int =
   c.removeLastEof
@@ -1725,10 +1790,14 @@ proc genExpr*(c: PCtx; n: PNode, requiresValue = true): int =
   var d: TDest = -1
   c.gen(n, d)
   if d < 0:
-    if requiresValue: internalError(n.info, "no destination set")
+    if requiresValue:
+      globalError(n.info, errGenerated, "VM problem: dest register is not set")
     d = 0
   c.gABC(n, opcEof, d)
 
+  #echo renderTree(n)
+  #c.echoCode(result)
+
 proc genParams(c: PCtx; params: PNode) =
   # res.sym.position is already 0
   c.prc.slots[0] = (inUse: true, kind: slotFixedVar)
@@ -1744,6 +1813,14 @@ proc finalJumpTarget(c: PCtx; pc, diff: int) =
   c.code[pc] = ((oldInstr.uint32 and 0xffff'u32).uint32 or
                 uint32(diff+wordExcess) shl 16'u32).TInstr
 
+proc genGenericParams(c: PCtx; gp: PNode) =
+  var base = c.prc.maxSlots
+  for i in 0.. <gp.len:
+    var param = gp.sons[i].sym
+    param.position = base + i # XXX: fix this earlier; make it consistent with templates
+    c.prc.slots[base + i] = (inUse: true, kind: slotFixedLet)
+  c.prc.maxSlots = base + gp.len
+
 proc optimizeJumps(c: PCtx; start: int) =
   const maxIterations = 10
   for i in start .. <c.code.len:
@@ -1808,6 +1885,13 @@ proc genProc(c: PCtx; s: PSym): int =
     c.prc = p
     # iterate over the parameters and allocate space for them:
     genParams(c, s.typ.n)
+
+    # allocate additional space for any generically bound parameters
+    if s.kind == skMacro and
+       sfImmediate notin s.flags and
+       s.ast[genericParamsPos].kind != nkEmpty:
+      genGenericParams(c, s.ast[genericParamsPos])
+
     if tfCapturesEnv in s.typ.flags:
       #let env = s.ast.sons[paramsPos].lastSon.sym
       #assert env.position == 2

compiler/vmhooks.nim

@@ -30,6 +30,24 @@ proc setResult*(a: VmArgs; v: string) =
   s[a.ra].node = newNode(nkStrLit)
   s[a.ra].node.strVal = v
 
+proc setResult*(a: VmArgs; n: PNode) =
+  var s: seq[TFullReg]
+  move(s, cast[seq[TFullReg]](a.slots))
+  if s[a.ra].kind != rkNode:
+    myreset(s[a.ra])
+    s[a.ra].kind = rkNode
+  s[a.ra].node = n
+
+proc setResult*(a: VmArgs; v: seq[string]) =
+  var s: seq[TFullReg]
+  move(s, cast[seq[TFullReg]](a.slots))
+  if s[a.ra].kind != rkNode:
+    myreset(s[a.ra])
+    s[a.ra].kind = rkNode
+  var n = newNode(nkBracket)
+  for x in v: n.add newStrNode(nkStrLit, x)
+  s[a.ra].node = n
+
 template getX(k, field) {.immediate, dirty.} =
   doAssert i < a.rc-1
   let s = cast[seq[TFullReg]](a.slots)

compiler/vmmarshal.nim

@@ -36,8 +36,8 @@ proc getField(n: PNode; position: int): PSym =
 proc storeAny(s: var string; t: PType; a: PNode; stored: var IntSet)
 
 proc storeObj(s: var string; typ: PType; x: PNode; stored: var IntSet) =
-  internalAssert x.kind in {nkObjConstr, nkPar}
+  internalAssert x.kind == nkObjConstr
-  let start = ord(x.kind == nkObjConstr)
+  let start = 1
   for i in countup(start, sonsLen(x) - 1):
     if i > start: s.add(", ")
     var it = x.sons[i]
@@ -205,18 +205,23 @@ proc loadAny(p: var JsonParser, t: PType,
   of tyObject:
     if p.kind != jsonObjectStart: raiseParseErr(p, "'{' expected for an object")
     next(p)
-    result = newNode(nkPar)
+    result = newNode(nkObjConstr)
-    result.sons = @[]
+    result.sons = @[newNode(nkEmpty)]
     while p.kind != jsonObjectEnd and p.kind != jsonEof:
       if p.kind != jsonString:
         raiseParseErr(p, "string expected for a field name")
-      let field = lookupInRecord(t.n, getIdent(p.str))
+      let ident = getIdent(p.str)
+      let field = lookupInRecord(t.n, ident)
       if field.isNil:
         raiseParseErr(p, "unknown field for object of type " & typeToString(t))
       next(p)
-      if field.position >= result.sons.len:
+      let pos = field.position + 1
-        setLen(result.sons, field.position+1)
+      if pos >= result.sons.len:
-      result.sons[field.position] = loadAny(p, field.typ, tab)
+        setLen(result.sons, pos + 1)
+      let fieldNode = newNode(nkExprColonExpr)
+      fieldNode.addSon(newSymNode(newSym(skField, ident, nil, unknownLineInfo())))
+      fieldNode.addSon(loadAny(p, field.typ, tab))
+      result.sons[pos] = fieldNode
     if p.kind == jsonObjectEnd: next(p)
     else: raiseParseErr(p, "'}' end of object expected")
   of tySet:
@@ -234,7 +239,7 @@ proc loadAny(p: var JsonParser, t: PType,
       result = newNode(nkNilLit)
       next(p)
     of jsonInt:
-      result = tab[p.getInt]
+      result = tab.getOrDefault(p.getInt)
       if result.isNil:
         raiseParseErr(p, "cannot load object with address " & $p.getInt)
       next(p)

compiler/vmops.nim

@@ -46,7 +46,7 @@ template wrap2svoid(op) {.immediate, dirty.} =
 
 proc getCurrentExceptionMsgWrapper(a: VmArgs) {.nimcall.} =
   setResult(a, if a.currentException.isNil: ""
-               else: a.currentException.sons[2].strVal)
+               else: a.currentException.sons[3].skipColon.strVal)
 
 proc registerAdditionalOps*(c: PCtx) =
   wrap1f(sqrt)

compiler/wordrecg.nim

@@ -42,10 +42,10 @@ type
     wImmediate, wConstructor, wDestructor, wDelegator, wOverride,
     wImportCpp, wImportObjC,
     wImportCompilerProc,
-    wImportc, wExportc, wIncompleteStruct, wRequiresInit,
+    wImportc, wExportc, wExportNims, wIncompleteStruct, wRequiresInit,
     wAlign, wNodecl, wPure, wSideeffect, wHeader,
     wNosideeffect, wGcSafe, wNoreturn, wMerge, wLib, wDynlib,
-    wCompilerproc, wProcVar,
+    wCompilerproc, wProcVar, wBase,
     wFatal, wError, wWarning, wHint, wLine, wPush, wPop, wDefine, wUndef,
     wLinedir, wStacktrace, wLinetrace, wLink, wCompile,
     wLinksys, wDeprecated, wVarargs, wCallconv, wBreakpoint, wDebugger,
@@ -55,7 +55,7 @@ type
     wFloatchecks, wNanChecks, wInfChecks,
     wAssertions, wPatterns, wWarnings,
     wHints, wOptimization, wRaises, wWrites, wReads, wSize, wEffects, wTags,
-    wDeadCodeElim, wSafecode, wNoForward,
+    wDeadCodeElim, wSafecode, wNoForward, wNoRewrite,
     wPragma,
     wCompileTime, wNoInit,
     wPassc, wPassl, wBorrow, wDiscardable,
@@ -82,6 +82,7 @@ type
     wStdIn, wStdOut, wStdErr,
 
     wInOut, wByCopy, wByRef, wOneWay,
+    wBitsize,
 
   TSpecialWords* = set[TSpecialWord]
 
@@ -125,10 +126,12 @@ const
 
     "immediate", "constructor", "destructor", "delegator", "override",
     "importcpp", "importobjc",
-    "importcompilerproc", "importc", "exportc", "incompletestruct",
+    "importcompilerproc", "importc", "exportc", "exportnims",
+    "incompletestruct",
     "requiresinit", "align", "nodecl", "pure", "sideeffect",
     "header", "nosideeffect", "gcsafe", "noreturn", "merge", "lib", "dynlib",
-    "compilerproc", "procvar", "fatal", "error", "warning", "hint", "line",
+    "compilerproc", "procvar", "base",
+    "fatal", "error", "warning", "hint", "line",
     "push", "pop", "define", "undef", "linedir", "stacktrace", "linetrace",
     "link", "compile", "linksys", "deprecated", "varargs",
     "callconv", "breakpoint", "debugger", "nimcall", "stdcall",
@@ -139,7 +142,7 @@ const
 
     "assertions", "patterns", "warnings", "hints",
     "optimization", "raises", "writes", "reads", "size", "effects", "tags",
-    "deadcodeelim", "safecode", "noforward",
+    "deadcodeelim", "safecode", "noforward", "norewrite",
     "pragma",
     "compiletime", "noinit",
     "passc", "passl", "borrow", "discardable", "fieldchecks",
@@ -167,6 +170,7 @@ const
     "stdin", "stdout", "stderr",
 
     "inout", "bycopy", "byref", "oneway",
+    "bitsize",
     ]
 
 proc findStr*(a: openArray[string], s: string): int =

compiler/writetracking.nim

@@ -0,0 +1,272 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements the write tracking analysis. Read my block post for
+## a basic description of the algorithm and ideas.
+## The algorithm operates in 2 phases:
+##
+##   * Collecting information about assignments (and pass-by-var calls).
+##   * Computing an aliasing relation based on the assignments. This relation
+##     is then used to compute the 'writes' and 'escapes' effects.
+
+import intsets, idents, ast, astalgo, trees, renderer, msgs, types
+
+const
+  debug = false
+
+type
+  AssignToResult = enum
+    asgnNil,   # 'nil' is fine
+    asgnNew,   # 'new(result)'
+    asgnOther  # result = fooBar # not a 'new' --> 'result' might not 'new'
+  NewLocation = enum
+    newNone,
+    newLit,
+    newCall
+  RootInfo = enum
+    rootIsResultOrParam,
+    rootIsHeapAccess,
+    rootIsSym,
+    markAsWrittenTo,
+    markAsEscaping
+
+  Assignment = object # \
+    # Note that the transitive closures MUST be computed in
+    # phase 2 of the algorithm.
+    dest, src: seq[ptr TSym] # we use 'ptr' here to save RC ops and GC cycles
+    destNoTc, srcNoTc: int # length of 'dest', 'src' without the
+                           # transitive closure
+    destInfo: set[RootInfo]
+    info: TLineInfo
+
+  W = object # WriteTrackContext
+    owner: PSym
+    returnsNew: AssignToResult # assignments to 'result'
+    assignments: seq[Assignment] # list of all assignments in this proc
+
+proc allRoots(n: PNode; result: var seq[ptr TSym]; info: var set[RootInfo]) =
+  case n.kind
+  of nkSym:
+    if n.sym.kind in {skParam, skVar, skTemp, skLet, skResult, skForVar}:
+      if n.sym.kind in {skResult, skParam}: incl(info, rootIsResultOrParam)
+      result.add(cast[ptr TSym](n.sym))
+  of nkHiddenDeref, nkDerefExpr:
+    incl(info, rootIsHeapAccess)
+    allRoots(n.sons[0], result, info)
+  of nkDotExpr, nkBracketExpr, nkCheckedFieldExpr,
+      nkHiddenAddr, nkObjUpConv, nkObjDownConv:
+    allRoots(n.sons[0], result, info)
+  of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv, nkConv,
+      nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr, nkOfBranch,
+      nkElifBranch, nkElse, nkExceptBranch, nkFinally, nkCast:
+    allRoots(n.lastSon, result, info)
+  of nkCallKinds:
+    if getMagic(n) == mSlice:
+      allRoots(n.sons[1], result, info)
+    else:
+      # we do significantly better here by using the available escape
+      # information:
+      if n.sons[0].typ.isNil: return
+      var typ = n.sons[0].typ
+      if typ != nil:
+        typ = skipTypes(typ, abstractInst)
+        if typ.kind != tyProc: typ = nil
+        else: assert(sonsLen(typ) == sonsLen(typ.n))
+
+      for i in 1 ..< n.len:
+        let it = n.sons[i]
+        if typ != nil and i < sonsLen(typ):
+          assert(typ.n.sons[i].kind == nkSym)
+          let paramType = typ.n.sons[i]
+          if paramType.typ.isCompileTimeOnly: continue
+          if sfEscapes in paramType.sym.flags or paramType.typ.kind == tyVar:
+            allRoots(it, result, info)
+        else:
+          allRoots(it, result, info)
+  else:
+    for i in 0..<n.safeLen:
+      allRoots(n.sons[i], result, info)
+
+proc addAsgn(a: var Assignment; dest, src: PNode; destInfo: set[RootInfo]) =
+  a.dest = @[]
+  a.src = @[]
+  a.destInfo = destInfo
+  allRoots(dest, a.dest, a.destInfo)
+  if dest.kind == nkSym: incl(a.destInfo, rootIsSym)
+  if src != nil:
+    var dummy: set[RootInfo]
+    allRoots(src, a.src, dummy)
+  a.destNoTc = a.dest.len
+  a.srcNoTc = a.src.len
+  a.info = dest.info
+  #echo "ADDING ", dest.info, " ", a.destInfo
+
+proc srcHasSym(a: Assignment; x: ptr TSym): bool =
+  for i in 0 ..< a.srcNoTc:
+    if a.src[i] == x: return true
+
+proc returnsNewExpr*(n: PNode): NewLocation =
+  case n.kind
+  of nkCharLit..nkInt64Lit, nkStrLit..nkTripleStrLit,
+      nkFloatLit..nkFloat64Lit, nkNilLit:
+    result = newLit
+  of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv,
+      nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr, nkOfBranch,
+      nkElifBranch, nkElse, nkExceptBranch, nkFinally, nkCast:
+    result = returnsNewExpr(n.lastSon)
+  of nkCurly, nkBracket, nkPar, nkObjConstr, nkClosure,
+      nkIfExpr, nkIfStmt, nkWhenStmt, nkCaseStmt, nkTryStmt:
+    result = newLit
+    for i in ord(n.kind == nkObjConstr) .. <n.len:
+      let x = returnsNewExpr(n.sons[i])
+      case x
+      of newNone: return newNone
+      of newLit: discard
+      of newCall: result = newCall
+  of nkCallKinds:
+    if n.sons[0].typ != nil and tfReturnsNew in n.sons[0].typ.flags:
+      result = newCall
+  else:
+    result = newNone
+
+proc deps(w: var W; dest, src: PNode; destInfo: set[RootInfo]) =
+  # let x = (localA, localB)
+  # compute 'returnsNew' property:
+  let retNew = if src.isNil: newNone else: returnsNewExpr(src)
+  if dest.kind == nkSym and dest.sym.kind == skResult:
+    if retNew != newNone:
+      if w.returnsNew != asgnOther: w.returnsNew = asgnNew
+    else:
+      w.returnsNew = asgnOther
+  # mark the dependency, but
+  # rule out obviously innocent assignments like 'somebool = true'
+  if dest.kind == nkSym and retNew == newLit: discard
+  else:
+    let L = w.assignments.len
+    w.assignments.setLen(L+1)
+    addAsgn(w.assignments[L], dest, src, destInfo)
+
+proc depsArgs(w: var W; n: PNode) =
+  if n.sons[0].typ.isNil: return
+  var typ = skipTypes(n.sons[0].typ, abstractInst)
+  if typ.kind != tyProc: return
+  # echo n.info, " ", n, " ", w.owner.name.s, " ", typeToString(typ)
+  assert(sonsLen(typ) == sonsLen(typ.n))
+  for i in 1 ..< n.len:
+    let it = n.sons[i]
+    if i < sonsLen(typ):
+      assert(typ.n.sons[i].kind == nkSym)
+      let paramType = typ.n.sons[i]
+      if paramType.typ.isCompileTimeOnly: continue
+      var destInfo: set[RootInfo] = {}
+      if sfWrittenTo in paramType.sym.flags or paramType.typ.kind == tyVar:
+        # p(f(x, y), X, g(h, z))
+        destInfo.incl markAsWrittenTo
+      if sfEscapes in paramType.sym.flags:
+        destInfo.incl markAsEscaping
+      if destInfo != {}:
+        deps(w, it, nil, destInfo)
+
+proc deps(w: var W; n: PNode) =
+  case n.kind
+  of nkLetSection, nkVarSection:
+    for child in n:
+      let last = lastSon(child)
+      if last.kind == nkEmpty: continue
+      if child.kind == nkVarTuple and last.kind == nkPar:
+        internalAssert child.len-2 == last.len
+        for i in 0 .. child.len-3:
+          deps(w, child.sons[i], last.sons[i], {})
+      else:
+        for i in 0 .. child.len-3:
+          deps(w, child.sons[i], last, {})
+  of nkAsgn, nkFastAsgn:
+    deps(w, n.sons[0], n.sons[1], {})
+  else:
+    for i in 0 ..< n.safeLen:
+      deps(w, n.sons[i])
+    if n.kind in nkCallKinds:
+      if getMagic(n) in {mNew, mNewFinalize, mNewSeq}:
+        # may not look like an assignment, but it is:
+        deps(w, n.sons[1], newNodeIT(nkObjConstr, n.info, n.sons[1].typ), {})
+      else:
+        depsArgs(w, n)
+
+proc possibleAliases(w: var W; result: var seq[ptr TSym]) =
+  # this is an expensive fixpoint iteration. We could speed up this analysis
+  # by a smarter data-structure but we wait until profiling shows us it's
+  # expensive. Usually 'w.assignments' is small enough.
+  var alreadySeen = initIntSet()
+  template addNoDup(x) =
+    if not alreadySeen.containsOrIncl(x.id): result.add x
+  for x in result: alreadySeen.incl x.id
+
+  var todo = 0
+  while todo < result.len:
+    let x = result[todo]
+    inc todo
+    for a in mitems(w.assignments):
+      #if a.srcHasSym(x):
+      #  # y = f(..., x, ...)
+      #  for i in 0 ..< a.destNoTc: addNoDup a.dest[i]
+      if a.destNoTc > 0 and a.dest[0] == x and rootIsSym in a.destInfo:
+        # x = f(..., y, ....)
+        for i in 0 ..< a.srcNoTc: addNoDup a.src[i]
+
+proc markWriteOrEscape(w: var W) =
+  ## Both 'writes' and 'escapes' effects ultimately only care
+  ## about *parameters*.
+  ## However, due to aliasing, even locals that might not look as parameters
+  ## have to count as parameters if they can alias a parameter:
+  ##
+  ## .. code-block:: nim
+  ##   proc modifies(n: Node) {.writes: [n].} =
+  ##     let x = n
+  ##     x.data = "abc"
+  ##
+  ## We call a symbol *parameter-like* if it is a parameter or can alias a
+  ## parameter.
+  ## Let ``p``, ``q`` be *parameter-like* and ``x``, ``y`` be general
+  ## expressions.
+  ##
+  ## A write then looks like ``p[] = x``.
+  ## An escape looks like ``p[] = q`` or more generally
+  ## like ``p[] = f(q)`` where ``f`` can forward ``q``.
+  for a in mitems(w.assignments):
+    if a.destInfo != {}:
+      possibleAliases(w, a.dest)
+
+    if {rootIsHeapAccess, markAsWrittenTo} * a.destInfo != {}:
+      for p in a.dest:
+        if p.kind == skParam and p.owner == w.owner:
+          incl(p.flags, sfWrittenTo)
+
+    if {rootIsResultOrParam, rootIsHeapAccess, markAsEscaping}*a.destInfo != {}:
+      var destIsParam = false
+      for p in a.dest:
+        if p.kind in {skResult, skParam} and p.owner == w.owner:
+          destIsParam = true
+          break
+      if destIsParam:
+        possibleAliases(w, a.src)
+        for p in a.src:
+          if p.kind == skParam and p.owner == w.owner:
+            incl(p.flags, sfEscapes)
+
+proc trackWrites*(owner: PSym; body: PNode) =
+  var w: W
+  w.owner = owner
+  w.assignments = @[]
+  # Phase 1: Collect and preprocess any assignments in the proc body:
+  deps(w, body)
+  # Phase 2: Compute the 'writes' and 'escapes' effects:
+  markWriteOrEscape(w)
+  if w.returnsNew != asgnOther and not isEmptyType(owner.typ.sons[0]) and
+      containsGarbageCollectedRef(owner.typ.sons[0]):
+    incl(owner.typ.flags, tfReturnsNew)

config/nim.cfg

@@ -27,30 +27,22 @@ mips.linux.gcc.linkerexe = "mips-openwrt-linux-gcc"
   cs:partial
 @end
 
-path="$lib/core"
+path="$lib/deprecated/core"
-
+path="$lib/deprecated/pure"
-path="$lib/pure"
 path="$lib/pure/collections"
 path="$lib/pure/concurrency"
 path="$lib/impure"
 path="$lib/wrappers"
-# path="$lib/wrappers/cairo"
+path="$lib/wrappers/linenoise"
-# path="$lib/wrappers/gtk"
-# path="$lib/wrappers/lua"
-# path="$lib/wrappers/opengl"
-path="$lib/wrappers/pcre"
-path="$lib/wrappers/readline"
-path="$lib/wrappers/sdl"
-# path="$lib/wrappers/x11"
-path="$lib/wrappers/zip"
-path="$lib/wrappers/libffi"
 path="$lib/windows"
 path="$lib/posix"
 path="$lib/js"
 path="$lib/pure/unidecode"
+path="$lib/arch"
+path="$lib/core"
+path="$lib/pure"
 
 @if nimbabel:
-  babelpath="$home/.babel/pkgs/"
   nimblepath="$home/.nimble/pkgs/"
 @end
 
@@ -78,6 +70,7 @@ path="$lib/pure/unidecode"
     gcc.options.linker = "-ldl"
     gcc.cpp.options.linker = "-ldl"
     clang.options.linker = "-ldl"
+    clang.cpp.options.linker = "-ldl"
     tcc.options.linker = "-ldl"
   @end
   @if bsd or haiku:

config/nimdoc.cfg

@@ -86,6 +86,9 @@ $moduledesc
 $content
 """
 
+doc.listing_start = "<pre class = \"listing\">"
+doc.listing_end = "</pre>"
+
 # * $analytics: Google analytics location, includes <script> tags
 doc.file = """<?xml version="1.0" encoding="utf-8" ?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
@@ -142,7 +145,7 @@ body {
   font-weight: 400;
   font-size: 14px;
   line-height: 20px;
-  color: #2d2d2d;
+  color: #666;
   background-color: rgba(252, 248, 244, 0.75); }
 
 /* Skeleton grid */
@@ -578,7 +581,7 @@ pre {
   box-sizing: border-box;
   min-width: calc(100% - 19.5px);
   padding: 9.5px;
-  margin: 0 10px 0px 10px;
+  margin: 0.25em 10px 0.25em 10px;
   font-size: 14px;
   line-height: 20px;
   white-space: pre !important;
@@ -981,6 +984,10 @@ div.align-right {
 
 /* div.align-center * { */
 /*   text-align: left } */
+
+ul.simple > li {
+  margin-bottom: 0.5em }
+
 ol.simple, ul.simple {
   margin-bottom: 1em; }
 

contributing.rst

@@ -0,0 +1,226 @@
+Writing tests
+=============
+
+Not all the tests follow this scheme, feel free to change the ones
+that don't. Always leave the code cleaner than you found it.
+
+Stdlib
+------
+
+If you change the stdlib (anything under ``lib/``), put a test in the
+file you changed. Add the tests under an ``when isMainModule:``
+condition so they only get executed when the tester is building the
+file. Each test should be in a separate ``block:`` statement, such that
+each has its own scope. Use boolean conditions and ``doAssert`` for the
+testing by itself, don't rely on echo statements or similar.
+
+Sample test:
+
+.. code-block:: nim
+
+  when isMainModule:
+    block: # newSeqWith tests
+      var seq2D = newSeqWith(4, newSeq[bool](2))
+      seq2D[0][0] = true
+      seq2D[1][0] = true
+      seq2D[0][1] = true
+      doAssert seq2D == @[@[true, true], @[true, false],
+                          @[false, false], @[false, false]]
+
+Compiler
+--------
+
+The tests for the compiler work differently, they are all located in
+``tests/``. Each test has its own file, which is different from the
+stdlib tests. All test files are prefixed with ``t``. If you want to
+create a file for import into another test only, use the prefix ``m``.
+
+At the beginning of every test is the expected side of the test.
+Possible keys are:
+
+- output: The expected output, most likely via ``echo``
+- exitcode: Exit code of the test (via ``exit(number)``)
+- errormsg: The expected error message
+- file: The file the errormsg
+- line: The line the errormsg was produced at
+
+An example for a test:
+
+.. code-block:: nim
+
+  discard """
+    errormsg: "type mismatch: got (PTest)"
+  """
+
+  type
+    PTest = ref object
+
+  proc test(x: PTest, y: int) = nil
+
+  var buf: PTest
+  buf.test()
+
+Running tests
+=============
+
+You can run the tests with
+
+::
+
+  ./koch tests
+
+which will run a good subset of tests. Some tests may fail. If you
+only want to see the output of failing tests, go for
+
+::
+
+  ./koch tests --failing all
+
+You can also run only a single category of tests. A category is a subdirectory
+in the ``tests`` directory. There are a couple of special categories; for a
+list of these, see ``tests/testament/categories.nim``, at the bottom.
+
+::
+
+  ./koch tests c lib
+
+Comparing tests
+===============
+
+Because some tests fail in the current ``devel`` branch, not every fail
+after your change is necessarily caused by your changes.
+
+The tester can compare two test runs. First, you need to create the
+reference test. You'll also need to the commit id, because that's what
+the tester needs to know in order to compare the two.
+
+::
+
+  git checkout devel
+  DEVEL_COMMIT=$(git rev-parse HEAD)
+  ./koch tests
+
+Then switch over to your changes and run the tester again.
+
+::
+
+  git checkout your-changes
+  ./koch tests
+
+Then you can ask the tester to create a ``testresults.html`` which will
+tell you if any new tests passed/failed.
+
+::
+
+  ./koch tests --print html $DEVEL_COMMIT
+
+
+Deprecation
+===========
+
+Backward compatibility is important, so if you are renaming a proc or
+a type, you can use
+
+
+.. code-block:: nim
+
+  {.deprecated: [oldName: new_name].}
+
+Or you can simply use
+
+.. code-block:: nim
+
+  proc oldProc() {.deprecated.}
+
+to mark a symbol as deprecated. Works for procs/types/vars/consts,
+etc. Note that currently the ``deprecated`` statement does not work well with
+overloading so for routines the latter variant is better.
+
+
+`Deprecated <http://nim-lang.org/docs/manual.html#pragmas-deprecated-pragma>`_
+pragma in the manual.
+
+
+Documentation
+=============
+
+When contributing new procedures, be sure to add documentation, especially if
+the procedure is exported from the module. Documentation begins on the line
+following the ``proc`` definition, and is prefixed by ``##`` on each line.
+
+Code examples are also encouraged. The RestructuredText Nim uses has a special
+syntax for including examples.
+
+.. code-block:: nim
+
+  proc someproc*(): string =
+    ## Return "something"
+    ##
+    ## .. code-block:: nim
+    ##
+    ##  echo someproc() # "something"
+    result = "something" # single-hash comments do not produce documentation
+
+The ``.. code-block:: nim`` followed by a newline and an indentation instructs the
+``nim doc`` and ``nim doc2`` commands to produce syntax-highlighted example code with
+the documentation.
+
+When forward declaration is used, the documentation should be included with the
+first appearance of the proc.
+
+.. code-block:: nim
+
+  proc hello*(): string
+    ## Put documentation here
+  proc nothing() = discard
+  proc hello*(): string =
+    ## Ignore this
+    echo "hello"
+
+The preferred documentation style is to begin with a capital letter and use
+the imperative (command) form. That is, between:
+
+.. code-block:: nim
+
+  proc hello*(): string =
+    # Return "hello"
+    result = "hello"
+or
+
+.. code-block:: nim
+
+  proc hello*(): string =
+    # says hello
+    result = "hello"
+
+the first is preferred.
+
+The Git stuff
+=============
+
+General commit rules
+--------------------
+
+1. All changes introduced by the commit (diff lines) must be related to the
+   subject of the commit.
+
+   If you change some other unrelated to the subject parts of the file, because
+   your editor reformatted automatically the code or whatever different reason,
+   this should be excluded from the commit.
+
+   *Tip:* Never commit everything as is using ``git commit -a``, but review
+   carefully your changes with ``git add -p``.
+
+2. Changes should not introduce any trailing whitespace.
+
+   Always check your changes for whitespace errors using ``git diff --check``
+   or add following ``pre-commit`` hook:
+
+   .. code-block:: sh
+
+      #!/bin/sh
+      git diff --check --cached || exit $?
+
+3. Describe your commit and use your common sense.
+
+.. include:: docstyle.rst

contributors.txt

@@ -1,4 +1,4 @@
-Comex
+Comex
 Eric Doughty-Papassideris
 Simon Hafner
 Keita Haga
@@ -6,8 +6,7 @@ Grzegorz Adam Hankiewicz
 Philippe Lhoste
 Zahary Karadjov
 Mario Ray Mahardhika
-Alex Mitchell
+Alexander Mitchell-Robinson (Amrykid)
 Dominik Picheta
 Jonathan Plona
-Alexander Rødseth
+Alexander Rødseth
-