lazy loading of body ast implemented

compiler/ast.nim

@@ -333,6 +333,11 @@ type
                           # mean: never)
   TSymKinds* = set[TSymKind]
 
+const
+  routineKinds* = {skProc, skMethod, skIterator, skConverter,
+    skMacro, skTemplate}
+
+type
   TMagic* = enum # symbols that require compiler magic:
     mNone, mDefined, mDefinedInScope, mLow, mHigh, mSizeOf, mIs, mOf,
     mEcho, mShallowCopy, mSlurp,
@@ -579,7 +584,7 @@ const
   genericParamsPos* = 1
   paramsPos* = 2
   pragmasPos* = 3
-  codePos* = 4
+  bodyPos* = 4       # position of body; use rodread.getBody() instead!
   resultPos* = 5
   dispatcherPos* = 6 # caution: if method has no 'result' it can be position 5!
 

compiler/ccgexprs.nim

@@ -1734,7 +1734,7 @@ proc expr(p: BProc, e: PNode, d: var TLoc) =
     var sym = e.sym
     case sym.Kind
     of skMethod:
-      if sym.ast.sons[codePos].kind == nkEmpty:
+      if sym.getBody.kind == nkEmpty:
         # we cannot produce code for the dispatcher yet:
         fillProcLoc(sym)
         genProcPrototype(p.module, sym)

compiler/ccgstmts.nim

@@ -702,7 +702,7 @@ proc genStmts(p: BProc, t: PNode) =
           (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or
           (prc.kind == skMethod): 
         # we have not only the header: 
-        if t.sons[codePos].kind != nkEmpty or lfDynamicLib in prc.loc.flags: 
+        if prc.getBody.kind != nkEmpty or lfDynamicLib in prc.loc.flags: 
           genProc(p.module, prc)
   else: internalError(t.info, "genStmts(" & $t.kind & ')')
   

compiler/cgen.nim

@@ -544,7 +544,7 @@ proc genProcAux(m: BModule, prc: PSym) =
   for i in countup(1, sonsLen(prc.typ.n) - 1): 
     var param = prc.typ.n.sons[i].sym
     assignParam(p, param)
-  genStmts(p, prc.ast.sons[codePos]) # modifies p.locals, p.init, etc.
+  genStmts(p, prc.getBody) # modifies p.locals, p.init, etc.
   var generatedProc: PRope
   if sfPure in prc.flags: 
     generatedProc = ropeff("$1 {$n$2$3$4}$n", "define $1 {$n$2$3$4}$n",

compiler/cgmeth.nim

@@ -95,7 +95,7 @@ proc methodDef*(s: PSym, fromCache: bool) =
     # we can't inline the dispatcher itself (for now):
     if disp.typ.callConv == ccInline: disp.typ.callConv = ccDefault
     disp.ast = copyTree(s.ast)
-    disp.ast.sons[codePos] = ast.emptyNode
+    disp.ast.sons[bodyPos] = ast.emptyNode
     if s.typ.sons[0] != nil: 
       disp.ast.sons[resultPos].sym = copySym(s.ast.sons[resultPos].sym)
     attachDispatcher(s, newSymNode(disp))
@@ -183,7 +183,7 @@ proc genDispatcher(methods: TSymSeq, relevantCols: TIntSet): PSym =
       addSon(disp, a)
     else:
       disp = ret
-  result.ast.sons[codePos] = disp
+  result.ast.sons[bodyPos] = disp
 
 proc generateMethodDispatchers*(): PNode = 
   result = newNode(nkStmtList)

compiler/ecmasgen.nim

@@ -1309,7 +1309,7 @@ proc genProc(oldProc: var TProc, prc: PSym, r: var TCompRes) =
     gen(p, prc.ast.sons[resultPos], a)
     if a.com != nil: appf(returnStmt, "$1;$n", [a.com])
     returnStmt = ropef("return $1;$n", [a.res])
-  genStmt(p, prc.ast.sons[codePos], r)
+  genStmt(p, prc.getBody, r)
   r.com = ropef("function $1($2) {$n$3$4$5}$n", 
                 [name, header, resultAsgn, genProcBody(p, prc, r), returnStmt])
   r.res = nil  
@@ -1360,7 +1360,7 @@ proc genStmt(p: var TProc, n: PNode, r: var TCompRes) =
   of nkProcDef, nkMethodDef, nkConverterDef: 
     if (n.sons[genericParamsPos].kind == nkEmpty): 
       var prc = n.sons[namePos].sym
-      if (prc.ast.sons[codePos].kind != nkEmpty) and not (lfNoDecl in prc.loc.flags): 
+      if lfNoDecl notin prc.loc.flags and prc.getBody.kind != nkEmpty:
         genProc(p, prc, r)
       else:
         discard mangleName(prc)

compiler/evals.nim

@@ -16,7 +16,7 @@
 import 
   strutils, magicsys, lists, options, ast, astalgo, trees, treetab, nimsets, 
   msgs, os, condsyms, idents, renderer, types, passes, semfold, transf, 
-  parser, ropes
+  parser, ropes, rodread
 
 type 
   PStackFrame* = ref TStackFrame
@@ -448,7 +448,8 @@ proc evalSwap(c: PEvalContext, n: PNode): PNode =
 proc evalSym(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode = 
   var s = n.sym
   case s.kind
-  of skProc, skConverter, skMacro: result = s.ast.sons[codePos]
+  of skProc, skConverter, skMacro: 
+    result = s.getBody
   of skVar, skForVar, skTemp, skResult:
     if sfGlobal notin s.flags:
       result = evalVariable(c.tos, s, flags)
@@ -621,15 +622,16 @@ proc evalReturn(c: PEvalContext, n: PNode): PNode =
 
 proc evalProc(c: PEvalContext, n: PNode): PNode = 
   if n.sons[genericParamsPos].kind == nkEmpty: 
+    var s = n.sons[namePos].sym
     if (resultPos < sonsLen(n)) and (n.sons[resultPos].kind != nkEmpty): 
       var v = n.sons[resultPos].sym
       result = getNullValue(v.typ, n.info)
       IdNodeTablePut(c.tos.mapping, v, result)
-      result = evalAux(c, n.sons[codePos], {})
+      result = evalAux(c, s.getBody, {})
       if result.kind == nkReturnToken: 
         result = IdNodeTableGet(c.tos.mapping, v)
     else:
-      result = evalAux(c, n.sons[codePos], {})
+      result = evalAux(c, s.getBody, {})
       if result.kind == nkReturnToken: 
         result = emptyNode
   else: 
@@ -852,7 +854,7 @@ proc evalTemplate(n: PNode, sym: PSym): PNode =
 
   # replace each param by the corresponding node:
   var args = evalTemplateArgs(n, sym)
-  result = evalTemplateAux(sym.ast.sons[codePos], args, sym)
+  result = evalTemplateAux(sym.getBody, args, sym)
 
   dec(evalTemplateCounter)
   
@@ -1273,7 +1275,7 @@ proc evalMacroCall*(c: PEvalContext, n: PNode, sym: PSym): PNode =
   s.params[0] = newNodeIT(nkNilLit, n.info, sym.typ.sons[0])
   s.params[1] = n
   pushStackFrame(c, s)
-  discard eval(c, sym.ast.sons[codePos])
+  discard eval(c, sym.getBody)
   result = s.params[0]
   popStackFrame(c)
   if cyclicTree(result): GlobalError(n.info, errCyclicTree)

compiler/nversion.nim

@@ -18,5 +18,5 @@ const
   VersionPatch* = 13
   VersionAsString* = $VersionMajor & "." & $VersionMinor & "." & $VersionPatch
 
-  RodFileVersion* = "1030"       # modify this if the rod-format changes!
+  RodFileVersion* = "1031"       # modify this if the rod-format changes!
 

compiler/passaux.nim

@@ -42,7 +42,7 @@ proc cleanUp(c: PPassContext, n: PNode): PNode =
     if n.sons[namePos].kind == nkSym: 
       var s = n.sons[namePos].sym
       if sfDeadCodeElim notin getModule(s).flags and not astNeeded(s): 
-        s.ast.sons[codePos] = ast.emptyNode # free the memory
+        s.ast.sons[bodyPos] = ast.emptyNode # free the memory
   else: 
     nil
 

compiler/renderer.nim

@@ -780,7 +780,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     gsub(g, n.sons[pragmasPos])
     put(g, tkSpaces, Space)
     putWithSpace(g, tkEquals, "=")
-    gsub(g, n.sons[codePos])
+    gsub(g, n.sons[bodyPos])
   of nkConstDef, nkIdentDefs: 
     gcomma(g, n, 0, - 3)
     var L = sonsLen(n)

compiler/rodread.nim

@@ -169,7 +169,22 @@ proc decodeLineInfo(r: PRodReader, info: var TLineInfo) =
         inc(r.pos)
         info = newLineInfo(r.files[decodeVInt(r.s, r.pos)], info.line, info.col)
 
-proc decodeNode(r: PRodReader, fInfo: TLineInfo): PNode = 
+proc skipNode(r: PRodReader) =
+  assert r.s[r.pos] == '('
+  var par = 0
+  var pos = r.pos+1
+  while true:
+    case r.s[pos]
+    of ')':
+      if par == 0: break
+      dec par
+    of '(': inc par
+    else: nil
+    inc pos
+  r.pos = pos+1 # skip ')'
+
+proc decodeNodeLazyBody(r: PRodReader, fInfo: TLineInfo, 
+                        belongsTo: PSym): PNode = 
   result = nil
   if r.s[r.pos] == '(': 
     inc(r.pos)
@@ -216,12 +231,22 @@ proc decodeNode(r: PRodReader, fInfo: TLineInfo): PNode =
       else: 
         internalError(result.info, "decodeNode: nkSym")
     else:
+      var i = 0
       while r.s[r.pos] != ')': 
-        addSonNilAllowed(result, decodeNode(r, result.info))
+        if belongsTo != nil and i == bodyPos:
+          addSonNilAllowed(result, nil)
+          belongsTo.offset = r.pos
+          skipNode(r)
+        else:
+          addSonNilAllowed(result, decodeNodeLazyBody(r, result.info, nil))
+        inc i
     if r.s[r.pos] == ')': inc(r.pos)
     else: internalError(result.info, "decodeNode")
   else: 
-    InternalError(result.info, "decodeNode " & r.s[r.pos])
+    InternalError(fInfo, "decodeNode " & r.s[r.pos])
+
+proc decodeNode(r: PRodReader, fInfo: TLineInfo): PNode =
+  result = decodeNodeLazyBody(r, fInfo, nil)
   
 proc decodeLoc(r: PRodReader, loc: var TLoc, info: TLineInfo) = 
   if r.s[r.pos] == '<': 
@@ -381,9 +406,10 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym =
   if r.s[r.pos] == '%': 
     inc(r.pos)
     result.position = decodeVInt(r.s, r.pos)
-  else: 
+  elif result.kind notin routineKinds:
     result.position = 0
-    # BUGFIX: this may have been misused as reader index!
+    # BUGFIX: this may have been misused as reader index! But we still
+    # need it for routines as the body is loaded lazily.
   if r.s[r.pos] == '`': 
     inc(r.pos)
     result.offset = decodeVInt(r.s, r.pos)
@@ -391,7 +417,11 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym =
     result.offset = - 1
   decodeLoc(r, result.loc, result.info)
   result.annex = decodeLib(r, info)
-  if r.s[r.pos] == '(': result.ast = decodeNode(r, result.info)
+  if r.s[r.pos] == '(':
+    if result.kind in routineKinds:
+      result.ast = decodeNodeLazyBody(r, result.info, result)
+    else:
+      result.ast = decodeNode(r, result.info)
   #echo "decoded: ", ident.s, "}"
 
 proc skipSection(r: PRodReader) = 
@@ -815,5 +845,21 @@ proc loadStub(s: PSym) =
     InternalError(rs.info, "loadStub: wrong ID") 
   #MessageOut('loaded stub: ' + s.name.s);
   
+proc getBody*(s: PSym): PNode =
+  ## retrieves the AST's body of `s`. If `s` has been loaded from a rod-file
+  ## it may perform an expensive reload operation. Otherwise it's a simple
+  ## accessor.
+  assert s.kind in routineKinds
+  result = s.ast.sons[bodyPos]
+  if result == nil:
+    assert s.offset != 0
+    var r = gMods[s.position].rd
+    var oldPos = r.pos
+    r.pos = s.offset
+    result = decodeNode(r, s.info)
+    r.pos = oldPos
+    s.ast.sons[bodyPos] = result
+    s.offset = 0
+  
 InitIdTable(gTypeTable)
 InitStrTable(rodCompilerProcs)

compiler/rodwrite.nim

@@ -526,7 +526,9 @@ proc process(c: PPassContext, n: PNode): PNode =
   of nkProcDef, nkMethodDef, nkIteratorDef, nkConverterDef: 
     var s = n.sons[namePos].sym
     if s == nil: InternalError(n.info, "rodwrite.process")
-    if n.sons[codePos].kind != nkEmpty or s.magic != mNone or
+    if n.sons[bodyPos] == nil:
+      InternalError(n.info, "rodwrite.process: body is nil")
+    if n.sons[bodyPos].kind != nkEmpty or s.magic != mNone or
         sfForward notin s.flags:
       addInterfaceSym(w, s)
   of nkVarSection:

compiler/sem.nim

@@ -130,7 +130,7 @@ proc addCodeForGenerics(c: PContext, n: PNode) =
   for i in countup(c.generics.lastGenericIdx, Len(c.generics.generics) - 1):
     var prc = c.generics.generics[i].instSym
     if prc.kind in {skProc, skMethod, skConverter} and prc.magic == mNone: 
-      if prc.ast == nil or prc.ast.sons[codePos] == nil: 
+      if prc.ast == nil or prc.ast.sons[bodyPos] == nil: 
         InternalError(prc.info, "no code for " & prc.name.s)
       addSon(n, prc.ast)
   c.generics.lastGenericIdx = Len(c.generics.generics)

compiler/semgnrc.nim

@@ -243,7 +243,7 @@ proc semGenericStmt(c: PContext, n: PNode,
         addDecl(c, newSymS(skUnknown, getIdentNode(a.sons[j]), c))
   of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, 
      nkIteratorDef, nkLambda: 
-    checkSonsLen(n, codePos + 1)
+    checkSonsLen(n, bodyPos + 1)
     addDecl(c, newSymS(skUnknown, getIdentNode(n.sons[0]), c))
     openScope(c.tab)
     n.sons[genericParamsPos] = semGenericStmt(c, n.sons[genericParamsPos], 
@@ -253,7 +253,8 @@ proc semGenericStmt(c: PContext, n: PNode,
         addDecl(c, newSym(skUnknown, getIdent("result"), nil))
       n.sons[paramsPos] = semGenericStmt(c, n.sons[paramsPos], flags, toBind)
     n.sons[pragmasPos] = semGenericStmt(c, n.sons[pragmasPos], flags, toBind)
-    n.sons[codePos] = semGenericStmtScope(c, n.sons[codePos], flags, toBind)
+    var s = n.sons[namePos].sym
+    n.sons[bodyPos] = semGenericStmtScope(c, s.getBody, flags, toBind)
     closeScope(c.tab)
   else: 
     for i in countup(0, sonsLen(n) - 1): 

compiler/seminst.nim

@@ -70,17 +70,17 @@ proc removeDefaultParamValues(n: PNode) =
         a.sons[L-1] = ast.emptyNode
 
 proc instantiateBody(c: PContext, n: PNode, result: PSym) =
-  if n.sons[codePos].kind != nkEmpty:
+  if n.sons[bodyPos].kind != nkEmpty:
     # add it here, so that recursive generic procs are possible:
     addDecl(c, result)
     pushProcCon(c, result)
     if result.kind in {skProc, skMethod, skConverter}: 
       addResult(c, result.typ.sons[0], n.info)
       addResultNode(c, n)
-    n.sons[codePos] = semStmtScope(c, n.sons[codePos])
+    n.sons[bodyPos] = semStmtScope(c, n.sons[bodyPos])
     if result.kind == skIterator:
       # XXX Bad hack for tests/titer2:
-      n.sons[codePos] = transform(c.module, n.sons[codePos])
+      n.sons[bodyPos] = transform(c.module, n.sons[bodyPos])
     #echo "code instantiated ", result.name.s
     excl(result.flags, sfForward)
     popProcCon(c)
@@ -92,7 +92,7 @@ proc fixupInstantiatedSymbols(c: PContext, s: PSym) =
       pushInfoContext(oldPrc.info)
       openScope(c.tab)
       var n = oldPrc.ast
-      n.sons[codePos] = copyTree(s.ast.sons[codePos])
+      n.sons[bodyPos] = copyTree(s.getBody)
       if n.sons[paramsPos].kind != nkEmpty: addParams(c, oldPrc.typ.n)
       instantiateBody(c, n, oldPrc)
       closeScope(c.tab)
@@ -118,6 +118,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   result = copySym(fn, false)
   incl(result.flags, sfFromGeneric)
   result.owner = getCurrOwner().owner
+  # careful! we copy the whole AST including the possibly nil body!
   var n = copyTree(fn.ast)
   result.ast = n
   pushOwner(result)
@@ -146,6 +147,8 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
     c.generics.generics.add(entry)
     if n.sons[pragmasPos].kind != nkEmpty:
       pragma(c, result, n.sons[pragmasPos], allRoutinePragmas)
+    if isNil(n.sons[bodyPos]):
+      n.sons[bodyPos] = copyTree(fn.getBody)
     instantiateBody(c, n, result)
     sideEffectsCheck(c, result)
   else:

compiler/semstmts.nim

@@ -576,7 +576,7 @@ proc semBorrow(c: PContext, n: PNode, s: PSym) =
   var b = SearchForBorrowProc(c, s, c.tab.tos - 2)
   if b != nil: 
     # store the alias:
-    n.sons[codePos] = newSymNode(b)
+    n.sons[bodyPos] = newSymNode(b)
   else:
     LocalError(n.info, errNoSymbolToBorrowFromFound) 
   
@@ -594,7 +594,7 @@ proc addResultNode(c: PContext, n: PNode) =
   
 proc semLambda(c: PContext, n: PNode): PNode = 
   result = n
-  checkSonsLen(n, codePos + 1)
+  checkSonsLen(n, bodyPos + 1)
   var s = newSym(skProc, getIdent":anonymous", getCurrOwner())
   s.info = n.info
   s.ast = n
@@ -615,12 +615,12 @@ proc semLambda(c: PContext, n: PNode): PNode =
   if n.sons[pragmasPos].kind != nkEmpty:
     pragma(c, s, n.sons[pragmasPos], lambdaPragmas)
   s.options = gOptions
-  if n.sons[codePos].kind != nkEmpty: 
+  if n.sons[bodyPos].kind != nkEmpty: 
     if sfImportc in s.flags: 
-      LocalError(n.sons[codePos].info, errImplOfXNotAllowed, s.name.s)
+      LocalError(n.sons[bodyPos].info, errImplOfXNotAllowed, s.name.s)
     pushProcCon(c, s)
     addResult(c, s.typ.sons[0], n.info)
-    n.sons[codePos] = semStmtScope(c, n.sons[codePos])
+    n.sons[bodyPos] = semStmtScope(c, n.sons[bodyPos])
     addResultNode(c, n)
     popProcCon(c)
   else: 
@@ -633,7 +633,7 @@ proc semLambda(c: PContext, n: PNode): PNode =
 proc semProcAux(c: PContext, n: PNode, kind: TSymKind, 
                 validPragmas: TSpecialWords): PNode = 
   result = n
-  checkSonsLen(n, codePos + 1)
+  checkSonsLen(n, bodyPos + 1)
   var s = semIdentDef(c, n.sons[0], kind)
   n.sons[namePos] = newSymNode(s)
   s.ast = n
@@ -695,10 +695,10 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     popOwner()
     pushOwner(s)
   s.options = gOptions
-  if n.sons[codePos].kind != nkEmpty: 
+  if n.sons[bodyPos].kind != nkEmpty: 
     # for DLL generation it is annoying to check for sfImportc!
     if sfBorrow in s.flags: 
-      LocalError(n.sons[codePos].info, errImplOfXNotAllowed, s.name.s)
+      LocalError(n.sons[bodyPos].info, errImplOfXNotAllowed, s.name.s)
     if n.sons[genericParamsPos].kind == nkEmpty: 
       ParamsTypeCheck(c, s.typ)
       pushProcCon(c, s)
@@ -706,18 +706,18 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
         addResult(c, s.typ.sons[0], n.info)
       if sfImportc notin s.flags: 
         # no semantic checking for importc:
-        n.sons[codePos] = semStmtScope(c, n.sons[codePos])
+        n.sons[bodyPos] = semStmtScope(c, n.sons[bodyPos])
       if s.typ.sons[0] != nil and kind != skIterator: addResultNode(c, n)
       popProcCon(c)
     else: 
       if s.typ.sons[0] != nil and kind != skIterator:
         addDecl(c, newSym(skUnknown, getIdent"result", nil))
       var toBind = initIntSet()
-      n.sons[codePos] = semGenericStmtScope(c, n.sons[codePos], {}, toBind)
+      n.sons[bodyPos] = semGenericStmtScope(c, n.sons[bodyPos], {}, toBind)
       fixupInstantiatedSymbols(c, s)
     if sfImportc in s.flags: 
       # so we just ignore the body after semantic checking for importc:
-      n.sons[codePos] = ast.emptyNode
+      n.sons[bodyPos] = ast.emptyNode
   else: 
     if proto != nil: LocalError(n.info, errImplOfXexpected, proto.name.s)
     if {sfImportc, sfBorrow} * s.flags == {}: incl(s.flags, sfForward)
@@ -732,7 +732,7 @@ proc semIterator(c: PContext, n: PNode): PNode =
   var t = s.typ
   if t.sons[0] == nil: 
     LocalError(n.info, errXNeedsReturnType, "iterator")
-  if n.sons[codePos].kind == nkEmpty and s.magic == mNone: 
+  if n.sons[bodyPos].kind == nkEmpty and s.magic == mNone: 
     LocalError(n.info, errImplOfXexpected, s.name.s)
   
 proc semProc(c: PContext, n: PNode): PNode = 
@@ -758,7 +758,7 @@ proc semMethod(c: PContext, n: PNode): PNode =
 
 proc semConverterDef(c: PContext, n: PNode): PNode = 
   if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "converter")
-  checkSonsLen(n, codePos + 1)
+  checkSonsLen(n, bodyPos + 1)
   if n.sons[genericParamsPos].kind != nkEmpty: 
     LocalError(n.info, errNoGenericParamsAllowedForX, "converter")
   result = semProcAux(c, n, skConverter, converterPragmas)
@@ -769,7 +769,7 @@ proc semConverterDef(c: PContext, n: PNode): PNode =
   addConverter(c, s)
 
 proc semMacroDef(c: PContext, n: PNode): PNode = 
-  checkSonsLen(n, codePos + 1)
+  checkSonsLen(n, bodyPos + 1)
   if n.sons[genericParamsPos].kind != nkEmpty: 
     LocalError(n.info, errNoGenericParamsAllowedForX, "macro")
   result = semProcAux(c, n, skMacro, macroPragmas)
@@ -777,7 +777,7 @@ proc semMacroDef(c: PContext, n: PNode): PNode =
   var t = s.typ
   if t.sons[0] == nil: LocalError(n.info, errXNeedsReturnType, "macro")
   if sonsLen(t) != 2: LocalError(n.info, errXRequiresOneArgument, "macro")
-  if n.sons[codePos].kind == nkEmpty:
+  if n.sons[bodyPos].kind == nkEmpty:
     LocalError(n.info, errImplOfXexpected, s.name.s)
   
 proc evalInclude(c: PContext, n: PNode): PNode = 

compiler/semtempl.nim

@@ -67,7 +67,7 @@ proc evalTemplate*(c: PContext, n: PNode, sym: PSym): PNode =
   if evalTemplateCounter <= 100: 
     # replace each param by the corresponding node:
     args = evalTemplateArgs(c, n, sym)
-    result = evalTemplateAux(sym.ast.sons[codePos], args, sym)
+    result = evalTemplateAux(sym.getBody, args, sym)
     dec(evalTemplateCounter)
   else:
     GlobalError(n.info, errTemplateInstantiationTooNested)
@@ -186,15 +186,15 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
       s.typ.n.sons[0] = newNodeIT(nkType, n.info, s.typ.sons[0])
   addParams(c, s.typ.n)       # resolve parameters:
   var toBind = initIntSet()
-  n.sons[codePos] = resolveTemplateParams(c, n.sons[codePos], false, toBind)
+  n.sons[bodyPos] = resolveTemplateParams(c, n.sons[bodyPos], false, toBind)
-  if not (s.typ.sons[0].kind in {tyStmt, tyTypeDesc}): 
+  if s.typ.sons[0].kind notin {tyStmt, tyTypeDesc}:
-    n.sons[codePos] = transformToExpr(n.sons[codePos]) 
+    n.sons[bodyPos] = transformToExpr(n.sons[bodyPos]) 
     # only parameters are resolved, no type checking is performed
   closeScope(c.tab)
   popOwner()
   s.ast = n
   result = n
-  if n.sons[codePos].kind == nkEmpty: 
+  if n.sons[bodyPos].kind == nkEmpty: 
     LocalError(n.info, errImplOfXexpected, s.name.s)
   # add identifier of template as a last step to not allow recursive templates:
   addInterfaceDecl(c, s)

compiler/semthreads.nim

@@ -59,7 +59,7 @@
 
 import
   ast, astalgo, strutils, hashes, options, msgs, idents, types, os,
-  renderer, tables
+  renderer, tables, rodread
 
 type
   TThreadOwner = enum
@@ -188,8 +188,8 @@ proc analyseCall(c: PProcCtx, n: PNode): TThreadOwner =
       var formal = skipTypes(prc.typ, abstractInst).n.sons[i].sym 
       newCtx.mapping[formal.id] = call.args[i-1]
     pushInfoContext(n.info)
-    result = analyse(newCtx, prc.ast.sons[codePos])
+    result = analyse(newCtx, prc.getBody)
-    if prc.ast.sons[codePos].kind == nkEmpty and 
+    if prc.ast.sons[bodyPos].kind == nkEmpty and 
        {sfNoSideEffect, sfThread, sfImportc} * prc.flags == {}:
       Message(n.info, warnAnalysisLoophole, renderTree(n))
       if result == toUndefined: result = toNil
@@ -372,7 +372,7 @@ proc analyseThreadProc*(prc: PSym) =
   for i in 1 .. formals.len-1:
     var formal = formals.sons[i].sym 
     c.mapping[formal.id] = toTheirs # thread receives foreign data!
-  discard analyse(c, prc.ast.sons[codePos])
+  discard analyse(c, prc.getBody)
 
 proc needsGlobalAnalysis*: bool =
   result = gGlobalOptions * {optThreads, optThreadAnalysis} == 

compiler/transf.nim

@@ -164,7 +164,7 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
   var tc = c.transCon
   if sfBorrow in n.sym.flags: 
     # simply exchange the symbol:
-    b = n.sym.ast.sons[codePos]
+    b = n.sym.getBody
     if b.kind != nkSym: internalError(n.info, "wrong AST for borrowed symbol")
     b = newSymNode(b.sym)
     b.info = n.info
@@ -493,7 +493,7 @@ proc transformFor(c: PTransf, n: PNode): PTransNode =
       assert(skipTypes(formal.typ, abstractInst).kind == tyVar)
       IdNodeTablePut(newC.mapping, formal, arg)
       # XXX BUG still not correct if the arg has a side effect!
-  var body = newC.owner.ast.sons[codePos]
+  var body = newC.owner.getBody
   pushInfoContext(n.info)
   inc(c.inlining)
   add(result, transform(c, body))
@@ -551,8 +551,9 @@ proc transformLambda(c: PTransf, n: PNode): PNode =
   result = n
   if n.sons[namePos].kind != nkSym: InternalError(n.info, "transformLambda")
   var s = n.sons[namePos].sym
-  var closure = newNodeI(nkRecList, n.sons[codePos].info)
+  var closure = newNodeI(nkRecList, n.info)
-  gatherVars(c, n.sons[codePos], marked, s, closure) 
+  var body = s.getBody
+  gatherVars(c, body, marked, s, closure) 
   # add closure type to the param list (even if closure is empty!):
   var cl = newType(tyObject, s)
   cl.n = closure
@@ -570,7 +571,7 @@ proc transformLambda(c: PTransf, n: PNode): PNode =
       IdNodeTablePut(newC.mapping, closure.sons[i].sym, 
                      indirectAccess(param, closure.sons[i].sym))
     pushTransCon(c, newC)
-    n.sons[codePos] = transform(c, n.sons[codePos]).pnode
+    n.sons[bodyPos] = transform(c, body).pnode
     popTransCon(c)
 
 proc transformCase(c: PTransf, n: PNode): PTransNode = 
@@ -666,7 +667,8 @@ proc transform(c: PTransf, n: PNode): PTransNode =
   of nkBracketExpr: 
     result = transformArrayAccess(c, n)
   of nkLambda: 
-    n.sons[codePos] = PNode(transform(c, n.sons[codePos]))
+    var s = n.sons[namePos].sym
+    n.sons[bodyPos] = PNode(transform(c, s.getBody))
     result = PTransNode(n)
     when false: result = transformLambda(c, n)
   of nkForStmt: 
@@ -675,8 +677,9 @@ proc transform(c: PTransf, n: PNode): PTransNode =
     result = transformCase(c, n)
   of nkProcDef, nkMethodDef, nkIteratorDef, nkMacroDef, nkConverterDef: 
     if n.sons[genericParamsPos].kind == nkEmpty: 
-      n.sons[codePos] = PNode(transform(c, n.sons[codePos]))
+      var s = n.sons[namePos].sym
-      if n.kind == nkMethodDef: methodDef(n.sons[namePos].sym, false)
+      n.sons[bodyPos] = PNode(transform(c, s.getBody))
+      if n.kind == nkMethodDef: methodDef(s, false)
     result = PTransNode(n)
   of nkContinueStmt:
     result = PTransNode(newNode(nkBreakStmt))

tests/tester.nim

@@ -272,42 +272,53 @@ proc run(r: var TResults, dir, options: string) =
 const
   rodfilesDir = "tests/rodfiles"
 
-proc delNimCache() = removeDir(rodfilesDir / "nimcache")
+proc delNimCache() =
+  try:
+    removeDir(rodfilesDir / "nimcache")
+  except EOS:
+    nil
+    
 proc plusCache(options: string): string = return options & " --symbolFiles:on"
 
 proc runRodFiles(r: var TResults, options: string) =
+  template test(filename: expr): stmt =
+    runSingleTest(r, rodfilesDir / filename, options)
+  
   var options = options.plusCache
   delNimCache()
   
   # test basic recompilation scheme:
-  runSingleTest(r, rodfilesDir / "hallo", options)
+  test "hallo"
-  runSingleTest(r, rodfilesDir / "hallo", options)
+  test "hallo"
   # test incremental type information:
-  runSingleTest(r, rodfilesDir / "hallo2", options)
+  test "hallo2"
   delNimCache()
   
   # test type converters:
-  runSingleTest(r, rodfilesDir / "aconv", options)
+  test "aconv"
-  runSingleTest(r, rodfilesDir / "bconv", options)
+  test "bconv"
   delNimCache()
   
   # test G, A, B example from the documentation; test init sections:
-  runSingleTest(r, rodfilesDir / "deada", options)
+  test "deada"
-  runSingleTest(r, rodfilesDir / "deada2", options)
+  test "deada2"
   delNimCache()
   
   # test method generation:
-  runSingleTest(r, rodfilesDir / "bmethods", options)
+  test "bmethods"
-  runSingleTest(r, rodfilesDir / "bmethods2", options)
+  test "bmethods2"
   delNimCache()
   
 
 proc compileRodFiles(r: var TResults, options: string) =
+  template test(filename: expr): stmt =
+    compileSingleTest(r, rodfilesDir / filename, options)
+    
   var options = options.plusCache
   delNimCache()
   # test DLL interfacing:
-  compileSingleTest(r, rodfilesDir / "gtkex1", options)
+  test "gtkex1"
-  compileSingleTest(r, rodfilesDir / "gtkex2", options)
+  test "gtkex2"
   delNimCache()
 
 # -----------------------------------------------------------------------------

todo.txt

@@ -4,13 +4,6 @@ Version 0.8.14
 - optimize unused constants away (affected by HLO)
 - fix actors.nim; test with different thread var implementations
 - dead code elim for JS backend
-
-
-incremental compilation
------------------------
-
-- the loading has to be MUCH lazier! --> next version: We should re-load
-  symbol.ast.sons[codePos] lazily
 - implement lib/pure/memfiles properly
 
 
@@ -38,6 +31,8 @@ version 0.9.0
 
 Bugs
 ----
+- bug: s[1..n] = @[] produces wrong C code
+- bug: template t(f_no_Type): stmt = ... crashes the compiler
 - bug: generic assign still buggy
   - Optimization: If we use a temporary for the result anyway the code gen
     should make use of this fact to generate better code...
@@ -54,6 +49,7 @@ Bugs
 - bug: stress testing basic method example (eval example) 
   without ``-d:release`` leaks memory; good way to figure out how a 
   fixed amount of stack can hold an arbitrary number of GC roots!
+- bug: osproc.execProcess() should raise an exception if the exit code is not 0
 
 
 version 0.9.XX