merged the persistent scopes work with the delayed proc compilation strategy

compiler/ast.nim

@@ -280,6 +280,9 @@ const
       
   sfHoist* = sfVolatile ## proc return value can be hoisted
 
+  sfNoForward* = sfRegister
+    # forward declarations are not required (per module)
+
 const
   # getting ready for the future expr/stmt merge
   nkWhen* = nkWhenStmt
@@ -604,7 +607,14 @@ type
                               # simple ref count here
   
   PInstantiation* = ref TInstantiation
-      
+ 
+  TScope* = object
+    depthLevel*: int
+    symbols*: TStrTable
+    parent*: PScope
+
+  PScope* = ref TScope
+
   PLib* = ref TLib
   TSym* {.acyclic.} = object of TIdObj
     # proc and type instantiations are cached in the generic symbol
@@ -613,12 +623,14 @@ type
       typeInstCache*: seq[PType]
     of routineKinds:
       procInstCache*: seq[PInstantiation]
+      scope*: PScope          # the scope where the proc was defined
     of skModule:
       # modules keep track of the generic symbols they use from other modules.
       # this is because in incremental compilation, when a module is about to
       # be replaced with a newer version, we must decrement the usage count
       # of all previously used generics.
       usedGenerics*: seq[PInstantiation]
+      tab*: TStrTable         # interface table for modules
     else: nil
 
     magic*: TMagic
@@ -627,7 +639,6 @@ type
     info*: TLineInfo
     owner*: PSym
     flags*: TSymFlags
-    tab*: TStrTable           # interface table for modules
     ast*: PNode               # syntax tree of proc, iterator, etc.:
                               # the whole proc including header; this is used
                               # for easy generation of proper error messages
@@ -1049,7 +1060,8 @@ proc copySym(s: PSym, keepId: bool = false): PSym =
     when debugIds: RegisterId(result)
   result.flags = s.flags
   result.magic = s.magic
-  copyStrTable(result.tab, s.tab)
+  if s.kind == skModule:
+    copyStrTable(result.tab, s.tab)
   result.options = s.options
   result.position = s.position
   result.loc = s.loc

compiler/cgen.nim

@@ -842,8 +842,10 @@ proc requestConstImpl(p: BProc, sym: PSym) =
     if sfExportc in sym.flags and generatedHeader != nil:
       app(generatedHeader.s[cfsData], headerDecl)
 
+proc isActivated(prc: PSym): bool = prc.typ != nil
+
 proc genProc(m: BModule, prc: PSym) = 
-  if sfBorrow in prc.flags: return 
+  if sfBorrow in prc.flags or not isActivated(prc): return
   fillProcLoc(prc)
   if {sfForward, sfFromGeneric} * prc.flags != {}: addForwardedProc(m, prc)
   else: 

compiler/lookups.nim

@@ -45,15 +45,13 @@ proc addUniqueSym*(scope: PScope, s: PSym): TResult =
     result = Success
 
 proc openScope*(c: PContext): PScope {.discardable.} =
-  inc c.scopeDepth
   result = PScope(parent: c.currentScope,
                   symbols: newStrTable(),
-                  depthLevel: c.scopeDepth)
+                  depthLevel: c.scopeDepth + 1)
   c.currentScope = result
 
 proc rawCloseScope*(c: PContext) =
   c.currentScope = c.currentScope.parent
-  dec c.scopeDepth
 
 proc closeScope*(c: PContext) =
   ensureNoMissingOrUnusedSymbols(c.currentScope)

compiler/passes.nim

@@ -177,6 +177,7 @@ proc processModule(module: PSym, stream: PLLStream, rd: PRodReader) =
       s = stream
     while true: 
       openParsers(p, fileIdx, s)
+      var code = p.parseAll
 
       if sfSystemModule notin module.flags:
         # XXX what about caching? no processing then? what if I change the 
@@ -186,10 +187,7 @@ proc processModule(module: PSym, stream: PLLStream, rd: PRodReader) =
         processImplicits implicitImports, nkImportStmt, a
         processImplicits implicitIncludes, nkIncludeStmt, a
 
-      while true: 
+      discard processTopLevelStmt(code, a)
-        var n = parseTopLevelStmt(p)
-        if n.kind == nkEmpty: break 
-        if not processTopLevelStmt(n, a): break
 
       closeParsers(p)
       if s.kind != llsStdIn: break 

compiler/pragmas.nim

@@ -42,7 +42,7 @@ const
     wFatal, wDefine, wUndef, wCompile, wLink, wLinkSys, wPure, wPush, wPop,
     wBreakpoint, wWatchpoint, wPassL, wPassC, wDeadCodeElim, wDeprecated,
     wFloatChecks, wInfChecks, wNanChecks, wPragma, wEmit, wUnroll,
-    wLinearScanEnd, wPatterns, wEffects}
+    wLinearScanEnd, wPatterns, wEffects, wNoForward}
   lambdaPragmas* = {FirstCallConv..LastCallConv, wImportc, wExportc, wNodecl, 
     wNosideEffect, wSideEffect, wNoreturn, wDynLib, wHeader, 
     wDeprecated, wExtern, wThread, wImportcpp, wImportobjc, wNoStackFrame,
@@ -182,7 +182,11 @@ proc onOff(c: PContext, n: PNode, op: TOptions) =
 proc pragmaDeadCodeElim(c: PContext, n: PNode) = 
   if IsTurnedOn(c, n): incl(c.module.flags, sfDeadCodeElim)
   else: excl(c.module.flags, sfDeadCodeElim)
-  
+
+proc pragmaNoForward(c: PContext, n: PNode) =
+  if IsTurnedOn(c, n): incl(c.module.flags, sfNoForward)
+  else: excl(c.module.flags, sfNoForward)
+
 proc processCallConv(c: PContext, n: PNode) = 
   if (n.kind == nkExprColonExpr) and (n.sons[1].kind == nkIdent): 
     var sw = whichKeyword(n.sons[1].ident)
@@ -552,6 +556,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int,
           noVal(it)
           incl(sym.flags, sfThread)
         of wDeadCodeElim: pragmaDeadCodeElim(c, it)
+        of wNoForward: pragmaNoForward(c, it)
         of wMagic: processMagic(c, it, sym)
         of wCompileTime: 
           noVal(it)

compiler/procfind.nim

@@ -30,8 +30,9 @@ proc equalGenericParams(procA, procB: PNode): bool =
       if not ExprStructuralEquivalent(a.ast, b.ast): return
   result = true
 
-proc SearchForProc*(c: PContext, scope: PScope, fn: PSym): PSym = 
+proc SearchForProc*(c: PContext, scope: PScope, fn: PSym): PSym =
-  # Searchs for the fn in the symbol table. If the parameter lists are exactly
+  # Searchs for a forward declaration or a "twin" symbol of fn
+  # in the symbol table. If the parameter lists are exactly
   # the same the sym in the symbol table is returned, else nil.
   var it: TIdentIter
   result = initIdentIter(it, scope.symbols, fn.Name)

compiler/sem.nim

@@ -99,7 +99,7 @@ proc commonType*(x, y: PType): PType =
         result.addSonSkipIntLit(r)
 
 proc isTopLevel(c: PContext): bool {.inline.} = 
-  result = c.scopeDepth <= 2
+  result = c.currentScope.depthLevel <= 2
 
 proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym = 
   result = newSym(kind, considerAcc(n), getCurrOwner(), n.info)
@@ -281,7 +281,6 @@ proc RecoverContext(c: PContext) =
   # faster than wrapping every stack operation in a 'try finally' block and 
   # requires far less code.
   c.currentScope = c.topLevelScope
-  c.scopeDepth = 2 # importTable and top-level scope
   while getCurrOwner().kind != skModule: popOwner()
   while c.p != nil and c.p.owner.kind != skModule: c.p = c.p.next
 

compiler/semcall.nim

@@ -32,6 +32,8 @@ proc sameMethodDispatcher(a, b: PSym): bool =
       # be disambiguated by the programmer; this way the right generic is
       # instantiated.
   
+proc determineType(c: PContext, s: PSym)
+
 proc resolveOverloads(c: PContext, n, orig: PNode, 
                       filter: TSymKinds): TCandidate =
   var initialBinding: PNode
@@ -58,6 +60,7 @@ proc resolveOverloads(c: PContext, n, orig: PNode,
 
   while sym != nil:
     if sym.kind in filter:
+      determineType(c, sym)
       initCandidate(z, sym, initialBinding, o.lastOverloadScope)
       z.calleeSym = sym
       matches(c, n, orig, z)

compiler/semdata.nim

@@ -44,13 +44,6 @@ type
     efLValue, efWantIterator, efInTypeof, efWantStmt, efDetermineType,
     efAllowDestructor
   TExprFlags* = set[TExprFlag]
-    
-  TScope* = object
-    depthLevel*: int
-    symbols*: TStrTable
-    parent*: PScope
-
-  PScope* = ref TScope
 
   PContext* = ref TContext
   TContext* = object of TPassContext # a context represents a module
@@ -58,7 +51,6 @@ type
     currentScope*: PScope      # current scope
     importTable*: PScope       # scope for all imported symbols
     topLevelScope*: PScope     # scope for all top-level symbols
-    scopeDepth*: int           # number of open scopes
     p*: PProcCon               # procedure context
     friendModule*: PSym        # current friend module; may access private data;
                                # this is used so that generic instantiations
@@ -112,6 +104,10 @@ proc makeVarType*(c: PContext, baseType: PType): PType
 proc newTypeS*(kind: TTypeKind, c: PContext): PType
 proc fillTypeS*(dest: PType, kind: TTypeKind, c: PContext)
 
+proc scopeDepth*(c: PContext): int {.inline.} =
+  result = if c.currentScope != nil: c.currentScope.depthLevel
+           else: 0
+
 # owner handling:
 proc getCurrOwner*(): PSym
 proc PushOwner*(owner: PSym)

compiler/semexprs.nim

@@ -1374,7 +1374,7 @@ proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   msgs.gErrorMax = high(int)
   
   # open a scope for temporary symbol inclusions:
-  let oldTos = c.scopeDepth
+  let oldScope = c.currentScope
   openScope(c)
   let oldOwnerLen = len(gOwners)
   let oldGenerics = c.generics
@@ -1398,7 +1398,7 @@ proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   c.p = oldProcCon
   msgs.setInfoContextLen(oldContextLen)
   setlen(gOwners, oldOwnerLen)
-  while c.scopeDepth > oldTos: rawCloseScope(c)
+  c.currentScope = oldScope
   dec c.InCompilesContext
   dec msgs.gSilence
   msgs.gErrorCounter = oldErrorCount

compiler/semstmts.nim

@@ -885,17 +885,46 @@ proc maybeAddResult(c: PContext, s: PSym, n: PNode) =
     addResult(c, s.typ.sons[0], n.info, s.kind)
     addResultNode(c, n)
 
-proc semProcAux(c: PContext, n: PNode, kind: TSymKind, 
+type
-                validPragmas: TSpecialWords): PNode = 
+  TProcCompilationSteps = enum
+    stepRegisterSymbol,
+    stepDetermineType,
+    stepCompileBody
+
+proc isForwardDecl(s: PSym): bool =
+  InternalAssert s.kind == skProc
+  result = s.ast[bodyPos].kind != nkEmpty
+
+proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
+                validPragmas: TSpecialWords,
+                phase = stepRegisterSymbol): PNode =
   result = semProcAnnotation(c, n)
   if result != nil: return result
   result = n
   checkSonsLen(n, bodyPos + 1)
-  var s = semIdentDef(c, n.sons[0], kind)
+  var s: PSym
-  n.sons[namePos] = newSymNode(s)
+  var typeIsDetermined = false
-  s.ast = n
+  if n[namePos].kind != nkSym:
+    assert phase == stepRegisterSymbol
+    s = semIdentDef(c, n.sons[0], kind)
+    n.sons[namePos] = newSymNode(s)
+    s.ast = n
+    s.scope = c.currentScope
+
+    if sfNoForward in c.module.flags and
+       sfSystemModule notin c.module.flags:
+      addInterfaceOverloadableSymAt(c, c.currentScope, s)
+      return
+  else:
+    s = n[namePos].sym
+    typeIsDetermined = s.typ == nil
+    if typeIsDetermined: assert phase == stepCompileBody
+    else: assert phase == stepDetermineType
+  # before compiling the proc body, set as current the scope
+  # where the proc was declared
+  let oldScope = c.currentScope
+  c.currentScope = s.scope
   pushOwner(s)
-  var outerScope = c.currentScope
   openScope(c)
   var gp: PNode
   if n.sons[genericParamsPos].kind != nkEmpty: 
@@ -919,15 +948,17 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     n.sons[patternPos] = semPattern(c, n.sons[patternPos])
   if s.kind == skIterator: s.typ.flags.incl(tfIterator)
   
-  var proto = SearchForProc(c, outerScope, s)
+  var proto = SearchForProc(c, s.scope, s)
   if proto == nil: 
     s.typ.callConv = lastOptionEntry(c).defaultCC
     # add it here, so that recursive procs are possible:
     if sfGenSym in s.flags: nil
-    elif kind in OverloadableSyms: 
+    elif kind in OverloadableSyms:
-      addInterfaceOverloadableSymAt(c, outerScope, s)
+      if not typeIsDetermined:
-    else: 
+        addInterfaceOverloadableSymAt(c, s.scope, s)
-      addInterfaceDeclAt(c, outerScope, s)
+    else:
+      if not typeIsDetermined:
+        addInterfaceDeclAt(c, s.scope, s)
     if n.sons[pragmasPos].kind != nkEmpty:
       pragma(c, s, n.sons[pragmasPos], validPragmas)
     else:
@@ -989,10 +1020,16 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     elif sfBorrow in s.flags: semBorrow(c, n, s)
   sideEffectsCheck(c, s)
   closeScope(c)           # close scope for parameters
+  c.currentScope = oldScope
   popOwner()
   if n.sons[patternPos].kind != nkEmpty:
     c.patterns.add(s)
-  
+
+proc determineType(c: PContext, s: PSym) =
+  if s.typ != nil: return
+  #if s.magic != mNone: return
+  discard semProcAux(c, s.ast, s.kind, {}, stepDetermineType)
+
 proc semIterator(c: PContext, n: PNode): PNode =
   result = semProcAux(c, n, skIterator, iteratorPragmas)
   var s = result.sons[namePos].sym
@@ -1055,7 +1092,7 @@ proc semMacroDef(c: PContext, n: PNode): PNode =
   if n.sons[bodyPos].kind == nkEmpty:
     LocalError(n.info, errImplOfXexpected, s.name.s)
   
-proc evalInclude(c: PContext, n: PNode): PNode = 
+proc evalInclude(c: PContext, n: PNode): PNode =
   result = newNodeI(nkStmtList, n.info)
   addSon(result, n)
   for i in countup(0, sonsLen(n) - 1): 

compiler/wordrecg.nim

@@ -53,7 +53,7 @@ type
     wFloatchecks, wNanChecks, wInfChecks,
     wAssertions, wPatterns, wWarnings,
     wHints, wOptimization, wRaises, wWrites, wReads, wSize, wEffects, wTags,
-    wDeadCodeElim, wSafecode, 
+    wDeadCodeElim, wSafecode, wNoForward,
     wPragma,
     wCompileTime, wNoInit,
     wPassc, wPassl, wBorrow, wDiscardable,
@@ -135,7 +135,7 @@ const
 
     "assertions", "patterns", "warnings", "hints", 
     "optimization", "raises", "writes", "reads", "size", "effects", "tags",
-    "deadcodeelim", "safecode", 
+    "deadcodeelim", "safecode", "noforward",
     "pragma",
     "compiletime", "noinit",
     "passc", "passl", "borrow", "discardable", "fieldchecks",