IC: next steps (#16550)

* cleanups
* ast.nim: cleanups
* IC: no more sym.tab field, stored externally in the module graph
* nimble compiles again
* rodfiles: store bitwidth of integers and the endianness in the cookie because we serialize 'int' directly
* rodfiles: added compilerproc and export sections
* rodfiles: added all the missing sections
* rodfiles: track the missing information
* IC: architecture for lazy loading of proc bodies
* make tests green again
* completed the lazy loading of proc bodies
* symbol lookup integration, part 1
* symbol lookup integration, part 2
* symbol lookup integration, part 3
* make tcompilerapi work again
* rodfiles: fixed config change handling

compiler/ast.nim

@@ -833,18 +833,6 @@ type
       procInstCache*: seq[PInstantiation]
       gcUnsafetyReason*: PSym  # for better error messages wrt gcsafe
       transformedBody*: PNode  # cached body after transf pass
-    of skModule, skPackage:
-      # modules keep track of the generic symbols they use from other modules.
-      # this is because in incremental compilation, when a module is about to
-      # be replaced with a newer version, we must decrement the usage count
-      # of all previously used generics.
-      # For 'import as' we copy the module symbol but shallowCopy the 'tab'
-      # and set the 'usedGenerics' to ... XXX gah! Better set module.name
-      # instead? But this doesn't work either. --> We need an skModuleAlias?
-      # No need, just leave it as skModule but set the owner accordingly and
-      # check for the owner when touching 'usedGenerics'.
-      usedGenerics*: seq[PInstantiation]
-      tab*: TStrTable         # interface table for modules
     of skLet, skVar, skField, skForVar:
       guard*: PSym
       bitsize*: int
@@ -1455,8 +1443,6 @@ proc copySym*(s: PSym; id: ItemId): PSym =
   result.typ = s.typ
   result.flags = s.flags
   result.magic = s.magic
-  if s.kind == skModule:
-    copyStrTable(result.tab, s.tab)
   result.options = s.options
   result.position = s.position
   result.loc = s.loc
@@ -1474,13 +1460,10 @@ proc createModuleAlias*(s: PSym, id: ItemId, newIdent: PIdent, info: TLineInfo;
   result.ast = s.ast
   #result.id = s.id # XXX figure out what to do with the ID.
   result.flags = s.flags
-  system.shallowCopy(result.tab, s.tab)
   result.options = s.options
   result.position = s.position
   result.loc = s.loc
   result.annex = s.annex
-  # XXX once usedGenerics is used, ensure module aliases keep working!
-  assert s.usedGenerics.len == 0
 
 proc initStrTable*(x: var TStrTable) =
   x.counter = 0
@@ -1915,8 +1898,6 @@ template incompleteType*(t: PType): bool =
 template typeCompleted*(s: PSym) =
   incl s.flags, sfNoForward
 
-template getBody*(s: PSym): PNode = s.ast[bodyPos]
-
 template detailedInfo*(sym: PSym): string =
   sym.name.s
 

compiler/ccgexprs.nim

@@ -1292,7 +1292,7 @@ proc rawGenNew(p: BProc, a: var TLoc, sizeExpr: Rope; needsInit: bool) =
     genAssignment(p, a, b, {})
   else:
     let ti = genTypeInfoV1(p.module, typ, a.lode.info)
-    if bt.destructor != nil and not isTrivialProc(bt.destructor):
+    if bt.destructor != nil and not isTrivialProc(p.module.g.graph, bt.destructor):
       # the prototype of a destructor is ``=destroy(x: var T)`` and that of a
       # finalizer is: ``proc (x: ref T) {.nimcall.}``. We need to check the calling
       # convention at least:
@@ -2204,7 +2204,7 @@ proc genDestroy(p: BProc; n: PNode) =
     else: discard "nothing to do"
   else:
     let t = n[1].typ.skipTypes(abstractVar)
-    if t.destructor != nil and t.destructor.ast[bodyPos].len != 0:
+    if t.destructor != nil and getBody(p.module.g.graph, t.destructor).len != 0:
       internalError(p.config, n.info, "destructor turned out to be not trivial")
     discard "ignore calls to the default destructor"
 

compiler/ccgtypes.nim

@@ -36,14 +36,6 @@ proc mangleField(m: BModule; name: PIdent): string =
   if isKeyword(name):
     result.add "_0"
 
-when false:
-  proc hashOwner(s: PSym): SigHash =
-    var m = s
-    while m.kind != skModule: m = m.owner
-    let p = m.owner
-    assert p.kind == skPackage
-    result = gDebugInfo.register(p.name.s, m.name.s)
-
 proc mangleName(m: BModule; s: PSym): Rope =
   result = s.loc.r
   if result == nil:
@@ -1313,11 +1305,11 @@ proc genTypeInfo2Name(m: BModule; t: PType): Rope =
     it = it[0]
   result = makeCString(res)
 
-proc isTrivialProc(s: PSym): bool {.inline.} = s.ast[bodyPos].len == 0
+proc isTrivialProc(g: ModuleGraph; s: PSym): bool {.inline.} = getBody(g, s).len == 0
 
 proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp): Rope =
   let theProc = t.attachedOps[op]
-  if theProc != nil and not isTrivialProc(theProc):
+  if theProc != nil and not isTrivialProc(m.g.graph, theProc):
     # the prototype of a destructor is ``=destroy(x: var T)`` and that of a
     # finalizer is: ``proc (x: ref T) {.nimcall.}``. We need to check the calling
     # convention at least:

compiler/commands.nim

@@ -801,7 +801,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectNoArg(conf, switch, arg, pass, info)
     helpOnError(conf, pass)
   of "symbolfiles": discard "ignore for backwards compat"
-  of "incremental":
+  of "incremental", "ic":
+    if pass in {passCmd2, passPP}:
       case arg.normalize
       of "on": conf.symbolFiles = v2Sf
       of "off": conf.symbolFiles = disabledSf

compiler/evaltempl.nim

@@ -187,7 +187,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   ctx.instID = instID[]
   ctx.idgen = idgen
 
-  let body = tmpl.getBody
+  let body = tmpl.ast[bodyPos]
   #echo "instantion of ", renderTree(body, {renderIds})
   if isAtom(body):
     result = newNodeI(nkPar, body.info)

compiler/ic/packed_ast.nim

@@ -16,28 +16,6 @@ import std / [hashes, tables, strtabs, md5]
 import bitabs
 import ".." / [ast, options]
 
-const
-  localNamePos* = 0
-  localExportMarkerPos* = 1
-  localPragmaPos* = 2
-  localTypePos* = 3
-  localValuePos* = 4
-
-  typeNamePos* = 0
-  typeExportMarkerPos* = 1
-  typeGenericParamsPos* = 2
-  typePragmaPos* = 3
-  typeBodyPos* = 4
-
-  routineNamePos* = 0
-  routineExportMarkerPos* = 1
-  routinePatternPos* = 2
-  routineGenericParamsPos* = 3
-  routineParamsPos* = 4
-  routineResultPos* = 5
-  routinePragmasPos* = 6
-  routineBodyPos* = 7
-
 const
   nkModuleRef* = nkNone # pair of (ModuleId, SymId)
 
@@ -55,7 +33,6 @@ type
   TypeId* = PackedItemId
 
 const
-  nilTypeId* = PackedItemId(module: LitId(0), item: -1.int32)
   nilItemId* = PackedItemId(module: LitId(0), item: -1.int32)
 
 const

compiler/ic/rodfiles.nim

@@ -18,6 +18,14 @@ type
     depsSection
     integersSection
     floatsSection
+    exportsSection
+    reexportsSection
+    compilerProcsSection
+    trmacrosSection
+    convertersSection
+    methodsSection
+    pureEnumsSection
+    macroUsagesSection
     topLevelSection
     bodiesSection
     symsSection
@@ -36,26 +44,30 @@ type
 const
   RodVersion = 1
   cookie = [byte(0), byte('R'), byte('O'), byte('D'),
-            byte(0), byte(0), byte(0), byte(RodVersion)]
+            byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)]
+
+proc setError(f: var RodFile; err: RodFileError) {.inline.} =
+  f.err = err
+  #raise newException(IOError, "IO error")
 
 proc storePrim*(f: var RodFile; s: string) =
   if f.err != ok: return
   if s.len >= high(int32):
-    f.err = tooBig
+    setError f, tooBig
     return
   var lenPrefix = int32(s.len)
   if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    f.err = ioFailure
+    setError f, ioFailure
   else:
     if s.len != 0:
       if writeBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
-        f.err = ioFailure
+        setError f, ioFailure
 
 proc storePrim*[T](f: var RodFile; x: T) =
   if f.err != ok: return
   when supportsCopyMem(T):
     if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
-      f.err = ioFailure
+      setError f, ioFailure
   elif T is tuple:
     for y in fields(x):
       storePrim(f, y)
@@ -65,11 +77,11 @@ proc storePrim*[T](f: var RodFile; x: T) =
 proc storeSeq*[T](f: var RodFile; s: seq[T]) =
   if f.err != ok: return
   if s.len >= high(int32):
-    f.err = tooBig
+    setError f, tooBig
     return
   var lenPrefix = int32(s.len)
   if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    f.err = ioFailure
+    setError f, ioFailure
   else:
     for i in 0..<s.len:
       storePrim(f, s[i])
@@ -78,18 +90,18 @@ proc loadPrim*(f: var RodFile; s: var string) =
   if f.err != ok: return
   var lenPrefix = int32(0)
   if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    f.err = ioFailure
+    setError f, ioFailure
   else:
     s = newString(lenPrefix)
     if lenPrefix > 0:
       if readBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
-        f.err = ioFailure
+        setError f, ioFailure
 
 proc loadPrim*[T](f: var RodFile; x: var T) =
   if f.err != ok: return
   when supportsCopyMem(T):
     if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
-      f.err = ioFailure
+      setError f, ioFailure
   elif T is tuple:
     for y in fields(x):
       loadPrim(f, y)
@@ -100,7 +112,7 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
   if f.err != ok: return
   var lenPrefix = int32(0)
   if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    f.err = ioFailure
+    setError f, ioFailure
   else:
     s = newSeq[T](lenPrefix)
     for i in 0..<lenPrefix:
@@ -109,15 +121,15 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
 proc storeHeader*(f: var RodFile) =
   if f.err != ok: return
   if f.f.writeBytes(cookie, 0, cookie.len) != cookie.len:
-    f.err = ioFailure
+    setError f, ioFailure
 
 proc loadHeader*(f: var RodFile) =
   if f.err != ok: return
   var thisCookie: array[cookie.len, byte]
   if f.f.readBytes(thisCookie, 0, thisCookie.len) != thisCookie.len:
-    f.err = ioFailure
+    setError f, ioFailure
   elif thisCookie != cookie:
-    f.err = wrongHeader
+    setError f, wrongHeader
 
 proc storeSection*(f: var RodFile; s: RodSection) =
   if f.err != ok: return
@@ -129,15 +141,15 @@ proc loadSection*(f: var RodFile; expected: RodSection) =
   if f.err != ok: return
   var s: RodSection
   loadPrim(f, s)
-  if expected != s:
+  if expected != s and f.err == ok:
-    f.err = wrongSection
+    setError f, wrongSection
 
 proc create*(filename: string): RodFile =
   if not open(result.f, filename, fmWrite):
-    result.err = ioFailure
+    setError result, ioFailure
 
 proc close*(f: var RodFile) = close(f.f)
 
 proc open*(filename: string): RodFile =
   if not open(result.f, filename, fmRead):
-    result.err = ioFailure
+    setError result, ioFailure

compiler/ic/to_packed_ast.nim

@@ -14,8 +14,6 @@ import ".." / [ast, idents, lineinfos, msgs, ropes, options,
 
 from std / os import removeFile, isAbsolute
 
-when not defined(release): import ".." / astalgo # debug()
-
 type
   PackedConfig* = object
     backend: TBackend
@@ -32,6 +30,11 @@ type
     bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position.
     hidden*: PackedTree # instantiated generics and other trees not directly in the source code.
     #producedGenerics*: Table[GenericKey, SymId]
+    exports*: seq[(LitId, int32)]
+    reexports*: seq[(LitId, PackedItemId)]
+    compilerProcs*, trmacros*, converters*, pureEnums*: seq[(LitId, int32)]
+    methods*: seq[(LitId, PackedItemId, int32)]
+    macroUsages*: seq[(PackedItemId, PackedLineInfo)]
     sh*: Shared
     cfg: PackedConfig
 
@@ -61,19 +64,28 @@ proc definedSymbolsAsString(config: ConfigRef): string =
     result.add ' '
     result.add d
 
-proc rememberConfig(c: var PackedEncoder; config: ConfigRef) =
+proc rememberConfig(c: var PackedEncoder; config: ConfigRef; pc: PackedConfig) =
   c.m.definedSymbols = definedSymbolsAsString(config)
-
+  #template rem(x) =
-  template rem(x) =
+  #  c.m.cfg.x = config.x
-    c.m.cfg.x = config.x
+  #primConfigFields rem
-  primConfigFields rem
+  c.m.cfg = pc
 
 proc configIdentical(m: PackedModule; config: ConfigRef): bool =
   result = m.definedSymbols == definedSymbolsAsString(config)
+  #if not result:
+  #  echo "A ", m.definedSymbols, " ", definedSymbolsAsString(config)
   template eq(x) =
     result = result and m.cfg.x == config.x
+    #if not result:
+    #  echo "B ", m.cfg.x, " ", config.x
   primConfigFields eq
 
+proc rememberStartupConfig*(dest: var PackedConfig, config: ConfigRef) =
+  template rem(x) =
+    dest.x = config.x
+  primConfigFields rem
+
 proc hashFileCached(conf: ConfigRef; fileIdx: FileIndex): string =
   result = msgs.getHash(conf, fileIdx)
   if result.len == 0:
@@ -104,7 +116,7 @@ proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
       return false
   result = true
 
-proc initEncoder*(c: var PackedEncoder; m: PSym; config: ConfigRef) =
+proc initEncoder*(c: var PackedEncoder; m: PSym; config: ConfigRef; pc: PackedConfig) =
   ## setup a context for serializing to packed ast
   c.m.sh = Shared()
   c.thisModule = m.itemId.module
@@ -122,12 +134,45 @@ proc initEncoder*(c: var PackedEncoder; m: PSym; config: ConfigRef) =
       msgs.setHash(config, thisNimFile, h)
   c.m.includes.add((toLitId(thisNimFile, c), h)) # the module itself
 
+  rememberConfig(c, config, pc)
+
 proc addIncludeFileDep*(c: var PackedEncoder; f: FileIndex) =
   c.m.includes.add((toLitId(f, c), hashFileCached(c.config, f)))
 
 proc addImportFileDep*(c: var PackedEncoder; f: FileIndex) =
   c.m.imports.add toLitId(f, c)
 
+proc addExported*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  c.m.exports.add((nameId, s.itemId.item))
+
+proc addConverter*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  c.m.converters.add((nameId, s.itemId.item))
+
+proc addTrmacro*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  c.m.trmacros.add((nameId, s.itemId.item))
+
+proc addPureEnum*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  assert s.kind == skType
+  c.m.pureEnums.add((nameId, s.itemId.item))
+
+proc addMethod*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  discard "to do"
+  # c.m.methods.add((nameId, s.itemId.item))
+
+proc addReexport*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  c.m.reexports.add((nameId, PackedItemId(module: toLitId(s.itemId.module.FileIndex, c),
+                                          item: s.itemId.item)))
+
+proc addCompilerProc*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  c.m.compilerProcs.add((nameId, s.itemId.item))
+
 proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder)
 proc toPackedSym*(s: PSym; c: var PackedEncoder): PackedItemId
 proc toPackedType(t: PType; c: var PackedEncoder): PackedItemId
@@ -197,7 +242,7 @@ template storeNode(dest, src, field) =
 
 proc toPackedType(t: PType; c: var PackedEncoder): PackedItemId =
   ## serialize a ptype
-  if t.isNil: return nilTypeId
+  if t.isNil: return nilItemId
 
   if t.uniqueId.module != c.thisModule:
     # XXX Assert here that it already was serialized in the foreign module!
@@ -376,88 +421,49 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
   f.loadPrim m.definedSymbols
   f.loadPrim m.cfg
 
-  if not configIdentical(m, config):
+  if f.err == ok and not configIdentical(m, config):
     f.err = configMismatch
 
-  f.loadSection stringsSection
+  template loadSeqSection(section, data) {.dirty.} =
-  f.load m.sh.strings
+    f.loadSection section
+    f.loadSeq data
 
-  f.loadSection checkSumsSection
+  template loadTabSection(section, data) {.dirty.} =
-  f.loadSeq m.includes
+    f.loadSection section
+    f.load data
+
+  loadTabSection stringsSection, m.sh.strings
+
+  loadSeqSection checkSumsSection, m.includes
   if not includesIdentical(m, config):
     f.err = includeFileChanged
 
-  f.loadSection depsSection
+  loadSeqSection depsSection, m.imports
-  f.loadSeq m.imports
 
-  f.loadSection integersSection
+  loadTabSection integersSection, m.sh.integers
-  f.load m.sh.integers
+  loadTabSection floatsSection, m.sh.floats
-  f.loadSection floatsSection
-  f.load m.sh.floats
 
-  f.loadSection topLevelSection
+  loadSeqSection exportsSection, m.exports
-  f.loadSeq m.topLevel.nodes
 
-  f.loadSection bodiesSection
+  loadSeqSection reexportsSection, m.reexports
-  f.loadSeq m.bodies.nodes
 
-  f.loadSection symsSection
+  loadSeqSection compilerProcsSection, m.compilerProcs
-  f.loadSeq m.sh.syms
 
-  f.loadSection typesSection
+  loadSeqSection trmacrosSection, m.trmacros
-  f.loadSeq m.sh.types
+
+  loadSeqSection convertersSection, m.converters
+  loadSeqSection methodsSection, m.methods
+  loadSeqSection pureEnumsSection, m.pureEnums
+  loadSeqSection macroUsagesSection, m.macroUsages
+
+  loadSeqSection topLevelSection, m.topLevel.nodes
+  loadSeqSection bodiesSection, m.bodies.nodes
+  loadSeqSection symsSection, m.sh.syms
+  loadSeqSection typesSection, m.sh.types
 
   close(f)
   result = f.err
 
-type
-  ModuleStatus* = enum
-    undefined,
-    loading,
-    loaded,
-    outdated
-
-  LoadedModule* = object
-    status: ModuleStatus
-    symsInit, typesInit: bool
-    fromDisk: PackedModule
-    syms: seq[PSym] # indexed by itemId
-    types: seq[PType]
-
-  PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex
-
-proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef;
-                    fileIdx: FileIndex): bool =
-  let m = int(fileIdx)
-  if m >= g.len:
-    g.setLen(m+1)
-
-  case g[m].status
-  of undefined:
-    g[m].status = loading
-    let fullpath = msgs.toFullPath(conf, fileIdx)
-    let rod = toRodFile(conf, AbsoluteFile fullpath)
-    let err = loadRodFile(rod, g[m].fromDisk, conf)
-    if err == ok:
-      result = false
-      # check its dependencies:
-      for dep in g[m].fromDisk.imports:
-        let fid = toFileIndex(dep, g[m].fromDisk, conf)
-        # Warning: we need to traverse the full graph, so
-        # do **not use break here**!
-        if needsRecompile(g, conf, fid):
-          result = true
-
-      g[m].status = if result: outdated else: loaded
-    else:
-      loadError(err, rod)
-      g[m].status = outdated
-      result = true
-  of loading, loaded:
-    result = false
-  of outdated:
-    result = true
-
 # -------------------------------------------------------------------------
 
 proc storeError(err: RodFileError; filename: AbsoluteFile) =
@@ -465,7 +471,7 @@ proc storeError(err: RodFileError; filename: AbsoluteFile) =
   removeFile(filename.string)
 
 proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder) =
-  rememberConfig(encoder, encoder.config)
+  #rememberConfig(encoder, encoder.config)
 
   var f = rodfiles.create(filename.string)
   f.storeHeader()
@@ -473,40 +479,50 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder) =
   f.storePrim encoder.m.definedSymbols
   f.storePrim encoder.m.cfg
 
-  f.storeSection stringsSection
+  template storeSeqSection(section, data) {.dirty.} =
-  f.store encoder.m.sh.strings
+    f.storeSection section
+    f.storeSeq data
 
-  f.storeSection checkSumsSection
+  template storeTabSection(section, data) {.dirty.} =
-  f.storeSeq encoder.m.includes
+    f.storeSection section
+    f.store data
 
-  f.storeSection depsSection
+  storeTabSection stringsSection, encoder.m.sh.strings
-  f.storeSeq encoder.m.imports
 
-  f.storeSection integersSection
+  storeSeqSection checkSumsSection, encoder.m.includes
-  f.store encoder.m.sh.integers
 
-  f.storeSection floatsSection
+  storeSeqSection depsSection, encoder.m.imports
-  f.store encoder.m.sh.floats
 
-  f.storeSection topLevelSection
+  storeTabSection integersSection, encoder.m.sh.integers
-  f.storeSeq encoder.m.topLevel.nodes
+  storeTabSection floatsSection, encoder.m.sh.floats
 
-  f.storeSection bodiesSection
+  storeSeqSection exportsSection, encoder.m.exports
-  f.storeSeq encoder.m.bodies.nodes
 
-  f.storeSection symsSection
+  storeSeqSection reexportsSection, encoder.m.reexports
-  f.storeSeq encoder.m.sh.syms
 
-  f.storeSection typesSection
+  storeSeqSection compilerProcsSection, encoder.m.compilerProcs
-  f.storeSeq encoder.m.sh.types
+
+  storeSeqSection trmacrosSection, encoder.m.trmacros
+  storeSeqSection convertersSection, encoder.m.converters
+  storeSeqSection methodsSection, encoder.m.methods
+  storeSeqSection pureEnumsSection, encoder.m.pureEnums
+  storeSeqSection macroUsagesSection, encoder.m.macroUsages
+
+  storeSeqSection topLevelSection, encoder.m.topLevel.nodes
+
+  storeSeqSection bodiesSection, encoder.m.bodies.nodes
+  storeSeqSection symsSection, encoder.m.sh.syms
+
+  storeSeqSection typesSection, encoder.m.sh.types
   close(f)
   if f.err != ok:
-    loadError(f.err, filename)
+    storeError(f.err, filename)
 
-  when true:
+  when false:
     # basic loader testing:
     var m2: PackedModule
     discard loadRodFile(filename, m2, encoder.config)
+    echo "loaded ", filename.string
 
 # ----------------------------------------------------------------------------
 
@@ -516,7 +532,25 @@ type
     lastLit*: LitId
     lastFile*: FileIndex # remember the last lookup entry.
     config*: ConfigRef
-    ident: IdentCache
+    cache: IdentCache
+
+type
+  ModuleStatus* = enum
+    undefined,
+    loading,
+    loaded,
+    outdated
+
+  LoadedModule* = object
+    status*: ModuleStatus
+    symsInit, typesInit: bool
+    fromDisk: PackedModule
+    syms: seq[PSym] # indexed by itemId
+    types: seq[PType]
+    module*: PSym # the one true module symbol.
+    iface: Table[PIdent, seq[PackedItemId]] # PackedItemId so that it works with reexported symbols too
+
+  PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex
 
 proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): PType
 proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): PSym
@@ -546,7 +580,7 @@ proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph;
   of nkEmpty, nkNilLit, nkType:
     discard
   of nkIdent:
-    result.ident = getIdent(c.ident, g[c.thisModule].fromDisk.sh.strings[n.litId])
+    result.ident = getIdent(c.cache, g[c.thisModule].fromDisk.sh.strings[n.litId])
   of nkSym:
     result.sym = loadSym(c, g, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand))
   of directIntLit:
@@ -567,6 +601,31 @@ proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph;
     for n0 in sonsReadonly(tree, n):
       result.add loadNodes(c, g, tree, n0)
 
+proc loadProcHeader(c: var PackedDecoder; g: var PackedModuleGraph;
+                    tree: PackedTree; n: NodePos): PNode =
+  # do not load the body of the proc. This will be done later in
+  # getProcBody, if required.
+  let k = n.kind
+  result = newNodeIT(k, translateLineInfo(c, g, n.info),
+    loadType(c, g, n.typ))
+  result.flags = n.flags
+  assert k in {nkProcDef, nkMethodDef, nkIteratorDef, nkFuncDef, nkConverterDef}
+  var i = 0
+  for n0 in sonsReadonly(tree, n):
+    if i != bodyPos:
+      result.add loadNodes(c, g, tree, n0)
+    else:
+      result.add nil
+    inc i
+
+proc loadProcBody(c: var PackedDecoder; g: var PackedModuleGraph;
+                  tree: PackedTree; n: NodePos): PNode =
+  var i = 0
+  for n0 in sonsReadonly(tree, n):
+    if i == bodyPos:
+      result = loadNodes(c, g, tree, n0)
+    inc i
+
 proc moduleIndex*(c: var PackedDecoder; g: var PackedModuleGraph;
                   s: PackedItemId): int32 {.inline.} =
   result = if s.module == LitId(0): c.thisModule
@@ -579,13 +638,17 @@ proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
     info: translateLineInfo(c, g, s.info),
     options: s.options,
     position: s.position,
-    name: getIdent(c.ident, g[si].fromDisk.sh.strings[s.name])
+    name: getIdent(c.cache, g[si].fromDisk.sh.strings[s.name])
   )
 
 template loadAstBody(p, field) =
   if p.field != emptyNodeId:
     result.field = loadNodes(c, g, g[si].fromDisk.bodies, NodePos p.field)
 
+template loadAstBodyLazy(p, field) =
+  if p.field != emptyNodeId:
+    result.field = loadProcHeader(c, g, g[si].fromDisk.bodies, NodePos p.field)
+
 proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph;
              si, item: int32; l: PackedLib): PLib =
   # XXX: hack; assume a zero LitId means the PackedLib is all zero (empty)
@@ -600,6 +663,9 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
                        s: PackedSym; si, item: int32; result: PSym) =
   result.typ = loadType(c, g, s.typ)
   loadAstBody(s, constraint)
+  if result.kind in {skProc, skFunc, skIterator, skConverter, skMethod}:
+    loadAstBodyLazy(s, ast)
+  else:
     loadAstBody(s, ast)
   result.annex = loadLib(c, g, si, item, s.annex)
   when hasFFI:
@@ -615,7 +681,7 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
     result.loc.r = rope externalName
 
 proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): PSym =
-  if s == nilTypeId:
+  if s == nilItemId:
     result = nil
   else:
     let si = moduleIndex(c, g, s)
@@ -626,10 +692,16 @@ proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): P
 
     if g[si].syms[s.item] == nil:
       let packed = addr(g[si].fromDisk.sh.syms[s.item])
+
+      if packed.kind != skModule:
         result = symHeaderFromPacked(c, g, packed[], si, s.item)
         # store it here early on, so that recursions work properly:
         g[si].syms[s.item] = result
         symBodyFromPacked(c, g, packed[], si, s.item, result)
+      else:
+        result = g[si].module
+        assert result != nil
+
     else:
       result = g[si].syms[s.item]
 
@@ -654,7 +726,7 @@ proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
     result.methods.add((gen, loadSym(c, g, id)))
 
 proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): PType =
-  if t == nilTypeId:
+  if t == nilItemId:
     result = nil
   else:
     let si = moduleIndex(c, g, t)
@@ -672,15 +744,142 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId):
     else:
       result = g[si].types[t.item]
 
+proc setupLookupTables(m: var LoadedModule; conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex) =
+  m.iface = initTable[PIdent, seq[PackedItemId]]()
+  for e in m.fromDisk.exports:
+    let nameLit = e[0]
+    m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(PackedItemId(module: LitId(0), item: e[1]))
+  for re in m.fromDisk.reexports:
+    let nameLit = re[0]
+    m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(re[1])
 
-when false:
+  let filename = AbsoluteFile toFullPath(conf, fileIdx)
-  proc initGenericKey*(s: PSym; types: seq[PType]): GenericKey =
+  # We cannot call ``newSym`` here, because we have to circumvent the ID
-    result.module = s.owner.itemId.module
+  # mechanism, which we do in order to assign each module a persistent ID.
-    result.name = s.name.s
+  m.module = PSym(kind: skModule, itemId: ItemId(module: int32(fileIdx), item: 0'i32),
-    result.types = mapIt types: hashType(it, {CoType, CoDistinct}).MD5Digest
+                  name: getIdent(cache, splitFile(filename).name),
+                  info: newLineInfo(fileIdx, 1, 1))
+
+proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
+                    fileIdx: FileIndex): bool =
+  let m = int(fileIdx)
+  if m >= g.len:
+    g.setLen(m+1)
+
+  case g[m].status
+  of undefined:
+    g[m].status = loading
+    let fullpath = msgs.toFullPath(conf, fileIdx)
+    let rod = toRodFile(conf, AbsoluteFile fullpath)
+    let err = loadRodFile(rod, g[m].fromDisk, conf)
+    if err == ok:
+      result = false
+      # check its dependencies:
+      for dep in g[m].fromDisk.imports:
+        let fid = toFileIndex(dep, g[m].fromDisk, conf)
+        # Warning: we need to traverse the full graph, so
+        # do **not use break here**!
+        if needsRecompile(g, conf, cache, fid):
+          result = true
+
+      if not result:
+        setupLookupTables(g[m], conf, cache, fileIdx)
+      g[m].status = if result: outdated else: loaded
+    else:
+      loadError(err, rod)
+      g[m].status = outdated
+      result = true
+  of loading, loaded:
+    result = false
+  of outdated:
+    result = true
+
+proc moduleFromRodFile*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
+                        fileIdx: FileIndex): PSym =
+  ## Returns 'nil' if the module needs to be recompiled.
+  if needsRecompile(g, conf, cache, fileIdx):
+    result = nil
+  else:
+    result = g[int fileIdx].module
+    assert result != nil
+
+template setupDecoder() {.dirty.} =
+  var decoder = PackedDecoder(
+    thisModule: int32(module),
+    lastLit: LitId(0),
+    lastFile: FileIndex(-1),
+    config: config,
+    cache: cache)
+
+proc loadProcBody*(config: ConfigRef, cache: IdentCache;
+                   g: var PackedModuleGraph; s: PSym): PNode =
+  let mId = s.itemId.module
+  var decoder = PackedDecoder(
+    thisModule: mId,
+    lastLit: LitId(0),
+    lastFile: FileIndex(-1),
+    config: config,
+    cache: cache)
+  let pos = g[mId].fromDisk.sh.syms[s.itemId.item].ast
+  assert pos != emptyNodeId
+  result = loadProcBody(decoder, g, g[mId].fromDisk.bodies, NodePos pos)
+
+type
+  RodIter* = object
+    decoder: PackedDecoder
+    values: seq[PackedItemId]
+    i: int
+
+proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache;
+                  g: var PackedModuleGraph; module: FileIndex;
+                  name: PIdent): PSym =
+  it.decoder = PackedDecoder(
+    thisModule: int32(module),
+    lastLit: LitId(0),
+    lastFile: FileIndex(-1),
+    config: config,
+    cache: cache)
+  it.values = g[int module].iface.getOrDefault(name)
+  it.i = 0
+  if it.i < it.values.len:
+    result = loadSym(it.decoder, g, it.values[it.i])
+    inc it.i
+
+proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache;
+                         g: var PackedModuleGraph; module: FileIndex): PSym =
+  it.decoder = PackedDecoder(
+    thisModule: int32(module),
+    lastLit: LitId(0),
+    lastFile: FileIndex(-1),
+    config: config,
+    cache: cache)
+  it.values = @[]
+  for v in g[int module].iface.values:
+    it.values.add v
+  it.i = 0
+  if it.i < it.values.len:
+    result = loadSym(it.decoder, g, it.values[it.i])
+    inc it.i
+
+proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym =
+  if it.i < it.values.len:
+    result = loadSym(it.decoder, g, it.values[it.i])
+    inc it.i
+
+iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache;
+                           g: var PackedModuleGraph; module: FileIndex;
+                           name: PIdent): PSym =
+  setupDecoder()
+  let values = g[int module].iface.getOrDefault(name)
+  for pid in values:
+    let s = loadSym(decoder, g, pid)
+    assert s != nil
+    yield s
+
+proc interfaceSymbol*(config: ConfigRef, cache: IdentCache;
+                      g: var PackedModuleGraph; module: FileIndex;
+                      name: PIdent): PSym =
+  setupDecoder()
+  let values = g[int module].iface.getOrDefault(name)
+  result = loadSym(decoder, g, values[0])
 
-  proc addGeneric*(m: var Module; c: var PackedEncoder; key: GenericKey; s: PSym) =
-    ## add a generic to the module
-    if key notin m.generics:
-      m.generics[key] = toPackedSym(s, m.ast, c)
-      toPackedNode(s.ast, m.ast, c)

compiler/idents.nim

@@ -113,3 +113,8 @@ proc newIdentCache*(): IdentCache =
 proc whichKeyword*(id: PIdent): TSpecialWord =
   if id.id < 0: result = wInvalid
   else: result = TSpecialWord(id.id)
+
+proc hash*(x: PIdent): Hash {.inline.} = x.h
+proc `==`*(a, b: PIdent): bool {.inline.} =
+  if a.isNil or b.isNil: result = system.`==`(a, b)
+  else: result = a.id == b.id

compiler/importer.nim

@@ -11,7 +11,8 @@
 
 import
   intsets, ast, astalgo, msgs, options, idents, lookups,
-  semdata, modulepaths, sigmatch, lineinfos, sets
+  semdata, modulepaths, sigmatch, lineinfos, sets,
+  modulegraphs
 
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
@@ -108,7 +109,7 @@ proc rawImportSymbol(c: PContext, s, origin: PSym; importSet: var IntSet) =
 
 proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
   let ident = lookups.considerQuotedIdent(c, n)
-  let s = strTableGet(fromMod.tab, ident)
+  let s = someSym(c.graph, fromMod, ident)
   if s == nil:
     errorUndeclaredIdentifier(c, n.info, ident.s)
   else:
@@ -118,16 +119,16 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
     # for an enumeration we have to add all identifiers
     if multiImport:
       # for a overloadable syms add all overloaded routines
-      var it: TIdentIter
+      var it: ModuleIter
-      var e = initIdentIter(it, fromMod.tab, s.name)
+      var e = initModuleIter(it, c.graph, fromMod, s.name)
       while e != nil:
         if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3")
         if s.kind in ExportableSymKinds:
           rawImportSymbol(c, e, fromMod, importSet)
-        e = nextIdentIter(it, fromMod.tab)
+        e = nextModuleIter(it, c.graph)
     else:
       rawImportSymbol(c, s, fromMod, importSet)
-    suggestSym(c.config, n.info, s, c.graph.usageSym, false)
+    suggestSym(c.graph, n.info, s, c.graph.usageSym, false)
 
 proc addImport(c: PContext; im: sink ImportedModule) =
   for i in 0..high(c.imports):
@@ -176,18 +177,6 @@ proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
   c.addImport ImportedModule(m: fromMod, mode: importExcept, exceptSet: exceptSet)
   addUnnamedIt(c, fromMod, it.id notin exceptSet)
 
-  when false:
-    var i: TTabIter
-    var s = initTabIter(i, fromMod.tab)
-    while s != nil:
-      if s.kind != skModule:
-        if s.kind != skEnumField:
-          if s.kind notin ExportableSymKinds:
-            internalError(c.config, s.info, "importAllSymbols: " & $s.kind & " " & s.name.s)
-          if exceptSet.isNil or s.name.id notin exceptSet:
-            rawImportSymbol(c, s, fromMod)
-      s = nextIter(i, fromMod.tab)
-
 proc importAllSymbols*(c: PContext, fromMod: PSym) =
   c.addImport ImportedModule(m: fromMod, mode: importAll)
   addUnnamedIt(c, fromMod, true)
@@ -256,7 +245,7 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
         message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated")
       else:
         message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated")
-    suggestSym(c.config, n.info, result, c.graph.usageSym, false)
+    suggestSym(c.graph, n.info, result, c.graph.usageSym, false)
     importStmtResult.add newSymNode(result, n.info)
     #newStrNode(toFullPath(c.config, f), n.info)
 

compiler/jsgen.nim

@@ -1426,7 +1426,7 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
     if lfNoDecl in s.loc.flags or s.magic != mNone or
        {sfImportc, sfInfixCall} * s.flags != {}:
       discard
-    elif s.kind == skMethod and s.getBody.kind == nkEmpty:
+    elif s.kind == skMethod and getBody(p.module.graph, s).kind == nkEmpty:
       # we cannot produce code for the dispatcher yet:
       discard
     elif sfForward in s.flags:

compiler/lookups.nim

@@ -11,7 +11,8 @@
 
 import
   intsets, ast, astalgo, idents, semdata, types, msgs, options,
-  renderer, nimfix/prettybase, lineinfos, strutils
+  renderer, nimfix/prettybase, lineinfos, strutils,
+  modulegraphs
 
 proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope)
 
@@ -107,8 +108,9 @@ proc localSearchInScope*(c: PContext, s: PIdent): PSym =
     scope = scope.parent
     result = strTableGet(scope.symbols, s)
 
-proc initIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule; name: PIdent): PSym =
+proc initIdentIter(ti: var ModuleIter; marked: var IntSet; im: ImportedModule; name: PIdent;
-  result = initIdentIter(ti, im.m.tab, name)
+                   g: ModuleGraph): PSym =
+  result = initModuleIter(ti, g, im.m, name)
   while result != nil:
     let b =
       case im.mode
@@ -117,11 +119,12 @@ proc initIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule; n
       of importExcept: name.id notin im.exceptSet
     if b and not containsOrIncl(marked, result.id):
       return result
-    result = nextIdentIter(ti, im.m.tab)
+    result = nextModuleIter(ti, g)
 
-proc nextIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule): PSym =
+proc nextIdentIter(ti: var ModuleIter; marked: var IntSet; im: ImportedModule;
+                   g: ModuleGraph): PSym =
   while true:
-    result = nextIdentIter(ti, im.m.tab)
+    result = nextModuleIter(ti, g)
     if result == nil: return nil
     case im.mode
     of importAll:
@@ -134,17 +137,17 @@ proc nextIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule):
       if result.name.id notin im.exceptSet and not containsOrIncl(marked, result.id):
         return result
 
-iterator symbols(im: ImportedModule; marked: var IntSet; name: PIdent): PSym =
+iterator symbols(im: ImportedModule; marked: var IntSet; name: PIdent; g: ModuleGraph): PSym =
-  var ti: TIdentIter
+  var ti: ModuleIter
-  var candidate = initIdentIter(ti, marked, im, name)
+  var candidate = initIdentIter(ti, marked, im, name, g)
   while candidate != nil:
     yield candidate
-    candidate = nextIdentIter(ti, marked, im)
+    candidate = nextIdentIter(ti, marked, im, g)
 
 iterator importedItems*(c: PContext; name: PIdent): PSym =
   var marked = initIntSet()
   for im in c.imports.mitems:
-    for s in symbols(im, marked, name):
+    for s in symbols(im, marked, name, c.graph):
       yield s
 
 proc allPureEnumFields(c: PContext; name: PIdent): seq[PSym] =
@@ -169,15 +172,15 @@ iterator allSyms*(c: PContext): (PSym, int, bool) =
   dec scopeN
   isLocal = false
   for im in c.imports.mitems:
-    for s in im.m.tab.data:
+    for s in modulegraphs.allSyms(c.graph, im.m):
-      if s != nil:
+      assert s != nil
       yield (s, scopeN, isLocal)
 
 proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PSym =
   var marked = initIntSet()
   result = nil
   for im in c.imports.mitems:
-    for s in symbols(im, marked, name):
+    for s in symbols(im, marked, name, c.graph):
       if result == nil:
         result = s
       else:
@@ -214,7 +217,7 @@ proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSy
   if result.len == 0:
     var marked = initIntSet()
     for im in c.imports.mitems:
-      for s in symbols(im, marked, s):
+      for s in symbols(im, marked, s, c.graph):
         if s.kind in filter:
           result.add s
 
@@ -240,6 +243,7 @@ type
     oimSymChoiceLocalLookup
   TOverloadIter* = object
     it*: TIdentIter
+    mit*: ModuleIter
     m*: PSym
     mode*: TOverloadIterMode
     symChoiceIndex*: int
@@ -306,7 +310,7 @@ proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) =
 proc addInterfaceDeclAux(c: PContext, sym: PSym) =
   if sfExported in sym.flags:
     # add to interface:
-    if c.module != nil: strTableAdd(c.module.tab, sym)
+    if c.module != nil: exportSym(c, sym)
     else: internalError(c.config, sym.info, "addInterfaceDeclAux")
 
 proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) =
@@ -475,7 +479,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
         if m == c.module:
           result = strTableGet(c.topLevelScope.symbols, ident).skipAlias(n, c.config)
         else:
-          result = strTableGet(m.tab, ident).skipAlias(n, c.config)
+          result = someSym(c.graph, m, ident).skipAlias(n, c.config)
         if result == nil and checkUndeclared in flags:
           fixSpelling(n[1], ident, searchInScopes)
           errorUndeclaredIdentifier(c, n[1].info, ident.s)
@@ -509,7 +513,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         scope = scope.parent
         if scope == nil:
           for i in 0..c.imports.high:
-            result = initIdentIter(o.it, o.marked, c.imports[i], ident).skipAlias(n, c.config)
+            result = initIdentIter(o.mit, o.marked, c.imports[i], ident, c.graph).skipAlias(n, c.config)
             if result != nil:
               o.currentScope = nil
               o.importIdx = i
@@ -535,7 +539,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
                                  ident).skipAlias(n, c.config)
           o.mode = oimSelfModule
         else:
-          result = initIdentIter(o.it, o.m.tab, ident).skipAlias(n, c.config)
+          result = initModuleIter(o.mit, c.graph, o.m, ident).skipAlias(n, c.config)
       else:
         noidentError(c.config, n[1], n)
         result = errorSym(c, n[1])
@@ -568,7 +572,7 @@ proc nextOverloadIterImports(o: var TOverloadIter, c: PContext, n: PNode): PSym
   var idx = o.importIdx+1
   o.importIdx = c.imports.len # assume the other imported modules lack this symbol too
   while idx < c.imports.len:
-    result = initIdentIter(o.it, o.marked, c.imports[idx], o.it.name).skipAlias(n, c.config)
+    result = initIdentIter(o.mit, o.marked, c.imports[idx], o.it.name, c.graph).skipAlias(n, c.config)
     if result != nil:
       # oh, we were wrong, some other module had the symbol, so remember that:
       o.importIdx = idx
@@ -578,7 +582,7 @@ proc nextOverloadIterImports(o: var TOverloadIter, c: PContext, n: PNode): PSym
 proc symChoiceExtension(o: var TOverloadIter; c: PContext; n: PNode): PSym =
   assert o.currentScope == nil
   while o.importIdx < c.imports.len:
-    result = initIdentIter(o.it, o.marked, c.imports[o.importIdx], o.it.name).skipAlias(n, c.config)
+    result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph).skipAlias(n, c.config)
     #while result != nil and result.id in o.marked:
     #  result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx])
     if result != nil:
@@ -602,12 +606,12 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         else:
           o.importIdx = 0
           if c.imports.len > 0:
-            result = initIdentIter(o.it, o.marked, c.imports[o.importIdx], o.it.name).skipAlias(n, c.config)
+            result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph).skipAlias(n, c.config)
             if result == nil:
               result = nextOverloadIterImports(o, c, n)
           break
     elif o.importIdx < c.imports.len:
-      result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx]).skipAlias(n, c.config)
+      result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph).skipAlias(n, c.config)
       if result == nil:
         result = nextOverloadIterImports(o, c, n)
     else:
@@ -615,7 +619,7 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
   of oimSelfModule:
     result = nextIdentIter(o.it, c.topLevelScope.symbols).skipAlias(n, c.config)
   of oimOtherModule:
-    result = nextIdentIter(o.it, o.m.tab).skipAlias(n, c.config)
+    result = nextModuleIter(o.mit, c.graph).skipAlias(n, c.config)
   of oimSymChoice:
     if o.symChoiceIndex < n.len:
       result = n[o.symChoiceIndex].sym
@@ -654,7 +658,7 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         incl o.marked, result.id
 
     elif o.importIdx < c.imports.len:
-      result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx]).skipAlias(n, c.config)
+      result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph).skipAlias(n, c.config)
       #assert result.id notin o.marked
       #while result != nil and result.id in o.marked:
       #  result = nextIdentIter(o.it, c.imports[o.importIdx]).skipAlias(n, c.config)

compiler/magicsys.nim

@@ -26,7 +26,7 @@ proc newSysType(g: ModuleGraph; kind: TTypeKind, size: int): PType =
   result.align = size.int16
 
 proc getSysSym*(g: ModuleGraph; info: TLineInfo; name: string): PSym =
-  result = strTableGet(g.systemModule.tab, getIdent(g.cache, name))
+  result = systemModuleSym(g, getIdent(g.cache, name))
   if result == nil:
     localError(g.config, info, "system module needs: " & name)
     result = newSym(skError, getIdent(g.cache, name), nextSymId(g.idgen), g.systemModule, g.systemModule.info, {})
@@ -34,15 +34,12 @@ proc getSysSym*(g: ModuleGraph; info: TLineInfo; name: string): PSym =
   if result.kind == skAlias: result = result.owner
 
 proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSym =
-  var ti: TIdentIter
   let id = getIdent(g.cache, name)
-  var r = initIdentIter(ti, g.systemModule.tab, id)
+  for r in systemModuleSyms(g, id):
-  while r != nil:
     if r.magic == m:
       # prefer the tyInt variant:
       if r.typ[0] != nil and r.typ[0].kind == tyInt: return r
       result = r
-    r = nextIdentIter(ti, g.systemModule.tab)
   if result != nil: return result
   localError(g.config, info, "system module needs: " & name)
   result = newSym(skError, id, nextSymId(g.idgen), g.systemModule, g.systemModule.info, {})

compiler/modulegraphs.nim

@@ -10,25 +10,11 @@
 ## This module implements the module graph data structure. The module graph
 ## represents a complete Nim project. Single modules can either be kept in RAM
 ## or stored in a rod-file.
-##
-## The caching of modules is critical for 'nimsuggest' and is tricky to get
-## right. If module E is being edited, we need autocompletion (and type
-## checking) for E but we don't want to recompile depending
-## modules right away for faster turnaround times. Instead we mark the module's
-## dependencies as 'dirty'. Let D be a dependency of E. If D is dirty, we
-## need to recompile it and all of its dependencies that are marked as 'dirty'.
-## 'nimsuggest sug' actually is invoked for the file being edited so we know
-## its content changed and there is no need to compute any checksums.
-## Instead of a recursive algorithm, we use an iterative algorithm:
-##
-## - If a module gets recompiled, its dependencies need to be updated.
-## - Its dependent module stays the same.
-##
 
-import ast, intsets, tables, options, lineinfos, hashes, idents,
+import ast, astalgo, intsets, tables, options, lineinfos, hashes, idents,
   btrees, md5
 
-# import ic / packed_ast
+import ic / to_packed_ast
 
 type
   SigHash* = distinct MD5Digest
@@ -39,9 +25,12 @@ type
     converters*: seq[PSym]
     patterns*: seq[PSym]
     pureEnums*: seq[PSym]
+    interf: TStrTable
 
   ModuleGraph* = ref object
     ifaces*: seq[Iface]  ## indexed by int32 fileIdx
+    packed: PackedModuleGraph
+    startupPackedConfig*: PackedConfig
     packageSyms*: TStrTable
     deps*: IntSet # the dependency graph or potentially its transitive closure.
     importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies
@@ -64,6 +53,7 @@ type
     sysTypes*: array[TTypeKind, PType]
     compilerprocs*: TStrTable
     exposed*: TStrTable
+    packageTypes*: TStrTable
     intTypeCache*: array[-5..64, PType]
     opContains*, opNot*: PSym
     emptyNode*: PNode
@@ -132,6 +122,62 @@ proc toBase64a(s: cstring, len: int): string =
     result.add cb64[a shr 2]
     result.add cb64[(a and 3) shl 4]
 
+template semtab*(m: PSym; g: ModuleGraph): TStrTable =
+  g.ifaces[m.position].interf
+
+proc cachedModule(g: ModuleGraph; m: PSym): bool {.inline.} =
+  m.position < g.packed.len and g.packed[m.position].status == loaded
+
+type
+  ModuleIter* = object
+    fromRod: bool
+    modIndex: int
+    ti: TIdentIter
+    rodIt: RodIter
+
+proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent): PSym =
+  assert m.kind == skModule
+  mi.modIndex = m.position
+  mi.fromRod = mi.modIndex < g.packed.len and g.packed[mi.modIndex].status == loaded
+  if mi.fromRod:
+    result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name)
+  else:
+    result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interf, name)
+
+proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym =
+  if mi.fromRod:
+    result = nextRodIter(mi.rodIt, g.packed)
+  else:
+    result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interf)
+
+iterator allSyms*(g: ModuleGraph; m: PSym): PSym =
+  if cachedModule(g, m):
+    var rodIt: RodIter
+    var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position)
+    while r != nil:
+      yield r
+      r = nextRodIter(rodIt, g.packed)
+  else:
+    for s in g.ifaces[m.position].interf.data:
+      if s != nil:
+        yield s
+
+proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym =
+  if cachedModule(g, m):
+    result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name)
+  else:
+    result = strTableGet(g.ifaces[m.position].interf, name)
+
+proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym =
+  result = someSym(g, g.systemModule, name)
+
+iterator systemModuleSyms*(g: ModuleGraph; name: PIdent): PSym =
+  var mi: ModuleIter
+  var r = initModuleIter(mi, g, g.systemModule, name)
+  while r != nil:
+    yield r
+    r = nextModuleIter(mi, g)
+
 proc `$`*(u: SigHash): string =
   toBase64a(cast[cstring](unsafeAddr u), sizeof(u))
 
@@ -186,6 +232,7 @@ proc registerModule*(g: ModuleGraph; m: PSym) =
   if m.position >= g.ifaces.len:
     setLen(g.ifaces, m.position + 1)
   g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[])
+  initStrTable(g.ifaces[m.position].interf)
 
 proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result = ModuleGraph()
@@ -202,6 +249,7 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.methods = @[]
   initStrTable(result.compilerprocs)
   initStrTable(result.exposed)
+  initStrTable(result.packageTypes)
   result.opNot = createMagic(result, "not", mNot)
   result.opContains = createMagic(result, "contains", mInSet)
   result.emptyNode = newNode(nkEmpty)
@@ -226,7 +274,10 @@ proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.exposed)
 
 proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
-  if fileIdx.int32 >= 0 and fileIdx.int32 < g.ifaces.len:
+  if fileIdx.int32 >= 0:
+    if fileIdx.int32 < g.packed.len and g.packed[fileIdx.int32].status == loaded:
+      result = g.packed[fileIdx.int32].module
+    elif fileIdx.int32 < g.ifaces.len:
       result = g.ifaces[fileIdx.int32].module
 
 proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b
@@ -280,3 +331,18 @@ proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
 
 proc isDirty*(g: ModuleGraph; m: PSym): bool =
   result = g.suggestMode and sfDirty in m.flags
+
+proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} =
+  result = s.ast[bodyPos]
+  if result == nil and g.config.symbolFiles in {readOnlySf, v2Sf}:
+    result = loadProcBody(g.config, g.cache, g.packed, s)
+    s.ast[bodyPos] = result
+  assert result != nil
+
+proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex): PSym =
+  ## Returns 'nil' if the module needs to be recompiled.
+  if g.config.symbolFiles in {readOnlySf, v2Sf}:
+    result = moduleFromRodFile(g.packed, g.config, g.cache, fileIdx)
+
+proc configComplete*(g: ModuleGraph) =
+  rememberStartupConfig(g.startupPackedConfig, g.config)

compiler/modules.nim

@@ -31,12 +31,21 @@ proc getPackage(graph: ModuleGraph; fileIdx: FileIndex): PSym =
     pck = getPackageName(graph.config, filename.string)
     pck2 = if pck.len > 0: pck else: "unknown"
     pack = getIdent(graph.cache, pck2)
-  var packSym = graph.packageSyms.strTableGet(pack)
+  result = graph.packageSyms.strTableGet(pack)
-  if packSym == nil:
+  if result == nil:
-    packSym = newSym(skPackage, getIdent(graph.cache, pck2), packageId(), nil, info)
+    result = newSym(skPackage, getIdent(graph.cache, pck2), packageId(), nil, info)
-    initStrTable(packSym.tab)
+    #initStrTable(packSym.tab)
-    graph.packageSyms.strTableAdd(packSym)
+    graph.packageSyms.strTableAdd(result)
   else:
+    # we now produce a fake Nimble package instead
+    # to resolve the conflicts:
+    let pck3 = fakePackageName(graph.config, filename)
+    # this makes the new `packSym`'s owner be the original `packSym`
+    result = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), result, info)
+    #initStrTable(packSym.tab)
+    graph.packageSyms.strTableAdd(result)
+
+    when false:
       let existing = strTableGet(packSym.tab, name)
       if existing != nil and existing.info.fileIndex != info.fileIndex:
         when false:
@@ -50,24 +59,21 @@ proc getPackage(graph: ModuleGraph; fileIdx: FileIndex): PSym =
           let pck3 = fakePackageName(graph.config, filename)
           # this makes the new `packSym`'s owner be the original `packSym`
           packSym = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), packSym, info)
-        initStrTable(packSym.tab)
+          #initStrTable(packSym.tab)
           graph.packageSyms.strTableAdd(packSym)
-  result = packSym
 
 proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
   let packSym = getPackage(graph, fileIdx)
   result.owner = packSym
   result.position = int fileIdx
 
-  graph.registerModule(result)
+  #initStrTable(result.tab(graph))
-
-  initStrTable(result.tab)
   when false:
     strTableAdd(result.tab, result) # a module knows itself
     # This is now implemented via
     #   c.moduleScope.addSym(module) # a module knows itself
     # in sem.nim, around line 527
-  strTableAdd(packSym.tab, result)
+  #strTableAdd(packSym.tab, result)
 
 proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
   let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
@@ -79,6 +85,7 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
   if not isNimIdentifier(result.name.s):
     rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s)
   partialInitModule(result, graph, fileIdx, filename)
+  graph.registerModule(result)
 
 proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym =
   var flags = flags
@@ -92,21 +99,20 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
       elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput)
     discard processModule(graph, result, idGeneratorFromModule(result), s)
   if result == nil:
+    result = moduleFromRodFile(graph, fileIdx)
     let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
-    when false:
+    if result == nil:
-      # XXX entry point for module loading from the rod file
-      result = loadModuleSym(graph, fileIdx, filename)
-    when true:
       result = newModule(graph, fileIdx)
       result.flags.incl flags
       registerModule(graph, result)
+      processModuleAux()
     else:
       partialInitModule(result, graph, fileIdx, filename)
-    processModuleAux()
+      # XXX replay the pragmas here!
   elif graph.isDirty(result):
     result.flags.excl sfDirty
     # reset module fields:
-    initStrTable(result.tab)
+    initStrTable(result.semtab(graph))
     result.ast = nil
     processModuleAux()
     graph.markClientsDirty(fileIdx)
@@ -152,6 +158,8 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
   connectCallbacks(graph)
   let conf = graph.config
   wantMainModule(conf)
+  configComplete(graph)
+
   let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
   let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
   conf.projectMainIdx2 = projectFile

compiler/nimeval.nim

@@ -24,11 +24,8 @@ type
 iterator exportedSymbols*(i: Interpreter): PSym =
   assert i != nil
   assert i.mainModule != nil, "no main module selected"
-  var it: TTabIter
+  for s in modulegraphs.allSyms(i.graph, i.mainModule):
-  var s = initTabIter(it, i.mainModule.tab)
-  while s != nil:
     yield s
-    s = nextIter(it, i.mainModule.tab)
 
 proc selectUniqueSymbol*(i: Interpreter; name: string;
                          symKinds: set[TSymKind] = {skLet, skVar}): PSym =
@@ -37,14 +34,14 @@ proc selectUniqueSymbol*(i: Interpreter; name: string;
   assert i != nil
   assert i.mainModule != nil, "no main module selected"
   let n = getIdent(i.graph.cache, name)
-  var it: TIdentIter
+  var it: ModuleIter
-  var s = initIdentIter(it, i.mainModule.tab, n)
+  var s = initModuleIter(it, i.graph, i.mainModule, n)
   result = nil
   while s != nil:
     if s.kind in symKinds:
       if result == nil: result = s
       else: return nil # ambiguous
-    s = nextIdentIter(it, i.mainModule.tab)
+    s = nextModuleIter(it, i.graph)
 
 proc selectRoutine*(i: Interpreter; name: string): PSym =
   ## Selects a declared routine (proc/func/etc) from the main module.
@@ -70,7 +67,7 @@ proc evalScript*(i: Interpreter; scriptStream: PLLStream = nil) =
   ## This can also be used to *reload* the script.
   assert i != nil
   assert i.mainModule != nil, "no main module selected"
-  initStrTable(i.mainModule.tab)
+  initStrTable(i.mainModule.semtab(i.graph))
   i.mainModule.ast = nil
 
   let s = if scriptStream != nil: scriptStream

compiler/packagehandling.nim

@@ -44,7 +44,8 @@ proc fakePackageName*(conf: ConfigRef; path: AbsoluteFile): string =
   # in different directory get different name and they can be
   # placed in a directory.
   # foo-#head/../bar becomes @foo-@hhead@s..@sbar
-  result = "@m" & relativeTo(path, conf.projectPath).string.multiReplace({$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"})
+  result = "@m" & relativeTo(path, conf.projectPath).string.multiReplace(
+    {$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"})
 
 proc demanglePackageName*(path: string): string =
   result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@c": ":"})

compiler/passes.nim

@@ -110,6 +110,12 @@ proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
 proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
   result = module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")
 
+proc partOfStdlib(x: PSym): bool =
+  var it = x.owner
+  while it != nil and it.kind == skPackage and it.owner != nil:
+    it = it.owner
+  result = it != nil and it.name.s == "stdlib"
+
 proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
                     stream: PLLStream): bool {.discardable.} =
   if graph.stopCompile(): return true
@@ -131,7 +137,7 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
   while true:
     openParser(p, fileIdx, s, graph.cache, graph.config)
 
-    if module.owner == nil or module.owner.name.s != "stdlib" or module.name.s == "distros":
+    if not partOfStdlib(module) or module.name.s == "distros":
       # XXX what about caching? no processing then? what if I change the
       # modules to include between compilation runs? we'd need to track that
       # in ROD files. I think we should enable this feature only

compiler/plugins/itersgen.nim

@@ -9,7 +9,7 @@
 
 ## Plugin to transform an inline iterator into a data structure.
 
-import ".." / [ast, lookups, semdata, lambdalifting, msgs]
+import ".." / [ast, modulegraphs, lookups, semdata, lambdalifting, msgs]
 
 proc iterToProcImpl*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkStmtList, n.info)
@@ -29,7 +29,7 @@ proc iterToProcImpl*(c: PContext, n: PNode): PNode =
     localError(c.config, n[2].info,
         "type must be a non-generic ref|ptr to object with state field")
     return
-  let body = liftIterToProc(c.graph, iter.sym, iter.sym.getBody, t, c.idgen)
+  let body = liftIterToProc(c.graph, iter.sym, getBody(c.graph, iter.sym), t, c.idgen)
 
   let prc = newSym(skProc, n[3].ident, nextSymId c.idgen, iter.sym.owner, iter.sym.info)
   prc.typ = copyType(iter.sym.typ, nextTypeId c.idgen, prc)

compiler/pragmas.nim

@@ -14,6 +14,8 @@ import
   wordrecg, ropes, options, strutils, extccomp, math, magicsys, trees,
   types, lookups, lineinfos, pathutils, linter
 
+from ic / to_packed_ast import addCompilerProc
+
 const
   FirstCallConv* = wNimcall
   LastCallConv* = wNoconv
@@ -702,6 +704,8 @@ proc markCompilerProc(c: PContext; s: PSym) =
   incl(s.flags, sfCompilerProc)
   incl(s.flags, sfUsed)
   registerCompilerProc(c.graph, s)
+  if c.config.symbolFiles != disabledSf:
+    addCompilerProc(c.encoder, s)
 
 proc deprecatedStmt(c: PContext; outerPragma: PNode) =
   let pragma = outerPragma[1]

compiler/sem.nim

@@ -19,7 +19,8 @@ import
   lowerings, plugins/active, lineinfos, strtabs, int128,
   isolation_check, typeallowed
 
-from modulegraphs import ModuleGraph, PPassContext, onUse, onDef, onDefResolveForward
+from modulegraphs import ModuleGraph, PPassContext, onUse, onDef, onDefResolveForward,
+  systemModuleSym, semtab, getBody, someSym, allSyms
 
 when defined(nimfix):
   import nimfix/prettybase

compiler/semdata.nim

@@ -268,7 +268,7 @@ proc newContext*(graph: ModuleGraph; module: PSym): PContext =
   result.typesWithOps = @[]
   result.features = graph.config.features
   if graph.config.symbolFiles != disabledSf:
-    initEncoder result.encoder, module, graph.config
+    initEncoder result.encoder, module, graph.config, graph.startupPackedConfig
 
 proc addIncludeFileDep*(c: PContext; f: FileIndex) =
   if c.config.symbolFiles != disabledSf:
@@ -286,15 +286,29 @@ proc inclSym(sq: var seq[PSym], s: PSym) =
 proc addConverter*(c: PContext, conv: PSym) =
   inclSym(c.converters, conv)
   inclSym(c.graph.ifaces[c.module.position].converters, conv)
-  #addConverter(c.graph, c.module, conv) # upcoming
+  if c.config.symbolFiles != disabledSf:
+    addConverter(c.encoder, conv)
 
 proc addPureEnum*(c: PContext, e: PSym) =
   inclSym(c.graph.ifaces[c.module.position].pureEnums, e)
+  if c.config.symbolFiles != disabledSf:
+    addPureEnum(c.encoder, e)
 
 proc addPattern*(c: PContext, p: PSym) =
   inclSym(c.patterns, p)
   inclSym(c.graph.ifaces[c.module.position].patterns, p)
-  #addPattern(c.graph, c.module, p) # upcoming
+  if c.config.symbolFiles != disabledSf:
+    addTrmacro(c.encoder, p)
+
+proc exportSym*(c: PContext; s: PSym) =
+  strTableAdd(c.module.semtab(c.graph), s)
+  if c.config.symbolFiles != disabledSf:
+    addExported(c.encoder, s)
+
+proc reexportSym*(c: PContext; s: PSym) =
+  strTableAdd(c.module.semtab(c.graph), s)
+  if c.config.symbolFiles != disabledSf:
+    addReexport(c.encoder, s)
 
 proc newLib*(kind: TLibKind): PLib =
   new(result)
@@ -489,4 +503,4 @@ proc storeRodNode*(c: PContext, n: PNode) =
 
 proc saveRodFile*(c: PContext) =
   if c.config.symbolFiles != disabledSf:
-    saveRodFile(toRodFile(c.config, c.filename.AbsoluteFile), c.encoder)
+    saveRodFile(toRodFile(c.config, AbsoluteFile toFullPath(c.config, FileIndex c.module.position)), c.encoder)

compiler/semexprs.nim

@@ -1014,7 +1014,7 @@ proc semDirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
 proc buildEchoStmt(c: PContext, n: PNode): PNode =
   # we MUST not check 'n' for semantics again here! But for now we give up:
   result = newNodeI(nkCall, n.info)
-  var e = strTableGet(c.graph.systemModule.tab, getIdent(c.cache, "echo"))
+  let e = systemModuleSym(c.graph, getIdent(c.cache, "echo"))
   if e != nil:
     result.add(newSymNode(e))
   else:
@@ -1897,7 +1897,7 @@ proc lookUpForDeclared(c: PContext, n: PNode, onlyCurrentScope: bool): PSym =
       if m == c.module:
         result = strTableGet(c.topLevelScope.symbols, ident)
       else:
-        result = strTableGet(m.tab, ident)
+        result = someSym(c.graph, m, ident)
   of nkSym:
     result = n.sym
   of nkOpenSymChoice, nkClosedSymChoice:
@@ -2504,7 +2504,7 @@ proc semBlock(c: PContext, n: PNode; flags: TExprFlags): PNode =
     elif labl.owner == nil:
       labl.owner = c.p.owner
     n[0] = newSymNode(labl, n[0].info)
-    suggestSym(c.config, n[0].info, labl, c.graph.usageSym)
+    suggestSym(c.graph, n[0].info, labl, c.graph.usageSym)
     styleCheckDef(c.config, labl)
     onDef(n[0].info, labl)
   n[1] = semExpr(c, n[1], flags)
@@ -2522,15 +2522,12 @@ proc semExportExcept(c: PContext, n: PNode): PNode =
   let exceptSet = readExceptSet(c, n)
   let exported = moduleName.sym
   result = newNodeI(nkExportStmt, n.info)
-  strTableAdd(c.module.tab, exported)
+  reexportSym(c, exported)
-  var i: TTabIter
+  for s in allSyms(c.graph, exported):
-  var s = initTabIter(i, exported.tab)
-  while s != nil:
     if s.kind in ExportableSymKinds+{skModule} and
        s.name.id notin exceptSet and sfError notin s.flags:
-      strTableAdd(c.module.tab, s)
+      reexportSym(c, s)
       result.add newSymNode(s, n.info)
-    s = nextIter(i, exported.tab)
   markUsed(c, n.info, exported)
 
 proc semExport(c: PContext, n: PNode): PNode =
@@ -2548,15 +2545,12 @@ proc semExport(c: PContext, n: PNode): PNode =
       localError(c.config, a.info, errGenerated, "cannot export: " & renderTree(a))
     elif s.kind == skModule:
       # forward everything from that module:
-      strTableAdd(c.module.tab, s)
+      reexportSym(c, s)
-      var ti: TTabIter
+      for it in allSyms(c.graph, s):
-      var it = initTabIter(ti, s.tab)
-      while it != nil:
         if it.kind in ExportableSymKinds+{skModule}:
-          strTableAdd(c.module.tab, it)
+          reexportSym(c, it)
           result.add newSymNode(it, a.info)
           specialSyms(c, it)
-        it = nextIter(ti, s.tab)
       markUsed(c, n.info, s)
     else:
       while s != nil:
@@ -2565,7 +2559,7 @@ proc semExport(c: PContext, n: PNode): PNode =
             "; enum field cannot be exported individually")
         if s.kind in ExportableSymKinds+{skModule} and sfError notin s.flags:
           result.add(newSymNode(s, a.info))
-          strTableAdd(c.module.tab, s)
+          reexportSym(c, s)
           markUsed(c, n.info, s)
           specialSyms(c, s)
           if s.kind == skType and sfPure notin s.flags:
@@ -2575,7 +2569,7 @@ proc semExport(c: PContext, n: PNode): PNode =
                 var e = etyp.n[j].sym
                 if e.kind != skEnumField:
                   internalError(c.config, s.info, "rawImportSymbol")
-                strTableAdd(c.module.tab, e)
+                reexportSym(c, e)
 
         s = nextOverloadIter(o, c, a)
 

compiler/semfields.nim

@@ -106,7 +106,7 @@ proc semForFields(c: PContext, n: PNode, m: TMagic): PNode =
   # so that 'break' etc. work as expected, we produce
   # a 'while true: stmt; break' loop ...
   result = newNodeI(nkWhileStmt, n.info, 2)
-  var trueSymbol = strTableGet(c.graph.systemModule.tab, getIdent(c.cache, "true"))
+  var trueSymbol = systemModuleSym(c.graph, getIdent(c.cache, "true"))
   if trueSymbol == nil:
     localError(c.config, n.info, "system needs: 'true'")
     trueSymbol = newSym(skUnknown, getIdent(c.cache, "true"), nextSymId c.idgen, getCurrOwner(c), n.info)

compiler/semgnrc.nim

@@ -481,7 +481,7 @@ proc semGenericStmt(c: PContext, n: PNode,
       if sfGenSym in s.flags and s.ast == nil:
         body = n[bodyPos]
       else:
-        body = s.getBody
+        body = getBody(c.graph, s)
     else: body = n[bodyPos]
     n[bodyPos] = semGenericStmtScope(c, body, flags, ctx)
     closeScope(c)

compiler/seminst.nim

@@ -160,7 +160,7 @@ proc fixupInstantiatedSymbols(c: PContext, s: PSym) =
       pushInfoContext(c.config, oldPrc.info)
       openScope(c)
       var n = oldPrc.ast
-      n[bodyPos] = copyTree(s.getBody)
+      n[bodyPos] = copyTree(getBody(c.graph, s))
       instantiateBody(c, n, oldPrc.typ.n, oldPrc, s)
       closeScope(c)
       popInfoContext(c.config)
@@ -383,7 +383,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
     if n[pragmasPos].kind != nkEmpty:
       pragma(c, result, n[pragmasPos], allRoutinePragmas)
     if isNil(n[bodyPos]):
-      n[bodyPos] = copyTree(fn.getBody)
+      n[bodyPos] = copyTree(getBody(c.graph, fn))
     if c.inGenericContext == 0:
       instantiateBody(c, n, fn.typ.n, result, fn)
     sideEffectsCheck(c, result)

compiler/semstmts.nim

@@ -66,7 +66,7 @@ proc semBreakOrContinue(c: PContext, n: PNode): PNode =
         x.info = n.info
         incl(s.flags, sfUsed)
         n[0] = x
-        suggestSym(c.config, x.info, s, c.graph.usageSym)
+        suggestSym(c.graph, x.info, s, c.graph.usageSym)
         onUse(x.info, s)
       else:
         localError(c.config, n.info, errInvalidControlFlowX % s.name.s)
@@ -332,7 +332,7 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym =
       discard
     result = n.info
   let info = getLineInfo(n)
-  suggestSym(c.config, info, result, c.graph.usageSym)
+  suggestSym(c.graph, info, result, c.graph.usageSym)
 
 proc checkNilable(c: PContext; v: PSym) =
   if {sfGlobal, sfImportc} * v.flags == {sfGlobal} and v.typ.requiresInit:
@@ -1055,14 +1055,14 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) =
     if pkg.isNil or pkg.kind != skPackage:
       localError(c.config, name.info, "unknown package name: " & pkgName.s)
     else:
-      let typsym = pkg.tab.strTableGet(typName)
+      let typsym = c.graph.packageTypes.strTableGet(typName)
       if typsym.isNil:
         s = semIdentDef(c, name[1], skType)
         onDef(name[1].info, s)
         s.typ = newTypeS(tyObject, c)
         s.typ.sym = s
         s.flags.incl sfForward
-        pkg.tab.strTableAdd s
+        c.graph.packageTypes.strTableAdd s
         addInterfaceDecl(c, s)
       elif typsym.kind == skType and sfForward in typsym.flags:
         s = typsym
@@ -1088,7 +1088,7 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) =
       if not isTopLevel(c) or pkg.isNil:
         localError(c.config, name.info, "only top level types in a package can be 'package'")
       else:
-        let typsym = pkg.tab.strTableGet(s.name)
+        let typsym = c.graph.packageTypes.strTableGet(s.name)
         if typsym != nil:
           if sfForward notin typsym.flags or sfNoForward notin typsym.flags:
             typeCompleted(typsym)
@@ -1956,7 +1956,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     if not comesFromShadowScope:
       excl(proto.flags, sfForward)
       incl(proto.flags, sfWasForwarded)
-    suggestSym(c.config, s.info, proto, c.graph.usageSym)
+    suggestSym(c.graph, s.info, proto, c.graph.usageSym)
     closeScope(c)         # close scope with wrong parameter symbols
     openScope(c)          # open scope for old (correct) parameter symbols
     if proto.ast[genericParamsPos].kind != nkEmpty:

compiler/semtempl.nim

@@ -250,7 +250,7 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode =
     else: result = newSymNode(s, n.info)
     # Issue #12832
     when defined(nimsuggest):
-      suggestSym(c.config, n.info, s, c.graph.usageSym, false)
+      suggestSym(c.graph, n.info, s, c.graph.usageSym, false)
     if {optStyleHint, optStyleError} * c.config.globalOptions != {}:
       styleCheckUse(c.config, n.info, s)
 

compiler/semtypes.nim

@@ -140,7 +140,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
     if result.sym != nil and sfExported in result.sym.flags:
       incl(e.flags, sfUsed)
       incl(e.flags, sfExported)
-      if not isPure: strTableAdd(c.module.tab, e)
+      if not isPure: exportSym(c, e)
     result.n.add symNode
     styleCheckDef(c.config, e)
     onDef(e.info, e)
@@ -779,7 +779,7 @@ proc semRecordNodeAux(c: PContext, n: PNode, check: var IntSet, pos: var int,
                    n[i][1].info
                  else:
                    n[i].info
-      suggestSym(c.config, info, f, c.graph.usageSym)
+      suggestSym(c.graph, info, f, c.graph.usageSym)
       f.typ = typ
       f.position = pos
       f.options = c.config.options

compiler/sighashes.nim

@@ -371,7 +371,7 @@ proc symBodyDigest*(graph: ModuleGraph, sym: PSym): SigHash =
   if sym.ast != nil:
     md5Init(c)
     c.md5Update(cast[cstring](result.addr), sizeof(result))
-    hashBodyTree(graph, c, sym.ast[bodyPos])
+    hashBodyTree(graph, c, getBody(graph, sym))
     c.md5Final(result.MD5Digest)
     graph.symBodyHashes[sym.id] = result
 

compiler/suggest.nim

@@ -56,10 +56,10 @@ proc findDocComment(n: PNode): PNode =
   elif n.kind in {nkAsgn, nkFastAsgn} and n.len == 2:
     result = findDocComment(n[1])
 
-proc extractDocComment(s: PSym): string =
+proc extractDocComment(g: ModuleGraph; s: PSym): string =
   var n = findDocComment(s.ast)
   if n.isNil and s.kind in routineKinds and s.ast != nil:
-    n = findDocComment(s.ast[bodyPos])
+    n = findDocComment(getBody(g, s))
   if not n.isNil:
     result = n.comment.replace("\n##", "\n").strip
   else:
@@ -117,7 +117,7 @@ proc getTokenLenFromSource(conf: ConfigRef; ident: string; info: TLineInfo): int
     elif sourceIdent != ident:
       result = 0
 
-proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo;
+proc symToSuggest(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo;
                   quality: range[0..100]; prefix: PrefixMatch;
                   inTypeContext: bool; scope: int;
                   useSuppliedInfo = false): Suggest =
@@ -136,7 +136,7 @@ proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info
       if u.fileIndex == info.fileIndex: inc c
     result.localUsages = c
   result.symkind = byte s.kind
-  if optIdeTerse notin conf.globalOptions:
+  if optIdeTerse notin g.config.globalOptions:
     result.qualifiedPath = @[]
     if not isLocal and s.kind != skModule:
       let ow = s.owner
@@ -156,20 +156,20 @@ proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info
     else:
       result.forth = ""
     when defined(nimsuggest) and not defined(noDocgen) and not defined(leanCompiler):
-      result.doc = s.extractDocComment
+      result.doc = extractDocComment(g, s)
   let infox =
     if useSuppliedInfo or section in {ideUse, ideHighlight, ideOutline}:
       info
     else:
       s.info
-  result.filePath = toFullPath(conf, infox)
+  result.filePath = toFullPath(g.config, infox)
   result.line = toLinenumber(infox)
   result.column = toColumn(infox)
-  result.version = conf.suggestVersion
+  result.version = g.config.suggestVersion
   result.tokenLen = if section != ideHighlight:
                       s.name.s.len
                     else:
-                      getTokenLenFromSource(conf, s.name.s, infox)
+                      getTokenLenFromSource(g.config, s.name.s, infox)
 
 proc `$`*(suggest: Suggest): string =
   result = $suggest.section
@@ -261,7 +261,7 @@ proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} =
 proc suggestField(c: PContext, s: PSym; f: PNode; info: TLineInfo; outputs: var Suggestions) =
   var pm: PrefixMatch
   if filterSym(s, f, pm) and fieldVisible(c, s):
-    outputs.add(symToSuggest(c.config, s, isLocal=true, ideSug, info, 100, pm, c.inTypeContext > 0, 0))
+    outputs.add(symToSuggest(c.graph, s, isLocal=true, ideSug, info, 100, pm, c.inTypeContext > 0, 0))
 
 proc getQuality(s: PSym): range[0..100] =
   if s.typ != nil and s.typ.len > 1:
@@ -275,7 +275,7 @@ template wholeSymTab(cond, section: untyped) {.dirty.} =
     let it = item
     var pm: PrefixMatch
     if cond:
-      outputs.add(symToSuggest(c.config, it, isLocal = isLocal, section, info, getQuality(it),
+      outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, section, info, getQuality(it),
                                 pm, c.inTypeContext > 0, scopeN))
 
 proc suggestSymList(c: PContext, list, f: PNode; info: TLineInfo, outputs: var Suggestions) =
@@ -346,7 +346,7 @@ proc suggestEverything(c: PContext, n, f: PNode, outputs: var Suggestions) =
   for (it, scopeN, isLocal) in allSyms(c):
     var pm: PrefixMatch
     if filterSym(it, f, pm):
-      outputs.add(symToSuggest(c.config, it, isLocal = isLocal, ideSug, n.info, 0, pm,
+      outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, ideSug, n.info, 0, pm,
                                 c.inTypeContext > 0, scopeN))
 
 proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions) =
@@ -365,10 +365,10 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions)
         let m = c.graph.importModuleCallback(c.graph, c.module, fileInfoIdx(c.config, fullPath))
         if m == nil: typ = nil
         else:
-          for it in items(n.sym.tab):
+          for it in allSyms(c.graph, n.sym):
             if filterSym(it, field, pm):
-              outputs.add(symToSuggest(c.config, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -100))
+              outputs.add(symToSuggest(c.graph, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -100))
-          outputs.add(symToSuggest(c.config, m, isLocal=false, ideMod, n.info, 100, PrefixMatch.None,
+          outputs.add(symToSuggest(c.graph, m, isLocal=false, ideMod, n.info, 100, PrefixMatch.None,
             c.inTypeContext > 0, -99))
 
   if typ == nil:
@@ -378,11 +378,11 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions)
         # all symbols accessible, because we are in the current module:
         for it in items(c.topLevelScope.symbols):
           if filterSym(it, field, pm):
-            outputs.add(symToSuggest(c.config, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -99))
+            outputs.add(symToSuggest(c.graph, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -99))
       else:
-        for it in items(n.sym.tab):
+        for it in allSyms(c.graph, n.sym):
           if filterSym(it, field, pm):
-            outputs.add(symToSuggest(c.config, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -99))
+            outputs.add(symToSuggest(c.graph, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -99))
     else:
       # fallback:
       suggestEverything(c, n, field, outputs)
@@ -440,27 +440,27 @@ when defined(nimsuggest):
       if infoB.infoToInt == infoAsInt: return
     s.allUsages.add(info)
 
-proc findUsages(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym) =
+proc findUsages(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym) =
-  if conf.suggestVersion == 1:
+  if g.config.suggestVersion == 1:
-    if usageSym == nil and isTracked(info, conf.m.trackPos, s.name.s.len):
+    if usageSym == nil and isTracked(info, g.config.m.trackPos, s.name.s.len):
       usageSym = s
-      suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideUse, info, 100, PrefixMatch.None, false, 0))
+      suggestResult(g.config, symToSuggest(g, s, isLocal=false, ideUse, info, 100, PrefixMatch.None, false, 0))
     elif s == usageSym:
-      if conf.lastLineInfo != info:
+      if g.config.lastLineInfo != info:
-        suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideUse, info, 100, PrefixMatch.None, false, 0))
+        suggestResult(g.config, symToSuggest(g, s, isLocal=false, ideUse, info, 100, PrefixMatch.None, false, 0))
-      conf.lastLineInfo = info
+      g.config.lastLineInfo = info
 
 when defined(nimsuggest):
-  proc listUsages*(conf: ConfigRef; s: PSym) =
+  proc listUsages*(g: ModuleGraph; s: PSym) =
     #echo "usages ", s.allUsages.len
     for info in s.allUsages:
       let x = if info == s.info and info.col == s.info.col: ideDef else: ideUse
-      suggestResult(conf, symToSuggest(conf, s, isLocal=false, x, info, 100, PrefixMatch.None, false, 0))
+      suggestResult(g.config, symToSuggest(g, s, isLocal=false, x, info, 100, PrefixMatch.None, false, 0))
 
-proc findDefinition(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym) =
+proc findDefinition(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym) =
   if s.isNil: return
-  if isTracked(info, conf.m.trackPos, s.name.s.len) or (s == usageSym and sfForward notin s.flags):
+  if isTracked(info, g.config.m.trackPos, s.name.s.len) or (s == usageSym and sfForward notin s.flags):
-    suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0, useSuppliedInfo = s == usageSym))
+    suggestResult(g.config, symToSuggest(g, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0, useSuppliedInfo = s == usageSym))
     if sfForward notin s.flags:
       suggestQuit()
     else:
@@ -472,8 +472,9 @@ proc ensureIdx[T](x: var T, y: int) =
 proc ensureSeq[T](x: var seq[T]) =
   if x == nil: newSeq(x, 0)
 
-proc suggestSym*(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym; isDecl=true) {.inline.} =
+proc suggestSym*(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym; isDecl=true) {.inline.} =
   ## misnamed: should be 'symDeclared'
+  let conf = g.config
   when defined(nimsuggest):
     if conf.suggestVersion == 0:
       if s.allUsages.len == 0:
@@ -482,15 +483,15 @@ proc suggestSym*(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym;
         s.addNoDup(info)
 
     if conf.ideCmd == ideUse:
-      findUsages(conf, info, s, usageSym)
+      findUsages(g, info, s, usageSym)
     elif conf.ideCmd == ideDef:
-      findDefinition(conf, info, s, usageSym)
+      findDefinition(g, info, s, usageSym)
     elif conf.ideCmd == ideDus and s != nil:
       if isTracked(info, conf.m.trackPos, s.name.s.len):
-        suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0))
+        suggestResult(conf, symToSuggest(g, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0))
-      findUsages(conf, info, s, usageSym)
+      findUsages(g, info, s, usageSym)
     elif conf.ideCmd == ideHighlight and info.fileIndex == conf.m.trackPos.fileIndex:
-      suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideHighlight, info, 100, PrefixMatch.None, false, 0))
+      suggestResult(conf, symToSuggest(g, s, isLocal=false, ideHighlight, info, 100, PrefixMatch.None, false, 0))
     elif conf.ideCmd == ideOutline and isDecl:
       # if a module is included then the info we have is inside the include and
       # we need to walk up the owners until we find the outer most module,
@@ -503,7 +504,7 @@ proc suggestSym*(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym;
         parentModule = parentModule.owner
 
       if parentFileIndex == conf.m.trackPos.fileIndex:
-        suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideOutline, info, 100, PrefixMatch.None, false, 0))
+        suggestResult(conf, symToSuggest(g, s, isLocal=false, ideOutline, info, 100, PrefixMatch.None, false, 0))
 
 proc extractPragma(s: PSym): PNode =
   if s.kind in routineKinds:
@@ -572,7 +573,7 @@ proc markUsed(c: PContext; info: TLineInfo; s: PSym) =
 
     if sfError in s.flags: userError(conf, info, s)
   when defined(nimsuggest):
-    suggestSym(conf, info, s, c.graph.usageSym, false)
+    suggestSym(c.graph, info, s, c.graph.usageSym, false)
   if {optStyleHint, optStyleError} * conf.globalOptions != {}:
     styleCheckUse(conf, info, s)
   markOwnerModuleAsUsed(c, s)
@@ -658,7 +659,7 @@ proc suggestSentinel*(c: PContext) =
   for (it, scopeN, isLocal) in allSyms(c):
     var pm: PrefixMatch
     if filterSymNoOpr(it, nil, pm):
-      outputs.add(symToSuggest(c.config, it, isLocal = isLocal, ideSug,
+      outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, ideSug,
           newLineInfo(c.config.m.trackPos.fileIndex, 0, -1), 0,
           PrefixMatch.None, false, scopeN))
 

compiler/transf.nim

@@ -126,7 +126,7 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
   var tc = c.transCon
   if sfBorrow in s.flags and s.kind in routineKinds:
     # simply exchange the symbol:
-    b = s.getBody
+    b = getBody(c.graph, s)
     if b.kind != nkSym: internalError(c.graph.config, n.info, "wrong AST for borrowed symbol")
     b = newSymNode(b.sym, n.info)
   elif c.inlining > 0:
@@ -594,7 +594,7 @@ proc findWrongOwners(c: PTransf, n: PNode) =
   else:
     for i in 0..<n.safeLen: findWrongOwners(c, n[i])
 
-proc isSimpleIteratorVar(iter: PSym): bool =
+proc isSimpleIteratorVar(c: PTransf; iter: PSym): bool =
   proc rec(n: PNode; owner: PSym; dangerousYields: var int) =
     case n.kind
     of nkEmpty..nkNilLit: discard
@@ -607,7 +607,7 @@ proc isSimpleIteratorVar(iter: PSym): bool =
       for c in n: rec(c, owner, dangerousYields)
 
   var dangerousYields = 0
-  rec(iter.ast[bodyPos], iter, dangerousYields)
+  rec(getBody(c.graph, iter), iter, dangerousYields)
   result = dangerousYields == 0
 
 proc transformFor(c: PTransf, n: PNode): PNode =
@@ -650,7 +650,7 @@ proc transformFor(c: PTransf, n: PNode): PNode =
       for j in 0..<n[i].len-1:
         addVar(v, copyTree(n[i][j])) # declare new vars
     else:
-      if n[i].kind == nkSym and isSimpleIteratorVar(iter):
+      if n[i].kind == nkSym and isSimpleIteratorVar(c, iter):
         incl n[i].sym.flags, sfCursor
       addVar(v, copyTree(n[i])) # declare new vars
   stmtList.add(v)
@@ -1087,12 +1087,12 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; cache: bool):
 
   if prc.transformedBody != nil:
     result = prc.transformedBody
-  elif nfTransf in prc.ast[bodyPos].flags or prc.kind in {skTemplate}:
+  elif nfTransf in getBody(g, prc).flags or prc.kind in {skTemplate}:
-    result = prc.ast[bodyPos]
+    result = getBody(g, prc)
   else:
     prc.transformedBody = newNode(nkEmpty) # protects from recursion
     var c = openTransf(g, prc.getModule, "", idgen)
-    result = liftLambdas(g, prc, prc.ast[bodyPos], c.tooEarly, c.idgen)
+    result = liftLambdas(g, prc, getBody(g, prc), c.tooEarly, c.idgen)
     result = processTransf(c, result, prc)
     liftDefer(c, result)
     result = liftLocalsIfRequested(prc, result, g.cache, g.config, c.idgen)
@@ -1108,6 +1108,7 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; cache: bool):
       prc.transformedBody = result
     else:
       prc.transformedBody = nil
+    # XXX Rodfile support for transformedBody!
 
   #if prc.name.s == "main":
   #  echo "transformed into ", renderTree(result, {renderIds})

compiler/vm.nim

@@ -1204,7 +1204,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
           VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[ptr UncheckedArray[TFullReg]](addr regs[0]),
                  currentException: c.currentExceptionA,
                  currentLineInfo: c.debug[pc]))
-      elif importcCond(prc):
+      elif importcCond(c, prc):
         if compiletimeFFI notin c.config.features:
           globalError(c.config, c.debug[pc], "VM not allowed to do FFI, see `compiletimeFFI`")
         # we pass 'tos.slots' instead of 'regs' so that the compiler can keep

compiler/vmgen.nim

@@ -31,6 +31,8 @@ import
   strutils, ast, types, msgs, renderer, vmdef,
   intsets, magicsys, options, lowerings, lineinfos, transf
 
+from modulegraphs import getBody
+
 const
   debugEchoCode* = defined(nimVMDebug)
 
@@ -1570,11 +1572,11 @@ proc genTypeLit(c: PCtx; t: PType; dest: var TDest) =
   n.typ = t
   genLit(c, n, dest)
 
-proc importcCond*(s: PSym): bool {.inline.} =
+proc importcCond*(c: PCtx; s: PSym): bool {.inline.} =
   ## return true to importc `s`, false to execute its body instead (refs #8405)
   if sfImportc in s.flags:
     if s.kind in routineKinds:
-      return s.ast[bodyPos].kind == nkEmpty
+      return getBody(c.graph, s).kind == nkEmpty
 
 proc importcSym(c: PCtx; info: TLineInfo; s: PSym) =
   when hasFFI:
@@ -1615,7 +1617,7 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
     elif s.position == 0:
       cannotEval(c, n)
     if s.position == 0:
-      if importcCond(s) or isImportcVar: c.importcSym(n.info, s)
+      if importcCond(c, s) or isImportcVar: c.importcSym(n.info, s)
       else: genGlobalInit(c, n, s)
     if dest < 0: dest = c.getTemp(n.typ)
     assert s.typ != nil
@@ -1841,7 +1843,7 @@ proc genVarSection(c: PCtx; n: PNode) =
       checkCanEval(c, a[0])
       if s.isGlobal:
         if s.position == 0:
-          if importcCond(s): c.importcSym(a.info, s)
+          if importcCond(c, s): c.importcSym(a.info, s)
           else:
             let sa = getNullValue(s.typ, a.info, c.config)
             #if s.ast.isNil: getNullValue(s.typ, a.info)
@@ -1964,6 +1966,7 @@ proc matches(s: PSym; x: string): bool =
     if s == nil or (y[^i].cmpIgnoreStyle(s.name.s) != 0 and y[^i] != "*"):
       return false
     s = if sfFromGeneric in s.flags: s.owner.owner else: s.owner
+    while s != nil and s.kind == skPackage and s.owner != nil: s = s.owner
   result = true
 
 proc procIsCallback(c: PCtx; s: PSym): bool =
@@ -1989,7 +1992,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
       if s.kind == skIterator and s.typ.callConv == TCallingConvention.ccClosure:
         globalError(c.config, n.info, "Closure iterators are not supported by VM!")
       if procIsCallback(c, s): discard
-      elif importcCond(s): c.importcSym(n.info, s)
+      elif importcCond(c, s): c.importcSym(n.info, s)
       genLit(c, n, dest)
     of skConst:
       let constVal = if s.ast != nil: s.ast else: s.typ.n

nimsuggest/nimsuggest.nim

@@ -195,7 +195,7 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int;
   if conf.ideCmd in {ideUse, ideDus}:
     let u = if conf.suggestVersion != 1: graph.symFromInfo(conf.m.trackPos) else: graph.usageSym
     if u != nil:
-      listUsages(conf, u)
+      listUsages(graph, u)
     else:
       localError(conf, conf.m.trackPos, "found no symbol at this position " & (conf $ conf.m.trackPos))