IC: next steps (#16550)

* cleanups
* ast.nim: cleanups
* IC: no more sym.tab field, stored externally in the module graph
* nimble compiles again
* rodfiles: store bitwidth of integers and the endianness in the cookie because we serialize 'int' directly
* rodfiles: added compilerproc and export sections
* rodfiles: added all the missing sections
* rodfiles: track the missing information
* IC: architecture for lazy loading of proc bodies
* make tests green again
* completed the lazy loading of proc bodies
* symbol lookup integration, part 1
* symbol lookup integration, part 2
* symbol lookup integration, part 3
* make tcompilerapi work again
* rodfiles: fixed config change handling

compiler/modulegraphs.nim

@@ -10,25 +10,11 @@
 ## This module implements the module graph data structure. The module graph
 ## represents a complete Nim project. Single modules can either be kept in RAM
 ## or stored in a rod-file.
-##
-## The caching of modules is critical for 'nimsuggest' and is tricky to get
-## right. If module E is being edited, we need autocompletion (and type
-## checking) for E but we don't want to recompile depending
-## modules right away for faster turnaround times. Instead we mark the module's
-## dependencies as 'dirty'. Let D be a dependency of E. If D is dirty, we
-## need to recompile it and all of its dependencies that are marked as 'dirty'.
-## 'nimsuggest sug' actually is invoked for the file being edited so we know
-## its content changed and there is no need to compute any checksums.
-## Instead of a recursive algorithm, we use an iterative algorithm:
-##
-## - If a module gets recompiled, its dependencies need to be updated.
-## - Its dependent module stays the same.
-##
 
-import ast, intsets, tables, options, lineinfos, hashes, idents,
+import ast, astalgo, intsets, tables, options, lineinfos, hashes, idents,
   btrees, md5
 
-# import ic / packed_ast
+import ic / to_packed_ast
 
 type
   SigHash* = distinct MD5Digest
@@ -39,9 +25,12 @@ type
     converters*: seq[PSym]
     patterns*: seq[PSym]
     pureEnums*: seq[PSym]
+    interf: TStrTable
 
   ModuleGraph* = ref object
     ifaces*: seq[Iface]  ## indexed by int32 fileIdx
+    packed: PackedModuleGraph
+    startupPackedConfig*: PackedConfig
     packageSyms*: TStrTable
     deps*: IntSet # the dependency graph or potentially its transitive closure.
     importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies
@@ -64,6 +53,7 @@ type
     sysTypes*: array[TTypeKind, PType]
     compilerprocs*: TStrTable
     exposed*: TStrTable
+    packageTypes*: TStrTable
     intTypeCache*: array[-5..64, PType]
     opContains*, opNot*: PSym
     emptyNode*: PNode
@@ -132,6 +122,62 @@ proc toBase64a(s: cstring, len: int): string =
     result.add cb64[a shr 2]
     result.add cb64[(a and 3) shl 4]
 
+template semtab*(m: PSym; g: ModuleGraph): TStrTable =
+  g.ifaces[m.position].interf
+
+proc cachedModule(g: ModuleGraph; m: PSym): bool {.inline.} =
+  m.position < g.packed.len and g.packed[m.position].status == loaded
+
+type
+  ModuleIter* = object
+    fromRod: bool
+    modIndex: int
+    ti: TIdentIter
+    rodIt: RodIter
+
+proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent): PSym =
+  assert m.kind == skModule
+  mi.modIndex = m.position
+  mi.fromRod = mi.modIndex < g.packed.len and g.packed[mi.modIndex].status == loaded
+  if mi.fromRod:
+    result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name)
+  else:
+    result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interf, name)
+
+proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym =
+  if mi.fromRod:
+    result = nextRodIter(mi.rodIt, g.packed)
+  else:
+    result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interf)
+
+iterator allSyms*(g: ModuleGraph; m: PSym): PSym =
+  if cachedModule(g, m):
+    var rodIt: RodIter
+    var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position)
+    while r != nil:
+      yield r
+      r = nextRodIter(rodIt, g.packed)
+  else:
+    for s in g.ifaces[m.position].interf.data:
+      if s != nil:
+        yield s
+
+proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym =
+  if cachedModule(g, m):
+    result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name)
+  else:
+    result = strTableGet(g.ifaces[m.position].interf, name)
+
+proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym =
+  result = someSym(g, g.systemModule, name)
+
+iterator systemModuleSyms*(g: ModuleGraph; name: PIdent): PSym =
+  var mi: ModuleIter
+  var r = initModuleIter(mi, g, g.systemModule, name)
+  while r != nil:
+    yield r
+    r = nextModuleIter(mi, g)
+
 proc `$`*(u: SigHash): string =
   toBase64a(cast[cstring](unsafeAddr u), sizeof(u))
 
@@ -186,6 +232,7 @@ proc registerModule*(g: ModuleGraph; m: PSym) =
   if m.position >= g.ifaces.len:
     setLen(g.ifaces, m.position + 1)
   g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[])
+  initStrTable(g.ifaces[m.position].interf)
 
 proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result = ModuleGraph()
@@ -202,6 +249,7 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.methods = @[]
   initStrTable(result.compilerprocs)
   initStrTable(result.exposed)
+  initStrTable(result.packageTypes)
   result.opNot = createMagic(result, "not", mNot)
   result.opContains = createMagic(result, "contains", mInSet)
   result.emptyNode = newNode(nkEmpty)
@@ -226,8 +274,11 @@ proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.exposed)
 
 proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
-  if fileIdx.int32 >= 0 and fileIdx.int32 < g.ifaces.len:
-    result = g.ifaces[fileIdx.int32].module
+  if fileIdx.int32 >= 0:
+    if fileIdx.int32 < g.packed.len and g.packed[fileIdx.int32].status == loaded:
+      result = g.packed[fileIdx.int32].module
+    elif fileIdx.int32 < g.ifaces.len:
+      result = g.ifaces[fileIdx.int32].module
 
 proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b
 
@@ -280,3 +331,18 @@ proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
 
 proc isDirty*(g: ModuleGraph; m: PSym): bool =
   result = g.suggestMode and sfDirty in m.flags
+
+proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} =
+  result = s.ast[bodyPos]
+  if result == nil and g.config.symbolFiles in {readOnlySf, v2Sf}:
+    result = loadProcBody(g.config, g.cache, g.packed, s)
+    s.ast[bodyPos] = result
+  assert result != nil
+
+proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex): PSym =
+  ## Returns 'nil' if the module needs to be recompiled.
+  if g.config.symbolFiles in {readOnlySf, v2Sf}:
+    result = moduleFromRodFile(g.packed, g.config, g.cache, fileIdx)
+
+proc configComplete*(g: ModuleGraph) =
+  rememberStartupConfig(g.startupPackedConfig, g.config)