big steps torwards an efficient, simple IC implementation (#16543)

* reworked ID handling
* the packed AST now has its own ID mechanism
* basic serialization code works
* extract rodfiles to its own module
* rodfiles: store and compare configs
* rodfiles: store dependencies
* store config at the end
* precise dependency tracking
* dependency tracking for rodfiles
* completed loading of PSym, PType, etc
* removed dead code
* bugfix: do not realloc seqs when taking addr into an element
* make IC opt-in for now
* makes tcompilerapi green again
* final cleanups

Co-authored-by: Andy Davidoff <github@andy.disruptek.com>

compiler/ast.nim

@@ -1074,16 +1074,33 @@ template id*(a: PIdObj): int =
   (x.itemId.module.int shl moduleShift) + x.itemId.item.int
 
 type
-  IdGenerator* = ref ItemId # unfortunately, we really need the 'shared mutable' aspect here.
+  IdGenerator* = ref object # unfortunately, we really need the 'shared mutable' aspect here.
+    module*: int32
+    symId*: int32
+    typeId*: int32
+
+proc hash*(x: ItemId): Hash =
+  var h: Hash = hash(x.module)
+  h = h !& hash(x.item)
+  result = !$h
 
 const
   PackageModuleId* = -3'i32
 
 proc idGeneratorFromModule*(m: PSym): IdGenerator =
   assert m.kind == skModule
-  result = IdGenerator(module: m.itemId.module, item: m.itemId.item)
+  result = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0)
 
-proc nextId*(x: IdGenerator): ItemId {.inline.} =
+proc nextSymId*(x: IdGenerator): ItemId {.inline.} =
+  inc x.symId
+  result = ItemId(module: x.module, item: x.symId)
+
+proc nextTypeId*(x: IdGenerator): ItemId {.inline.} =
+  inc x.typeId
+  result = ItemId(module: x.module, item: x.typeId)
+
+when false:
+  proc nextId*(x: IdGenerator): ItemId {.inline.} =
     inc x.item
     result = x[]
 
@@ -1831,7 +1848,7 @@ proc toVar*(typ: PType; kind: TTypeKind; idgen: IdGenerator): PType =
   ## returned. Otherwise ``typ`` is simply returned as-is.
   result = typ
   if typ.kind != kind:
-    result = newType(kind, nextId(idgen), typ.owner)
+    result = newType(kind, nextTypeId(idgen), typ.owner)
     rawAddSon(result, typ)
 
 proc toRef*(typ: PType; idgen: IdGenerator): PType =
@@ -1839,7 +1856,7 @@ proc toRef*(typ: PType; idgen: IdGenerator): PType =
   ## returned. Otherwise ``typ`` is simply returned as-is.
   result = typ
   if typ.skipTypes({tyAlias, tyGenericInst}).kind == tyObject:
-    result = newType(tyRef, nextId(idgen), typ.owner)
+    result = newType(tyRef, nextTypeId(idgen), typ.owner)
     rawAddSon(result, typ)
 
 proc toObject*(typ: PType): PType =

compiler/ccgexprs.nim

@@ -1772,7 +1772,7 @@ proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   else: internalError(p.config, e.info, "genArrayLen()")
 
 proc makePtrType(baseType: PType; idgen: IdGenerator): PType =
-  result = newType(tyPtr, nextId idgen, baseType.owner)
+  result = newType(tyPtr, nextTypeId idgen, baseType.owner)
   addSonSkipIntLit(result, baseType, idgen)
 
 proc makeAddr(n: PNode; idgen: IdGenerator): PNode =

compiler/ccgtypes.nim

@@ -1267,9 +1267,9 @@ proc genArrayInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
 
 proc fakeClosureType(m: BModule; owner: PSym): PType =
   # we generate the same RTTI as for a tuple[pointer, ref tuple[]]
-  result = newType(tyTuple, nextId m.idgen, owner)
+  result = newType(tyTuple, nextTypeId m.idgen, owner)
-  result.rawAddSon(newType(tyPointer, nextId m.idgen, owner))
+  result.rawAddSon(newType(tyPointer, nextTypeId m.idgen, owner))
-  var r = newType(tyRef, nextId m.idgen, owner)
+  var r = newType(tyRef, nextTypeId m.idgen, owner)
   let obj = createObj(m.g.graph, m.idgen, owner, owner.info, final=false)
   r.rawAddSon(obj)
   result.rawAddSon(r)
@@ -1396,9 +1396,9 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope =
   result = prefixTI.rope & result & ")".rope
 
 proc openArrayToTuple(m: BModule; t: PType): PType =
-  result = newType(tyTuple, nextId m.idgen, t.owner)
+  result = newType(tyTuple, nextTypeId m.idgen, t.owner)
-  let p = newType(tyPtr, nextId m.idgen, t.owner)
+  let p = newType(tyPtr, nextTypeId m.idgen, t.owner)
-  let a = newType(tyUncheckedArray, nextId m.idgen, t.owner)
+  let a = newType(tyUncheckedArray, nextTypeId m.idgen, t.owner)
   a.add t.lastSon
   p.add a
   result.add p

compiler/cgmeth.nim

@@ -108,10 +108,10 @@ proc attachDispatcher(s: PSym, dispatcher: PNode) =
     s.ast[dispatcherPos] = dispatcher
 
 proc createDispatcher(s: PSym; idgen: IdGenerator): PSym =
-  var disp = copySym(s, nextId(idgen))
+  var disp = copySym(s, nextSymId(idgen))
   incl(disp.flags, sfDispatcher)
   excl(disp.flags, sfExported)
-  disp.typ = copyType(disp.typ, nextId(idgen), disp.typ.owner)
+  disp.typ = copyType(disp.typ, nextTypeId(idgen), disp.typ.owner)
   # we can't inline the dispatcher itself (for now):
   if disp.typ.callConv == ccInline: disp.typ.callConv = ccNimCall
   disp.ast = copyTree(s.ast)
@@ -119,7 +119,7 @@ proc createDispatcher(s: PSym; idgen: IdGenerator): PSym =
   disp.loc.r = nil
   if s.typ[0] != nil:
     if disp.ast.len > resultPos:
-      disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym, nextId(idgen))
+      disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym, nextSymId(idgen))
     else:
       # We've encountered a method prototype without a filled-in
       # resultPos slot. We put a placeholder in there that will

compiler/closureiters.nim

@@ -177,7 +177,7 @@ proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode =
   ctx.newStateAssgn(newIntTypeNode(stateNo, ctx.g.getSysType(TLineInfo(), tyInt)))
 
 proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
-  result = newSym(skVar, getIdent(ctx.g.cache, name), nextId(ctx.idgen), ctx.fn, ctx.fn.info)
+  result = newSym(skVar, getIdent(ctx.g.cache, name), nextSymId(ctx.idgen), ctx.fn, ctx.fn.info)
   result.typ = typ
   assert(not typ.isNil)
 
@@ -1118,9 +1118,9 @@ proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode=
       n[i] = ctx.skipThroughEmptyStates(n[i])
 
 proc newArrayType(g: ModuleGraph; n: int, t: PType; idgen: IdGenerator; owner: PSym): PType =
-  result = newType(tyArray, nextId(idgen), owner)
+  result = newType(tyArray, nextTypeId(idgen), owner)
 
-  let rng = newType(tyRange, nextId(idgen), owner)
+  let rng = newType(tyRange, nextTypeId(idgen), owner)
   rng.n = newTree(nkRange, g.newIntLit(owner.info, 0), g.newIntLit(owner.info, n))
   rng.rawAddSon(t)
 
@@ -1314,7 +1314,7 @@ proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
         let idefs = copyNode(it)
         for v in 0..it.len-3:
           if it[v].kind == nkSym:
-            let x = copySym(it[v].sym, nextId(c.idgen))
+            let x = copySym(it[v].sym, nextSymId(c.idgen))
             c.tab[it[v].sym.id] = x
             idefs.add newSymNode(x)
           else:
@@ -1393,9 +1393,9 @@ proc transformClosureIterator*(g: ModuleGraph; idgen: IdGenerator; fn: PSym, n:
     # Lambda lifting was not done yet. Use temporary :state sym, which will
     # be handled specially by lambda lifting. Local temp vars (if needed)
     # should follow the same logic.
-    ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), nextId(idgen), fn, fn.info)
+    ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), nextSymId(idgen), fn, fn.info)
     ctx.stateVarSym.typ = g.createClosureIterStateType(fn, idgen)
-  ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), nextId(idgen), fn, fn.info)
+  ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), nextSymId(idgen), fn, fn.info)
   var pc = PreprocessContext(finallys: @[], config: g.config, idgen: idgen)
   var n = preprocess(pc, n.toStmtList)
   #echo "transformed into ", n

compiler/commands.nim

@@ -31,7 +31,6 @@ import
   wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos,
   pathutils, strtabs
 
-from incremental import nimIncremental
 from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect
 
 # but some have deps to imported modules. Yay.
@@ -799,10 +798,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     helpOnError(conf, pass)
   of "symbolfiles": discard "ignore for backwards compat"
   of "incremental":
-    when not nimIncremental:
-      localError(conf, info, "the compiler was not built with " &
-        "incremental compilation features; bootstrap with " &
-        "-d:nimIncremental to enable")
     case arg.normalize
     of "on": conf.symbolFiles = v2Sf
     of "off": conf.symbolFiles = disabledSf

compiler/enumtostr.nim

@@ -2,15 +2,15 @@
 import ast, idents, lineinfos, modulegraphs, magicsys
 
 proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym =
-  result = newSym(skProc, getIdent(g.cache, "$"), nextId idgen, t.owner, info)
+  result = newSym(skProc, getIdent(g.cache, "$"), nextSymId idgen, t.owner, info)
 
-  let dest = newSym(skParam, getIdent(g.cache, "e"), nextId idgen, result, info)
+  let dest = newSym(skParam, getIdent(g.cache, "e"), nextSymId idgen, result, info)
   dest.typ = t
 
-  let res = newSym(skResult, getIdent(g.cache, "result"), nextId idgen, result, info)
+  let res = newSym(skResult, getIdent(g.cache, "result"), nextSymId idgen, result, info)
   res.typ = getSysType(g, info, tyString)
 
-  result.typ = newType(tyProc, nextId idgen, t.owner)
+  result.typ = newType(tyProc, nextTypeId idgen, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
   result.typ.n.add newNodeI(nkEffectList, info)
@@ -63,15 +63,15 @@ proc searchObjCase(t: PType; field: PSym): PNode =
   doAssert result != nil
 
 proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym =
-  result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), nextId idgen, t.owner, info)
+  result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), nextSymId idgen, t.owner, info)
 
-  let dest = newSym(skParam, getIdent(g.cache, "e"), nextId idgen, result, info)
+  let dest = newSym(skParam, getIdent(g.cache, "e"), nextSymId idgen, result, info)
   dest.typ = field.typ
 
-  let res = newSym(skResult, getIdent(g.cache, "result"), nextId idgen, result, info)
+  let res = newSym(skResult, getIdent(g.cache, "result"), nextSymId idgen, result, info)
   res.typ = getSysType(g, info, tyUInt8)
 
-  result.typ = newType(tyProc, nextId idgen, t.owner)
+  result.typ = newType(tyProc, nextTypeId idgen, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
   result.typ.n.add newNodeI(nkEffectList, info)

compiler/evaltempl.nim

@@ -49,7 +49,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
         internalAssert c.config, sfGenSym in s.flags or s.kind == skType
         var x = PSym(idTableGet(c.mapping, s))
         if x == nil:
-          x = copySym(s, nextId(c.idgen))
+          x = copySym(s, nextSymId(c.idgen))
           # sem'check needs to set the owner properly later, see bug #9476
           x.owner = nil # c.genSymOwner
           #if x.kind == skParam and x.owner.kind == skModule:

compiler/ic/bitabs.nim

@@ -1,7 +1,7 @@
 ## A BiTable is a table that can be seen as an optimized pair
 ## of (Table[LitId, Val], Table[Val, LitId]).
 
-import hashes
+import hashes, rodfiles
 
 type
   LitId* = distinct uint32
@@ -30,7 +30,9 @@ proc mustRehash(length, counter: int): bool {.inline.} =
   result = (length * 2 < counter * 3) or (length - counter < 4)
 
 const
-  idStart = 256 # Ids do not start with 0 but with this value. The IR needs it.
+  idStart = 256 ##
+  ## Ids do not start with 0 but with this value. The IR needs it.
+  ## TODO: explain why
 
 template idToIdx(x: LitId): int = x.int - idStart
 
@@ -94,6 +96,21 @@ proc `[]`*[T](t: BiTable[T]; LitId: LitId): lent T {.inline.} =
   assert idx < t.vals.len
   result = t.vals[idx]
 
+proc hash*[T](t: BiTable[T]): Hash =
+  ## as the keys are hashes of the values, we simply use them instead
+  var h: Hash = 0
+  for i, n in pairs t.keys:
+    h = h !& hash((i, n))
+  result = !$h
+
+proc store*[T](f: var RodFile; t: BiTable[T]) =
+  storeSeq(f, t.vals)
+  storeSeq(f, t.keys)
+
+proc load*[T](f: var RodFile; t: var BiTable[T]) =
+  loadSeq(f, t.vals)
+  loadSeq(f, t.keys)
+
 when isMainModule:
 
   var t: BiTable[string]
@@ -113,7 +130,35 @@ when isMainModule:
 
   for i in 0 ..< 100_000:
     assert t.getOrIncl($i & "___" & $i).idToIdx == i + 4
+  echo "begin"
   echo t.vals.len
 
   echo t.vals[0]
   echo t.vals[1004]
+
+  echo "middle"
+
+  var tf: BiTable[float]
+
+  discard tf.getOrIncl(0.4)
+  discard tf.getOrIncl(16.4)
+  discard tf.getOrIncl(32.4)
+  echo getKeyId(tf, 32.4)
+
+  var f2 = open("testblah.bin", fmWrite)
+  echo store(f2, tf)
+  f2.close
+
+  var f1 = open("testblah.bin", fmRead)
+
+  var t2: BiTable[float]
+
+  echo f1.load(t2)
+  echo t2.vals.len
+
+  echo getKeyId(t2, 32.4)
+
+  echo "end"
+
+
+  f1.close

compiler/ic/design.rst

@@ -29,14 +29,13 @@ mechanism needs to be implemented that we could get wrong. ModuleIds
 are rod-file specific too.
 
 
-Configuration setup changes
----------------------------
-
-For a MVP these are not detected. Later the configuration will be
-stored in every `.rod` file.
-
 
 Global state
 ------------
 
 Global persistent state will be kept in a project specific `.rod` file.
+
+Rod File Format
+---------------
+
+It's a simple binary file format. `rodfiles.nim` contains some details.

compiler/ic/from_packed_ast.nim

@@ -1,12 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2020 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-import std / [hashes, tables]
-import bitabs
-import ".." / [ast, lineinfos, options, pathutils]

compiler/ic/packed_ast.nim

@@ -12,9 +12,9 @@
 ## use this representation directly in all the transformations,
 ## it is superior.
 
-import std / [hashes, tables]
+import std / [hashes, tables, strtabs, md5]
 import bitabs
-import ".." / [ast, lineinfos, options, pathutils]
+import ".." / [ast, options]
 
 const
   localNamePos* = 0
@@ -39,16 +39,29 @@ const
   routineBodyPos* = 7
 
 const
-  nkModuleRef = nkNone # pair of (ModuleId, SymId)
+  nkModuleRef* = nkNone # pair of (ModuleId, SymId)
 
 type
   SymId* = distinct int32
-  TypeId* = distinct int32
   ModuleId* = distinct int32
   NodePos* = distinct int
 
   NodeId* = distinct int32
 
+  PackedItemId* = object
+    module*: LitId       # 0 if it's this module
+    item*: int32         # same as the in-memory representation
+
+  TypeId* = PackedItemId
+
+const
+  nilTypeId* = PackedItemId(module: LitId(0), item: -1.int32)
+  nilItemId* = PackedItemId(module: LitId(0), item: -1.int32)
+
+const
+  emptyNodeId* = NodeId(-1)
+
+type
   PackedLineInfo* = object
     line*: uint16
     col*: int16
@@ -64,13 +77,13 @@ type
   PackedSym* = object
     kind*: TSymKind
     name*: LitId
-    typeId*: TypeId
+    typ*: TypeId
     flags*: TSymFlags
     magic*: TMagic
     info*: PackedLineInfo
-    ast*: NodePos
+    ast*: NodeId
-    owner*: ItemId
+    owner*: PackedItemId
-    guard*: ItemId
+    guard*: PackedItemId
     bitsize*: int
     alignment*: int # for alignment
     options*: TOptions
@@ -84,25 +97,25 @@ type
 
   PackedType* = object
     kind*: TTypeKind
-    nodekind*: TNodeKind
+    callConv*: TCallingConvention
+    #nodekind*: TNodeKind
     flags*: TTypeFlags
-    types*: int32
+    types*: seq[TypeId]
-    nodes*: int32
+    n*: NodeId
-    methods*: int32
+    methods*: seq[(int, PackedItemId)]
-    nodeflags*: TNodeFlags
+    #nodeflags*: TNodeFlags
-    info*: PackedLineInfo
+    sym*: PackedItemId
-    sym*: ItemId
+    owner*: PackedItemId
-    owner*: ItemId
+    attachedOps*: array[TTypeAttachedOp, PackedItemId]
-    attachedOps*: array[TTypeAttachedOp, ItemId]
     size*: BiggestInt
     align*: int16
     paddingAtEnd*: int16
     lockLevel*: TLockLevel # lock level as required for deadlock checking
     # not serialized: loc*: TLoc because it is backend-specific
     typeInst*: TypeId
-    nonUniqueId*: ItemId
+    nonUniqueId*: int32
 
-  Node* = object     # 20 bytes
+  PackedNode* = object     # 20 bytes
     kind*: TNodeKind
     flags*: TNodeFlags
     operand*: int32  # for kind in {nkSym, nkSymDef}: SymId
@@ -115,72 +128,73 @@ type
   ModulePhase* = enum
     preLookup, lookedUpTopLevelStmts
 
-  Module* = object
+  GenericKey* = object
+    module*: int32
     name*: string
-    file*: AbsoluteFile
+    types*: seq[MD5Digest] # is this a joke?
-    ast*: PackedTree
-    phase*: ModulePhase
-    iface*: Table[string, seq[SymId]] # 'seq' because of overloading
 
-  Program* = ref object
+  PackedTree* = object ## usually represents a full Nim module
-    modules*: seq[Module]
+    nodes*: seq[PackedNode]
+    #sh*: Shared
 
   Shared* = ref object # shared between different versions of 'Module'.
                        # (though there is always exactly one valid
                        # version of a module)
     syms*: seq[PackedSym]
-    types*: seq[seq[Node]]
+    types*: seq[PackedType]
     strings*: BiTable[string] # we could share these between modules.
     integers*: BiTable[BiggestInt]
     floats*: BiTable[BiggestFloat]
-    config*: ConfigRef
+    #config*: ConfigRef
-    #thisModule*: ModuleId
-    #program*: Program
 
-  PackedTree* = object ## usually represents a full Nim module
+proc hash*(key: GenericKey): Hash =
-    nodes*: seq[Node]
+  var h: Hash = 0
-    toPosition*: Table[SymId, NodePos]
+  h = h !& hash(key.module)
-    sh*: Shared
+  h = h !& hash(key.name)
+  h = h !& hash(key.types)
+  result = !$h
 
 proc `==`*(a, b: SymId): bool {.borrow.}
 proc hash*(a: SymId): Hash {.borrow.}
 
 proc `==`*(a, b: NodePos): bool {.borrow.}
-proc `==`*(a, b: TypeId): bool {.borrow.}
+#proc `==`*(a, b: TypeId): bool {.borrow.}
-proc `==`*(a, b: ModuleId): bool {.borrow.}
+proc `==`*(a, b: NodeId): bool {.borrow.}
 
-proc declareSym*(tree: var PackedTree; kind: TSymKind;
+proc newTreeFrom*(old: PackedTree): PackedTree =
+  result.nodes = @[]
+  when false: result.sh = old.sh
+
+when false:
+  proc declareSym*(tree: var PackedTree; kind: TSymKind;
                   name: LitId; info: PackedLineInfo): SymId =
     result = SymId(tree.sh.syms.len)
     tree.sh.syms.add PackedSym(kind: kind, name: name, flags: {}, magic: mNone, info: info)
 
-proc newTreeFrom*(old: PackedTree): PackedTree =
+  proc litIdFromName*(tree: PackedTree; name: string): LitId =
-  result.nodes = @[]
-  result.sh = old.sh
-
-proc litIdFromName*(tree: PackedTree; name: string): LitId =
     result = tree.sh.strings.getOrIncl(name)
 
-proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) =
+  proc add*(tree: var PackedTree; kind: TNodeKind; token: string; info: PackedLineInfo) =
-  tree.nodes.add Node(kind: kind, operand: int32 getOrIncl(tree.sh.strings, token), info: info)
+    tree.nodes.add PackedNode(kind: kind, info: info,
+                              operand: int32 getOrIncl(tree.sh.strings, token))
 
-proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) =
+  proc add*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo) =
-  tree.nodes.add Node(kind: kind, operand: 0, info: info)
+    tree.nodes.add PackedNode(kind: kind, operand: 0, info: info)
 
 proc throwAwayLastNode*(tree: var PackedTree) =
   tree.nodes.setLen(tree.nodes.len-1)
 
 proc addIdent*(tree: var PackedTree; s: LitId; info: PackedLineInfo) =
-  tree.nodes.add Node(kind: nkIdent, operand: int32(s), info: info)
+  tree.nodes.add PackedNode(kind: nkIdent, operand: int32(s), info: info)
 
-proc addSym*(tree: var PackedTree; s: SymId; info: PackedLineInfo) =
+proc addSym*(tree: var PackedTree; s: int32; info: PackedLineInfo) =
-  tree.nodes.add Node(kind: nkSym, operand: int32(s), info: info)
+  tree.nodes.add PackedNode(kind: nkSym, operand: s, info: info)
 
 proc addModuleId*(tree: var PackedTree; s: ModuleId; info: PackedLineInfo) =
-  tree.nodes.add Node(kind: nkInt32Lit, operand: int32(s), info: info)
+  tree.nodes.add PackedNode(kind: nkInt32Lit, operand: int32(s), info: info)
 
 proc addSymDef*(tree: var PackedTree; s: SymId; info: PackedLineInfo) =
-  tree.nodes.add Node(kind: nkSym, operand: int32(s), info: info)
+  tree.nodes.add PackedNode(kind: nkSym, operand: int32(s), info: info)
 
 proc isAtom*(tree: PackedTree; pos: int): bool {.inline.} = tree.nodes[pos].kind <= nkNilLit
 
@@ -194,7 +208,8 @@ proc copyTree*(dest: var PackedTree; tree: PackedTree; n: NodePos) =
   for i in 0..<L:
     dest.nodes[d+i] = tree.nodes[pos+i]
 
-proc copySym*(dest: var PackedTree; tree: PackedTree; s: SymId): SymId =
+when false:
+  proc copySym*(dest: var PackedTree; tree: PackedTree; s: SymId): SymId =
     result = SymId(dest.sh.syms.len)
     assert int(s) < tree.sh.syms.len
     let oldSym = tree.sh.syms[s.int]
@@ -206,11 +221,12 @@ type
 when false:
   proc prepare*(tree: var PackedTree; kind: TNodeKind; info: PackedLineInfo): PatchPos =
     result = PatchPos tree.nodes.len
-    tree.nodes.add Node(kind: kind, operand: 0, info: info)
+    tree.nodes.add PackedNode(kind: kind, operand: 0, info: info)
 
 proc prepare*(tree: var PackedTree; kind: TNodeKind; flags: TNodeFlags; typeId: TypeId; info: PackedLineInfo): PatchPos =
   result = PatchPos tree.nodes.len
-  tree.nodes.add Node(kind: kind, flags: flags, operand: 0, typeId: typeId, info: info)
+  tree.nodes.add PackedNode(kind: kind, flags: flags, operand: 0, info: info,
+                            typeId: typeId)
 
 proc prepare*(dest: var PackedTree; source: PackedTree; sourcePos: NodePos): PatchPos =
   result = PatchPos dest.nodes.len
@@ -224,7 +240,8 @@ proc patch*(tree: var PackedTree; pos: PatchPos) =
 
 proc len*(tree: PackedTree): int {.inline.} = tree.nodes.len
 
-proc `[]`*(tree: PackedTree; i: int): lent Node {.inline.} = tree.nodes[i]
+proc `[]`*(tree: PackedTree; i: int): lent PackedNode {.inline.} =
+  tree.nodes[i]
 
 proc nextChild(tree: PackedTree; pos: var int) {.inline.} =
   if tree.nodes[pos].kind > nkNilLit:
@@ -247,7 +264,8 @@ iterator sons*(dest: var PackedTree; tree: PackedTree; n: NodePos): NodePos =
   for x in sonsReadonly(tree, n): yield x
   patch dest, patchPos
 
-iterator isons*(dest: var PackedTree; tree: PackedTree; n: NodePos): (int, NodePos) =
+iterator isons*(dest: var PackedTree; tree: PackedTree;
+                n: NodePos): (int, NodePos) =
   var i = 0
   for ch0 in sons(dest, tree, n):
     yield (i, ch0)
@@ -301,10 +319,19 @@ proc hasAtLeastXsons*(tree: PackedTree; n: NodePos; x: int): bool =
       if count >= x: return true
   return false
 
-proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} = NodePos(n.int+1)
+proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} =
-proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} = tree.nodes[n.int].kind
+  NodePos(n.int+1)
-proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} = LitId tree.nodes[n.int].operand
+proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} =
-proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} = tree.nodes[n.int].info
+  tree.nodes[n.int].kind
+proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} =
+  LitId tree.nodes[n.int].operand
+proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} =
+  tree.nodes[n.int].info
+
+template typ*(n: NodePos): PackedItemId =
+  tree.nodes[n.int].typeId
+template flags*(n: NodePos): TNodeFlags =
+  tree.nodes[n.int].flags
 
 proc span(tree: PackedTree; pos: int): int {.inline.} =
   if isAtom(tree, pos): 1 else: tree.nodes[pos].operand
@@ -330,7 +357,9 @@ proc ithSon*(tree: PackedTree; n: NodePos; i: int): NodePos =
       inc count
   assert false, "node has no i-th child"
 
-proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} = tree.sh.strings[lit]
+when false:
+  proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} =
+    tree.sh.strings[lit]
 
 template kind*(n: NodePos): TNodeKind = tree.nodes[n.int].kind
 template info*(n: NodePos): PackedLineInfo = tree.nodes[n.int].info
@@ -340,16 +369,17 @@ template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].operand
 
 proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1)
 
-proc strLit*(tree: PackedTree; n: NodePos): lent string =
+when false:
+  proc strLit*(tree: PackedTree; n: NodePos): lent string =
     assert n.kind == nkStrLit
     result = tree.sh.strings[LitId tree.nodes[n.int].operand]
 
-proc strVal*(tree: PackedTree; n: NodePos): string =
+  proc strVal*(tree: PackedTree; n: NodePos): string =
     assert n.kind == nkStrLit
     result = tree.sh.strings[LitId tree.nodes[n.int].operand]
     #result = cookedStrLit(raw)
 
-proc filenameVal*(tree: PackedTree; n: NodePos): string =
+  proc filenameVal*(tree: PackedTree; n: NodePos): string =
     case n.kind
     of nkStrLit:
       result = strVal(tree, n)
@@ -360,7 +390,7 @@ proc filenameVal*(tree: PackedTree; n: NodePos): string =
     else:
       result = ""
 
-proc identAsStr*(tree: PackedTree; n: NodePos): lent string =
+  proc identAsStr*(tree: PackedTree; n: NodePos): lent string =
     assert n.kind == nkIdent
     result = tree.sh.strings[LitId tree.nodes[n.int].operand]
 
@@ -380,7 +410,8 @@ const
   externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit}
   directIntLit* = nkInt32Lit
 
-proc toString*(tree: PackedTree; n: NodePos; nesting: int; result: var string) =
+proc toString*(tree: PackedTree; n: NodePos; sh: Shared; nesting: int;
+               result: var string) =
   let pos = n.int
   if result.len > 0 and result[^1] notin {' ', '\n'}:
     result.add ' '
@@ -390,38 +421,39 @@ proc toString*(tree: PackedTree; n: NodePos; nesting: int; result: var string) =
   of nkNone, nkEmpty, nkNilLit, nkType: discard
   of nkIdent, nkStrLit..nkTripleStrLit:
     result.add " "
-    result.add tree.sh.strings[LitId tree.nodes[pos].operand]
+    result.add sh.strings[LitId tree.nodes[pos].operand]
   of nkSym:
     result.add " "
-    result.add tree.sh.strings[tree.sh.syms[tree.nodes[pos].operand].name]
+    result.add sh.strings[sh.syms[tree.nodes[pos].operand].name]
   of directIntLit:
     result.add " "
     result.addInt tree.nodes[pos].operand
   of externSIntLit:
     result.add " "
-    result.addInt tree.sh.integers[LitId tree.nodes[pos].operand]
+    result.addInt sh.integers[LitId tree.nodes[pos].operand]
   of externUIntLit:
     result.add " "
-    result.add $cast[uint64](tree.sh.integers[LitId tree.nodes[pos].operand])
+    result.add $cast[uint64](sh.integers[LitId tree.nodes[pos].operand])
   else:
     result.add "(\n"
     for i in 1..(nesting+1)*2: result.add ' '
     for child in sonsReadonly(tree, n):
-      toString(tree, child, nesting + 1, result)
+      toString(tree, child, sh, nesting + 1, result)
     result.add "\n"
     for i in 1..nesting*2: result.add ' '
     result.add ")"
     #for i in 1..nesting*2: result.add ' '
 
 
-proc toString*(tree: PackedTree; n: NodePos): string =
+proc toString*(tree: PackedTree; n: NodePos; sh: Shared): string =
   result = ""
-  toString(tree, n, 0, result)
+  toString(tree, n, sh, 0, result)
 
-proc debug*(tree: PackedTree) =
+proc debug*(tree: PackedTree; sh: Shared) =
-  stdout.write toString(tree, NodePos 0)
+  stdout.write toString(tree, NodePos 0, sh)
 
-proc identIdImpl(tree: PackedTree; n: NodePos): LitId =
+when false:
+  proc identIdImpl(tree: PackedTree; n: NodePos): LitId =
     if n.kind == nkIdent:
       result = n.litId
     elif n.kind == nkSym:
@@ -429,7 +461,7 @@ proc identIdImpl(tree: PackedTree; n: NodePos): LitId =
     else:
       result = LitId(0)
 
-template identId*(n: NodePos): LitId = identIdImpl(tree, n)
+  template identId*(n: NodePos): LitId = identIdImpl(tree, n)
 
 template copyInto*(dest, n, body) =
   let patchPos = prepare(dest, tree, n)
@@ -441,7 +473,8 @@ template copyIntoKind*(dest, kind, info, body) =
   body
   patch dest, patchPos
 
-proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool =
+when false:
+  proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool =
     let litId = tree.sh.strings.getKeyId(pragma)
     if litId == LitId(0):
       return false
@@ -453,9 +486,76 @@ proc hasPragma*(tree: PackedTree; n: NodePos; pragma: string): bool =
       elif ch0.identId == litId:
         return true
 
+proc getNodeId*(tree: PackedTree): NodeId {.inline.} = NodeId tree.nodes.len
+
 when false:
   proc produceError*(dest: var PackedTree; tree: PackedTree; n: NodePos; msg: string) =
     let patchPos = prepare(dest, nkError, n.info)
     dest.add nkStrLit, msg, n.info
     copyTree(dest, tree, n)
     patch dest, patchPos
+
+  proc hash*(table: StringTableRef): Hash =
+    ## XXX: really should be introduced into strtabs...
+    var h: Hash = 0
+    for pair in pairs table:
+      h = h !& hash(pair)
+    result = !$h
+
+  proc hash*(config: ConfigRef): Hash =
+    ## XXX: vet and/or extend this
+    var h: Hash = 0
+    h = h !& hash(config.selectedGC)
+    h = h !& hash(config.features)
+    h = h !& hash(config.legacyFeatures)
+    h = h !& hash(config.configVars)
+    h = h !& hash(config.symbols)
+    result = !$h
+
+  # XXX: lazy hashes for now
+  type
+    LazyHashes = PackedSym or PackedType or PackedLib or
+                PackedLineInfo or PackedTree or PackedNode
+
+  proc hash*(sh: Shared): Hash
+  proc hash*(s: LazyHashes): Hash
+  proc hash*(s: seq[LazyHashes]): Hash
+
+  proc hash*(s: LazyHashes): Hash =
+    var h: Hash = 0
+    for k, v in fieldPairs(s):
+      h = h !& hash((k, v))
+    result = !$h
+
+  proc hash*(s: seq[LazyHashes]): Hash =
+    ## critically, we need to hash the indices alongside their values
+    var h: Hash = 0
+    for i, n in pairs s:
+      h = h !& hash((i, n))
+    result = !$h
+
+  proc hash*(sh: Shared): Hash =
+    ## might want to edit this...
+    # XXX: these have too many references
+    when false:
+      var h: Hash = 0
+      h = h !& hash(sh.syms)
+      h = h !& hash(sh.types)
+      h = h !& hash(sh.strings)
+      h = h !& hash(sh.integers)
+      h = h !& hash(sh.floats)
+      h = h !& hash(sh.config)
+      result = !$h
+
+  proc hash*(m: Module): Hash =
+    var h: Hash = 0
+    h = h !& hash(m.name)
+    h = h !& hash(m.ast)
+    result = !$h
+
+  template safeItemId*(x: typed; f: untyped): ItemId =
+    ## yield a valid ItemId value for the field of a nillable type
+    if x.isNil:
+      nilItemId
+    else:
+      x.`f`

compiler/ic/rodfiles.nim

@@ -0,0 +1,143 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+from typetraits import supportsCopyMem
+
+type
+  RodSection* = enum
+    versionSection
+    configSection
+    stringsSection
+    checkSumsSection
+    depsSection
+    integersSection
+    floatsSection
+    topLevelSection
+    bodiesSection
+    symsSection
+    typesSection
+
+  RodFileError* = enum
+    ok, tooBig, ioFailure, wrongHeader, wrongSection, configMismatch,
+    includeFileChanged
+
+  RodFile* = object
+    f*: File
+    currentSection*: RodSection # for error checking
+    err*: RodFileError # little experiment to see if this works
+                       # better than exceptions.
+
+const
+  RodVersion = 1
+  cookie = [byte(0), byte('R'), byte('O'), byte('D'),
+            byte(0), byte(0), byte(0), byte(RodVersion)]
+
+proc storePrim*(f: var RodFile; s: string) =
+  if f.err != ok: return
+  if s.len >= high(int32):
+    f.err = tooBig
+    return
+  var lenPrefix = int32(s.len)
+  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
+    f.err = ioFailure
+  else:
+    if s.len != 0:
+      if writeBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
+        f.err = ioFailure
+
+proc storePrim*[T](f: var RodFile; x: T) =
+  if f.err != ok: return
+  when supportsCopyMem(T):
+    if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
+      f.err = ioFailure
+  elif T is tuple:
+    for y in fields(x):
+      storePrim(f, y)
+  else:
+    {.error: "unsupported type for 'storePrim'".}
+
+proc storeSeq*[T](f: var RodFile; s: seq[T]) =
+  if f.err != ok: return
+  if s.len >= high(int32):
+    f.err = tooBig
+    return
+  var lenPrefix = int32(s.len)
+  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
+    f.err = ioFailure
+  else:
+    for i in 0..<s.len:
+      storePrim(f, s[i])
+
+proc loadPrim*(f: var RodFile; s: var string) =
+  if f.err != ok: return
+  var lenPrefix = int32(0)
+  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
+    f.err = ioFailure
+  else:
+    s = newString(lenPrefix)
+    if lenPrefix > 0:
+      if readBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
+        f.err = ioFailure
+
+proc loadPrim*[T](f: var RodFile; x: var T) =
+  if f.err != ok: return
+  when supportsCopyMem(T):
+    if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
+      f.err = ioFailure
+  elif T is tuple:
+    for y in fields(x):
+      loadPrim(f, y)
+  else:
+    {.error: "unsupported type for 'loadPrim'".}
+
+proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
+  if f.err != ok: return
+  var lenPrefix = int32(0)
+  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
+    f.err = ioFailure
+  else:
+    s = newSeq[T](lenPrefix)
+    for i in 0..<lenPrefix:
+      loadPrim(f, s[i])
+
+proc storeHeader*(f: var RodFile) =
+  if f.err != ok: return
+  if f.f.writeBytes(cookie, 0, cookie.len) != cookie.len:
+    f.err = ioFailure
+
+proc loadHeader*(f: var RodFile) =
+  if f.err != ok: return
+  var thisCookie: array[cookie.len, byte]
+  if f.f.readBytes(thisCookie, 0, thisCookie.len) != thisCookie.len:
+    f.err = ioFailure
+  elif thisCookie != cookie:
+    f.err = wrongHeader
+
+proc storeSection*(f: var RodFile; s: RodSection) =
+  if f.err != ok: return
+  assert f.currentSection == pred s
+  f.currentSection = s
+  storePrim(f, s)
+
+proc loadSection*(f: var RodFile; expected: RodSection) =
+  if f.err != ok: return
+  var s: RodSection
+  loadPrim(f, s)
+  if expected != s:
+    f.err = wrongSection
+
+proc create*(filename: string): RodFile =
+  if not open(result.f, filename, fmWrite):
+    result.err = ioFailure
+
+proc close*(f: var RodFile) = close(f.f)
+
+proc open*(filename: string): RodFile =
+  if not open(result.f, filename, fmRead):
+    result.err = ioFailure

compiler/ic/to_packed_ast.nim

@@ -7,84 +7,680 @@
 #    distribution, for details about the copyright.
 #
 
-import std / [hashes, tables]
+import std / [hashes, tables, intsets, sha1]
-import packed_ast, bitabs
+import packed_ast, bitabs, rodfiles
-import ".." / [ast, idents, lineinfos, options, pathutils, msgs]
+import ".." / [ast, idents, lineinfos, msgs, ropes, options,
+  pathutils, condsyms]
+
+from std / os import removeFile, isAbsolute
+
+when not defined(release): import ".." / astalgo # debug()
 
 type
-  Context = object
+  PackedConfig* = object
-    thisModule: int32
+    backend: TBackend
-    lastFile: FileIndex # remember the last lookup entry.
+    selectedGC: TGCMode
-    lastLit: LitId
+    cCompiler: TSystemCC
-    filenames: Table[FileIndex, LitId]
+    options: TOptions
+    globalOptions: TGlobalOptions
 
-proc toLitId(x: FileIndex; ir: var PackedTree; c: var Context): LitId =
+  PackedModule* = object ## the parts of a PackedEncoder that are part of the .rod file
+    definedSymbols: string
+    includes: seq[(LitId, string)] # first entry is the module filename itself
+    imports: seq[LitId] # the modules this module depends on
+    topLevel*: PackedTree  # top level statements
+    bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position.
+    hidden*: PackedTree # instantiated generics and other trees not directly in the source code.
+    #producedGenerics*: Table[GenericKey, SymId]
+    sh*: Shared
+    cfg: PackedConfig
+
+  PackedEncoder* = object
+    m: PackedModule
+    thisModule*: int32
+    lastFile*: FileIndex # remember the last lookup entry.
+    lastLit*: LitId
+    filenames*: Table[FileIndex, LitId]
+    pendingTypes*: seq[PType]
+    pendingSyms*: seq[PSym]
+    typeMarker*: IntSet #Table[ItemId, TypeId]  # ItemId.item -> TypeId
+    symMarker*: IntSet #Table[ItemId, SymId]    # ItemId.item -> SymId
+    config*: ConfigRef
+
+template primConfigFields(fn: untyped) {.dirty.} =
+  fn backend
+  fn selectedGC
+  fn cCompiler
+  fn options
+  fn globalOptions
+
+proc definedSymbolsAsString(config: ConfigRef): string =
+  result = newStringOfCap(200)
+  result.add "config"
+  for d in definedSymbolNames(config.symbols):
+    result.add ' '
+    result.add d
+
+proc rememberConfig(c: var PackedEncoder; config: ConfigRef) =
+  c.m.definedSymbols = definedSymbolsAsString(config)
+
+  template rem(x) =
+    c.m.cfg.x = config.x
+  primConfigFields rem
+
+proc configIdentical(m: PackedModule; config: ConfigRef): bool =
+  result = m.definedSymbols == definedSymbolsAsString(config)
+  template eq(x) =
+    result = result and m.cfg.x == config.x
+  primConfigFields eq
+
+proc hashFileCached(conf: ConfigRef; fileIdx: FileIndex): string =
+  result = msgs.getHash(conf, fileIdx)
+  if result.len == 0:
+    let fullpath = msgs.toFullPath(conf, fileIdx)
+    result = $secureHashFile(fullpath)
+    msgs.setHash(conf, fileIdx, result)
+
+proc toLitId(x: FileIndex; c: var PackedEncoder): LitId =
+  ## store a file index as a literal
   if x == c.lastFile:
     result = c.lastLit
   else:
     result = c.filenames.getOrDefault(x)
     if result == LitId(0):
-      let p = msgs.toFullPath(ir.sh.config, x)
+      let p = msgs.toFullPath(c.config, x)
-      result = getOrIncl(ir.sh.strings, p)
+      result = getOrIncl(c.m.sh.strings, p)
       c.filenames[x] = result
     c.lastFile = x
     c.lastLit = result
+    assert result != LitId(0)
 
-proc toPackedInfo(x: TLineInfo; ir: var PackedTree; c: var Context): PackedLineInfo =
+proc toFileIndex(x: LitId; m: PackedModule; config: ConfigRef): FileIndex =
-  PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, ir, c))
+  result = msgs.fileInfoIdx(config, AbsoluteFile m.sh.strings[x])
 
-proc toPackedType(t: PType; ir: var PackedTree; c: var Context): TypeId =
+proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
-  result = TypeId(0)
+  for it in mitems(m.includes):
+    if hashFileCached(config, toFileIndex(it[0], m, config)) != it[1]:
+      return false
+  result = true
 
-proc toPackedSym(s: PSym; ir: var PackedTree; c: var Context): SymId =
+proc initEncoder*(c: var PackedEncoder; m: PSym; config: ConfigRef) =
-  result = SymId(0)
+  ## setup a context for serializing to packed ast
+  c.m.sh = Shared()
+  c.thisModule = m.itemId.module
+  c.config = config
+  c.m.bodies = newTreeFrom(c.m.topLevel)
+  c.m.hidden = newTreeFrom(c.m.topLevel)
 
-proc toPackedSymNode(n: PNode; ir: var PackedTree; c: var Context) =
+  let thisNimFile = FileIndex c.thisModule
+  var h = msgs.getHash(config, thisNimFile)
+  if h.len == 0:
+    let fullpath = msgs.toFullPath(config, thisNimFile)
+    if isAbsolute(fullpath):
+      # For NimScript compiler API support the main Nim file might be from a stream.
+      h = $secureHashFile(fullpath)
+      msgs.setHash(config, thisNimFile, h)
+  c.m.includes.add((toLitId(thisNimFile, c), h)) # the module itself
+
+proc addIncludeFileDep*(c: var PackedEncoder; f: FileIndex) =
+  c.m.includes.add((toLitId(f, c), hashFileCached(c.config, f)))
+
+proc addImportFileDep*(c: var PackedEncoder; f: FileIndex) =
+  c.m.imports.add toLitId(f, c)
+
+proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder)
+proc toPackedSym*(s: PSym; c: var PackedEncoder): PackedItemId
+proc toPackedType(t: PType; c: var PackedEncoder): PackedItemId
+
+proc flush(c: var PackedEncoder) =
+  ## serialize any pending types or symbols from the context
+  while true:
+    if c.pendingTypes.len > 0:
+      discard toPackedType(c.pendingTypes.pop, c)
+    elif c.pendingSyms.len > 0:
+      discard toPackedSym(c.pendingSyms.pop, c)
+    else:
+      break
+
+proc toLitId(x: string; c: var PackedEncoder): LitId =
+  ## store a string as a literal
+  result = getOrIncl(c.m.sh.strings, x)
+
+proc toLitId(x: BiggestInt; c: var PackedEncoder): LitId =
+  ## store an integer as a literal
+  result = getOrIncl(c.m.sh.integers, x)
+
+proc toPackedInfo(x: TLineInfo; c: var PackedEncoder): PackedLineInfo =
+  PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c))
+
+proc safeItemId(s: PSym; c: var PackedEncoder): PackedItemId {.inline.} =
+  ## given a symbol, produce an ItemId with the correct properties
+  ## for local or remote symbols, packing the symbol as necessary
+  if s == nil:
+    result = nilItemId
+  elif s.itemId.module == c.thisModule:
+    result = PackedItemId(module: LitId(0), item: s.itemId.item)
+  else:
+    result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c),
+                          item: s.itemId.item)
+
+proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder) =
+  ## add a remote symbol reference to the tree
+  let info = n.info.toPackedInfo(c)
+  ir.nodes.add PackedNode(kind: nkModuleRef, operand: 2.int32,  # 2 kids...
+                          typeId: toPackedType(n.typ, c), info: info)
+  ir.nodes.add PackedNode(kind: nkInt32Lit, info: info,
+                          operand: toLitId(n.sym.itemId.module.FileIndex, c).int32)
+  ir.nodes.add PackedNode(kind: nkInt32Lit, info: info,
+                          operand: n.sym.itemId.item)
+
+proc addMissing(c: var PackedEncoder; p: PSym) =
+  ## consider queuing a symbol for later addition to the packed tree
+  if p != nil and p.itemId.module == c.thisModule:
+    if p.itemId.item notin c.symMarker:
+      c.pendingSyms.add p
+
+proc addMissing(c: var PackedEncoder; p: PType) =
+  ## consider queuing a type for later addition to the packed tree
+  if p != nil and p.uniqueId.module == c.thisModule:
+    if p.uniqueId.item notin c.typeMarker:
+      c.pendingTypes.add p
+
+template storeNode(dest, src, field) =
+  var nodeId: NodeId
+  if src.field != nil:
+    nodeId = getNodeId(c.m.bodies)
+    toPackedNode(src.field, c.m.bodies, c)
+  else:
+    nodeId = emptyNodeId
+  dest.field = nodeId
+
+proc toPackedType(t: PType; c: var PackedEncoder): PackedItemId =
+  ## serialize a ptype
+  if t.isNil: return nilTypeId
+
+  if t.uniqueId.module != c.thisModule:
+    # XXX Assert here that it already was serialized in the foreign module!
+    # it is a foreign type:
+    return PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c), item: t.uniqueId.item)
+
+  if not c.typeMarker.containsOrIncl(t.uniqueId.item):
+    if t.uniqueId.item >= c.m.sh.types.len:
+      setLen c.m.sh.types, t.uniqueId.item+1
+
+    var p = PackedType(kind: t.kind, flags: t.flags, callConv: t.callConv,
+      size: t.size, align: t.align, nonUniqueId: t.itemId.item,
+      paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel)
+    storeNode(p, t, n)
+
+    for op, s in pairs t.attachedOps:
+      c.addMissing s
+      p.attachedOps[op] = s.safeItemId(c)
+
+    p.typeInst = t.typeInst.toPackedType(c)
+    for kid in items t.sons:
+      p.types.add kid.toPackedType(c)
+    for i, s in items t.methods:
+      c.addMissing s
+      p.methods.add (i, s.safeItemId(c))
+    c.addMissing t.sym
+    p.sym = t.sym.safeItemId(c)
+    c.addMissing t.owner
+    p.owner = t.owner.safeItemId(c)
+
+    # fill the reserved slot, nothing else:
+    c.m.sh.types[t.uniqueId.item] = p
+
+  result = PackedItemId(module: LitId(0), item: t.uniqueId.item)
+
+proc toPackedLib(l: PLib; c: var PackedEncoder): PackedLib =
+  ## the plib hangs off the psym via the .annex field
+  if l.isNil: return
+  result.kind = l.kind
+  result.generated = l.generated
+  result.isOverriden = l.isOverriden
+  result.name = toLitId($l.name, c)
+  storeNode(result, l, path)
+
+proc toPackedSym*(s: PSym; c: var PackedEncoder): PackedItemId =
+  ## serialize a psym
+  if s.isNil: return nilItemId
+
+  if s.itemId.module != c.thisModule:
+    # XXX Assert here that it already was serialized in the foreign module!
+    # it is a foreign symbol:
+    return PackedItemId(module: toLitId(s.itemId.module.FileIndex, c), item: s.itemId.item)
+
+  if not c.symMarker.containsOrIncl(s.itemId.item):
+    if s.itemId.item >= c.m.sh.syms.len:
+      setLen c.m.sh.syms, s.itemId.item+1
+
+    var p = PackedSym(kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c), magic: s.magic,
+      position: s.position, offset: s.offset, options: s.options,
+      name: s.name.s.toLitId(c))
+
+    storeNode(p, s, ast)
+    storeNode(p, s, constraint)
+
+    if s.kind in {skLet, skVar, skField, skForVar}:
+      c.addMissing s.guard
+      p.guard = s.guard.safeItemId(c)
+      p.bitsize = s.bitsize
+      p.alignment = s.alignment
+
+    p.externalName = toLitId(if s.loc.r.isNil: "" else: $s.loc.r, c)
+    c.addMissing s.typ
+    p.typ = s.typ.toPackedType(c)
+    c.addMissing s.owner
+    p.owner = s.owner.safeItemId(c)
+    p.annex = toPackedLib(s.annex, c)
+    when hasFFI:
+      p.cname = toLitId(s.cname, c)
+
+    # fill the reserved slot, nothing else:
+    c.m.sh.syms[s.itemId.item] = p
+
+  result = PackedItemId(module: LitId(0), item: s.itemId.item)
+
+proc toSymNode(n: PNode; ir: var PackedTree; c: var PackedEncoder) =
+  ## store a local or remote psym reference in the tree
   assert n.kind == nkSym
-  let t = toPackedType(n.typ, ir, c)
+  template s: PSym = n.sym
-
+  let id = s.toPackedSym(c).item
-  if n.sym.itemId.module == c.thisModule:
+  if s.itemId.module == c.thisModule:
     # it is a symbol that belongs to the module we're currently
     # packing:
-    let sid = toPackedSym(n.sym, ir, c)
+    ir.addSym(id, toPackedInfo(n.info, c))
-    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32(sid),
-      typeId: t, info: toPackedInfo(n.info, ir, c))
   else:
     # store it as an external module reference:
-    #  nkModuleRef
+    addModuleRef(n, ir, c)
-    discard
-
-
-proc toPackedNode*(n: PNode; ir: var PackedTree; c: var Context) =
-  template toP(x: TLineInfo): PackedLineInfo = toPackedInfo(x, ir, c)
 
+proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder) =
+  ## serialize a node into the tree
+  if n.isNil: return
+  let info = toPackedInfo(n.info, c)
   case n.kind
-  of nkNone, nkEmpty, nkNilLit:
+  of nkNone, nkEmpty, nkNilLit, nkType:
-    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: 0,
+    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags, operand: 0,
-      typeId: toPackedType(n.typ, ir, c), info: toP n.info)
+                            typeId: toPackedType(n.typ, c), info: info)
   of nkIdent:
-    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.strings, n.ident.s),
+    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-      typeId: toPackedType(n.typ, ir, c), info: toP n.info)
+                            operand: int32 getOrIncl(c.m.sh.strings, n.ident.s),
+                            typeId: toPackedType(n.typ, c), info: info)
   of nkSym:
-    toPackedSymNode(n, ir, c)
+    toSymNode(n, ir, c)
   of directIntLit:
-    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32(n.intVal),
+    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-      typeId: toPackedType(n.typ, ir, c), info: toP n.info)
+                            operand: int32(n.intVal),
+                            typeId: toPackedType(n.typ, c), info: info)
   of externIntLit:
-    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.integers, n.intVal),
+    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-      typeId: toPackedType(n.typ, ir, c), info: toP n.info)
+                            operand: int32 getOrIncl(c.m.sh.integers, n.intVal),
+                            typeId: toPackedType(n.typ, c), info: info)
   of nkStrLit..nkTripleStrLit:
-    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.strings, n.strVal),
+    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-      typeId: toPackedType(n.typ, ir, c), info: toP n.info)
+                            operand: int32 getOrIncl(c.m.sh.strings, n.strVal),
+                            typeId: toPackedType(n.typ, c), info: info)
   of nkFloatLit..nkFloat128Lit:
-    ir.nodes.add Node(kind: n.kind, flags: n.flags, operand: int32 getOrIncl(ir.sh.floats, n.floatVal),
+    ir.nodes.add PackedNode(kind: n.kind, flags: n.flags,
-      typeId: toPackedType(n.typ, ir, c), info: toP n.info)
+                            operand: int32 getOrIncl(c.m.sh.floats, n.floatVal),
+                            typeId: toPackedType(n.typ, c), info: info)
   else:
-    let patchPos = ir.prepare(n.kind, n.flags, toPackedType(n.typ, ir, c), toP n.info)
+    let patchPos = ir.prepare(n.kind, n.flags,
+                              toPackedType(n.typ, c), info)
     for i in 0..<n.len:
       toPackedNode(n[i], ir, c)
     ir.patch patchPos
 
-proc moduleToIr*(n: PNode; ir: var PackedTree; module: PSym) =
+  when false:
-  var c = Context(thisModule: module.itemId.module)
+    ir.flush c   # flush any pending types and symbols
-  toPackedNode(n, ir, c)
+
+proc toPackedNodeIgnoreProcDefs*(n: PNode, encoder: var PackedEncoder) =
+  case n.kind
+  of routineDefs:
+    # we serialize n[namePos].sym instead
+    if n[namePos].kind == nkSym:
+      discard toPackedSym(n[namePos].sym, encoder)
+    else:
+      toPackedNode(n, encoder.m.topLevel, encoder)
+  else:
+    toPackedNode(n, encoder.m.topLevel, encoder)
+
+proc toPackedNodeTopLevel*(n: PNode, encoder: var PackedEncoder) =
+  toPackedNodeIgnoreProcDefs(n, encoder)
+  flush encoder
+
+proc storePrim*(f: var RodFile; x: PackedType) =
+  for y in fields(x):
+    when y is seq:
+      storeSeq(f, y)
+    else:
+      storePrim(f, y)
+
+proc loadPrim*(f: var RodFile; x: var PackedType) =
+  for y in fields(x):
+    when y is seq:
+      loadSeq(f, y)
+    else:
+      loadPrim(f, y)
+
+proc loadError(err: RodFileError; filename: AbsoluteFile) =
+  echo "Error: ", $err, "\nloading file: ", filename.string
+
+proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef): RodFileError =
+  m.sh = Shared()
+  var f = rodfiles.open(filename.string)
+  f.loadHeader()
+  f.loadSection configSection
+
+  f.loadPrim m.definedSymbols
+  f.loadPrim m.cfg
+
+  if not configIdentical(m, config):
+    f.err = configMismatch
+
+  f.loadSection stringsSection
+  f.load m.sh.strings
+
+  f.loadSection checkSumsSection
+  f.loadSeq m.includes
+  if not includesIdentical(m, config):
+    f.err = includeFileChanged
+
+  f.loadSection depsSection
+  f.loadSeq m.imports
+
+  f.loadSection integersSection
+  f.load m.sh.integers
+  f.loadSection floatsSection
+  f.load m.sh.floats
+
+  f.loadSection topLevelSection
+  f.loadSeq m.topLevel.nodes
+
+  f.loadSection bodiesSection
+  f.loadSeq m.bodies.nodes
+
+  f.loadSection symsSection
+  f.loadSeq m.sh.syms
+
+  f.loadSection typesSection
+  f.loadSeq m.sh.types
+
+  close(f)
+  result = f.err
+
+type
+  ModuleStatus* = enum
+    undefined,
+    loading,
+    loaded,
+    outdated
+
+  LoadedModule* = object
+    status: ModuleStatus
+    symsInit, typesInit: bool
+    fromDisk: PackedModule
+    syms: seq[PSym] # indexed by itemId
+    types: seq[PType]
+
+  PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex
+
+proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef;
+                    fileIdx: FileIndex): bool =
+  let m = int(fileIdx)
+  if m >= g.len:
+    g.setLen(m+1)
+
+  case g[m].status
+  of undefined:
+    g[m].status = loading
+    let fullpath = msgs.toFullPath(conf, fileIdx)
+    let rod = toRodFile(conf, AbsoluteFile fullpath)
+    let err = loadRodFile(rod, g[m].fromDisk, conf)
+    if err == ok:
+      result = false
+      # check its dependencies:
+      for dep in g[m].fromDisk.imports:
+        let fid = toFileIndex(dep, g[m].fromDisk, conf)
+        # Warning: we need to traverse the full graph, so
+        # do **not use break here**!
+        if needsRecompile(g, conf, fid):
+          result = true
+
+      g[m].status = if result: outdated else: loaded
+    else:
+      loadError(err, rod)
+      g[m].status = outdated
+      result = true
+  of loading, loaded:
+    result = false
+  of outdated:
+    result = true
+
+# -------------------------------------------------------------------------
+
+proc storeError(err: RodFileError; filename: AbsoluteFile) =
+  echo "Error: ", $err, "; couldn't write to ", filename.string
+  removeFile(filename.string)
+
+proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder) =
+  rememberConfig(encoder, encoder.config)
+
+  var f = rodfiles.create(filename.string)
+  f.storeHeader()
+  f.storeSection configSection
+  f.storePrim encoder.m.definedSymbols
+  f.storePrim encoder.m.cfg
+
+  f.storeSection stringsSection
+  f.store encoder.m.sh.strings
+
+  f.storeSection checkSumsSection
+  f.storeSeq encoder.m.includes
+
+  f.storeSection depsSection
+  f.storeSeq encoder.m.imports
+
+  f.storeSection integersSection
+  f.store encoder.m.sh.integers
+
+  f.storeSection floatsSection
+  f.store encoder.m.sh.floats
+
+  f.storeSection topLevelSection
+  f.storeSeq encoder.m.topLevel.nodes
+
+  f.storeSection bodiesSection
+  f.storeSeq encoder.m.bodies.nodes
+
+  f.storeSection symsSection
+  f.storeSeq encoder.m.sh.syms
+
+  f.storeSection typesSection
+  f.storeSeq encoder.m.sh.types
+  close(f)
+  if f.err != ok:
+    loadError(f.err, filename)
+
+  when true:
+    # basic loader testing:
+    var m2: PackedModule
+    discard loadRodFile(filename, m2, encoder.config)
+
+# ----------------------------------------------------------------------------
+
+type
+  PackedDecoder* = object
+    thisModule*: int32
+    lastLit*: LitId
+    lastFile*: FileIndex # remember the last lookup entry.
+    config*: ConfigRef
+    ident: IdentCache
+
+proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): PType
+proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): PSym
+
+proc toFileIndexCached(c: var PackedDecoder; g: var PackedModuleGraph; f: LitId): FileIndex =
+  if c.lastLit == f:
+    result = c.lastFile
+  else:
+    result = toFileIndex(f, g[c.thisModule].fromDisk, c.config)
+    c.lastLit = f
+    c.lastFile = result
+
+proc translateLineInfo(c: var PackedDecoder; g: var PackedModuleGraph;
+                       x: PackedLineInfo): TLineInfo =
+  assert g[c.thisModule].status == loaded
+  result = TLineInfo(line: x.line, col: x.col,
+            fileIndex: toFileIndexCached(c, g, x.file))
+
+proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph;
+               tree: PackedTree; n: NodePos): PNode =
+  let k = n.kind
+  result = newNodeIT(k, translateLineInfo(c, g, n.info),
+    loadType(c, g, n.typ))
+  result.flags = n.flags
+
+  case k
+  of nkEmpty, nkNilLit, nkType:
+    discard
+  of nkIdent:
+    result.ident = getIdent(c.ident, g[c.thisModule].fromDisk.sh.strings[n.litId])
+  of nkSym:
+    result.sym = loadSym(c, g, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand))
+  of directIntLit:
+    result.intVal = tree.nodes[n.int].operand
+  of externIntLit:
+    result.intVal = g[c.thisModule].fromDisk.sh.integers[n.litId]
+  of nkStrLit..nkTripleStrLit:
+    result.strVal = g[c.thisModule].fromDisk.sh.strings[n.litId]
+  of nkFloatLit..nkFloat128Lit:
+    result.floatVal = g[c.thisModule].fromDisk.sh.floats[n.litId]
+  of nkModuleRef:
+    let (n1, n2) = sons2(tree, n)
+    assert n1.kind == nkInt32Lit
+    assert n2.kind == nkInt32Lit
+    transitionNoneToSym(result)
+    result.sym = loadSym(c, g, PackedItemId(module: n1.litId, item: tree.nodes[n2.int].operand))
+  else:
+    for n0 in sonsReadonly(tree, n):
+      result.add loadNodes(c, g, tree, n0)
+
+proc moduleIndex*(c: var PackedDecoder; g: var PackedModuleGraph;
+                  s: PackedItemId): int32 {.inline.} =
+  result = if s.module == LitId(0): c.thisModule
+           else: toFileIndexCached(c, g, s.module).int32
+
+proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
+                         s: PackedSym; si, item: int32): PSym =
+  result = PSym(itemId: ItemId(module: si, item: item),
+    kind: s.kind, magic: s.magic, flags: s.flags,
+    info: translateLineInfo(c, g, s.info),
+    options: s.options,
+    position: s.position,
+    name: getIdent(c.ident, g[si].fromDisk.sh.strings[s.name])
+  )
+
+template loadAstBody(p, field) =
+  if p.field != emptyNodeId:
+    result.field = loadNodes(c, g, g[si].fromDisk.bodies, NodePos p.field)
+
+proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph;
+             si, item: int32; l: PackedLib): PLib =
+  # XXX: hack; assume a zero LitId means the PackedLib is all zero (empty)
+  if l.name.int == 0:
+    result = nil
+  else:
+    result = PLib(generated: l.generated, isOverriden: l.isOverriden,
+                  kind: l.kind, name: rope g[si].fromDisk.sh.strings[l.name])
+    loadAstBody(l, path)
+
+proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
+                       s: PackedSym; si, item: int32; result: PSym) =
+  result.typ = loadType(c, g, s.typ)
+  loadAstBody(s, constraint)
+  loadAstBody(s, ast)
+  result.annex = loadLib(c, g, si, item, s.annex)
+  when hasFFI:
+    result.cname = g[si].fromDisk.sh.strings[s.cname]
+
+  if s.kind in {skLet, skVar, skField, skForVar}:
+    result.guard = loadSym(c, g, s.guard)
+    result.bitsize = s.bitsize
+    result.alignment = s.alignment
+  result.owner = loadSym(c, g, s.owner)
+  let externalName = g[si].fromDisk.sh.strings[s.externalName]
+  if externalName != "":
+    result.loc.r = rope externalName
+
+proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): PSym =
+  if s == nilTypeId:
+    result = nil
+  else:
+    let si = moduleIndex(c, g, s)
+    assert g[si].status == loaded
+    if not g[si].symsInit:
+      g[si].symsInit = true
+      setLen g[si].syms, g[si].fromDisk.sh.syms.len
+
+    if g[si].syms[s.item] == nil:
+      let packed = addr(g[si].fromDisk.sh.syms[s.item])
+      result = symHeaderFromPacked(c, g, packed[], si, s.item)
+      # store it here early on, so that recursions work properly:
+      g[si].syms[s.item] = result
+      symBodyFromPacked(c, g, packed[], si, s.item, result)
+    else:
+      result = g[si].syms[s.item]
+
+proc typeHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
+                          t: PackedType; si, item: int32): PType =
+  result = PType(itemId: ItemId(module: si, item: t.nonUniqueId), kind: t.kind,
+                flags: t.flags, size: t.size, align: t.align,
+                paddingAtEnd: t.paddingAtEnd, lockLevel: t.lockLevel,
+                uniqueId: ItemId(module: si, item: item))
+
+proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
+                        t: PackedType; si, item: int32; result: PType) =
+  result.sym = loadSym(c, g, t.sym)
+  result.owner = loadSym(c, g, t.owner)
+  for op, item in pairs t.attachedOps:
+    result.attachedOps[op] = loadSym(c, g, item)
+  result.typeInst = loadType(c, g, t.typeInst)
+  for son in items t.types:
+    result.sons.add loadType(c, g, son)
+  loadAstBody(t, n)
+  for gen, id in items t.methods:
+    result.methods.add((gen, loadSym(c, g, id)))
+
+proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): PType =
+  if t == nilTypeId:
+    result = nil
+  else:
+    let si = moduleIndex(c, g, t)
+    assert g[si].status == loaded
+    if not g[si].typesInit:
+      g[si].typesInit = true
+      setLen g[si].types, g[si].fromDisk.sh.types.len
+
+    if g[si].types[t.item] == nil:
+      let packed = addr(g[si].fromDisk.sh.types[t.item])
+      result = typeHeaderFromPacked(c, g, packed[], si, t.item)
+      # store it here early on, so that recursions work properly:
+      g[si].types[t.item] = result
+      typeBodyFromPacked(c, g, packed[], si, t.item, result)
+    else:
+      result = g[si].types[t.item]
+
+
+when false:
+  proc initGenericKey*(s: PSym; types: seq[PType]): GenericKey =
+    result.module = s.owner.itemId.module
+    result.name = s.name.s
+    result.types = mapIt types: hashType(it, {CoType, CoDistinct}).MD5Digest
+
+  proc addGeneric*(m: var Module; c: var PackedEncoder; key: GenericKey; s: PSym) =
+    ## add a generic to the module
+    if key notin m.generics:
+      m.generics[key] = toPackedSym(s, m.ast, c)
+      toPackedNode(s.ast, m.ast, c)

compiler/importer.nim

@@ -220,12 +220,13 @@ proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
     localError(c.config, n.info, "module alias must be an identifier")
   elif n[1].ident.id != realModule.name.id:
     # some misguided guy will write 'import abc.foo as foo' ...
-    result = createModuleAlias(realModule, nextId c.idgen, n[1].ident, realModule.info,
+    result = createModuleAlias(realModule, nextSymId c.idgen, n[1].ident, realModule.info,
                                c.config.options)
 
 proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
   let f = checkModuleName(c.config, n)
   if f != InvalidFileIdx:
+    addImportFileDep(c, f)
     let L = c.graph.importStack.len
     let recursion = c.graph.importStack.find(f)
     c.graph.importStack.add f

compiler/incremental.nim

@@ -1,198 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2018 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## Basic type definitions the module graph needs in order to support
-## incremental compilations.
-
-const nimIncremental* = defined(nimIncremental)
-
-import options, lineinfos
-
-when nimIncremental:
-  import ast, msgs, intsets, btrees, db_sqlite, std / sha1, pathutils
-  from strutils import parseInt
-  from os import isAbsolute
-
-  type
-    Writer* = object
-      sstack*: seq[PSym]          # a stack of symbols to process
-      tstack*: seq[PType]         # a stack of types to process
-      tmarks*, smarks*: IntSet
-      forwardedSyms*: seq[PSym]
-
-    Reader* = object
-      syms*: BTree[int, PSym]
-      types*: BTree[int, PType]
-
-    IncrementalCtx* = object
-      db*: DbConn
-      w*: Writer
-      r*: Reader
-      configChanged*: bool
-
-  proc init*(incr: var IncrementalCtx) =
-    incr.w.sstack = @[]
-    incr.w.tstack = @[]
-    incr.w.tmarks = initIntSet()
-    incr.w.smarks = initIntSet()
-    incr.w.forwardedSyms = @[]
-    incr.r.syms = initBTree[int, PSym]()
-    incr.r.types = initBTree[int, PType]()
-
-
-  proc hashFileCached*(conf: ConfigRef; fileIdx: FileIndex; fullpath: AbsoluteFile): string =
-    result = msgs.getHash(conf, fileIdx)
-    if result.len == 0 and isAbsolute(string fullpath):
-      result = $secureHashFile(string fullpath)
-      msgs.setHash(conf, fileIdx, result)
-
-  proc toDbFileId*(incr: var IncrementalCtx; conf: ConfigRef; fileIdx: FileIndex): int =
-    if fileIdx == FileIndex(-1): return -1
-    let fullpath = toFullPath(conf, fileIdx)
-    let row = incr.db.getRow(sql"select id, fullhash from filenames where fullpath = ?",
-      fullpath)
-    let id = row[0]
-    let fullhash = hashFileCached(conf, fileIdx, AbsoluteFile fullpath)
-    if id.len == 0:
-      result = int incr.db.insertID(sql"insert into filenames(nimid, fullpath, fullhash) values (?, ?, ?)",
-        int(fileIdx), fullpath, fullhash)
-    else:
-      if row[1] != fullhash:
-        incr.db.exec(sql"update filenames set fullhash = ? where fullpath = ?", fullhash, fullpath)
-      result = parseInt(id)
-
-  proc fromDbFileId*(incr: var IncrementalCtx; conf: ConfigRef; dbId: int): FileIndex =
-    if dbId == -1: return FileIndex(-1)
-    let fullpath = incr.db.getValue(sql"select fullpath from filenames where id = ?", dbId)
-    doAssert fullpath.len > 0, "cannot find file name for DB ID " & $dbId
-    result = fileInfoIdx(conf, AbsoluteFile fullpath)
-
-
-  proc addModuleDep*(incr: var IncrementalCtx; conf: ConfigRef;
-                     module, fileIdx: FileIndex;
-                     isIncludeFile: bool) =
-    if conf.symbolFiles != v2Sf: return
-
-    let a = toDbFileId(incr, conf, module)
-    let b = toDbFileId(incr, conf, fileIdx)
-
-    incr.db.exec(sql"insert into deps(module, dependency, isIncludeFile) values (?, ?, ?)",
-      a, b, ord(isIncludeFile))
-
-  # --------------- Database model ---------------------------------------------
-
-  proc createDb*(db: DbConn) =
-    db.exec(sql"""
-      create table if not exists controlblock(
-        idgen integer not null
-      );
-    """)
-
-    db.exec(sql"""
-      create table if not exists config(
-        config varchar(8000) not null
-      );
-    """)
-
-    db.exec(sql"""
-      create table if not exists filenames(
-        id integer primary key,
-        nimid integer not null,
-        fullpath varchar(8000) not null,
-        fullHash varchar(256) not null
-      );
-    """)
-    db.exec sql"create index if not exists FilenameIx on filenames(fullpath);"
-
-    db.exec(sql"""
-      create table if not exists modules(
-        id integer primary key,
-        nimid integer not null,
-        fullpath varchar(8000) not null,
-        interfHash varchar(256) not null,
-        fullHash varchar(256) not null,
-
-        created timestamp not null default (DATETIME('now'))
-      );""")
-    db.exec(sql"""create unique index if not exists SymNameIx on modules(fullpath);""")
-
-    db.exec(sql"""
-      create table if not exists deps(
-        id integer primary key,
-        module integer not null,
-        dependency integer not null,
-        isIncludeFile integer not null,
-        foreign key (module) references filenames(id),
-        foreign key (dependency) references filenames(id)
-      );""")
-    db.exec(sql"""create index if not exists DepsIx on deps(module);""")
-
-    db.exec(sql"""
-      create table if not exists types(
-        id integer primary key,
-        nimid integer not null,
-        module integer not null,
-        data blob not null,
-        foreign key (module) references module(id)
-      );
-    """)
-    db.exec sql"create index TypeByModuleIdx on types(module);"
-    db.exec sql"create index TypeByNimIdIdx on types(nimid);"
-
-    db.exec(sql"""
-      create table if not exists syms(
-        id integer primary key,
-        nimid integer not null,
-        module integer not null,
-        name varchar(256) not null,
-        data blob not null,
-        exported int not null,
-        foreign key (module) references module(id)
-      );
-    """)
-    db.exec sql"create index if not exists SymNameIx on syms(name);"
-    db.exec sql"create index SymByNameAndModuleIdx on syms(name, module);"
-    db.exec sql"create index SymByModuleIdx on syms(module);"
-    db.exec sql"create index SymByNimIdIdx on syms(nimid);"
-
-
-    db.exec(sql"""
-      create table if not exists toplevelstmts(
-        id integer primary key,
-        position integer not null,
-        module integer not null,
-        data blob not null,
-        foreign key (module) references module(id)
-      );
-    """)
-    db.exec sql"create index TopLevelStmtByModuleIdx on toplevelstmts(module);"
-    db.exec sql"create index TopLevelStmtByPositionIdx on toplevelstmts(position);"
-
-    db.exec(sql"""
-      create table if not exists statics(
-        id integer primary key,
-        module integer not null,
-        data blob not null,
-        foreign key (module) references module(id)
-      );
-    """)
-    db.exec sql"create index StaticsByModuleIdx on toplevelstmts(module);"
-    db.exec sql"insert into controlblock(idgen) values (0)"
-
-
-else:
-  type
-    IncrementalCtx* = object
-
-  template init*(incr: IncrementalCtx) = discard
-
-  template addModuleDep*(incr: var IncrementalCtx; conf: ConfigRef;
-                     module, fileIdx: FileIndex;
-                     isIncludeFile: bool) =
-    discard

compiler/injectdestructors.nim

@@ -62,7 +62,7 @@ template dbg(body) =
       body
 
 proc getTemp(c: var Con; s: var Scope; typ: PType; info: TLineInfo): PNode =
-  let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), nextId c.idgen, c.owner, info)
+  let sym = newSym(skTemp, getIdent(c.graph.cache, ":tmpD"), nextSymId c.idgen, c.owner, info)
   sym.typ = typ
   s.vars.add(sym)
   result = newSymNode(sym)
@@ -252,7 +252,7 @@ proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) =
   localError(c.graph.config, ri.info, errGenerated, m)
 
 proc makePtrType(c: var Con, baseType: PType): PType =
-  result = newType(tyPtr, nextId c.idgen, c.owner)
+  result = newType(tyPtr, nextTypeId c.idgen, c.owner)
   addSonSkipIntLit(result, baseType, c.idgen)
 
 proc genOp(c: var Con; op: PSym; dest: PNode): PNode =
@@ -415,7 +415,7 @@ proc destructiveMoveVar(n: PNode; c: var Con; s: var Scope): PNode =
   else:
     result = newNodeIT(nkStmtListExpr, n.info, n.typ)
 
-    var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), nextId c.idgen, c.owner, n.info)
+    var temp = newSym(skLet, getIdent(c.graph.cache, "blitTmp"), nextSymId c.idgen, c.owner, n.info)
     temp.typ = n.typ
     var v = newNodeI(nkLetSection, n.info)
     let tempAsNode = newSymNode(temp)

compiler/lambdalifting.nim

@@ -130,14 +130,14 @@ proc createClosureIterStateType*(g: ModuleGraph; iter: PSym; idgen: IdGenerator)
   var n = newNodeI(nkRange, iter.info)
   n.add newIntNode(nkIntLit, -1)
   n.add newIntNode(nkIntLit, 0)
-  result = newType(tyRange, nextId(idgen), iter)
+  result = newType(tyRange, nextTypeId(idgen), iter)
   result.n = n
   var intType = nilOrSysInt(g)
-  if intType.isNil: intType = newType(tyInt, nextId(idgen), iter)
+  if intType.isNil: intType = newType(tyInt, nextTypeId(idgen), iter)
   rawAddSon(result, intType)
 
 proc createStateField(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym =
-  result = newSym(skField, getIdent(g.cache, ":state"), nextId(idgen), iter, iter.info)
+  result = newSym(skField, getIdent(g.cache, ":state"), nextSymId(idgen), iter, iter.info)
   result.typ = createClosureIterStateType(g, iter, idgen)
 
 proc createEnvObj(g: ModuleGraph; idgen: IdGenerator; owner: PSym; info: TLineInfo): PType =
@@ -151,7 +151,7 @@ proc getClosureIterResult*(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym
     result = iter.ast[resultPos].sym
   else:
     # XXX a bit hacky:
-    result = newSym(skResult, getIdent(g.cache, ":result"), nextId(idgen), iter, iter.info, {})
+    result = newSym(skResult, getIdent(g.cache, ":result"), nextSymId(idgen), iter, iter.info, {})
     result.typ = iter.typ[0]
     incl(result.flags, sfUsed)
     iter.ast.add newSymNode(result)
@@ -259,7 +259,7 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PN
     addUniqueField(it.typ.skipTypes({tyOwned})[0], hp, g.cache, idgen)
     env = indirectAccess(newSymNode(it), hp, hp.info)
   else:
-    let e = newSym(skLet, iter.name, nextId(idgen), owner, n.info)
+    let e = newSym(skLet, iter.name, nextSymId(idgen), owner, n.info)
     e.typ = hp.typ
     e.flags = hp.flags
     env = newSymNode(e)
@@ -330,7 +330,7 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
                         info: TLineInfo): PType =
   result = c.ownerToType.getOrDefault(owner.id)
   if result.isNil:
-    result = newType(tyRef, nextId(c.idgen), owner)
+    result = newType(tyRef, nextTypeId(c.idgen), owner)
     let obj = createEnvObj(c.graph, c.idgen, owner, info)
     rawAddSon(result, obj)
     c.ownerToType[owner.id] = result
@@ -338,7 +338,7 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
 proc asOwnedRef(c: var DetectionPass; t: PType): PType =
   if optOwnedRefs in c.graph.config.globalOptions:
     assert t.kind == tyRef
-    result = newType(tyOwned, nextId(c.idgen), t.owner)
+    result = newType(tyOwned, nextTypeId(c.idgen), t.owner)
     result.flags.incl tfHasOwned
     result.rawAddSon t
   else:
@@ -347,7 +347,7 @@ proc asOwnedRef(c: var DetectionPass; t: PType): PType =
 proc getEnvTypeForOwnerUp(c: var DetectionPass; owner: PSym;
                           info: TLineInfo): PType =
   var r = c.getEnvTypeForOwner(owner, info)
-  result = newType(tyPtr, nextId(c.idgen), owner)
+  result = newType(tyPtr, nextTypeId(c.idgen), owner)
   rawAddSon(result, r.skipTypes({tyOwned, tyRef, tyPtr}))
 
 proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
@@ -375,7 +375,7 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
       if c.graph.config.selectedGC == gcDestructors and sfCursor notin upField.flags:
         localError(c.graph.config, dep.info, "internal error: up reference is not a .cursor")
   else:
-    let result = newSym(skField, upIdent, nextId(c.idgen), obj.owner, obj.owner.info)
+    let result = newSym(skField, upIdent, nextSymId(c.idgen), obj.owner, obj.owner.info)
     result.typ = fieldType
     when false:
       if c.graph.config.selectedGC == gcDestructors:
@@ -413,7 +413,7 @@ proc addClosureParam(c: var DetectionPass; fn: PSym; info: TLineInfo) =
   let owner = if fn.kind == skIterator: fn else: fn.skipGenericOwner
   let t = c.getEnvTypeForOwner(owner, info)
   if cp == nil:
-    cp = newSym(skParam, getIdent(c.graph.cache, paramName), nextId(c.idgen), fn, fn.info)
+    cp = newSym(skParam, getIdent(c.graph.cache, paramName), nextSymId(c.idgen), fn, fn.info)
     incl(cp.flags, sfFromGeneric)
     cp.typ = t
     addHiddenParam(fn, cp)
@@ -539,7 +539,7 @@ proc accessViaEnvParam(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   result = n
 
 proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType; info: TLineInfo; idgen: IdGenerator): PNode =
-  var v = newSym(skVar, getIdent(cache, envName), nextId(idgen), owner, info)
+  var v = newSym(skVar, getIdent(cache, envName), nextSymId(idgen), owner, info)
   v.flags = {sfShadowed, sfGeneratedOp}
   v.typ = typ
   result = newSymNode(v)
@@ -563,7 +563,7 @@ proc setupEnvVar(owner: PSym; d: var DetectionPass;
     result = newEnvVar(d.graph.cache, owner, asOwnedRef(d, envVarType), info, d.idgen)
     c.envVars[owner.id] = result
     if optOwnedRefs in d.graph.config.globalOptions:
-      var v = newSym(skVar, getIdent(d.graph.cache, envName & "Alt"), nextId d.idgen, owner, info)
+      var v = newSym(skVar, getIdent(d.graph.cache, envName & "Alt"), nextSymId d.idgen, owner, info)
       v.flags = {sfShadowed, sfGeneratedOp}
       v.typ = envVarType
       c.unownedEnvVars[owner.id] = newSymNode(v)
@@ -647,7 +647,7 @@ proc closureCreationForIter(iter: PNode;
                             d: var DetectionPass; c: var LiftingPass): PNode =
   result = newNodeIT(nkStmtListExpr, iter.info, iter.sym.typ)
   let owner = iter.sym.skipGenericOwner
-  var v = newSym(skVar, getIdent(d.graph.cache, envName), nextId(d.idgen), owner, iter.info)
+  var v = newSym(skVar, getIdent(d.graph.cache, envName), nextSymId(d.idgen), owner, iter.info)
   incl(v.flags, sfShadowed)
   v.typ = asOwnedRef(d, getHiddenParam(d.graph, iter.sym).typ)
   var vnode: PNode
@@ -937,7 +937,7 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; idgen: IdGenerator; owner: PSym):
     let iter = op.sym
 
     let hp = getHiddenParam(g, iter)
-    env = newSym(skLet, iter.name, nextId(idgen), owner, body.info)
+    env = newSym(skLet, iter.name, nextSymId(idgen), owner, body.info)
     env.typ = hp.typ
     env.flags = hp.flags
 

compiler/liftdestructors.nim

@@ -222,7 +222,7 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
     # for every field (dependent on dest.kind):
     #   `=` dest.field, src.field
     # =destroy(blob)
-    var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextId c.idgen, c.fn, c.info)
+    var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId c.idgen, c.fn, c.info)
     temp.typ = x.typ
     incl(temp.flags, sfFromGeneric)
     var v = newNodeI(nkVarSection, c.info)
@@ -412,7 +412,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       result = true
 
 proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
-  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextId(c.idgen), c.fn, c.info)
+  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info)
   temp.typ = getSysType(c.g, body.info, tyInt)
   incl(temp.flags, sfFromGeneric)
 
@@ -422,7 +422,7 @@ proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
   body.add v
 
 proc declareTempOf(c: var TLiftCtx; body: PNode; value: PNode): PNode =
-  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextId(c.idgen), c.fn, c.info)
+  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info)
   temp.typ = value.typ
   incl(temp.flags, sfFromGeneric)
 
@@ -899,17 +899,17 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
               info: TLineInfo; idgen: IdGenerator): PSym =
 
   let procname = getIdent(g.cache, AttachedOpToStr[kind])
-  result = newSym(skProc, procname, nextId(idgen), owner, info)
+  result = newSym(skProc, procname, nextSymId(idgen), owner, info)
-  let dest = newSym(skParam, getIdent(g.cache, "dest"), nextId(idgen), result, info)
+  let dest = newSym(skParam, getIdent(g.cache, "dest"), nextSymId(idgen), result, info)
   let src = newSym(skParam, getIdent(g.cache, if kind == attachedTrace: "env" else: "src"),
-                   nextId(idgen), result, info)
+                   nextSymId(idgen), result, info)
   dest.typ = makeVarType(typ.owner, typ, idgen)
   if kind == attachedTrace:
     src.typ = getSysType(g, info, tyPointer)
   else:
     src.typ = typ
 
-  result.typ = newProcType(info, nextId(idgen), owner)
+  result.typ = newProcType(info, nextTypeId(idgen), owner)
   result.typ.addParam dest
   if kind notin {attachedDestructor, attachedDispose}:
     result.typ.addParam src
@@ -917,7 +917,7 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
   if kind == attachedAsgn and g.config.selectedGC == gcOrc and
       cyclicType(typ.skipTypes(abstractInst)):
     let cycleParam = newSym(skParam, getIdent(g.cache, "cyclic"),
-                            nextId(idgen), result, info)
+                            nextSymId(idgen), result, info)
     cycleParam.typ = getSysType(g, info, tyBool)
     result.typ.addParam cycleParam
 
@@ -983,7 +983,7 @@ proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym,
   a.addMemReset = true
   let discrimantDest = result.typ.n[1].sym
 
-  let dst = newSym(skVar, getIdent(g.cache, "dest"), nextId(idgen), result, info)
+  let dst = newSym(skVar, getIdent(g.cache, "dest"), nextSymId(idgen), result, info)
   dst.typ = makePtrType(typ.owner, typ, idgen)
   let dstSym = newSymNode(dst)
   let d = newDeref(dstSym)

compiler/lookups.nim

@@ -227,7 +227,7 @@ proc errorSym*(c: PContext, n: PNode): PSym =
       considerQuotedIdent(c, m)
     else:
       getIdent(c.cache, "err:" & renderTree(m))
-  result = newSym(skError, ident, nextId(c.idgen), getCurrOwner(c), n.info, {})
+  result = newSym(skError, ident, nextSymId(c.idgen), getCurrOwner(c), n.info, {})
   result.typ = errorType(c)
   incl(result.flags, sfDiscardable)
   # pretend it's from the top level scope to prevent cascading errors:

compiler/lowerings.nim

@@ -71,7 +71,7 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: P
   let value = n.lastSon
   result = newNodeI(nkStmtList, n.info)
 
-  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextId(idgen),
+  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen),
                     owner, value.info, g.config.options)
   temp.typ = skipTypes(value.typ, abstractInst)
   incl(temp.flags, sfFromGeneric)
@@ -91,7 +91,7 @@ proc evalOnce*(g: ModuleGraph; value: PNode; idgen: IdGenerator; owner: PSym): P
   ## freely, multiple times. This is frequently required and such a builtin would also be
   ## handy to have in macros.nim. The value that can be reused is 'result.lastSon'!
   result = newNodeIT(nkStmtListExpr, value.info, value.typ)
-  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextId(idgen),
+  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen),
                     owner, value.info, g.config.options)
   temp.typ = skipTypes(value.typ, abstractInst)
   incl(temp.flags, sfFromGeneric)
@@ -117,7 +117,7 @@ proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; idgen: IdGenerator; o
   let value = n.lastSon
   result = newNodeI(nkStmtList, n.info)
 
-  var temp = newSym(skTemp, getIdent(g.cache, "_"), nextId(idgen), owner, value.info, owner.options)
+  var temp = newSym(skTemp, getIdent(g.cache, "_"), nextSymId(idgen), owner, value.info, owner.options)
   var v = newNodeI(nkLetSection, value.info)
   let tempAsNode = newSymNode(temp) #newIdentNode(getIdent(genPrefix & $temp.id), value.info)
 
@@ -135,7 +135,7 @@ proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; idgen: IdGenerator; o
 proc lowerSwap*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNode =
   result = newNodeI(nkStmtList, n.info)
   # note: cannot use 'skTemp' here cause we really need the copy for the VM :-(
-  var temp = newSym(skVar, getIdent(g.cache, genPrefix), nextId(idgen), owner, n.info, owner.options)
+  var temp = newSym(skVar, getIdent(g.cache, genPrefix), nextSymId(idgen), owner, n.info, owner.options)
   temp.typ = n[1].typ
   incl(temp.flags, sfFromGeneric)
   incl(temp.flags, sfGenSym)
@@ -154,7 +154,7 @@ proc lowerSwap*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PNod
   result.add newFastAsgnStmt(n[2], tempAsNode)
 
 proc createObj*(g: ModuleGraph; idgen: IdGenerator; owner: PSym, info: TLineInfo; final=true): PType =
-  result = newType(tyObject, nextId(idgen), owner)
+  result = newType(tyObject, nextTypeId(idgen), owner)
   if final:
     rawAddSon(result, nil)
     incl result.flags, tfFinal
@@ -162,7 +162,7 @@ proc createObj*(g: ModuleGraph; idgen: IdGenerator; owner: PSym, info: TLineInfo
     rawAddSon(result, getCompilerProc(g, "RootObj").typ)
   result.n = newNodeI(nkRecList, info)
   let s = newSym(skType, getIdent(g.cache, "Env_" & toFilename(g.config, info) & "_" & $owner.name.s),
-                  nextId(idgen),
+                  nextSymId(idgen),
                   owner, info, owner.options)
   incl s.flags, sfAnon
   s.typ = result
@@ -225,7 +225,7 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator) =
   # because of 'gensym' support, we have to mangle the name with its ID.
   # This is hacky but the clean solution is much more complex than it looks.
   var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len),
-                     nextId(idgen), s.owner, s.info, s.options)
+                     nextSymId(idgen), s.owner, s.info, s.options)
   field.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
   let t = skipIntLit(s.typ, idgen)
   field.typ = t
@@ -239,7 +239,7 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator) =
 proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym {.discardable.} =
   result = lookupInRecord(obj.n, s.itemId)
   if result == nil:
-    var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), nextId(idgen),
+    var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), nextSymId(idgen),
                        s.owner, s.info, s.options)
     field.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
     let t = skipIntLit(s.typ, idgen)
@@ -326,7 +326,7 @@ proc indirectAccess*(a, b: PSym, info: TLineInfo): PNode =
 proc genAddrOf*(n: PNode; idgen: IdGenerator; typeKind = tyPtr): PNode =
   result = newNodeI(nkAddr, n.info, 1)
   result[0] = n
-  result.typ = newType(typeKind, nextId(idgen), n.typ.owner)
+  result.typ = newType(typeKind, nextTypeId(idgen), n.typ.owner)
   result.typ.rawAddSon(n.typ)
 
 proc genDeref*(n: PNode; k = nkHiddenDeref): PNode =

compiler/magicsys.nim

@@ -21,7 +21,7 @@ proc registerSysType*(g: ModuleGraph; t: PType) =
   if g.sysTypes[t.kind] == nil: g.sysTypes[t.kind] = t
 
 proc newSysType(g: ModuleGraph; kind: TTypeKind, size: int): PType =
-  result = newType(kind, nextId(g.idgen), g.systemModule)
+  result = newType(kind, nextTypeId(g.idgen), g.systemModule)
   result.size = size
   result.align = size.int16
 
@@ -29,8 +29,8 @@ proc getSysSym*(g: ModuleGraph; info: TLineInfo; name: string): PSym =
   result = strTableGet(g.systemModule.tab, getIdent(g.cache, name))
   if result == nil:
     localError(g.config, info, "system module needs: " & name)
-    result = newSym(skError, getIdent(g.cache, name), nextId(g.idgen), g.systemModule, g.systemModule.info, {})
+    result = newSym(skError, getIdent(g.cache, name), nextSymId(g.idgen), g.systemModule, g.systemModule.info, {})
-    result.typ = newType(tyError, nextId(g.idgen), g.systemModule)
+    result.typ = newType(tyError, nextTypeId(g.idgen), g.systemModule)
   if result.kind == skAlias: result = result.owner
 
 proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSym =
@@ -45,8 +45,8 @@ proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSy
     r = nextIdentIter(ti, g.systemModule.tab)
   if result != nil: return result
   localError(g.config, info, "system module needs: " & name)
-  result = newSym(skError, id, nextId(g.idgen), g.systemModule, g.systemModule.info, {})
+  result = newSym(skError, id, nextSymId(g.idgen), g.systemModule, g.systemModule.info, {})
-  result.typ = newType(tyError, nextId(g.idgen), g.systemModule)
+  result.typ = newType(tyError, nextTypeId(g.idgen), g.systemModule)
 
 proc sysTypeFromName*(g: ModuleGraph; info: TLineInfo; name: string): PType =
   result = getSysSym(g, info, name).typ
@@ -101,12 +101,12 @@ proc getIntLitType*(g: ModuleGraph; literal: PNode): PType =
     result = g.intTypeCache[value.int]
     if result == nil:
       let ti = getSysType(g, literal.info, tyInt)
-      result = copyType(ti, nextId(g.idgen), ti.owner)
+      result = copyType(ti, nextTypeId(g.idgen), ti.owner)
       result.n = literal
       g.intTypeCache[value.int] = result
   else:
     let ti = getSysType(g, literal.info, tyInt)
-    result = copyType(ti, nextId(g.idgen), ti.owner)
+    result = copyType(ti, nextTypeId(g.idgen), ti.owner)
     result.n = literal
 
 proc getFloatLitType*(g: ModuleGraph; literal: PNode): PType =
@@ -116,7 +116,7 @@ proc getFloatLitType*(g: ModuleGraph; literal: PNode): PType =
 
 proc skipIntLit*(t: PType; id: IdGenerator): PType {.inline.} =
   if t.n != nil and t.kind in {tyInt, tyFloat}:
-    result = copyType(t, nextId(id), t.owner)
+    result = copyType(t, nextTypeId(id), t.owner)
     result.n = nil
   else:
     result = t

compiler/main.nim

@@ -19,7 +19,7 @@ import
   cgen, json, nversion,
   platform, nimconf, passaux, depends, vm,
   modules,
-  modulegraphs, tables, rod, lineinfos, pathutils, vmprofiler
+  modulegraphs, tables, lineinfos, pathutils, vmprofiler
 
 when not defined(leanCompiler):
   import jsgen, docgen, docgen2
@@ -137,7 +137,7 @@ proc commandInteractive(graph: ModuleGraph) =
   else:
     var m = graph.makeStdinModule()
     incl(m.flags, sfMainModule)
-    var idgen = IdGenerator(module: m.itemId.module, item: m.itemId.item)
+    var idgen = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0)
     let s = llStreamOpenStdIn(onPrompt = proc() = flushDot(graph.config))
     processModule(graph, m, idgen, s)
 
@@ -165,7 +165,6 @@ proc mainCommand*(graph: ModuleGraph) =
   let conf = graph.config
   let cache = graph.cache
 
-  setupModuleCache(graph)
   # In "nim serve" scenario, each command must reset the registered passes
   clearPasses(graph)
   conf.lastCmdTime = epochTime()

compiler/modulegraphs.nim

@@ -9,7 +9,7 @@
 
 ## This module implements the module graph data structure. The module graph
 ## represents a complete Nim project. Single modules can either be kept in RAM
-## or stored in a Sqlite database.
+## or stored in a rod-file.
 ##
 ## The caching of modules is critical for 'nimsuggest' and is tricky to get
 ## right. If module E is being edited, we need autocompletion (and type
@@ -26,7 +26,7 @@
 ##
 
 import ast, intsets, tables, options, lineinfos, hashes, idents,
-  incremental, btrees, md5
+  btrees, md5
 
 # import ic / packed_ast
 
@@ -67,7 +67,6 @@ type
     intTypeCache*: array[-5..64, PType]
     opContains*, opNot*: PSym
     emptyNode*: PNode
-    incr*: IncrementalCtx
     canonTypes*: Table[SigHash, PType]
     symBodyHashes*: Table[int, SigHash] # symId to digest mapping
     importModuleCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PSym {.nimcall.}
@@ -176,7 +175,7 @@ proc stopCompile*(g: ModuleGraph): bool {.inline.} =
   result = g.doStopCompile != nil and g.doStopCompile()
 
 proc createMagic*(g: ModuleGraph; name: string, m: TMagic): PSym =
-  result = newSym(skProc, getIdent(g.cache, name), nextId(g.idgen), nil, unknownLineInfo, {})
+  result = newSym(skProc, getIdent(g.cache, name), nextSymId(g.idgen), nil, unknownLineInfo, {})
   result.magic = m
   result.flags = {sfNeverRaises}
 
@@ -190,7 +189,7 @@ proc registerModule*(g: ModuleGraph; m: PSym) =
 
 proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result = ModuleGraph()
-  result.idgen = IdGenerator(module: -1'i32, item: 0'i32)
+  result.idgen = IdGenerator(module: -1'i32, symId: 0'i32, typeId: 0'i32)
   initStrTable(result.packageSyms)
   result.deps = initIntSet()
   result.importDeps = initTable[FileIndex, seq[FileIndex]]()
@@ -206,7 +205,6 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.opNot = createMagic(result, "not", mNot)
   result.opContains = createMagic(result, "contains", mInSet)
   result.emptyNode = newNode(nkEmpty)
-  init(result.incr)
   result.recordStmt = proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.} =
     discard
   result.cacheSeqs = initTable[string, PNode]()
@@ -235,7 +233,6 @@ proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b
 
 proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) =
   assert m.position == m.info.fileIndex.int32
-  addModuleDep(g.incr, g.config, m.info.fileIndex, dep, isIncludeFile = false)
   if g.suggestMode:
     g.deps.incl m.position.dependsOn(dep.int)
     # we compute the transitive closure later when querying the graph lazily.
@@ -243,7 +240,6 @@ proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) =
     #invalidTransitiveClosure = true
 
 proc addIncludeDep*(g: ModuleGraph; module, includeFile: FileIndex) =
-  addModuleDep(g.incr, g.config, module, includeFile, isIncludeFile = true)
   discard hasKeyOrPut(g.inclToMod, includeFile, module)
 
 proc parentModule*(g: ModuleGraph; fileIdx: FileIndex): FileIndex =

compiler/modules.nim

@@ -11,7 +11,7 @@
 
 import
   ast, astalgo, magicsys, msgs, options,
-  idents, lexer, passes, syntaxes, llstream, modulegraphs, rod,
+  idents, lexer, passes, syntaxes, llstream, modulegraphs,
   lineinfos, pathutils, tables
 
 proc resetSystemArtifacts*(g: ModuleGraph) =
@@ -93,8 +93,10 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
     discard processModule(graph, result, idGeneratorFromModule(result), s)
   if result == nil:
     let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
+    when false:
+      # XXX entry point for module loading from the rod file
       result = loadModuleSym(graph, fileIdx, filename)
-    if result == nil:
+    when true:
       result = newModule(graph, fileIdx)
       result.flags.incl flags
       registerModule(graph, result)

compiler/nilcheck.nim

@@ -906,24 +906,24 @@ proc infix(ctx: NilCheckerContext, l: PNode, r: PNode, magic: TMagic): PNode =
     else: ""
 
   var cache = newIdentCache()
-  var op = newSym(skVar, cache.getIdent(name), nextId ctx.idgen, nil, r.info)
+  var op = newSym(skVar, cache.getIdent(name), nextSymId ctx.idgen, nil, r.info)
 
   op.magic = magic
   result = nkInfix.newTree(
     newSymNode(op, r.info),
     l,
     r)
-  result.typ = newType(tyBool, nextId ctx.idgen, nil)
+  result.typ = newType(tyBool, nextTypeId ctx.idgen, nil)
 
 proc prefixNot(ctx: NilCheckerContext, node: PNode): PNode =
   var cache = newIdentCache()
-  var op = newSym(skVar, cache.getIdent("not"), nextId ctx.idgen, nil, node.info)
+  var op = newSym(skVar, cache.getIdent("not"), nextSymId ctx.idgen, nil, node.info)
 
   op.magic = mNot
   result = nkPrefix.newTree(
     newSymNode(op, node.info),
     node)
-  result.typ = newType(tyBool, nextId ctx.idgen, nil)
+  result.typ = newType(tyBool, nextTypeId ctx.idgen, nil)
 
 proc infixEq(ctx: NilCheckerContext, l: PNode, r: PNode): PNode =
   infix(ctx, l, r, mEqRef)

compiler/options.nim

@@ -722,6 +722,10 @@ proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile,
   result = subdir / RelativeFile f.string.splitPath.tail
   #echo "completeGeneratedFilePath(", f, ") = ", result
 
+proc toRodFile*(conf: ConfigRef; f: AbsoluteFile): AbsoluteFile =
+  result = changeFileExt(completeGeneratedFilePath(conf,
+    withPackageName(conf, f)), RodExt)
+
 proc rawFindFile(conf: ConfigRef; f: RelativeFile; suppressStdlib: bool): AbsoluteFile =
   for it in conf.searchPaths:
     if suppressStdlib and it.string.startsWith(conf.libpath.string):

compiler/passes.nim

@@ -13,7 +13,7 @@
 import
   options, ast, llstream, msgs,
   idents,
-  syntaxes, modulegraphs, reorder, rod,
+  syntaxes, modulegraphs, reorder,
   lineinfos, pathutils
 
 type
@@ -119,29 +119,6 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
     s: PLLStream
     fileIdx = module.fileIdx
   prepareConfigNotes(graph, module)
-  if module.id < 0:
-    # new module caching mechanism:
-    for i in 0..<graph.passes.len:
-      if not isNil(graph.passes[i].open) and not graph.passes[i].isFrontend:
-        a[i] = graph.passes[i].open(graph, module, idgen)
-      else:
-        a[i] = nil
-
-    if not graph.stopCompile():
-      let n = loadNode(graph, module)
-      var m = n
-      for i in 0..<graph.passes.len:
-        if not isNil(graph.passes[i].process) and not graph.passes[i].isFrontend:
-          m = graph.passes[i].process(a[i], m)
-          if isNil(m):
-            break
-
-    var m: PNode = nil
-    for i in 0..<graph.passes.len:
-      if not isNil(graph.passes[i].close) and not graph.passes[i].isFrontend:
-        m = graph.passes[i].close(graph, a[i], m)
-      a[i] = nil
-  else:
   openPasses(graph, a, module, idgen)
   if stream == nil:
     let filename = toFullPathConsiderDirty(graph.config, fileIdx)

compiler/plugins/itersgen.nim

@@ -31,8 +31,8 @@ proc iterToProcImpl*(c: PContext, n: PNode): PNode =
     return
   let body = liftIterToProc(c.graph, iter.sym, iter.sym.getBody, t, c.idgen)
 
-  let prc = newSym(skProc, n[3].ident, nextId c.idgen, iter.sym.owner, iter.sym.info)
+  let prc = newSym(skProc, n[3].ident, nextSymId c.idgen, iter.sym.owner, iter.sym.info)
-  prc.typ = copyType(iter.sym.typ, nextId c.idgen, prc)
+  prc.typ = copyType(iter.sym.typ, nextTypeId c.idgen, prc)
   excl prc.typ.flags, tfCapturesEnv
   prc.typ.n.add newSymNode(getEnvParam(iter.sym))
   prc.typ.rawAddSon t

compiler/plugins/locals.nim

@@ -26,7 +26,7 @@ proc semLocals*(c: PContext, n: PNode): PNode =
             {tyVarargs, tyOpenArray, tyTypeDesc, tyStatic, tyUntyped, tyTyped, tyEmpty}:
 
         if it.owner == owner:
-          var field = newSym(skField, it.name, nextId c.idgen, owner, n.info)
+          var field = newSym(skField, it.name, nextSymId c.idgen, owner, n.info)
           field.typ = it.typ.skipTypes({tyVar})
           field.position = counter
           inc(counter)

compiler/pragmas.nim

@@ -121,7 +121,7 @@ proc pragmaEnsures(c: PContext, n: PNode) =
     openScope(c)
     let o = getCurrOwner(c)
     if o.kind in routineKinds and o.typ != nil and o.typ.sons[0] != nil:
-      var s = newSym(skResult, getIdent(c.cache, "result"), nextId(c.idgen), o, n.info)
+      var s = newSym(skResult, getIdent(c.cache, "result"), nextSymId(c.idgen), o, n.info)
       s.typ = o.typ.sons[0]
       incl(s.flags, sfUsed)
       addDecl(c, s)
@@ -563,8 +563,6 @@ proc semAsmOrEmit*(con: PContext, n: PNode, marker: char): PNode =
         var e = searchInScopes(con, getIdent(con.cache, sub), amb)
         # XXX what to do here if 'amb' is true?
         if e != nil:
-          when false:
-            if e.kind == skStub: loadStub(e)
           incl(e.flags, sfUsed)
           result.add newSymNode(e)
         else:
@@ -638,7 +636,7 @@ proc processPragma(c: PContext, n: PNode, i: int) =
   elif it.safeLen != 2 or it[0].kind != nkIdent or it[1].kind != nkIdent:
     invalidPragma(c, n)
 
-  var userPragma = newSym(skTemplate, it[1].ident, nextId(c.idgen), nil, it.info, c.config.options)
+  var userPragma = newSym(skTemplate, it[1].ident, nextSymId(c.idgen), nil, it.info, c.config.options)
   userPragma.ast = newTreeI(nkPragma, n.info, n.sons[i+1..^1])
   strTableAdd(c.userPragmas, userPragma)
 
@@ -719,7 +717,7 @@ proc deprecatedStmt(c: PContext; outerPragma: PNode) =
       if dest == nil or dest.kind in routineKinds:
         localError(c.config, n.info, warnUser, "the .deprecated pragma is unreliable for routines")
       let src = considerQuotedIdent(c, n[0])
-      let alias = newSym(skAlias, src, nextId(c.idgen), dest, n[0].info, c.config.options)
+      let alias = newSym(skAlias, src, nextSymId(c.idgen), dest, n[0].info, c.config.options)
       incl(alias.flags, sfExported)
       if sfCompilerProc in dest.flags: markCompilerProc(c, alias)
       addInterfaceDecl(c, alias)
@@ -741,7 +739,7 @@ proc pragmaGuard(c: PContext; it: PNode; kind: TSymKind): PSym =
       # We return a dummy symbol; later passes over the type will repair it.
       # Generic instantiation needs to know about this too. But we're lazy
       # and perform the lookup on demand instead.
-      result = newSym(skUnknown, considerQuotedIdent(c, n), nextId(c.idgen), nil, n.info,
+      result = newSym(skUnknown, considerQuotedIdent(c, n), nextSymId(c.idgen), nil, n.info,
         c.config.options)
   else:
     result = qualifiedLookUp(c, n, {checkUndeclared})

compiler/rod.nim

@@ -1,31 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2017 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module implements the canonalization for the various caching mechanisms.
-
-import ast, lineinfos, incremental, modulegraphs, pathutils
-
-when not nimIncremental:
-  template setupModuleCache*(g: ModuleGraph) = discard
-  template storeNode*(g: ModuleGraph; module: PSym; n: PNode) = discard
-  template loadNode*(g: ModuleGraph; module: PSym): PNode = newNode(nkStmtList)
-
-  proc loadModuleSym*(g: ModuleGraph; fileIdx: FileIndex; fullpath: AbsoluteFile): (PSym) {.inline.} = nil
-
-  template addModuleDep*(g: ModuleGraph; module, fileIdx: FileIndex; isIncludeFile: bool) = discard
-
-  template storeRemaining*(g: ModuleGraph; module: PSym) = discard
-
-  #template registerModule*(g: ModuleGraph; module: PSym) = discard
-
-else:
-  include rodimpl
-
-  # idea for testing all this logic: *Always* load the AST from the DB, whether
-  # we already have it in RAM or not!

compiler/rodimpl.nim

@@ -1,950 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2018 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module implements the new compilation cache.
-
-import strutils, intsets, tables, ropes, db_sqlite, msgs, options,
-  renderer, rodutils, idents, astalgo, btrees, magicsys, cgmeth, extccomp,
-  btrees, trees, condsyms, nversion, pathutils
-
-## Todo:
-## - Dependency computation should use *signature* hashes in order to
-##   avoid recompiling dependent modules.
-## - Patch the rest of the compiler to do lazy loading of proc bodies.
-## - serialize the AST in a smarter way (avoid storing some ASTs twice!)
-
-template db(): DbConn = g.incr.db
-
-proc encodeConfig(g: ModuleGraph): string =
-  result = newStringOfCap(100)
-  result.add RodFileVersion
-  for d in definedSymbolNames(g.config.symbols):
-    result.add ' '
-    result.add d
-
-  template serialize(field) =
-    result.add ' '
-    result.add($g.config.field)
-
-  depConfigFields(serialize)
-
-proc needsRecompile(g: ModuleGraph; fileIdx: FileIndex; fullpath: AbsoluteFile;
-                    cycleCheck: var IntSet): bool =
-  let root = db.getRow(sql"select id, fullhash from filenames where fullpath = ?",
-    fullpath.string)
-  if root[0].len == 0: return true
-  if root[1] != hashFileCached(g.config, fileIdx, fullpath):
-    return true
-  # cycle detection: assume "not changed" is correct.
-  if cycleCheck.containsOrIncl(int fileIdx):
-    return false
-  # check dependencies (recursively):
-  for row in db.fastRows(sql"select fullpath from filenames where id in (select dependency from deps where module = ?)",
-                         root[0]):
-    let dep = AbsoluteFile row[0]
-    if needsRecompile(g, g.config.fileInfoIdx(dep), dep, cycleCheck):
-      return true
-  return false
-
-proc getModuleId(g: ModuleGraph; fileIdx: FileIndex; fullpath: AbsoluteFile): int =
-  ## Analyse the known dependency graph.
-  if g.config.symbolFiles == disabledSf: return getID()
-  when false:
-    if g.config.symbolFiles in {disabledSf, writeOnlySf} or
-      g.incr.configChanged:
-      return getID()
-  let module = g.incr.db.getRow(
-    sql"select id, fullHash, nimid from modules where fullpath = ?", string fullpath)
-  let currentFullhash = hashFileCached(g.config, fileIdx, fullpath)
-  if module[0].len == 0:
-    result = getID()
-    db.exec(sql"insert into modules(fullpath, interfHash, fullHash, nimid) values (?, ?, ?, ?)",
-      string fullpath, "", currentFullhash, result)
-  else:
-    result = parseInt(module[2])
-    if currentFullhash == module[1]:
-      # not changed, so use the cached AST:
-      doAssert(result != 0)
-      var cycleCheck = initIntSet()
-      if not needsRecompile(g, fileIdx, fullpath, cycleCheck):
-        if not g.incr.configChanged or g.config.symbolFiles == readOnlySf:
-          #echo "cached successfully! ", string fullpath
-          return -result
-      elif g.config.symbolFiles == readOnlySf:
-        internalError(g.config, "file needs to be recompiled: " & (string fullpath))
-    db.exec(sql"update modules set fullHash = ? where id = ?", currentFullhash, module[0])
-    db.exec(sql"delete from deps where module = ?", module[0])
-    db.exec(sql"delete from types where module = ?", module[0])
-    db.exec(sql"delete from syms where module = ?", module[0])
-    db.exec(sql"delete from toplevelstmts where module = ?", module[0])
-    db.exec(sql"delete from statics where module = ?", module[0])
-
-proc loadModuleSym*(g: ModuleGraph; fileIdx: FileIndex; fullpath: AbsoluteFile): PSym =
-  let id = getModuleId(g, fileIdx, fullpath)
-  result = g.incr.r.syms.getOrDefault(abs id)
-
-proc pushType(w: var Writer, t: PType) =
-  if not containsOrIncl(w.tmarks, t.uniqueId):
-    w.tstack.add(t)
-
-proc pushSym(w: var Writer, s: PSym) =
-  if not containsOrIncl(w.smarks, s.id):
-    w.sstack.add(s)
-
-template w: untyped = g.incr.w
-
-proc encodeNode(g: ModuleGraph; fInfo: TLineInfo, n: PNode,
-                result: var string) =
-  if n == nil:
-    # nil nodes have to be stored too:
-    result.add("()")
-    return
-  result.add('(')
-  encodeVInt(ord(n.kind), result)
-  # we do not write comments for now
-  # Line information takes easily 20% or more of the filesize! Therefore we
-  # omit line information if it is the same as the parent's line information:
-  if fInfo.fileIndex != n.info.fileIndex:
-    result.add('?')
-    encodeVInt(n.info.col, result)
-    result.add(',')
-    encodeVInt(int n.info.line, result)
-    result.add(',')
-    #encodeVInt(toDbFileId(g.incr, g.config, n.info.fileIndex), result)
-    encodeVInt(n.info.fileIndex.int, result)
-  elif fInfo.line != n.info.line:
-    result.add('?')
-    encodeVInt(n.info.col, result)
-    result.add(',')
-    encodeVInt(int n.info.line, result)
-  elif fInfo.col != n.info.col:
-    result.add('?')
-    encodeVInt(n.info.col, result)
-  # No need to output the file index, as this is the serialization of one
-  # file.
-  let f = n.flags * PersistentNodeFlags
-  if f != {}:
-    result.add('$')
-    encodeVInt(cast[int32](f), result)
-  if n.typ != nil:
-    result.add('^')
-    encodeVInt(n.typ.uniqueId, result)
-    pushType(w, n.typ)
-  case n.kind
-  of nkCharLit..nkUInt64Lit:
-    if n.intVal != 0:
-      result.add('!')
-      encodeVBiggestInt(n.intVal, result)
-  of nkFloatLit..nkFloat64Lit:
-    if n.floatVal != 0.0:
-      result.add('!')
-      encodeStr($n.floatVal, result)
-  of nkStrLit..nkTripleStrLit:
-    if n.strVal != "":
-      result.add('!')
-      encodeStr(n.strVal, result)
-  of nkIdent:
-    result.add('!')
-    encodeStr(n.ident.s, result)
-  of nkSym:
-    result.add('!')
-    encodeVInt(n.sym.id, result)
-    pushSym(w, n.sym)
-  else:
-    for i in 0..<n.len:
-      encodeNode(g, n.info, n[i], result)
-  result.add(')')
-
-proc encodeLoc(g: ModuleGraph; loc: TLoc, result: var string) =
-  var oldLen = result.len
-  result.add('<')
-  if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
-  if loc.storage != low(loc.storage):
-    result.add('*')
-    encodeVInt(ord(loc.storage), result)
-  if loc.flags != {}:
-    result.add('$')
-    encodeVInt(cast[int32](loc.flags), result)
-  if loc.lode != nil:
-    result.add('^')
-    encodeNode(g, unknownLineInfo, loc.lode, result)
-  if loc.r != nil:
-    result.add('!')
-    encodeStr($loc.r, result)
-  if oldLen + 1 == result.len:
-    # no data was necessary, so remove the '<' again:
-    setLen(result, oldLen)
-  else:
-    result.add('>')
-
-proc encodeType(g: ModuleGraph, t: PType, result: var string) =
-  if t == nil:
-    # nil nodes have to be stored too:
-    result.add("[]")
-    return
-  # we need no surrounding [] here because the type is in a line of its own
-  if t.kind == tyForward: internalError(g.config, "encodeType: tyForward")
-  # for the new rodfile viewer we use a preceding [ so that the data section
-  # can easily be disambiguated:
-  result.add('[')
-  encodeVInt(ord(t.kind), result)
-  result.add('+')
-  encodeVInt(t.uniqueId, result)
-  if t.id != t.uniqueId:
-    result.add('+')
-    encodeVInt(t.id, result)
-  if t.n != nil:
-    encodeNode(g, unknownLineInfo, t.n, result)
-  if t.flags != {}:
-    result.add('$')
-    encodeVInt(cast[int32](t.flags), result)
-  if t.callConv != low(t.callConv):
-    result.add('?')
-    encodeVInt(ord(t.callConv), result)
-  if t.owner != nil:
-    result.add('*')
-    encodeVInt(t.owner.id, result)
-    pushSym(w, t.owner)
-  if t.sym != nil:
-    result.add('&')
-    encodeVInt(t.sym.id, result)
-    pushSym(w, t.sym)
-  if t.size != - 1:
-    result.add('/')
-    encodeVBiggestInt(t.size, result)
-  if t.align != 2:
-    result.add('=')
-    encodeVInt(t.align, result)
-  if t.lockLevel.ord != UnspecifiedLockLevel.ord:
-    result.add('\14')
-    encodeVInt(t.lockLevel.int16, result)
-  if t.paddingAtEnd != 0:
-    result.add('\15')
-    encodeVInt(t.paddingAtEnd, result)
-  for a in t.attachedOps:
-    result.add('\16')
-    if a == nil:
-      encodeVInt(-1, result)
-    else:
-      encodeVInt(a.id, result)
-      pushSym(w, a)
-  for i, s in items(t.methods):
-    result.add('\19')
-    encodeVInt(i, result)
-    result.add('\20')
-    encodeVInt(s.id, result)
-    pushSym(w, s)
-  encodeLoc(g, t.loc, result)
-  if t.typeInst != nil:
-    result.add('\21')
-    encodeVInt(t.typeInst.uniqueId, result)
-    pushType(w, t.typeInst)
-  for i in 0..<t.len:
-    if t[i] == nil:
-      result.add("^()")
-    else:
-      result.add('^')
-      encodeVInt(t[i].uniqueId, result)
-      pushType(w, t[i])
-
-proc encodeLib(g: ModuleGraph, lib: PLib, info: TLineInfo, result: var string) =
-  result.add('|')
-  encodeVInt(ord(lib.kind), result)
-  result.add('|')
-  encodeStr($lib.name, result)
-  result.add('|')
-  encodeNode(g, info, lib.path, result)
-
-proc encodeInstantiations(g: ModuleGraph; s: seq[PInstantiation];
-                          result: var string) =
-  for t in s:
-    result.add('\15')
-    encodeVInt(t.sym.id, result)
-    pushSym(w, t.sym)
-    for tt in t.concreteTypes:
-      result.add('\17')
-      encodeVInt(tt.uniqueId, result)
-      pushType(w, tt)
-    result.add('\20')
-    encodeVInt(t.compilesId, result)
-
-proc encodeSym(g: ModuleGraph, s: PSym, result: var string) =
-  if s == nil:
-    # nil nodes have to be stored too:
-    result.add("{}")
-    return
-  # we need no surrounding {} here because the symbol is in a line of its own
-  encodeVInt(ord(s.kind), result)
-  result.add('+')
-  encodeVInt(s.id, result)
-  result.add('&')
-  encodeStr(s.name.s, result)
-  if s.typ != nil:
-    result.add('^')
-    encodeVInt(s.typ.uniqueId, result)
-    pushType(w, s.typ)
-  result.add('?')
-  if s.info.col != -1'i16: encodeVInt(s.info.col, result)
-  result.add(',')
-  encodeVInt(int s.info.line, result)
-  result.add(',')
-  #encodeVInt(toDbFileId(g.incr, g.config, s.info.fileIndex), result)
-  encodeVInt(s.info.fileIndex.int, result)
-  if s.owner != nil:
-    result.add('*')
-    encodeVInt(s.owner.id, result)
-    pushSym(w, s.owner)
-  if s.flags != {}:
-    result.add('$')
-    encodeVBiggestInt(cast[int64](s.flags), result)
-  if s.magic != mNone:
-    result.add('@')
-    encodeVInt(ord(s.magic), result)
-  result.add('!')
-  encodeVInt(cast[int32](s.options), result)
-  if s.position != 0:
-    result.add('%')
-    encodeVInt(s.position, result)
-  if s.offset != - 1:
-    result.add('`')
-    encodeVInt(s.offset, result)
-  encodeLoc(g, s.loc, result)
-  if s.annex != nil: encodeLib(g, s.annex, s.info, result)
-  if s.constraint != nil:
-    result.add('#')
-    encodeNode(g, unknownLineInfo, s.constraint, result)
-  case s.kind
-  of skType, skGenericParam:
-    for t in s.typeInstCache:
-      result.add('\14')
-      encodeVInt(t.uniqueId, result)
-      pushType(w, t)
-  of routineKinds:
-    encodeInstantiations(g, s.procInstCache, result)
-    if s.gcUnsafetyReason != nil:
-      result.add('\16')
-      encodeVInt(s.gcUnsafetyReason.id, result)
-      pushSym(w, s.gcUnsafetyReason)
-    if s.transformedBody != nil:
-      result.add('\24')
-      encodeNode(g, s.info, s.transformedBody, result)
-  of skModule, skPackage:
-    encodeInstantiations(g, s.usedGenerics, result)
-    # we don't serialize:
-    #tab*: TStrTable         # interface table for modules
-  of skLet, skVar, skField, skForVar:
-    if s.guard != nil:
-      result.add('\18')
-      encodeVInt(s.guard.id, result)
-      pushSym(w, s.guard)
-    if s.bitsize != 0:
-      result.add('\19')
-      encodeVInt(s.bitsize, result)
-  else: discard
-  # lazy loading will soon reload the ast lazily, so the ast needs to be
-  # the last entry of a symbol:
-  if s.ast != nil:
-    # we used to attempt to save space here by only storing a dummy AST if
-    # it is not necessary, but Nim's heavy compile-time evaluation features
-    # make that unfeasible nowadays:
-    encodeNode(g, s.info, s.ast, result)
-
-proc storeSym(g: ModuleGraph; s: PSym) =
-  if sfForward in s.flags and s.kind != skModule:
-    w.forwardedSyms.add s
-    return
-  var buf = newStringOfCap(160)
-  encodeSym(g, s, buf)
-  # XXX only store the name for exported symbols in order to speed up lookup
-  # times once we enable the skStub logic.
-  let m = getModule(s)
-  let mid = if m == nil: 0 else: abs(m.id)
-  db.exec(sql"insert into syms(nimid, module, name, data, exported) values (?, ?, ?, ?, ?)",
-    s.id, mid, s.name.s, buf, ord(sfExported in s.flags))
-
-proc storeType(g: ModuleGraph; t: PType) =
-  var buf = newStringOfCap(160)
-  encodeType(g, t, buf)
-  let m = if t.owner != nil: getModule(t.owner) else: nil
-  let mid = if m == nil: 0 else: abs(m.id)
-  db.exec(sql"insert into types(nimid, module, data) values (?, ?, ?)",
-    t.uniqueId, mid, buf)
-
-proc transitiveClosure(g: ModuleGraph) =
-  var i = 0
-  while true:
-    if i > 100_000:
-      doAssert false, "loop never ends!"
-    if w.sstack.len > 0:
-      let s = w.sstack.pop()
-      when false:
-        echo "popped ", s.name.s, " ", s.id
-      storeSym(g, s)
-    elif w.tstack.len > 0:
-      let t = w.tstack.pop()
-      storeType(g, t)
-      when false:
-        echo "popped type ", typeToString(t), " ", t.uniqueId
-    else:
-      break
-    inc i
-
-proc storeNode*(g: ModuleGraph; module: PSym; n: PNode) =
-  if g.config.symbolFiles == disabledSf: return
-  var buf = newStringOfCap(160)
-  encodeNode(g, module.info, n, buf)
-  db.exec(sql"insert into toplevelstmts(module, position, data) values (?, ?, ?)",
-    abs(module.id), module.offset, buf)
-  inc module.offset
-  transitiveClosure(g)
-
-proc recordStmt*(g: ModuleGraph; module: PSym; n: PNode) =
-  storeNode(g, module, n)
-
-proc storeFilename(g: ModuleGraph; fullpath: AbsoluteFile; fileIdx: FileIndex) =
-  let id = db.getValue(sql"select id from filenames where fullpath = ?", fullpath.string)
-  if id.len == 0:
-    let fullhash = hashFileCached(g.config, fileIdx, fullpath)
-    db.exec(sql"insert into filenames(nimid, fullpath, fullhash) values (?, ?, ?)",
-        int(fileIdx), fullpath.string, fullhash)
-
-proc storeRemaining*(g: ModuleGraph; module: PSym) =
-  if g.config.symbolFiles == disabledSf: return
-  var stillForwarded: seq[PSym] = @[]
-  for s in w.forwardedSyms:
-    if sfForward notin s.flags:
-      storeSym(g, s)
-    else:
-      stillForwarded.add s
-  swap w.forwardedSyms, stillForwarded
-  transitiveClosure(g)
-  var nimid = 0
-  for x in items(g.config.m.fileInfos):
-    storeFilename(g, x.fullPath, FileIndex(nimid))
-    inc nimid
-
-# ---------------- decoder -----------------------------------
-
-type
-  BlobReader = object
-    s: string
-    pos: int
-
-using
-  b: var BlobReader
-  g: ModuleGraph
-
-proc loadSym(g; id: int, info: TLineInfo): PSym
-proc loadType(g; id: int, info: TLineInfo): PType
-
-proc decodeLineInfo(g; b; info: var TLineInfo) =
-  if b.s[b.pos] == '?':
-    inc(b.pos)
-    if b.s[b.pos] == ',': info.col = -1'i16
-    else: info.col = int16(decodeVInt(b.s, b.pos))
-    if b.s[b.pos] == ',':
-      inc(b.pos)
-      if b.s[b.pos] == ',': info.line = 0'u16
-      else: info.line = uint16(decodeVInt(b.s, b.pos))
-      if b.s[b.pos] == ',':
-        inc(b.pos)
-        #info.fileIndex = fromDbFileId(g.incr, g.config, decodeVInt(b.s, b.pos))
-        info.fileIndex = FileIndex decodeVInt(b.s, b.pos)
-
-proc skipNode(b) =
-  # ')' itself cannot be part of a string literal so that this is correct.
-  assert b.s[b.pos] == '('
-  var par = 0
-  var pos = b.pos+1
-  while true:
-    case b.s[pos]
-    of ')':
-      if par == 0: break
-      dec par
-    of '(': inc par
-    else: discard
-    inc pos
-  b.pos = pos+1 # skip ')'
-
-proc decodeNodeLazyBody(g; b; fInfo: TLineInfo,
-                        belongsTo: PSym): PNode =
-  result = nil
-  if b.s[b.pos] == '(':
-    inc(b.pos)
-    if b.s[b.pos] == ')':
-      inc(b.pos)
-      return                  # nil node
-    result = newNodeI(TNodeKind(decodeVInt(b.s, b.pos)), fInfo)
-    decodeLineInfo(g, b, result.info)
-    if b.s[b.pos] == '$':
-      inc(b.pos)
-      result.flags = cast[TNodeFlags](int32(decodeVInt(b.s, b.pos)))
-    if b.s[b.pos] == '^':
-      inc(b.pos)
-      var id = decodeVInt(b.s, b.pos)
-      result.typ = loadType(g, id, result.info)
-    case result.kind
-    of nkCharLit..nkUInt64Lit:
-      if b.s[b.pos] == '!':
-        inc(b.pos)
-        result.intVal = decodeVBiggestInt(b.s, b.pos)
-    of nkFloatLit..nkFloat64Lit:
-      if b.s[b.pos] == '!':
-        inc(b.pos)
-        var fl = decodeStr(b.s, b.pos)
-        result.floatVal = parseFloat(fl)
-    of nkStrLit..nkTripleStrLit:
-      if b.s[b.pos] == '!':
-        inc(b.pos)
-        result.strVal = decodeStr(b.s, b.pos)
-      else:
-        result.strVal = ""
-    of nkIdent:
-      if b.s[b.pos] == '!':
-        inc(b.pos)
-        var fl = decodeStr(b.s, b.pos)
-        result.ident = g.cache.getIdent(fl)
-      else:
-        internalError(g.config, result.info, "decodeNode: nkIdent")
-    of nkSym:
-      if b.s[b.pos] == '!':
-        inc(b.pos)
-        var id = decodeVInt(b.s, b.pos)
-        result.sym = loadSym(g, id, result.info)
-      else:
-        internalError(g.config, result.info, "decodeNode: nkSym")
-    else:
-      var i = 0
-      while b.s[b.pos] != ')':
-        when false:
-          if belongsTo != nil and i == bodyPos:
-            addSonNilAllowed(result, nil)
-            belongsTo.offset = b.pos
-            skipNode(b)
-          else:
-            discard
-        addSonNilAllowed(result, decodeNodeLazyBody(g, b, result.info, nil))
-        inc i
-    if b.s[b.pos] == ')': inc(b.pos)
-    else: internalError(g.config, result.info, "decodeNode: ')' missing")
-  else:
-    internalError(g.config, fInfo, "decodeNode: '(' missing " & $b.pos)
-
-proc decodeNode(g; b; fInfo: TLineInfo): PNode =
-  result = decodeNodeLazyBody(g, b, fInfo, nil)
-
-proc decodeLoc(g; b; loc: var TLoc, info: TLineInfo) =
-  if b.s[b.pos] == '<':
-    inc(b.pos)
-    if b.s[b.pos] in {'0'..'9', 'a'..'z', 'A'..'Z'}:
-      loc.k = TLocKind(decodeVInt(b.s, b.pos))
-    else:
-      loc.k = low(loc.k)
-    if b.s[b.pos] == '*':
-      inc(b.pos)
-      loc.storage = TStorageLoc(decodeVInt(b.s, b.pos))
-    else:
-      loc.storage = low(loc.storage)
-    if b.s[b.pos] == '$':
-      inc(b.pos)
-      loc.flags = cast[TLocFlags](int32(decodeVInt(b.s, b.pos)))
-    else:
-      loc.flags = {}
-    if b.s[b.pos] == '^':
-      inc(b.pos)
-      loc.lode = decodeNode(g, b, info)
-      # rrGetType(b, decodeVInt(b.s, b.pos), info)
-    else:
-      loc.lode = nil
-    if b.s[b.pos] == '!':
-      inc(b.pos)
-      loc.r = rope(decodeStr(b.s, b.pos))
-    else:
-      loc.r = nil
-    if b.s[b.pos] == '>': inc(b.pos)
-    else: internalError(g.config, info, "decodeLoc " & b.s[b.pos])
-
-proc loadBlob(g; query: SqlQuery; id: int): BlobReader =
-  let blob = db.getValue(query, id)
-  if blob.len == 0:
-    internalError(g.config, "symbolfiles: cannot find ID " & $ id)
-  result = BlobReader(pos: 0)
-  shallowCopy(result.s, blob)
-  # ensure we can read without index checks:
-  result.s.add '\0'
-
-proc loadType(g; id: int; info: TLineInfo): PType =
-  result = g.incr.r.types.getOrDefault(id)
-  if result != nil: return result
-  var b = loadBlob(g, sql"select data from types where nimid = ?", id)
-
-  if b.s[b.pos] == '[':
-    inc(b.pos)
-    if b.s[b.pos] == ']':
-      inc(b.pos)
-      return                  # nil type
-  new(result)
-  result.kind = TTypeKind(decodeVInt(b.s, b.pos))
-  if b.s[b.pos] == '+':
-    inc(b.pos)
-    result.uniqueId = decodeVInt(b.s, b.pos)
-    setId(result.uniqueId)
-    #if debugIds: registerID(result)
-  else:
-    internalError(g.config, info, "decodeType: no id")
-  if b.s[b.pos] == '+':
-    inc(b.pos)
-    result.id = decodeVInt(b.s, b.pos)
-  else:
-    result.id = result.uniqueId
-  # here this also avoids endless recursion for recursive type
-  g.incr.r.types.add(result.uniqueId, result)
-  if b.s[b.pos] == '(': result.n = decodeNode(g, b, unknownLineInfo)
-  if b.s[b.pos] == '$':
-    inc(b.pos)
-    result.flags = cast[TTypeFlags](int32(decodeVInt(b.s, b.pos)))
-  if b.s[b.pos] == '?':
-    inc(b.pos)
-    result.callConv = TCallingConvention(decodeVInt(b.s, b.pos))
-  if b.s[b.pos] == '*':
-    inc(b.pos)
-    result.owner = loadSym(g, decodeVInt(b.s, b.pos), info)
-  if b.s[b.pos] == '&':
-    inc(b.pos)
-    result.sym = loadSym(g, decodeVInt(b.s, b.pos), info)
-  if b.s[b.pos] == '/':
-    inc(b.pos)
-    result.size = decodeVInt(b.s, b.pos)
-  else:
-    result.size = -1
-  if b.s[b.pos] == '=':
-    inc(b.pos)
-    result.align = decodeVInt(b.s, b.pos).int16
-  else:
-    result.align = 2
-
-  if b.s[b.pos] == '\14':
-    inc(b.pos)
-    result.lockLevel = decodeVInt(b.s, b.pos).TLockLevel
-  else:
-    result.lockLevel = UnspecifiedLockLevel
-
-  if b.s[b.pos] == '\15':
-    inc(b.pos)
-    result.paddingAtEnd = decodeVInt(b.s, b.pos).int16
-
-  for a in low(result.attachedOps)..high(result.attachedOps):
-    if b.s[b.pos] == '\16':
-      inc(b.pos)
-      let id = decodeVInt(b.s, b.pos)
-      if id >= 0:
-        result.attachedOps[a] = loadSym(g, id, info)
-
-  while b.s[b.pos] == '\19':
-    inc(b.pos)
-    let x = decodeVInt(b.s, b.pos)
-    doAssert b.s[b.pos] == '\20'
-    inc(b.pos)
-    let y = loadSym(g, decodeVInt(b.s, b.pos), info)
-    result.methods.add((x, y))
-  decodeLoc(g, b, result.loc, info)
-  if b.s[b.pos] == '\21':
-    inc(b.pos)
-    let d = decodeVInt(b.s, b.pos)
-    result.typeInst = loadType(g, d, info)
-  while b.s[b.pos] == '^':
-    inc(b.pos)
-    if b.s[b.pos] == '(':
-      inc(b.pos)
-      if b.s[b.pos] == ')': inc(b.pos)
-      else: internalError(g.config, info, "decodeType ^(" & b.s[b.pos])
-      rawAddSon(result, nil)
-    else:
-      let d = decodeVInt(b.s, b.pos)
-      result.sons.add loadType(g, d, info)
-
-proc decodeLib(g; b; info: TLineInfo): PLib =
-  result = nil
-  if b.s[b.pos] == '|':
-    new(result)
-    inc(b.pos)
-    result.kind = TLibKind(decodeVInt(b.s, b.pos))
-    if b.s[b.pos] != '|': internalError(g.config, "decodeLib: 1")
-    inc(b.pos)
-    result.name = rope(decodeStr(b.s, b.pos))
-    if b.s[b.pos] != '|': internalError(g.config, "decodeLib: 2")
-    inc(b.pos)
-    result.path = decodeNode(g, b, info)
-
-proc decodeInstantiations(g; b; info: TLineInfo;
-                          s: var seq[PInstantiation]) =
-  while b.s[b.pos] == '\15':
-    inc(b.pos)
-    var ii: PInstantiation
-    new ii
-    ii.sym = loadSym(g, decodeVInt(b.s, b.pos), info)
-    ii.concreteTypes = @[]
-    while b.s[b.pos] == '\17':
-      inc(b.pos)
-      ii.concreteTypes.add loadType(g, decodeVInt(b.s, b.pos), info)
-    if b.s[b.pos] == '\20':
-      inc(b.pos)
-      ii.compilesId = decodeVInt(b.s, b.pos)
-    s.add ii
-
-proc loadSymFromBlob(g; b; info: TLineInfo): PSym =
-  if b.s[b.pos] == '{':
-    inc(b.pos)
-    if b.s[b.pos] == '}':
-      inc(b.pos)
-      return                  # nil sym
-  var k = TSymKind(decodeVInt(b.s, b.pos))
-  var id: int
-  if b.s[b.pos] == '+':
-    inc(b.pos)
-    id = decodeVInt(b.s, b.pos)
-    setId(id)
-  else:
-    internalError(g.config, info, "decodeSym: no id")
-  var ident: PIdent
-  if b.s[b.pos] == '&':
-    inc(b.pos)
-    ident = g.cache.getIdent(decodeStr(b.s, b.pos))
-  else:
-    internalError(g.config, info, "decodeSym: no ident")
-  #echo "decoding: {", ident.s
-  result = PSym(id: id, kind: k, name: ident)
-  # read the rest of the symbol description:
-  g.incr.r.syms.add(result.id, result)
-  if b.s[b.pos] == '^':
-    inc(b.pos)
-    result.typ = loadType(g, decodeVInt(b.s, b.pos), info)
-  decodeLineInfo(g, b, result.info)
-  if b.s[b.pos] == '*':
-    inc(b.pos)
-    result.owner = loadSym(g, decodeVInt(b.s, b.pos), result.info)
-  if b.s[b.pos] == '$':
-    inc(b.pos)
-    result.flags = cast[TSymFlags](decodeVBiggestInt(b.s, b.pos))
-  if b.s[b.pos] == '@':
-    inc(b.pos)
-    result.magic = TMagic(decodeVInt(b.s, b.pos))
-  if b.s[b.pos] == '!':
-    inc(b.pos)
-    result.options = cast[TOptions](int32(decodeVInt(b.s, b.pos)))
-  if b.s[b.pos] == '%':
-    inc(b.pos)
-    result.position = decodeVInt(b.s, b.pos)
-  if b.s[b.pos] == '`':
-    inc(b.pos)
-    result.offset = decodeVInt(b.s, b.pos)
-  else:
-    result.offset = -1
-  decodeLoc(g, b, result.loc, result.info)
-  result.annex = decodeLib(g, b, info)
-  if b.s[b.pos] == '#':
-    inc(b.pos)
-    result.constraint = decodeNode(g, b, unknownLineInfo)
-  case result.kind
-  of skType, skGenericParam:
-    while b.s[b.pos] == '\14':
-      inc(b.pos)
-      result.typeInstCache.add loadType(g, decodeVInt(b.s, b.pos), result.info)
-  of routineKinds:
-    decodeInstantiations(g, b, result.info, result.procInstCache)
-    if b.s[b.pos] == '\16':
-      inc(b.pos)
-      result.gcUnsafetyReason = loadSym(g, decodeVInt(b.s, b.pos), result.info)
-    if b.s[b.pos] == '\24':
-      inc b.pos
-      result.transformedBody = decodeNode(g, b, result.info)
-      #result.transformedBody = nil
-  of skModule, skPackage:
-    decodeInstantiations(g, b, result.info, result.usedGenerics)
-  of skLet, skVar, skField, skForVar:
-    if b.s[b.pos] == '\18':
-      inc(b.pos)
-      result.guard = loadSym(g, decodeVInt(b.s, b.pos), result.info)
-    if b.s[b.pos] == '\19':
-      inc(b.pos)
-      result.bitsize = decodeVInt(b.s, b.pos).int16
-  else: discard
-
-  if b.s[b.pos] == '(':
-    #if result.kind in routineKinds:
-    #  result.ast = nil
-    #else:
-    result.ast = decodeNode(g, b, result.info)
-  if sfCompilerProc in result.flags:
-    registerCompilerProc(g, result)
-    #echo "loading ", result.name.s
-
-proc loadSym(g; id: int; info: TLineInfo): PSym =
-  result = g.incr.r.syms.getOrDefault(id)
-  if result != nil: return result
-  var b = loadBlob(g, sql"select data from syms where nimid = ?", id)
-  result = loadSymFromBlob(g, b, info)
-  doAssert id == result.id, "symbol ID is not consistent!"
-
-proc registerModule*(g; module: PSym) =
-  g.incr.r.syms.add(abs module.id, module)
-
-proc loadModuleSymTab(g; module: PSym) =
-  ## goal: fill  module.tab
-  g.incr.r.syms.add(module.id, module)
-  for row in db.fastRows(sql"select nimid, data from syms where module = ? and exported = 1", abs(module.id)):
-    let id = parseInt(row[0])
-    var s = g.incr.r.syms.getOrDefault(id)
-    if s == nil:
-      var b = BlobReader(pos: 0)
-      shallowCopy(b.s, row[1])
-      # ensure we can read without index checks:
-      b.s.add '\0'
-      s = loadSymFromBlob(g, b, module.info)
-    assert s != nil
-    if s.kind != skField:
-      strTableAdd(module.tab, s)
-  if sfSystemModule in module.flags:
-    g.systemModule = module
-
-proc replay(g: ModuleGraph; module: PSym; n: PNode) =
-  # XXX check if we need to replay nkStaticStmt here.
-  case n.kind
-  #of nkStaticStmt:
-    #evalStaticStmt(module, g, n[0], module)
-    #of nkVarSection, nkLetSection:
-    #  nkVarSections are already covered by the vmgen which produces nkStaticStmt
-  of nkMethodDef:
-    methodDef(g, n[namePos].sym, fromCache=true)
-  of nkCommentStmt:
-    # pragmas are complex and can be user-overriden via templates. So
-    # instead of using the original ``nkPragma`` nodes, we rely on the
-    # fact that pragmas.nim was patched to produce specialized recorded
-    # statements for us in the form of ``nkCommentStmt`` with (key, value)
-    # pairs. Ordinary nkCommentStmt nodes never have children so this is
-    # not ambiguous.
-    # Fortunately only a tiny subset of the available pragmas need to
-    # be replayed here. This is always a subset of ``pragmas.stmtPragmas``.
-    if n.len >= 2:
-      internalAssert g.config, n[0].kind == nkStrLit and n[1].kind == nkStrLit
-      case n[0].strVal
-      of "hint": message(g.config, n.info, hintUser, n[1].strVal)
-      of "warning": message(g.config, n.info, warnUser, n[1].strVal)
-      of "error": localError(g.config, n.info, errUser, n[1].strVal)
-      of "compile":
-        internalAssert g.config, n.len == 4 and n[2].kind == nkStrLit
-        let cname = AbsoluteFile n[1].strVal
-        var cf = Cfile(nimname: splitFile(cname).name, cname: cname,
-                       obj: AbsoluteFile n[2].strVal,
-                       flags: {CfileFlag.External},
-                       customArgs: n[3].strVal)
-        extccomp.addExternalFileToCompile(g.config, cf)
-      of "link":
-        extccomp.addExternalFileToLink(g.config, AbsoluteFile n[1].strVal)
-      of "passl":
-        extccomp.addLinkOption(g.config, n[1].strVal)
-      of "passc":
-        extccomp.addCompileOption(g.config, n[1].strVal)
-      of "localpassc":
-        extccomp.addLocalCompileOption(g.config, n[1].strVal, toFullPathConsiderDirty(g.config, module.info.fileIndex))
-      of "cppdefine":
-        options.cppDefine(g.config, n[1].strVal)
-      of "inc":
-        let destKey = n[1].strVal
-        let by = n[2].intVal
-        let v = getOrDefault(g.cacheCounters, destKey)
-        g.cacheCounters[destKey] = v+by
-      of "put":
-        let destKey = n[1].strVal
-        let key = n[2].strVal
-        let val = n[3]
-        if not contains(g.cacheTables, destKey):
-          g.cacheTables[destKey] = initBTree[string, PNode]()
-        if not contains(g.cacheTables[destKey], key):
-          g.cacheTables[destKey].add(key, val)
-        else:
-          internalError(g.config, n.info, "key already exists: " & key)
-      of "incl":
-        let destKey = n[1].strVal
-        let val = n[2]
-        if not contains(g.cacheSeqs, destKey):
-          g.cacheSeqs[destKey] = newTree(nkStmtList, val)
-        else:
-          block search:
-            for existing in g.cacheSeqs[destKey]:
-              if exprStructuralEquivalent(existing, val, strictSymEquality=true):
-                break search
-            g.cacheSeqs[destKey].add val
-      of "add":
-        let destKey = n[1].strVal
-        let val = n[2]
-        if not contains(g.cacheSeqs, destKey):
-          g.cacheSeqs[destKey] = newTree(nkStmtList, val)
-        else:
-          g.cacheSeqs[destKey].add val
-      else:
-        internalAssert g.config, false
-  of nkImportStmt:
-    for x in n:
-      internalAssert g.config, x.kind == nkSym
-      let modpath = AbsoluteFile toFullPath(g.config, x.sym.info)
-      let imported = g.importModuleCallback(g, module, fileInfoIdx(g.config, modpath))
-      internalAssert g.config, imported.id < 0
-  of nkStmtList, nkStmtListExpr:
-    for x in n: replay(g, module, x)
-  of nkExportStmt:
-    for x in n:
-      doAssert x.kind == nkSym
-      strTableAdd(module.tab, x.sym)
-  else: discard "nothing to do for this node"
-
-proc loadNode*(g: ModuleGraph; module: PSym): PNode =
-  loadModuleSymTab(g, module)
-  result = newNodeI(nkStmtList, module.info)
-  for row in db.rows(sql"select data from toplevelstmts where module = ? order by position asc",
-                        abs module.id):
-    var b = BlobReader(pos: 0)
-    # ensure we can read without index checks:
-    b.s = row[0] & '\0'
-    result.add decodeNode(g, b, module.info)
-  db.exec(sql"insert into controlblock(idgen) values (?)", gFrontEndId)
-  replay(g, module, result)
-
-proc setupModuleCache*(g: ModuleGraph) =
-  # historical note: there used to be a `rodfiles` dir with special tests
-  # for incremental compilation via symbol files. This was likely replaced by ic.
-  if g.config.symbolFiles == disabledSf: return
-  g.recordStmt = recordStmt
-  let dbfile = getNimcacheDir(g.config) / RelativeFile"rodfiles.db"
-  if g.config.symbolFiles == writeOnlySf:
-    removeFile(dbfile)
-  createDir getNimcacheDir(g.config)
-  let ec = encodeConfig(g)
-  if not fileExists(dbfile):
-    db = open(connection=string dbfile, user="nim", password="",
-              database="nim")
-    createDb(db)
-    db.exec(sql"insert into config(config) values (?)", ec)
-  else:
-    db = open(connection=string dbfile, user="nim", password="",
-              database="nim")
-    let oldConfig = db.getValue(sql"select config from config")
-    g.incr.configChanged = oldConfig != ec
-    # ensure the filename IDs stay consistent:
-    for row in db.rows(sql"select fullpath, nimid from filenames order by nimid"):
-      let id = fileInfoIdx(g.config, AbsoluteFile row[0])
-      doAssert id.int == parseInt(row[1])
-    db.exec(sql"update config set config = ?", ec)
-  db.exec(sql"pragma journal_mode=off")
-  # This MUST be turned off, otherwise it's way too slow even for testing purposes:
-  db.exec(sql"pragma SYNCHRONOUS=off")
-  db.exec(sql"pragma LOCKING_MODE=exclusive")
-  let lastId = db.getValue(sql"select max(idgen) from controlblock")
-  if lastId.len > 0:
-    idgen.setId(parseInt lastId)

compiler/sem.nim

@@ -16,7 +16,7 @@ import
   procfind, lookups, pragmas, passes, semdata, semtypinst, sigmatch,
   intsets, transf, vmdef, vm, aliases, cgmeth, lambdalifting,
   evaltempl, patterns, parampatterns, sempass2, linter, semmacrosanity,
-  lowerings, plugins/active, rod, lineinfos, strtabs, int128,
+  lowerings, plugins/active, lineinfos, strtabs, int128,
   isolation_check, typeallowed
 
 from modulegraphs import ModuleGraph, PPassContext, onUse, onDef, onDefResolveForward
@@ -124,8 +124,8 @@ proc commonType*(c: PContext; x, y: PType): PType =
     # turn any concrete typedesc into the abstract typedesc type
     if a.len == 0: result = a
     else:
-      result = newType(tyTypeDesc, nextId(c.idgen), a.owner)
+      result = newType(tyTypeDesc, nextTypeId(c.idgen), a.owner)
-      rawAddSon(result, newType(tyNone, nextId(c.idgen), a.owner))
+      rawAddSon(result, newType(tyNone, nextTypeId(c.idgen), a.owner))
   elif b.kind in {tyArray, tySet, tySequence} and
       a.kind == b.kind:
     # check for seq[empty] vs. seq[int]
@@ -137,7 +137,7 @@ proc commonType*(c: PContext; x, y: PType): PType =
       let aEmpty = isEmptyContainer(a[i])
       let bEmpty = isEmptyContainer(b[i])
       if aEmpty != bEmpty:
-        if nt.isNil: nt = copyType(a, nextId(c.idgen), a.owner)
+        if nt.isNil: nt = copyType(a, nextTypeId(c.idgen), a.owner)
         nt[i] = if aEmpty: b[i] else: a[i]
     if not nt.isNil: result = nt
     #elif b[idx].kind == tyEmpty: return x
@@ -176,7 +176,7 @@ proc commonType*(c: PContext; x, y: PType): PType =
       # ill-formed AST, no need for additional tyRef/tyPtr
       if k != tyNone and x.kind != tyGenericInst:
         let r = result
-        result = newType(k, nextId(c.idgen), r.owner)
+        result = newType(k, nextTypeId(c.idgen), r.owner)
         result.addSonSkipIntLit(r, c.idgen)
 
 proc endsInNoReturn(n: PNode): bool =
@@ -193,7 +193,7 @@ proc commonType*(c: PContext; x: PType, y: PNode): PType =
   commonType(c, x, y.typ)
 
 proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym =
-  result = newSym(kind, considerQuotedIdent(c, n), nextId c.idgen, getCurrOwner(c), n.info)
+  result = newSym(kind, considerQuotedIdent(c, n), nextSymId c.idgen, getCurrOwner(c), n.info)
   when defined(nimsuggest):
     suggestDecl(c, n, result)
 
@@ -216,7 +216,7 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
     # template; we must fix it here: see #909
     result.owner = getCurrOwner(c)
   else:
-    result = newSym(kind, considerQuotedIdent(c, n), nextId c.idgen, getCurrOwner(c), n.info)
+    result = newSym(kind, considerQuotedIdent(c, n), nextSymId c.idgen, getCurrOwner(c), n.info)
   #if kind in {skForVar, skLet, skVar} and result.owner.kind == skModule:
   #  incl(result.flags, sfGlobal)
   when defined(nimsuggest):
@@ -255,7 +255,7 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
 
 proc symFromType(c: PContext; t: PType, info: TLineInfo): PSym =
   if t.sym != nil: return t.sym
-  result = newSym(skType, getIdent(c.cache, "AnonType"), nextId c.idgen, t.owner, info)
+  result = newSym(skType, getIdent(c.cache, "AnonType"), nextSymId c.idgen, t.owner, info)
   result.flags.incl sfAnon
   result.typ = t
 
@@ -616,7 +616,7 @@ proc myProcess(context: PPassContext, n: PNode): PNode {.nosinks.} =
       else:
         result = newNodeI(nkEmpty, n.info)
       #if c.config.cmd == cmdIdeTools: findSuggest(c, n)
-  rod.storeNode(c.graph, c.module, result)
+  storeRodNode(c, result)
 
 proc reportUnusedModules(c: PContext) =
   for i in 0..high(c.unusedImports):
@@ -638,7 +638,7 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
     result.add(c.module.ast)
   popOwner(c)
   popProcCon(c)
-  storeRemaining(c.graph, c.module)
+  saveRodFile(c)
 
 const semPass* = makePass(myOpen, myProcess, myClose,
                           isFrontend = true)

compiler/semdata.nim

@@ -11,7 +11,9 @@
 
 import
   intsets, options, ast, astalgo, msgs, idents, renderer,
-  magicsys, vmdef, modulegraphs, lineinfos, sets
+  magicsys, vmdef, modulegraphs, lineinfos, sets, pathutils
+
+import ic / to_packed_ast
 
 type
   TOptionEntry* = object      # entries to put on a stack for pragma parsing
@@ -77,9 +79,9 @@ type
     case mode*: ImportMode
     of importAll: discard
     of importSet:
-      imported*: IntSet
+      imported*: IntSet          # of PIdent.id
     of importExcept:
-      exceptSet*: IntSet
+      exceptSet*: IntSet         # of PIdent.id
 
   PContext* = ref TContext
   TContext* = object of TPassContext # a context represents the module
@@ -140,6 +142,7 @@ type
     selfName*: PIdent
     cache*: IdentCache
     graph*: ModuleGraph
+    encoder*: PackedEncoder
     signatures*: TStrTable
     recursiveDep*: string
     suggestionsMade*: bool
@@ -264,6 +267,16 @@ proc newContext*(graph: ModuleGraph; module: PSym): PContext =
   initStrTable(result.signatures)
   result.typesWithOps = @[]
   result.features = graph.config.features
+  if graph.config.symbolFiles != disabledSf:
+    initEncoder result.encoder, module, graph.config
+
+proc addIncludeFileDep*(c: PContext; f: FileIndex) =
+  if c.config.symbolFiles != disabledSf:
+    addIncludeFileDep(c.encoder, f)
+
+proc addImportFileDep*(c: PContext; f: FileIndex) =
+  if c.config.symbolFiles != disabledSf:
+    addImportFileDep(c.encoder, f)
 
 proc inclSym(sq: var seq[PSym], s: PSym) =
   for i in 0..<sq.len:
@@ -273,6 +286,7 @@ proc inclSym(sq: var seq[PSym], s: PSym) =
 proc addConverter*(c: PContext, conv: PSym) =
   inclSym(c.converters, conv)
   inclSym(c.graph.ifaces[c.module.position].converters, conv)
+  #addConverter(c.graph, c.module, conv) # upcoming
 
 proc addPureEnum*(c: PContext, e: PSym) =
   inclSym(c.graph.ifaces[c.module.position].pureEnums, e)
@@ -280,6 +294,7 @@ proc addPureEnum*(c: PContext, e: PSym) =
 proc addPattern*(c: PContext, p: PSym) =
   inclSym(c.patterns, p)
   inclSym(c.graph.ifaces[c.module.position].patterns, p)
+  #addPattern(c.graph, c.module, p) # upcoming
 
 proc newLib*(kind: TLibKind): PLib =
   new(result)
@@ -291,10 +306,10 @@ proc addToLib*(lib: PLib, sym: PSym) =
   sym.annex = lib
 
 proc newTypeS*(kind: TTypeKind, c: PContext): PType =
-  result = newType(kind, nextId(c.idgen), getCurrOwner(c))
+  result = newType(kind, nextTypeId(c.idgen), getCurrOwner(c))
 
 proc makePtrType*(owner: PSym, baseType: PType; idgen: IdGenerator): PType =
-  result = newType(tyPtr, nextId(idgen), owner)
+  result = newType(tyPtr, nextTypeId(idgen), owner)
   addSonSkipIntLit(result, baseType, idgen)
 
 proc makePtrType*(c: PContext, baseType: PType): PType =
@@ -322,7 +337,7 @@ proc makeVarType*(owner: PSym, baseType: PType; idgen: IdGenerator; kind = tyVar
   if baseType.kind == kind:
     result = baseType
   else:
-    result = newType(kind, nextId(idgen), owner)
+    result = newType(kind, nextTypeId(idgen), owner)
     addSonSkipIntLit(result, baseType, idgen)
 
 proc makeTypeDesc*(c: PContext, typ: PType): PType =
@@ -338,7 +353,7 @@ proc makeTypeSymNode*(c: PContext, typ: PType, info: TLineInfo): PNode =
   incl typedesc.flags, tfCheckedForDestructor
   internalAssert(c.config, typ != nil)
   typedesc.addSonSkipIntLit(typ, c.idgen)
-  let sym = newSym(skType, c.cache.idAnon, nextId(c.idgen), getCurrOwner(c), info,
+  let sym = newSym(skType, c.cache.idAnon, nextSymId(c.idgen), getCurrOwner(c), info,
                    c.config.options).linkTo(typedesc)
   return newSymNode(sym, info)
 
@@ -349,12 +364,12 @@ proc makeTypeFromExpr*(c: PContext, n: PNode): PType =
 
 proc newTypeWithSons*(owner: PSym, kind: TTypeKind, sons: seq[PType];
                       idgen: IdGenerator): PType =
-  result = newType(kind, nextId(idgen), owner)
+  result = newType(kind, nextTypeId(idgen), owner)
   result.sons = sons
 
 proc newTypeWithSons*(c: PContext, kind: TTypeKind,
                       sons: seq[PType]): PType =
-  result = newType(kind, nextId(c.idgen), getCurrOwner(c))
+  result = newType(kind, nextTypeId(c.idgen), getCurrOwner(c))
   result.sons = sons
 
 proc makeStaticExpr*(c: PContext, n: PNode): PNode =
@@ -463,3 +478,15 @@ proc popCaseContext*(c: PContext) =
 
 proc setCaseContextIdx*(c: PContext, idx: int) =
   c.p.caseContext[^1].idx = idx
+
+template addExport*(c: PContext; s: PSym) =
+  ## convenience to export a symbol from the current module
+  addExport(c.graph, c.module, s)
+
+proc storeRodNode*(c: PContext, n: PNode) =
+  if c.config.symbolFiles != disabledSf:
+    toPackedNodeTopLevel(n, c.encoder)
+
+proc saveRodFile*(c: PContext) =
+  if c.config.symbolFiles != disabledSf:
+    saveRodFile(toRodFile(c.config, c.filename.AbsoluteFile), c.encoder)

compiler/semexprs.nim

@@ -1119,7 +1119,7 @@ proc readTypeParameter(c: PContext, typ: PType,
             # This seems semantically correct and then we'll be able
             # to return the section symbol directly here
             let foundType = makeTypeDesc(c, def[2].typ)
-            return newSymNode(copySym(def[0].sym, nextId c.idgen).linkTo(foundType), info)
+            return newSymNode(copySym(def[0].sym, nextSymId c.idgen).linkTo(foundType), info)
 
       of nkConstSection:
         for def in statement:
@@ -1144,7 +1144,7 @@ proc readTypeParameter(c: PContext, typ: PType,
             return c.graph.emptyNode
         else:
           let foundTyp = makeTypeDesc(c, rawTyp)
-          return newSymNode(copySym(tParam.sym, nextId c.idgen).linkTo(foundTyp), info)
+          return newSymNode(copySym(tParam.sym, nextSymId c.idgen).linkTo(foundTyp), info)
 
   return nil
 
@@ -1941,7 +1941,7 @@ proc expectString(c: PContext, n: PNode): string =
     localError(c.config, n.info, errStringLiteralExpected)
 
 proc newAnonSym(c: PContext; kind: TSymKind, info: TLineInfo): PSym =
-  result = newSym(kind, c.cache.idAnon, nextId c.idgen, getCurrOwner(c), info)
+  result = newSym(kind, c.cache.idAnon, nextSymId c.idgen, getCurrOwner(c), info)
 
 proc semExpandToAst(c: PContext, n: PNode): PNode =
   let macroCall = n[1]
@@ -2156,7 +2156,7 @@ proc semShallowCopy(c: PContext, n: PNode, flags: TExprFlags): PNode =
     result = semDirectOp(c, n, flags)
 
 proc createFlowVar(c: PContext; t: PType; info: TLineInfo): PType =
-  result = newType(tyGenericInvocation, nextId c.idgen, c.module)
+  result = newType(tyGenericInvocation, nextTypeId c.idgen, c.module)
   addSonSkipIntLit(result, magicsys.getCompilerProc(c.graph, "FlowVar").typ, c.idgen)
   addSonSkipIntLit(result, t, c.idgen)
   result = instGenericContainer(c, info, result, allowMetaTypes = false)
@@ -2631,7 +2631,7 @@ proc hoistParamsUsedInDefault(c: PContext, call, letSection, defExpr: var PNode)
     let paramPos = defExpr.sym.position + 1
 
     if call[paramPos].kind != nkSym:
-      let hoistedVarSym = newSym(skLet, getIdent(c.graph.cache, genPrefix), nextId c.idgen,
+      let hoistedVarSym = newSym(skLet, getIdent(c.graph.cache, genPrefix), nextSymId c.idgen,
                                  c.p.owner, letSection.info, c.p.owner.options)
       hoistedVarSym.typ = call[paramPos].typ
 

compiler/semfields.nim

@@ -109,7 +109,7 @@ proc semForFields(c: PContext, n: PNode, m: TMagic): PNode =
   var trueSymbol = strTableGet(c.graph.systemModule.tab, getIdent(c.cache, "true"))
   if trueSymbol == nil:
     localError(c.config, n.info, "system needs: 'true'")
-    trueSymbol = newSym(skUnknown, getIdent(c.cache, "true"), nextId c.idgen, getCurrOwner(c), n.info)
+    trueSymbol = newSym(skUnknown, getIdent(c.cache, "true"), nextSymId c.idgen, getCurrOwner(c), n.info)
     trueSymbol.typ = getSysType(c.graph, n.info, tyBool)
 
   result[0] = newSymNode(trueSymbol, n.info)

compiler/semfold.nim

@@ -17,7 +17,7 @@ import
 
 proc errorType*(g: ModuleGraph): PType =
   ## creates a type representing an error state
-  result = newType(tyError, nextId(g.idgen), g.owners[^1])
+  result = newType(tyError, nextTypeId(g.idgen), g.owners[^1])
   result.flags.incl tfCheckedForDestructor
 
 proc newIntNodeT*(intVal: Int128, n: PNode; g: ModuleGraph): PNode =
@@ -476,7 +476,7 @@ proc foldConStrStr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
 proc newSymNodeTypeDesc*(s: PSym; idgen: IdGenerator; info: TLineInfo): PNode =
   result = newSymNode(s, info)
   if s.typ.kind != tyTypeDesc:
-    result.typ = newType(tyTypeDesc, idgen.nextId, s.owner)
+    result.typ = newType(tyTypeDesc, idgen.nextTypeId, s.owner)
     result.typ.addSonSkipIntLit(s.typ, idgen)
   else:
     result.typ = s.typ

compiler/semgnrc.nim

@@ -472,7 +472,7 @@ proc semGenericStmt(c: PContext, n: PNode,
                                               flags, ctx)
     if n[paramsPos].kind != nkEmpty:
       if n[paramsPos][0].kind != nkEmpty:
-        addPrelimDecl(c, newSym(skUnknown, getIdent(c.cache, "result"), nextId c.idgen, nil, n.info))
+        addPrelimDecl(c, newSym(skUnknown, getIdent(c.cache, "result"), nextSymId c.idgen, nil, n.info))
       n[paramsPos] = semGenericStmt(c, n[paramsPos], flags, ctx)
     n[pragmasPos] = semGenericStmt(c, n[pragmasPos], flags, ctx)
     var body: PNode

compiler/seminst.nim

@@ -65,7 +65,7 @@ iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable): PSym
     if q.typ.kind notin {tyTypeDesc, tyGenericParam, tyStatic}+tyTypeClasses:
       continue
     let symKind = if q.typ.kind == tyStatic: skConst else: skType
-    var s = newSym(symKind, q.name, nextId c.idgen, getCurrOwner(c), q.info)
+    var s = newSym(symKind, q.name, nextSymId c.idgen, getCurrOwner(c), q.info)
     s.flags.incl {sfUsed, sfFromGeneric}
     var t = PType(idTableGet(pt, q.typ))
     if t == nil:
@@ -118,7 +118,7 @@ proc freshGenSyms(c: PContext; n: PNode, owner, orig: PSym, symMap: var TIdTable
       n.sym = x
     elif s.owner == nil or s.owner.kind == skPackage:
       #echo "copied this ", s.name.s
-      x = copySym(s, nextId c.idgen)
+      x = copySym(s, nextSymId c.idgen)
       x.owner = owner
       idTablePut(symMap, s, x)
       n.sym = x
@@ -200,7 +200,7 @@ proc instGenericContainer(c: PContext, info: TLineInfo, header: PType,
     var param: PSym
 
     template paramSym(kind): untyped =
-      newSym(kind, genParam.sym.name, nextId c.idgen, genericTyp.sym, genParam.sym.info)
+      newSym(kind, genParam.sym.name, nextSymId c.idgen, genericTyp.sym, genParam.sym.info)
 
     if genParam.kind == tyStatic:
       param = paramSym skConst
@@ -270,7 +270,7 @@ proc instantiateProcType(c: PContext, pt: TIdTable,
 
     internalAssert c.config, originalParams[i].kind == nkSym
     let oldParam = originalParams[i].sym
-    let param = copySym(oldParam, nextId c.idgen)
+    let param = copySym(oldParam, nextSymId c.idgen)
     param.owner = prc
     param.typ = result[i]
 
@@ -339,7 +339,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   c.matchedConcept = nil
   let oldScope = c.currentScope
   while not isTopLevel(c): c.currentScope = c.currentScope.parent
-  result = copySym(fn, nextId c.idgen)
+  result = copySym(fn, nextSymId c.idgen)
   incl(result.flags, sfFromGeneric)
   result.owner = fn
   result.ast = n
@@ -401,3 +401,6 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   dec(c.instCounter)
   c.matchedConcept = oldMatchedConcept
   if result.kind == skMethod: finishMethod(c, result)
+
+  # inform IC of the generic
+  #addGeneric(c.ic, result, entry.concreteTypes)

compiler/semmagic.nim

@@ -119,7 +119,7 @@ proc uninstantiate(t: PType): PType =
     else: t
 
 proc getTypeDescNode(c: PContext; typ: PType, sym: PSym, info: TLineInfo): PNode =
-  var resType = newType(tyTypeDesc, nextId c.idgen, sym)
+  var resType = newType(tyTypeDesc, nextTypeId c.idgen, sym)
   rawAddSon(resType, typ)
   result = toNode(resType, info)
 
@@ -160,7 +160,7 @@ proc evalTypeTrait(c: PContext; traitCall: PNode, operand: PType, context: PSym)
     result.info = traitCall.info
   of "arity":
     result = newIntNode(nkIntLit, operand.len - ord(operand.kind==tyProc))
-    result.typ = newType(tyInt, nextId c.idgen, context)
+    result.typ = newType(tyInt, nextTypeId c.idgen, context)
     result.info = traitCall.info
   of "genericHead":
     var arg = operand
@@ -172,7 +172,7 @@ proc evalTypeTrait(c: PContext; traitCall: PNode, operand: PType, context: PSym)
     #   result = toNode(resType, traitCall.info) # doesn't work yet
     else:
       localError(c.config, traitCall.info, "expected generic type, got: type $2 of kind $1" % [arg.kind.toHumanStr, typeToString(operand)])
-      result = newType(tyError, nextId c.idgen, context).toNode(traitCall.info)
+      result = newType(tyError, nextTypeId c.idgen, context).toNode(traitCall.info)
   of "stripGenericParams":
     result = uninstantiate(operand).toNode(traitCall.info)
   of "supportsCopyMem":
@@ -198,7 +198,7 @@ proc evalTypeTrait(c: PContext; traitCall: PNode, operand: PType, context: PSym)
     else:
       localError(c.config, traitCall.info,
         "distinctBase expects a distinct type as argument. The given type was " & typeToString(operand))
-      result = newType(tyError, nextId c.idgen, context).toNode(traitCall.info)
+      result = newType(tyError, nextTypeId c.idgen, context).toNode(traitCall.info)
   else:
     localError(c.config, traitCall.info, "unknown trait: " & s)
     result = newNodeI(nkEmpty, traitCall.info)
@@ -370,7 +370,7 @@ proc semUnown(c: PContext; n: PNode): PNode =
         elems[i] = unownedType(c, t[i])
         if elems[i] != t[i]: someChange = true
       if someChange:
-        result = newType(tyTuple, nextId c.idgen, t.owner)
+        result = newType(tyTuple, nextTypeId c.idgen, t.owner)
         # we have to use 'rawAddSon' here so that type flags are
         # properly computed:
         for e in elems: result.rawAddSon(e)
@@ -381,7 +381,7 @@ proc semUnown(c: PContext; n: PNode): PNode =
        tyGenericInst, tyAlias:
       let b = unownedType(c, t[^1])
       if b != t[^1]:
-        result = copyType(t, nextId c.idgen, t.owner)
+        result = copyType(t, nextTypeId c.idgen, t.owner)
         result[^1] = b
         result.flags.excl tfHasOwned
       else:
@@ -407,26 +407,26 @@ proc turnFinalizerIntoDestructor(c: PContext; orig: PSym; info: TLineInfo): PSym
       if n.sym == oldParam:
         result.sym = newParam
       elif n.sym.owner == orig:
-        result.sym = copySym(n.sym, nextId c.idgen)
+        result.sym = copySym(n.sym, nextSymId c.idgen)
         result.sym.owner = procSym
     for i in 0 ..< safeLen(n):
       result[i] = transform(c, procSym, n[i], old, fresh, oldParam, newParam)
     #if n.kind == nkDerefExpr and sameType(n[0].typ, old):
     #  result =
 
-  result = copySym(orig, nextId c.idgen)
+  result = copySym(orig, nextSymId c.idgen)
   result.info = info
   result.flags.incl sfFromGeneric
   result.owner = orig
   let origParamType = orig.typ[1]
   let newParamType = makeVarType(result, origParamType.skipTypes(abstractPtrs), c.idgen)
   let oldParam = orig.typ.n[1].sym
-  let newParam = newSym(skParam, oldParam.name, nextId c.idgen, result, result.info)
+  let newParam = newSym(skParam, oldParam.name, nextSymId c.idgen, result, result.info)
   newParam.typ = newParamType
   # proc body:
   result.ast = transform(c, result, orig.ast, origParamType, newParamType, oldParam, newParam)
   # proc signature:
-  result.typ = newProcType(result.info, nextId c.idgen, result)
+  result.typ = newProcType(result.info, nextTypeId c.idgen, result)
   result.typ.addParam newParam
 
 proc semQuantifier(c: PContext; n: PNode): PNode =

compiler/semparallel.nim

@@ -482,7 +482,7 @@ proc liftParallel*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; n: PNode): P
   checkArgs(a, body)
 
   var varSection = newNodeI(nkVarSection, n.info)
-  var temp = newSym(skTemp, getIdent(g.cache, "barrier"), nextId idgen, owner, n.info)
+  var temp = newSym(skTemp, getIdent(g.cache, "barrier"), nextSymId idgen, owner, n.info)
   temp.typ = magicsys.getCompilerProc(g, "Barrier").typ
   incl(temp.flags, sfFromGeneric)
   let tempNode = newSymNode(temp)

compiler/sempass2.nim

@@ -219,7 +219,7 @@ proc markGcUnsafe(a: PEffects; reason: PNode) =
       if reason.kind == nkSym:
         a.owner.gcUnsafetyReason = reason.sym
       else:
-        a.owner.gcUnsafetyReason = newSym(skUnknown, a.owner.name, nextId a.c.idgen,
+        a.owner.gcUnsafetyReason = newSym(skUnknown, a.owner.name, nextSymId a.c.idgen,
                                           a.owner, reason.info, {})
 
 when true:

compiler/semstmts.nim

@@ -410,7 +410,7 @@ proc fillPartialObject(c: PContext; n: PNode; typ: PType) =
     let y = considerQuotedIdent(c, n[1])
     let obj = x.typ.skipTypes(abstractPtrs)
     if obj.kind == tyObject and tfPartial in obj.flags:
-      let field = newSym(skField, getIdent(c.cache, y.s), nextId c.idgen, obj.sym, n[1].info)
+      let field = newSym(skField, getIdent(c.cache, y.s), nextSymId c.idgen, obj.sym, n[1].info)
       field.typ = skipIntLit(typ, c.idgen)
       field.position = obj.n.len
       obj.n.add newSymNode(field)
@@ -1278,7 +1278,7 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) =
       internalAssert c.config, st.lastSon.sym == nil
       incl st.flags, tfRefsAnonObj
       let obj = newSym(skType, getIdent(c.cache, s.name.s & ":ObjectType"),
-                       nextId c.idgen, getCurrOwner(c), s.info)
+                       nextSymId c.idgen, getCurrOwner(c), s.info)
       let symNode = newSymNode(obj)
       obj.ast = a.shallowCopy
       case a[0].kind
@@ -1449,7 +1449,7 @@ proc addResult(c: PContext, n: PNode, t: PType, owner: TSymKind) =
         localError(c.config, n.info, "incorrect result proc symbol")
       c.p.resultSym = n[resultPos].sym
     else:
-      var s = newSym(skResult, getIdent(c.cache, "result"), nextId c.idgen, getCurrOwner(c), n.info)
+      var s = newSym(skResult, getIdent(c.cache, "result"), nextSymId c.idgen, getCurrOwner(c), n.info)
       s.typ = t
       incl(s.flags, sfUsed)
       c.p.resultSym = s
@@ -1548,7 +1548,7 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
   checkSonsLen(n, bodyPos + 1, c.config)
   var s: PSym
   if n[namePos].kind != nkSym:
-    s = newSym(skProc, c.cache.idAnon, nextId c.idgen, getCurrOwner(c), n.info)
+    s = newSym(skProc, c.cache.idAnon, nextSymId c.idgen, getCurrOwner(c), n.info)
     s.ast = n
     n[namePos] = newSymNode(s)
   else:
@@ -1856,7 +1856,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
     assert phase == stepRegisterSymbol
 
     if n[namePos].kind == nkEmpty:
-      s = newSym(kind, c.cache.idAnon, nextId c.idgen, getCurrOwner(c), n.info)
+      s = newSym(kind, c.cache.idAnon, nextSymId c.idgen, getCurrOwner(c), n.info)
       incl(s.flags, sfUsed)
       isAnon = true
     else:
@@ -2011,7 +2011,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
         if s.kind == skMethod: semMethodPrototype(c, s, n)
       else:
         if (s.typ[0] != nil and kind != skIterator) or kind == skMacro:
-          addDecl(c, newSym(skUnknown, getIdent(c.cache, "result"), nextId c.idgen, nil, n.info))
+          addDecl(c, newSym(skUnknown, getIdent(c.cache, "result"), nextSymId c.idgen, nil, n.info))
 
         openScope(c)
         n[bodyPos] = semGenericStmt(c, n[bodyPos])
@@ -2149,6 +2149,7 @@ proc semMacroDef(c: PContext, n: PNode): PNode =
 proc incMod(c: PContext, n: PNode, it: PNode, includeStmtResult: PNode) =
   var f = checkModuleName(c.config, it)
   if f != InvalidFileIdx:
+    addIncludeFileDep(c, f)
     if containsOrIncl(c.includedFiles, f.int):
       localError(c.config, n.info, errRecursiveDependencyX % toMsgFilename(c.config, f))
     else:

compiler/semtempl.nim

@@ -176,7 +176,7 @@ proc onlyReplaceParams(c: var TemplCtx, n: PNode): PNode =
       result[i] = onlyReplaceParams(c, n[i])
 
 proc newGenSym(kind: TSymKind, n: PNode, c: var TemplCtx): PSym =
-  result = newSym(kind, considerQuotedIdent(c.c, n), nextId c.c.idgen, c.owner, n.info)
+  result = newSym(kind, considerQuotedIdent(c.c, n), nextSymId c.c.idgen, c.owner, n.info)
   incl(result.flags, sfGenSym)
   incl(result.flags, sfShadowed)
 

compiler/semtypes.nim

@@ -387,7 +387,7 @@ proc semTypeIdent(c: PContext, n: PNode): PSym =
         if result.typ.sym == nil:
           localError(c.config, n.info, errTypeExpected)
           return errorSym(c, n)
-        result = result.typ.sym.copySym(nextId c.idgen)
+        result = result.typ.sym.copySym(nextSymId c.idgen)
         result.typ = exactReplica(result.typ)
         result.typ.flags.incl tfUnresolved
 
@@ -965,7 +965,7 @@ proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind) =
   if kind == skMacro:
     let staticType = findEnforcedStaticType(param.typ)
     if staticType != nil:
-      var a = copySym(param, nextId c.idgen)
+      var a = copySym(param, nextSymId c.idgen)
       a.typ = staticType.base
       addDecl(c, a)
       #elif param.typ != nil and param.typ.kind == tyTypeDesc:
@@ -973,7 +973,7 @@ proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind) =
     else:
       # within a macro, every param has the type NimNode!
       let nn = getSysSym(c.graph, param.info, "NimNode")
-      var a = copySym(param, nextId c.idgen)
+      var a = copySym(param, nextSymId c.idgen)
       a.typ = nn.typ
       addDecl(c, a)
   else:
@@ -1003,7 +1003,7 @@ proc addImplicitGeneric(c: PContext; typeClass: PType, typId: PIdent;
 
   let owner = if typeClass.sym != nil: typeClass.sym
               else: getCurrOwner(c)
-  var s = newSym(skType, finalTypId, nextId c.idgen, owner, info)
+  var s = newSym(skType, finalTypId, nextSymId c.idgen, owner, info)
   if sfExplain in owner.flags: s.flags.incl sfExplain
   if typId == nil: s.flags.incl(sfAnon)
   s.linkTo(typeClass)
@@ -1109,7 +1109,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
 
   of tyGenericInst:
     if paramType.lastSon.kind == tyUserTypeClass:
-      var cp = copyType(paramType, nextId c.idgen, getCurrOwner(c))
+      var cp = copyType(paramType, nextTypeId c.idgen, getCurrOwner(c))
       cp.kind = tyUserTypeClassInst
       return addImplicitGeneric(c, cp, paramTypId, info, genericParams, paramName)
 
@@ -1146,7 +1146,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
   of tyUserTypeClasses, tyBuiltInTypeClass, tyCompositeTypeClass,
      tyAnd, tyOr, tyNot:
     result = addImplicitGeneric(c,
-        copyType(paramType, nextId c.idgen, getCurrOwner(c)), paramTypId,
+        copyType(paramType, nextTypeId c.idgen, getCurrOwner(c)), paramTypId,
         info, genericParams, paramName)
 
   of tyGenericParam:
@@ -1522,7 +1522,7 @@ proc semTypeExpr(c: PContext, n: PNode; prev: PType): PType =
 
 proc freshType(c: PContext; res, prev: PType): PType {.inline.} =
   if prev.isNil:
-    result = copyType(res, nextId c.idgen, res.owner)
+    result = copyType(res, nextTypeId c.idgen, res.owner)
   else:
     result = res
 
@@ -1579,7 +1579,7 @@ proc semTypeClass(c: PContext, n: PNode, prev: PType): PType =
 
     internalAssert c.config, dummyName.kind == nkIdent
     var dummyParam = newSym(if modifier == tyTypeDesc: skType else: skVar,
-                            dummyName.ident, nextId c.idgen, owner, param.info)
+                            dummyName.ident, nextSymId c.idgen, owner, param.info)
     dummyParam.typ = dummyType
     incl dummyParam.flags, sfUsed
     addDecl(c, dummyParam)
@@ -1836,7 +1836,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
     of mExpr:
       result = semTypeNode(c, n[0], nil)
       if result != nil:
-        result = copyType(result, nextId c.idgen, getCurrOwner(c))
+        result = copyType(result, nextTypeId c.idgen, getCurrOwner(c))
         for i in 1..<n.len:
           result.rawAddSon(semTypeNode(c, n[i], nil))
     of mDistinct:
@@ -2115,7 +2115,7 @@ proc semGenericParamList(c: PContext, n: PNode, father: PType = nil): PNode =
 
       for j in 0..<a.len-2:
         let finalType = if j == 0: typ
-                        else: copyType(typ, nextId c.idgen, typ.owner)
+                        else: copyType(typ, nextTypeId c.idgen, typ.owner)
                         # it's important the we create an unique
                         # type for each generic param. the index
                         # of the parameter will be stored in the

compiler/semtypinst.nim

@@ -279,7 +279,7 @@ proc replaceTypeVarsS(cl: var TReplTypeVars, s: PSym): PSym =
       var g: G[string]
 
   ]#
-  result = copySym(s, nextId cl.c.idgen)
+  result = copySym(s, nextSymId cl.c.idgen)
   incl(result.flags, sfFromGeneric)
   #idTablePut(cl.symMap, s, result)
   result.owner = s.owner
@@ -305,7 +305,7 @@ proc instCopyType*(cl: var TReplTypeVars, t: PType): PType =
   if cl.allowMetaTypes:
     result = t.exactReplica
   else:
-    result = copyType(t, nextId(cl.c.idgen), t.owner)
+    result = copyType(t, nextTypeId(cl.c.idgen), t.owner)
     #cl.typeMap.topLayer.idTablePut(result, t)
 
   if cl.allowMetaTypes: return
@@ -360,7 +360,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
   else:
     header = instCopyType(cl, t)
 
-  result = newType(tyGenericInst, nextId(cl.c.idgen), t[0].owner)
+  result = newType(tyGenericInst, nextTypeId(cl.c.idgen), t[0].owner)
   result.flags = header.flags
   # be careful not to propagate unnecessary flags here (don't use rawAddSon)
   result.sons = @[header[0]]

compiler/sigmatch.nim

@@ -728,7 +728,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
       if alreadyBound != nil: typ = alreadyBound
 
       template paramSym(kind): untyped =
-        newSym(kind, typeParamName, nextId(c.idgen), typeClass.sym, typeClass.sym.info, {})
+        newSym(kind, typeParamName, nextSymId(c.idgen), typeClass.sym, typeClass.sym.info, {})
 
       block addTypeParam:
         for prev in typeParams:
@@ -741,7 +741,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
         of tyStatic:
           param = paramSym skConst
           param.typ = typ.exactReplica
-          #copyType(typ, nextId(c.idgen), typ.owner)
+          #copyType(typ, nextTypeId(c.idgen), typ.owner)
           if typ.n == nil:
             param.typ.flags.incl tfInferrableStatic
           else:
@@ -749,7 +749,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
         of tyUnknown:
           param = paramSym skVar
           param.typ = typ.exactReplica
-          #copyType(typ, nextId(c.idgen), typ.owner)
+          #copyType(typ, nextTypeId(c.idgen), typ.owner)
         else:
           param = paramSym skType
           param.typ = if typ.isMetaType:
@@ -801,7 +801,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
     result = generateTypeInstance(c, m.bindings, typeClass.sym.info, ff)
   else:
     result = ff.exactReplica
-    #copyType(ff, nextId(c.idgen), ff.owner)
+    #copyType(ff, nextTypeId(c.idgen), ff.owner)
 
   result.n = checkedBody
 

compiler/sinkparameter_inference.nim

@@ -32,7 +32,7 @@ proc checkForSink*(config: ConfigRef; idgen: IdGenerator; owner: PSym; arg: PNod
       if sfWasForwarded notin owner.flags:
         let argType = arg.sym.typ
 
-        let sinkType = newType(tySink, nextId(idgen), owner)
+        let sinkType = newType(tySink, nextTypeId(idgen), owner)
         sinkType.size = argType.size
         sinkType.align = argType.align
         sinkType.paddingAtEnd = argType.paddingAtEnd

compiler/spawn.nim

@@ -55,7 +55,7 @@ proc typeNeedsNoDeepCopy(t: PType): bool =
 
 proc addLocalVar(g: ModuleGraph; varSection, varInit: PNode; idgen: IdGenerator; owner: PSym; typ: PType;
                  v: PNode; useShallowCopy=false): PSym =
-  result = newSym(skTemp, getIdent(g.cache, genPrefix), nextId idgen, owner, varSection.info,
+  result = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId idgen, owner, varSection.info,
                   owner.options)
   result.typ = typ
   incl(result.flags, sfFromGeneric)
@@ -169,7 +169,7 @@ proc createWrapperProc(g: ModuleGraph; f: PNode; threadParam, argsParam: PSym;
   params.add threadParam.newSymNode
   params.add argsParam.newSymNode
 
-  var t = newType(tyProc, nextId idgen, threadParam.owner)
+  var t = newType(tyProc, nextTypeId idgen, threadParam.owner)
   t.rawAddSon nil
   t.rawAddSon threadParam.typ
   t.rawAddSon argsParam.typ
@@ -189,7 +189,7 @@ proc createCastExpr(argsParam: PSym; objType: PType; idgen: IdGenerator): PNode
   result = newNodeI(nkCast, argsParam.info)
   result.add newNodeI(nkEmpty, argsParam.info)
   result.add newSymNode(argsParam)
-  result.typ = newType(tyPtr, nextId idgen, objType.owner)
+  result.typ = newType(tyPtr, nextTypeId idgen, objType.owner)
   result.typ.rawAddSon(objType)
 
 proc setupArgsForConcurrency(g: ModuleGraph; n: PNode; objType: PType;
@@ -211,7 +211,7 @@ proc setupArgsForConcurrency(g: ModuleGraph; n: PNode; objType: PType;
     #  localError(n[i].info, "'spawn'ed function cannot refer to 'ref'/closure")
 
     let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
-    var field = newSym(skField, fieldname, nextId idgen, objType.owner, n.info, g.config.options)
+    var field = newSym(skField, fieldname, nextSymId idgen, objType.owner, n.info, g.config.options)
     field.typ = argType
     objType.addField(field, g.cache, idgen)
     result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[i])
@@ -239,7 +239,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType;
     #  localError(n.info, "'spawn'ed function cannot refer to 'ref'/closure")
 
     let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
-    var field = newSym(skField, fieldname, nextId idgen, objType.owner, n.info, g.config.options)
+    var field = newSym(skField, fieldname, nextSymId idgen, objType.owner, n.info, g.config.options)
 
     if argType.kind in {tyVarargs, tyOpenArray}:
       # important special case: we always create a zero-copy slice:
@@ -247,7 +247,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType;
       slice.typ = n.typ
       slice[0] = newSymNode(createMagic(g, "slice", mSlice))
       slice[0].typ = getSysType(g, n.info, tyInt) # fake type
-      var fieldB = newSym(skField, tmpName, nextId idgen, objType.owner, n.info, g.config.options)
+      var fieldB = newSym(skField, tmpName, nextSymId idgen, objType.owner, n.info, g.config.options)
       fieldB.typ = getSysType(g, n.info, tyInt)
       objType.addField(fieldB, g.cache, idgen)
 
@@ -257,7 +257,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType;
         objType.addField(field, g.cache, idgen)
         result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
 
-        var fieldA = newSym(skField, tmpName, nextId idgen, objType.owner, n.info, g.config.options)
+        var fieldA = newSym(skField, tmpName, nextSymId idgen, objType.owner, n.info, g.config.options)
         fieldA.typ = getSysType(g, n.info, tyInt)
         objType.addField(fieldA, g.cache, idgen)
         result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldA), n[2])
@@ -332,9 +332,9 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp
   var fn = n[0]
   let
     name = (if fn.kind == nkSym: fn.sym.name.s else: genPrefix) & "Wrapper"
-    wrapperProc = newSym(skProc, getIdent(g.cache, name), nextId idgen, owner, fn.info, g.config.options)
+    wrapperProc = newSym(skProc, getIdent(g.cache, name), nextSymId idgen, owner, fn.info, g.config.options)
-    threadParam = newSym(skParam, getIdent(g.cache, "thread"), nextId idgen, wrapperProc, n.info, g.config.options)
+    threadParam = newSym(skParam, getIdent(g.cache, "thread"), nextSymId idgen, wrapperProc, n.info, g.config.options)
-    argsParam = newSym(skParam, getIdent(g.cache, "args"), nextId idgen, wrapperProc, n.info, g.config.options)
+    argsParam = newSym(skParam, getIdent(g.cache, "args"), nextSymId idgen, wrapperProc, n.info, g.config.options)
 
   wrapperProc.flags.incl sfInjectDestructors
   block:
@@ -347,7 +347,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp
   incl(objType.flags, tfFinal)
   let castExpr = createCastExpr(argsParam, objType, idgen)
 
-  var scratchObj = newSym(skVar, getIdent(g.cache, "scratch"), nextId idgen, owner, n.info, g.config.options)
+  var scratchObj = newSym(skVar, getIdent(g.cache, "scratch"), nextSymId idgen, owner, n.info, g.config.options)
   block:
     scratchObj.typ = objType
     incl(scratchObj.flags, sfFromGeneric)
@@ -364,7 +364,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp
                                                skFunc, skMethod, skConverter}):
     # for indirect calls we pass the function pointer in the scratchObj
     var argType = n[0].typ.skipTypes(abstractInst)
-    var field = newSym(skField, getIdent(g.cache, "fn"), nextId idgen, owner, n.info, g.config.options)
+    var field = newSym(skField, getIdent(g.cache, "fn"), nextSymId idgen, owner, n.info, g.config.options)
     field.typ = argType
     objType.addField(field, g.cache, idgen)
     result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0])
@@ -386,9 +386,9 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp
 
   var barrierAsExpr: PNode = nil
   if barrier != nil:
-    let typ = newType(tyPtr, nextId idgen, owner)
+    let typ = newType(tyPtr, nextTypeId idgen, owner)
     typ.rawAddSon(magicsys.getCompilerProc(g, "Barrier").typ)
-    var field = newSym(skField, getIdent(g.cache, "barrier"), nextId idgen, owner, n.info, g.config.options)
+    var field = newSym(skField, getIdent(g.cache, "barrier"), nextSymId idgen, owner, n.info, g.config.options)
     field.typ = typ
     objType.addField(field, g.cache, idgen)
     result.add newFastAsgnStmt(newDotExpr(scratchObj, field), barrier)
@@ -396,7 +396,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp
 
   var fvField, fvAsExpr: PNode = nil
   if spawnKind == srFlowVar:
-    var field = newSym(skField, getIdent(g.cache, "fv"), nextId idgen, owner, n.info, g.config.options)
+    var field = newSym(skField, getIdent(g.cache, "fv"), nextSymId idgen, owner, n.info, g.config.options)
     field.typ = retType
     objType.addField(field, g.cache, idgen)
     fvField = newDotExpr(scratchObj, field)
@@ -407,8 +407,8 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp
       result.add callCodegenProc(g, "nimFlowVarCreateSemaphore", fvField.info, fvField)
 
   elif spawnKind == srByVar:
-    var field = newSym(skField, getIdent(g.cache, "fv"), nextId idgen, owner, n.info, g.config.options)
+    var field = newSym(skField, getIdent(g.cache, "fv"), nextSymId idgen, owner, n.info, g.config.options)
-    field.typ = newType(tyPtr, nextId idgen, objType.owner)
+    field.typ = newType(tyPtr, nextTypeId idgen, objType.owner)
     field.typ.rawAddSon(retType)
     objType.addField(field, g.cache, idgen)
     fvAsExpr = indirectAccess(castExpr, field, n.info)

compiler/transf.nim

@@ -87,7 +87,7 @@ proc getCurrOwner(c: PTransf): PSym =
   else: result = c.module
 
 proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode =
-  let r = newSym(skTemp, getIdent(c.graph.cache, genPrefix), nextId(c.idgen), getCurrOwner(c), info)
+  let r = newSym(skTemp, getIdent(c.graph.cache, genPrefix), nextSymId(c.idgen), getCurrOwner(c), info)
   r.typ = typ #skipTypes(typ, {tyGenericInst, tyAlias, tySink})
   incl(r.flags, sfFromGeneric)
   let owner = getCurrOwner(c)
@@ -163,7 +163,7 @@ proc freshVar(c: PTransf; v: PSym): PNode =
   if owner.isIterator and not c.tooEarly:
     result = freshVarForClosureIter(c.graph, v, c.idgen, owner)
   else:
-    var newVar = copySym(v, nextId(c.idgen))
+    var newVar = copySym(v, nextSymId(c.idgen))
     incl(newVar.flags, sfFromGeneric)
     newVar.owner = owner
     result = newSymNode(newVar)
@@ -233,7 +233,7 @@ proc hasContinue(n: PNode): bool =
       if hasContinue(n[i]): return true
 
 proc newLabel(c: PTransf, n: PNode): PSym =
-  result = newSym(skLabel, nil, nextId(c.idgen), getCurrOwner(c), n.info)
+  result = newSym(skLabel, nil, nextSymId(c.idgen), getCurrOwner(c), n.info)
   result.name = getIdent(c.graph.cache, genPrefix)
 
 proc transformBlock(c: PTransf, n: PNode): PNode =

compiler/types.nim

@@ -1311,7 +1311,7 @@ proc baseOfDistinct*(t: PType; idgen: IdGenerator): PType =
   if t.kind == tyDistinct:
     result = t[0]
   else:
-    result = copyType(t, nextId idgen, t.owner)
+    result = copyType(t, nextTypeId idgen, t.owner)
     var parent: PType = nil
     var it = result
     while it.kind in {tyPtr, tyRef, tyOwned}:
@@ -1457,7 +1457,7 @@ proc takeType*(formal, arg: PType; idgen: IdGenerator): PType =
     result = formal
   elif formal.kind in {tyOpenArray, tyVarargs, tySequence} and
       arg.isEmptyContainer:
-    let a = copyType(arg.skipTypes({tyGenericInst, tyAlias}), nextId(idgen), arg.owner)
+    let a = copyType(arg.skipTypes({tyGenericInst, tyAlias}), nextTypeId(idgen), arg.owner)
     a[ord(arg.kind == tyArray)] = formal[0]
     result = a
   elif formal.kind in {tyTuple, tySet} and arg.kind == formal.kind:

compiler/vm.nim

@@ -688,7 +688,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       let s = regs[rb].node.strVal.addr # or `byaddr`
       if idx <% s[].len:
          # `makePtrType` not accessible from vm.nim
-        let typ = newType(tyPtr, nextId c.idgen, c.module.owner)
+        let typ = newType(tyPtr, nextTypeId c.idgen, c.module.owner)
         typ.add getSysType(c.graph, c.debug[pc], tyChar)
         let node = newNodeIT(nkIntLit, c.debug[pc], typ) # xxx nkPtrLit
         node.intVal = cast[int](s[][idx].addr)
@@ -1962,7 +1962,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
                  else: regs[rc].node.strVal
       if k < 0 or k > ord(high(TSymKind)):
         internalError(c.config, c.debug[pc], "request to create symbol of invalid kind")
-      var sym = newSym(k.TSymKind, getIdent(c.cache, name), nextId c.idgen, c.module.owner, c.debug[pc])
+      var sym = newSym(k.TSymKind, getIdent(c.cache, name), nextSymId c.idgen, c.module.owner, c.debug[pc])
       incl(sym.flags, sfGenSym)
       regs[ra].node = newSymNode(sym)
       regs[ra].node.flags.incl nfIsRef
@@ -2257,7 +2257,7 @@ const evalMacroLimit = 1000
 
 proc errorNode(idgen: IdGenerator; owner: PSym, n: PNode): PNode =
   result = newNodeI(nkEmpty, n.info)
-  result.typ = newType(tyError, nextId idgen, owner)
+  result.typ = newType(tyError, nextTypeId idgen, owner)
   result.typ.flags.incl tfCheckedForDestructor
 
 proc evalMacroCall*(module: PSym; idgen: IdGenerator; g: ModuleGraph; templInstCounter: ref int;

compiler/vmdeps.nim

@@ -26,7 +26,7 @@ proc opSlurp*(file: string, info: TLineInfo, module: PSym; conf: ConfigRef): str
 
 proc atomicTypeX(cache: IdentCache; name: string; m: TMagic; t: PType; info: TLineInfo;
                  idgen: IdGenerator): PNode =
-  let sym = newSym(skType, getIdent(cache, name), nextId(idgen), t.owner, info)
+  let sym = newSym(skType, getIdent(cache, name), nextSymId(idgen), t.owner, info)
   sym.magic = m
   sym.typ = t
   result = newSymNode(sym)
@@ -47,7 +47,7 @@ proc mapTypeToBracketX(cache: IdentCache; name: string; m: TMagic; t: PType; inf
   for i in 0..<t.len:
     if t[i] == nil:
       let void = atomicTypeX(cache, "void", mVoid, t, info, idgen)
-      void.typ = newType(tyVoid, nextId(idgen), t.owner)
+      void.typ = newType(tyVoid, nextTypeId(idgen), t.owner)
       result.add void
     else:
       result.add mapTypeToAstX(cache, t[i], info, idgen, inst)

compiler/vmmarshal.nim

@@ -220,7 +220,7 @@ proc loadAny(p: var JsonParser, t: PType,
       if pos >= result.len:
         setLen(result.sons, pos + 1)
       let fieldNode = newNode(nkExprColonExpr)
-      fieldNode.add newSymNode(newSym(skField, ident, nextId(idgen), nil, unknownLineInfo))
+      fieldNode.add newSymNode(newSym(skField, ident, nextSymId(idgen), nil, unknownLineInfo))
       fieldNode.add loadAny(p, field.typ, tab, cache, conf, idgen)
       result[pos] = fieldNode
     if p.kind == jsonObjectEnd: next(p)

tests/compilerapi/tcompilerapi.nim

@@ -18,7 +18,7 @@ import std / [os]
 
 proc initInterpreter(script: string): Interpreter =
   let std = findNimStdLibCompileTime()
-  result = createInterpreter(script , [std, parentDir(currentSourcePath),
+  result = createInterpreter(script, [std, parentDir(currentSourcePath),
     std / "pure", std / "core"])
 
 proc main() =