completed VM support for incremental compilations

compiler/macrocacheimpl.nim

@@ -11,39 +11,69 @@
 
 import lineinfos, ast, modulegraphs, vmdef, magicsys
 
-proc genCall3(g: ModuleGraph; m: TMagic; s: string; a, b, c: PNode): PNode =
-  newTree(nkStaticStmt, newTree(nkCall, createMagic(g, s, m).newSymNode, a, b, c))
-
-proc genCall2(g: ModuleGraph; m: TMagic; s: string; a, b: PNode): PNode =
-  newTree(nkStaticStmt, newTree(nkCall, createMagic(g, s, m).newSymNode, a, b))
-
-template nodeFrom(s: string): PNode =
-  var res = newStrNode(s, info)
-  res.typ = getSysType(g, info, tyString)
-  res
-
-template nodeFrom(i: BiggestInt): PNode =
-  var res = newIntNode(i, info)
-  res.typ = getSysType(g, info, tyInt)
-  res
-
-template nodeFrom(n: PNode): PNode = copyTree(n)
-
-template record(call) =
-  g.recordStmt(g, c.module, call)
-
 proc recordInc*(c: PCtx; info: TLineInfo; key: string; by: BiggestInt) =
-  let g = c.graph
+  var recorded = newNodeI(nkCommentStmt, info)
-  record genCall2(mNccInc, "inc", nodeFrom key, nodeFrom by)
+  recorded.add newStrNode("inc", info)
+  recorded.add newStrNode(key, info)
+  recorded.add newIntNode(nkIntLit, by)
+  c.graph.recordStmt(c.graph, c.module, recorded)
 
 proc recordPut*(c: PCtx; info: TLineInfo; key: string; k: string; val: PNode) =
-  let g = c.graph
+  var recorded = newNodeI(nkCommentStmt, info)
-  record genCall3(mNctPut, "[]=", nodeFrom key, nodeFrom k, nodeFrom val)
+  recorded.add newStrNode("put", info)
+  recorded.add newStrNode(key, info)
+  recorded.add newStrNode(k, info)
+  recorded.add copyTree(val)
+  c.graph.recordStmt(c.graph, c.module, recorded)
 
 proc recordAdd*(c: PCtx; info: TLineInfo; key: string; val: PNode) =
-  let g = c.graph
+  var recorded = newNodeI(nkCommentStmt, info)
-  record genCall2(mNcsAdd, "add", nodeFrom key, nodeFrom val)
+  recorded.add newStrNode("add", info)
+  recorded.add newStrNode(key, info)
+  recorded.add copyTree(val)
+  c.graph.recordStmt(c.graph, c.module, recorded)
 
 proc recordIncl*(c: PCtx; info: TLineInfo; key: string; val: PNode) =
-  let g = c.graph
+  var recorded = newNodeI(nkCommentStmt, info)
-  record genCall2(mNcsIncl, "incl", nodeFrom key, nodeFrom val)
+  recorded.add newStrNode("incl", info)
+  recorded.add newStrNode(key, info)
+  recorded.add copyTree(val)
+  c.graph.recordStmt(c.graph, c.module, recorded)
+
+when false:
+  proc genCall3(g: ModuleGraph; m: TMagic; s: string; a, b, c: PNode): PNode =
+    newTree(nkStaticStmt, newTree(nkCall, createMagic(g, s, m).newSymNode, a, b, c))
+
+  proc genCall2(g: ModuleGraph; m: TMagic; s: string; a, b: PNode): PNode =
+    newTree(nkStaticStmt, newTree(nkCall, createMagic(g, s, m).newSymNode, a, b))
+
+  template nodeFrom(s: string): PNode =
+    var res = newStrNode(s, info)
+    res.typ = getSysType(g, info, tyString)
+    res
+
+  template nodeFrom(i: BiggestInt): PNode =
+    var res = newIntNode(i, info)
+    res.typ = getSysType(g, info, tyInt)
+    res
+
+  template nodeFrom(n: PNode): PNode = copyTree(n)
+
+  template record(call) =
+    g.recordStmt(g, c.module, call)
+
+  proc recordInc*(c: PCtx; info: TLineInfo; key: string; by: BiggestInt) =
+    let g = c.graph
+    record genCall2(mNccInc, "inc", nodeFrom key, nodeFrom by)
+
+  proc recordPut*(c: PCtx; info: TLineInfo; key: string; k: string; val: PNode) =
+    let g = c.graph
+    record genCall3(mNctPut, "[]=", nodeFrom key, nodeFrom k, nodeFrom val)
+
+  proc recordAdd*(c: PCtx; info: TLineInfo; key: string; val: PNode) =
+    let g = c.graph
+    record genCall2(mNcsAdd, "add", nodeFrom key, nodeFrom val)
+
+  proc recordIncl*(c: PCtx; info: TLineInfo; key: string; val: PNode) =
+    let g = c.graph
+    record genCall2(mNcsIncl, "incl", nodeFrom key, nodeFrom val)

compiler/modulegraphs.nim

@@ -26,7 +26,7 @@
 ##
 
 import ast, intsets, tables, options, lineinfos, hashes, idents,
-  incremental
+  incremental, btrees
 
 type
   ModuleGraph* = ref object
@@ -59,6 +59,9 @@ type
     importModuleCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PSym {.nimcall.}
     includeFileCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PNode {.nimcall.}
     recordStmt*: proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.}
+    cacheSeqs*: Table[string, PNode] # state that is shared to suppor the 'macrocache' API
+    cacheCounters*: Table[string, BiggestInt]
+    cacheTables*: Table[string, BTree[string, PNode]]
 
 proc hash*(x: FileIndex): Hash {.borrow.}
 
@@ -90,6 +93,9 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   init(result.incr)
   result.recordStmt = proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.} =
     discard
+  result.cacheSeqs = initTable[string, PNode]()
+  result.cacheCounters = initTable[string, BiggestInt]()
+  result.cacheTables = initTable[string, BTree[string, PNode]]()
 
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(packageSyms)

compiler/rodimpl.nim

@@ -10,7 +10,8 @@
 ## This module implements the new compilation cache.
 
 import strutils, os, intsets, tables, ropes, db_sqlite, msgs, options, types,
-  renderer, rodutils, idents, astalgo, btrees, magicsys, cgmeth, extccomp, vm
+  renderer, rodutils, idents, astalgo, btrees, magicsys, cgmeth, extccomp,
+  btrees, trees
 
 ## Todo:
 ## - Make conditional symbols and the configuration part of a module's
@@ -757,9 +758,10 @@ proc loadModuleSymTab(g; module: PSym) =
     g.systemModule = module
 
 proc replay(g: ModuleGraph; module: PSym; n: PNode) =
+  # XXX check if we need to replay nkStaticStmt here.
   case n.kind
-  of nkStaticStmt:
+  #of nkStaticStmt:
-    evalStaticStmt(module, g, n[0], module)
+    #evalStaticStmt(module, g, n[0], module)
     #of nkVarSection, nkLetSection:
     #  nkVarSections are already covered by the vmgen which produces nkStaticStmt
   of nkMethodDef:
@@ -786,6 +788,39 @@ proc replay(g: ModuleGraph; module: PSym; n: PNode) =
         extccomp.addExternalFileToCompile(g.config, cf)
       of "link":
         extccomp.addExternalFileToLink(g.config, n[1].strVal)
+      of "inc":
+        let destKey = n[1].strVal
+        let by = n[2].intVal
+        let v = getOrDefault(g.cacheCounters, destKey)
+        g.cacheCounters[destKey] = v+by
+      of "put":
+        let destKey = n[1].strVal
+        let key = n[2].strVal
+        let val = n[3]
+        if not contains(g.cacheTables, destKey):
+          g.cacheTables[destKey] = initBTree[string, PNode]()
+        if not contains(g.cacheTables[destKey], key):
+          g.cacheTables[destKey].add(key, val)
+        else:
+          internalError(g.config, n.info, "key already exists: " & key)
+      of "incl":
+        let destKey = n[1].strVal
+        let val = n[2]
+        if not contains(g.cacheSeqs, destKey):
+          g.cacheSeqs[destKey] = newTree(nkStmtList, val)
+        else:
+          block search:
+            for existing in g.cacheSeqs[destKey]:
+              if exprStructuralEquivalent(existing, val, strictSymEquality=true):
+                break search
+            g.cacheSeqs[destKey].add val
+      of "add":
+        let destKey = n[1].strVal
+        let val = n[2]
+        if not contains(g.cacheSeqs, destKey):
+          g.cacheSeqs[destKey] = newTree(nkStmtList, val)
+        else:
+          g.cacheSeqs[destKey].add val
       else:
         internalAssert g.config, false
   of nkImportStmt:

compiler/vm.nim

@@ -1576,86 +1576,98 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcNccValue:
       decodeB(rkInt)
       let destKey = regs[rb].node.strVal
-      regs[ra].intVal = getOrDefault(c.cacheCounters, destKey)
+      regs[ra].intVal = getOrDefault(c.graph.cacheCounters, destKey)
     of opcNccInc:
-      let rb = instr.regB
+      let g = c.graph
       let destKey = regs[ra].node.strVal
-      let by = regs[rb].intVal
+      let by = regs[instr.regB].intVal
-      let v = getOrDefault(c.cacheCounters, destKey)
+      let v = getOrDefault(g.cacheCounters, destKey)
-      c.cacheCounters[destKey] = v+by
+      g.cacheCounters[destKey] = v+by
       recordInc(c, c.debug[pc], destKey, by)
     of opcNcsAdd:
+      let g = c.graph
       let destKey = regs[ra].node.strVal
       let val = regs[instr.regB].node
-      if not contains(c.cacheSeqs, destKey):
+      if not contains(g.cacheSeqs, destKey):
-        c.cacheSeqs[destKey] = newTree(nkStmtList, val)
+        g.cacheSeqs[destKey] = newTree(nkStmtList, val)
         # newNodeI(nkStmtList, c.debug[pc])
       else:
-        c.cacheSeqs[destKey].add val
+        g.cacheSeqs[destKey].add val
       recordAdd(c, c.debug[pc], destKey, val)
     of opcNcsIncl:
+      let g = c.graph
       let destKey = regs[ra].node.strVal
       let val = regs[instr.regB].node
-      if not contains(c.cacheSeqs, destKey):
+      if not contains(g.cacheSeqs, destKey):
-        c.cacheSeqs[destKey] = newTree(nkStmtList, val)
+        g.cacheSeqs[destKey] = newTree(nkStmtList, val)
       else:
         block search:
-          for existing in c.cacheSeqs[destKey]:
+          for existing in g.cacheSeqs[destKey]:
             if exprStructuralEquivalent(existing, val, strictSymEquality=true):
               break search
-          c.cacheSeqs[destKey].add val
+          g.cacheSeqs[destKey].add val
       recordIncl(c, c.debug[pc], destKey, val)
     of opcNcsLen:
+      let g = c.graph
       decodeB(rkInt)
       let destKey = regs[rb].node.strVal
       regs[ra].intVal =
-        if contains(c.cacheSeqs, destKey): c.cacheSeqs[destKey].len else: 0
+        if contains(g.cacheSeqs, destKey): g.cacheSeqs[destKey].len else: 0
     of opcNcsAt:
+      let g = c.graph
       decodeBC(rkNode)
       let idx = regs[rc].intVal
       let destKey = regs[rb].node.strVal
-      if contains(c.cacheSeqs, destKey) and idx <% c.cacheSeqs[destKey].len:
+      if contains(g.cacheSeqs, destKey) and idx <% g.cacheSeqs[destKey].len:
-        regs[ra].node = c.cacheSeqs[destKey][idx.int]
+        regs[ra].node = g.cacheSeqs[destKey][idx.int]
       else:
         stackTrace(c, tos, pc, errIndexOutOfBounds)
     of opcNctPut:
+      let g = c.graph
       let destKey = regs[ra].node.strVal
       let key = regs[instr.regB].node.strVal
       let val = regs[instr.regC].node
-      if not contains(c.cacheTables, destKey):
+      if not contains(g.cacheTables, destKey):
-        c.cacheTables[destKey] = initBTree[string, PNode]()
+        g.cacheTables[destKey] = initBTree[string, PNode]()
-      if not contains(c.cacheTables[destKey], key):
+      if not contains(g.cacheTables[destKey], key):
-        c.cacheTables[destKey].add(key, val)
+        g.cacheTables[destKey].add(key, val)
         recordPut(c, c.debug[pc], destKey, key, val)
       else:
         stackTrace(c, tos, pc, "key already exists: " & key)
     of opcNctLen:
+      let g = c.graph
       decodeB(rkInt)
       let destKey = regs[rb].node.strVal
       regs[ra].intVal =
-        if contains(c.cacheTables, destKey): c.cacheTables[destKey].len else: 0
+        if contains(g.cacheTables, destKey): g.cacheTables[destKey].len else: 0
     of opcNctGet:
+      let g = c.graph
       decodeBC(rkNode)
       let destKey = regs[rb].node.strVal
       let key = regs[rc].node.strVal
-      if contains(c.cacheTables, destKey):
+      if contains(g.cacheTables, destKey):
-        if contains(c.cacheTables[destKey], key):
+        if contains(g.cacheTables[destKey], key):
-          regs[ra].node = getOrDefault(c.cacheTables[destKey], key)
+          regs[ra].node = getOrDefault(g.cacheTables[destKey], key)
         else:
           stackTrace(c, tos, pc, "key does not exist: " & key)
       else:
         stackTrace(c, tos, pc, "key does not exist: " & destKey)
     of opcNctHasNext:
+      let g = c.graph
       decodeBC(rkInt)
       let destKey = regs[rb].node.strVal
       regs[ra].intVal =
-        btrees.hasNext(c.cacheTables[destKey], regs[rc].intVal.int) else: 0
+        if g.cacheTables.contains(destKey):
+          ord(btrees.hasNext(g.cacheTables[destKey], regs[rc].intVal.int))
+        else:
+          0
     of opcNctNext:
+      let g = c.graph
       decodeBC(rkNode)
       let destKey = regs[rb].node.strVal
       let index = regs[rc].intVal
-      if contains(c.cacheTables, destKey):
+      if contains(g.cacheTables, destKey):
-        let (k, v, nextIndex) = btrees.next(c.cacheTables[destKey], index.int)
+        let (k, v, nextIndex) = btrees.next(g.cacheTables[destKey], index.int)
         regs[ra].node = newTree(nkTupleConstr, newStrNode(k, c.debug[pc]), v,
                                 newIntNode(nkIntLit, nextIndex))
       else:

compiler/vmdef.nim

@@ -213,9 +213,6 @@ type
     config*: ConfigRef
     graph*: ModuleGraph
     oldErrorCount*: int
-    cacheSeqs*: Table[string, PNode]
-    cacheCounters*: Table[string, BiggestInt]
-    cacheTables*: Table[string, BTree[string, PNode]]
 
   TPosition* = distinct int
 
@@ -226,10 +223,7 @@ proc newCtx*(module: PSym; cache: IdentCache; g: ModuleGraph): PCtx =
     globals: newNode(nkStmtListExpr), constants: newNode(nkStmtList), types: @[],
     prc: PProc(blocks: @[]), module: module, loopIterations: MaxLoopIterations,
     comesFromHeuristic: unknownLineInfo(), callbacks: @[], errorFlag: "",
-    cache: cache, config: g.config, graph: g,
+    cache: cache, config: g.config, graph: g)
-    cacheSeqs: initTable[string, PNode]()
-    cacheCounters: initTable[string, BiggestInt]()
-    cacheTables: initTable[string, BTree[string, PNode]]())
 
 proc refresh*(c: PCtx, module: PSym) =
   c.module = module

compiler/vmgen.nim

@@ -804,12 +804,12 @@ proc genIntCast(c: PCtx; n: PNode; dest: var TDest) =
   else:
     globalError(c.config, n.info, "VM is only allowed to 'cast' between integers of same size")
 
-proc genVoidABC(c: PCtx, n: PNode, dest: TRegister, opcode: TOpcode)
+proc genVoidABC(c: PCtx, n: PNode, dest: TRegister, opcode: TOpcode) =
   unused(c, n, dest)
   var
-    tmp1 = c.genx(n.sons[1])
+    tmp1 = c.genx(n[1])
-    tmp2 = c.genx(n.sons[2])
+    tmp2 = c.genx(n[2])
-    tmp3 = c.genx(n.sons[3])
+    tmp3 = c.genx(n[3])
   c.gABC(n, opcode, tmp1, tmp2, tmp3)
   c.freeTemp(tmp1)
   c.freeTemp(tmp2)
@@ -1087,8 +1087,8 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
 
   of mNccValue: genUnaryABC(c, n, dest, opcNccValue)
   of mNccInc: genBinaryABC(c, n, dest, opcNccInc)
-  of mNcsAdd: genBinaryABC(c, n, dest, opcNcsAdd
+  of mNcsAdd: genBinaryABC(c, n, dest, opcNcsAdd)
-  of mNcsIncl: genBinaryABC(c, n, dest, opcNcsIncl
+  of mNcsIncl: genBinaryABC(c, n, dest, opcNcsIncl)
   of mNcsLen: genUnaryABC(c, n, dest, opcNcsLen)
   of mNcsAt: genBinaryABC(c, n, dest, opcNcsAt)
   of mNctPut: genVoidABC(c, n, dest, opcNctPut)