Compiler plugin for implementing incremental computation in user space (#10819)

This plugin provides essential building block for implementing incremental computations in your programs. The idea behind incremental computations is that if you do the same calculation multiple times but with slightly different inputs you don't have to recompute everything from scratch. Also you don't want to adopt special algorithms either, you would like to write your code in standard from scratch manner and get incrementality for free when it is possible.

The plugin computes the digest of the proc bodies, recursively hashing all called procs as well . Such digest with the digest of the argument values gives a good "name" for the result. Terminology loosely follows paper "Incremental Computation with Names" link below. It works well if you have no side effects in your computations. If you have global state in your computations then you will need problem specific workarounds to represent global state in set of "names" . SideEffect tracking in Nim also useful in this topic.

Classical examples:

Dashboard with ticking data. New data arrives non stop and you would like to update the dashboard recomputing only changed outputs.
Excel spreadsheet where user changes one cell and you would like to recompute all cells that are affected by the change, but do not want to recompute every cell in the spreadsheet.

compiler/modulegraphs.nim

@@ -26,9 +26,11 @@
 ##
 
 import ast, intsets, tables, options, lineinfos, hashes, idents,
-  incremental, btrees, sighashes
+  incremental, btrees, md5
 
 type
+  SigHash* = distinct Md5Digest
+
   ModuleGraph* = ref object
     modules*: seq[PSym]  ## indexed by int32 fileIdx
     packageSyms*: TStrTable
@@ -58,6 +60,7 @@ type
     emptyNode*: PNode
     incr*: IncrementalCtx
     canonTypes*: Table[SigHash, PType]
+    symBodyHashes*: Table[int, SigHash] # symId to digest mapping
     importModuleCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PSym {.nimcall.}
     includeFileCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PNode {.nimcall.}
     recordStmt*: proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.}
@@ -81,6 +84,52 @@ type
                  close: TPassClose,
                  isFrontend: bool]
 
+
+const
+  cb64 = [
+    "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N",
+    "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z",
+    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n",
+    "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
+    "0", "1", "2", "3", "4", "5", "6", "7", "8", "9a",
+    "9b", "9c"]
+
+proc toBase64a(s: cstring, len: int): string =
+  ## encodes `s` into base64 representation.
+  result = newStringOfCap(((len + 2) div 3) * 4)
+  result.add '_'
+  var i = 0
+  while i < len - 2:
+    let a = ord(s[i])
+    let b = ord(s[i+1])
+    let c = ord(s[i+2])
+    result.add cb64[a shr 2]
+    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
+    result.add cb64[((b and 0x0F) shl 2) or ((c and 0xC0) shr 6)]
+    result.add cb64[c and 0x3F]
+    inc(i, 3)
+  if i < len-1:
+    let a = ord(s[i])
+    let b = ord(s[i+1])
+    result.add cb64[a shr 2]
+    result.add cb64[((a and 3) shl 4) or ((b and 0xF0) shr 4)]
+    result.add cb64[((b and 0x0F) shl 2)]
+  elif i < len:
+    let a = ord(s[i])
+    result.add cb64[a shr 2]
+    result.add cb64[(a and 3) shl 4]
+
+proc `$`*(u: SigHash): string =
+  toBase64a(cast[cstring](unsafeAddr u), sizeof(u))
+
+proc `==`*(a, b: SigHash): bool =
+  result = equalMem(unsafeAddr a, unsafeAddr b, sizeof(a))
+
+proc hash*(u: SigHash): Hash =
+  result = 0
+  for x in 0..3:
+    result = (result shl 8) or u.MD5Digest[x].int
+
 proc hash*(x: FileIndex): Hash {.borrow.}
 
 when defined(nimfind):
@@ -140,6 +189,7 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.cacheCounters = initTable[string, BiggestInt]()
   result.cacheTables = initTable[string, BTree[string, PNode]]()
   result.canonTypes = initTable[SigHash, PType]()
+  result.symBodyHashes = initTable[int, SigHash]()
 
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.packageSyms)