[caas] first version that actually works (still has a lot of logical memory leaks on recompilation)

compiler/ccgmerge.nim

@@ -45,8 +45,11 @@ const
   ]
   NimMergeEndMark = "/*\tNIM_merge_END:*/"
 
+template mergeSectionsEnabled: expr =
+  {optCaasEnabled, optSymbolFiles} * gGlobalOptions != {}
+
 proc genSectionStart*(fs: TCFileSection): PRope =
-  if optSymbolFiles in gGlobalOptions:
+  if mergeSectionsEnabled:
     result = toRope(tnl)
     app(result, "/*\t")
     app(result, CFileSectionNames[fs])
@@ -54,11 +57,11 @@ proc genSectionStart*(fs: TCFileSection): PRope =
     app(result, tnl)
 
 proc genSectionEnd*(fs: TCFileSection): PRope =
-  if optSymbolFiles in gGlobalOptions:
+  if mergeSectionsEnabled:
     result = toRope(NimMergeEndMark & tnl)
 
 proc genSectionStart*(ps: TCProcSection): PRope =
-  if optSymbolFiles in gGlobalOptions:
+  if mergeSectionsEnabled:
     result = toRope(tnl)
     app(result, "/*\t")
     app(result, CProcSectionNames[ps])
@@ -66,7 +69,7 @@ proc genSectionStart*(ps: TCProcSection): PRope =
     app(result, tnl)
 
 proc genSectionEnd*(ps: TCProcSection): PRope =
-  if optSymbolFiles in gGlobalOptions:
+  if mergeSectionsEnabled:
     result = toRope(NimMergeEndMark & tnl)
 
 proc writeTypeCache(a: TIdTable, s: var string) =

compiler/ccgutils.nim

@@ -59,6 +59,12 @@ var
 proc initTypeTables() = 
   for i in countup(low(TTypeKind), high(TTypeKind)): InitIdTable(gTypeTable[i])
 
+proc resetCaches* =
+  ## XXX: fix that more properly
+  initTypeTables()
+  for i in low(gCanonicalTypes)..high(gCanonicalTypes):
+    gCanonicalTypes[i] = nil
+
 when false:
   proc echoStats*() =
     for i in countup(low(TTypeKind), high(TTypeKind)): 

compiler/cgen.nim

@@ -37,6 +37,10 @@ proc addForwardedProc(m: BModule, prc: PSym) =
   m.forwardedProcs.add(prc)
   inc(gForwardedProcsCounter)
 
+proc getCgenModule(s: PSym): BModule =
+  result = if s.position >= 0 and s.position < gModules.len: gModules[s.position]
+           else: nil
+
 proc findPendingModule(m: BModule, s: PSym): BModule = 
   var ms = getModule(s)
   result = gModules[ms.position]
@@ -1013,17 +1017,66 @@ proc rawNewModule(module: PSym, filename: string): BModule =
   result.nimTypesName = getTempName()
   result.PreventStackTrace = sfSystemModule in module.flags
 
+proc nullify[T](arr: var T) =
+  for i in low(arr)..high(arr):
+    arr[i] = nil
+
+proc resetModule*(m: var BModule) =
+  # between two compilations in CAAS mode, we can throw
+  # away all the data that was written to disk
+  InitLinkedList(m.headerFiles)
+  m.declaredProtos = initIntSet()
+  initIdTable(m.forwTypeCache)
+  m.initProc = newProc(nil, m)
+  m.initProc.options = gOptions
+  m.preInitProc = newProc(nil, m)
+  initNodeTable(m.dataCache)
+  m.typeStack = @[]
+  m.forwardedProcs = @[]
+  m.typeNodesName = getTempName()
+  m.nimTypesName = getTempName()
+  m.PreventStackTrace = sfSystemModule in m.module.flags
+  nullify m.s
+  m.usesThreadVars = false
+  m.typeNodes = 0
+  m.nimTypes = 0
+  nullify m.extensionLoaders
+  
+  # indicate that this is now cached module
+  # the cache will be invalidated by nullifying gModules
+  m.fromCache = true
+  
+  # we keep only the "merge info" information for the module
+  # and the properties that can't change:
+  # m.filename
+  # m.cfilename
+  # m.isHeaderFile
+  # m.module ?
+  # m.typeCache
+  # m.declaredThings
+  # m.typeInfoMarker
+  # m.labels
+  # m.FrameDeclared
+
+proc resetCgenModules* =
+  for m in cgenModules(): resetModule(m)
+
 proc rawNewModule(module: PSym): BModule =
   result = rawNewModule(module, module.filename)
 
 proc newModule(module: PSym): BModule =
+  result = getCgenModule(module)
+  if result == nil:
     result = rawNewModule(module)
-  if gModules.len <= module.position: gModules.setLen(module.position + 1)
+    growCache gModules, module.position
     gModules[module.position] = result
 
     if (optDeadCodeElim in gGlobalOptions): 
       if (sfDeadCodeElim in module.flags): 
         InternalError("added pending module twice: " & module.filename)
+  else:
+    echo "CGEN CACHED MODULE: ", result.filename
+    assert optCaasEnabled in gGlobalOptions
 
 proc myOpen(module: PSym): PPassContext = 
   result = newModule(module)
@@ -1057,6 +1110,7 @@ proc getCFile(m: BModule): string =
   result = changeFileExt(completeCFilePath(m.cfilename), cExt)
 
 proc myOpenCached(module: PSym, rd: PRodReader): PPassContext =
+  assert optSymbolFiles in gGlobalOptions
   var m = newModule(module)
   readMergeInfo(getCFile(m), m)
   result = m
@@ -1130,8 +1184,28 @@ proc writeModule(m: BModule, pending: bool) =
     # ``system.c`` but then compilation fails due to an error. This means
     # that ``system.o`` is missing, so we need to call the C compiler for it:
     addFileToCompile(cfilenoext)
+  
   addFileToLink(cfilenoext)
 
+proc updateCachedModule(m: BModule) =
+  let cfile = getCFile(m)
+  let cfilenoext = changeFileExt(cfile, "")
+  
+  if mergeRequired(m):
+    echo "MERGE REQUIRED FOR ", m.filename
+    mergeFiles(cfile, m)
+    genInitCode(m)
+    finishTypeDescriptions(m)
+    var code = genModule(m, cfilenoext)
+    writeRope(code, cfile)
+    addFileToCompile(cfilenoext)
+
+  addFileToLink(cfilenoext)
+
+proc cgenCaasUpdate* =
+  for m in cgenModules():
+    if m.fromCache: m.updateCachedModule
+
 proc myClose(b: PPassContext, n: PNode): PNode = 
   result = n
   if b == nil or passes.skipCodegen(n): return 
@@ -1151,16 +1225,12 @@ proc myClose(b: PPassContext, n: PNode): PNode =
     # order anyway)
     if generatedHeader != nil: finishModule(generatedHeader)
     while gForwardedProcsCounter > 0:
-      for i in countup(0, high(gModules)):
+      for m in cgenModules():
-        # some modules (like stdin) may exist only in memory
+        if not m.fromCache:
-        # they won't have a cgen BModule for them and we must
+          finishModule(m)
-        # skip them
+    for m in cgenModules():
-        if gModules[i] != nil:
+      if not m.fromCache:
-          finishModule(gModules[i])
+        writeModule(m, pending=true)
-    for i in countup(0, high(gModules)): 
-      # see above
-      if gModules[i] != nil:
-        writeModule(gModules[i], pending=true)
     writeMapping(gMapping)
     if generatedHeader != nil: writeHeader(generatedHeader)
 

compiler/cgendata.nim

@@ -141,3 +141,9 @@ proc newProc*(prc: PSym, module: BModule): BProc =
   newSeq(result.blocks, 1)
   result.nestedTryStmts = @[]
 
+iterator cgenModules*: var BModule =
+  for i in 0..high(gModules):
+    # some modules (like stdin) may exist only in memory.
+    # they won't have a cgen BModule for them and we must skip them.
+    if gModules[i] != nil: yield gModules[i]
+

compiler/commands.nim

@@ -23,6 +23,7 @@ type
 
 proc ProcessCommand*(switch: string, pass: TCmdLinePass)
 proc processSwitch*(switch, arg: string, pass: TCmdlinePass, info: TLineInfo)
+
 # implementation
 
 const

compiler/extccomp.nim

@@ -382,6 +382,15 @@ proc toObjFile*(filenameWithoutExt: string): string =
 proc addFileToCompile*(filename: string) =
   appendStr(toCompile, filename)
 
+proc resetCompilationLists* =
+  initLinkedList(toCompile)
+  ## XXX: we must associate these with their originating module
+  # when the module is loaded/unloaded it adds/removes its items
+  # That's because we still need to CRC check the external files
+  # Maybe we can do that in checkDep on the other hand?
+  initLinkedList(externalToCompile)
+  initLinkedList(toLink)
+
 proc footprint(filename: string): TCrc32 =
   result = crcFromFile(filename) >< 
       platform.OS[targetOS].name ><

compiler/main.nim

@@ -14,9 +14,9 @@ import
   llstream, strutils, ast, astalgo, lexer, syntaxes, renderer, options, msgs, 
   os, lists, condsyms, rodread, rodwrite, ropes, trees, times,
   wordrecg, sem, semdata, idents, passes, docgen, extccomp,
-  cgen, ecmasgen,
+  cgen, ecmasgen, cgendata,
   platform, nimconf, importer, passaux, depends, evals, types, idgen,
-  tables, docgen2, service, magicsys, parser, crc
+  tables, docgen2, service, magicsys, parser, crc, ccgutils
 
 const
   has_LLVM_Backend = false
@@ -29,11 +29,15 @@ proc MainCommand*()
 # ------------------ module handling -----------------------------------------
 
 type
+  TNeedRecompile = enum Maybe, No, Yes, Probing, Recompiled
+  TCrcStatus = enum crcNotTaken, crcCached, crcHasChanged, crcNotChanged
+
   TModuleInMemory = object
     compiledAt: float
-    crc: int
+    crc: TCrc32
     deps: seq[int32] ## XXX: slurped files are not currently tracked
-    needsRecompile: bool
+    needsRecompile: TNeedRecompile
+    crcStatus: TCrcStatus
 
 var
   gCompiledModules: seq[PSym] = @[]
@@ -53,24 +57,76 @@ template compiledAt(x: PSym): expr =
 template crc(x: PSym): expr =
   gMemCacheData[x.position].crc
 
-template addDep(x: Psym, dep: int32) =
+proc crcChanged(fileIdx: int32): bool =
-  gMemCacheData[x.position].deps.add(dep)
+  InternalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len
   
-proc checkDepMem(fileIdx: int32): bool =
+  template updateStatus =
+    gMemCacheData[fileIdx].crcStatus = if result: crcHasChanged
+                                       else: crcNotChanged
+    # echo "TESTING CRC: ", fileIdx.toFilename, " ", result
+    
+  case gMemCacheData[fileIdx].crcStatus:
+  of crcHasChanged:
+    result = true
+  of crcNotChanged:
+    result = false
+  of crcCached:
+    let newCrc = crcFromFile(fileIdx.toFilename)
+    result = newCrc != gMemCacheData[fileIdx].crc
+    gMemCacheData[fileIdx].crc = newCrc
+    updateStatus()
+  of crcNotTaken:
+    gMemCacheData[fileIdx].crc = crcFromFile(fileIdx.toFilename)
+    result = true
+    updateStatus()
+
+proc doCRC(fileIdx: int32) =
+  if gMemCacheData[fileIdx].crcStatus == crcNotTaken:
+    # echo "FIRST CRC: ", fileIdx.ToFilename
+    gMemCacheData[fileIdx].crc = crcFromFile(fileIdx.toFilename)
+
+proc safeAdd*[T](x: var seq[T], y: T) {.noSideEffect.} =
+  if x == nil: x = @[y]
+  else: x.add(y)
+
+proc safeAdd*(x: var string, y: char) =
+  if x == nil: x = ""
+  x.add(y)
+
+proc safeAdd*(x: var string, y: string) =
+  if x == nil: x = y
+  else: x.add(y)
+
+proc addDep(x: Psym, dep: int32) =
+  growCache gMemCacheData, dep
+  gMemCacheData[x.position].deps.safeAdd(dep)
+
+proc checkDepMem(fileIdx: int32): TNeedRecompile  =
   template markDirty =
-    gMemCacheData[fileIdx].needsRecompile = true
+    echo "HARD RESETTING ", fileIdx.toFilename
-    return true
+    gMemCacheData[fileIdx].needsRecompile = Yes
+    gCompiledModules[fileIdx] = nil
+    cgendata.gModules[fileIdx] = nil
+      
+    return Yes
+
+  if gMemCacheData[fileIdx].needsRecompile != Maybe:
+    return gMemCacheData[fileIdx].needsRecompile
 
   if optForceFullMake in gGlobalOptions or
-     curCaasCmd != lastCaasCmd: markDirty
+     curCaasCmd != lastCaasCmd or
-  
+     crcChanged(fileIdx): markDirty
-  let crc = crcFromFile(fileIdx.toFilename)
-  if crc != gMemCacheData[fileIdx].crc: markDirty
   
+  if gMemCacheData[fileIdx].deps != nil:
+    gMemCacheData[fileIdx].needsRecompile = Probing
     for dep in gMemCacheData[fileIdx].deps:
-    if checkDepMem(dep): markDirty
+      let d = checkDepMem(dep)
+      if d in { Yes, Recompiled }:
+        echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d
+        markDirty
   
-  return false
+  gMemCacheData[fileIdx].needsRecompile = No
+  return No
 
 proc newModule(fileIdx: int32): PSym =
   # We cannot call ``newSym`` here, because we have to circumvent the ID
@@ -98,6 +154,8 @@ proc newModule(fileIdx: int32): PSym =
 proc compileModule(fileIdx: int32, flags: TSymFlags): PSym =
   result = getModule(fileIdx)
   if result == nil:
+    growCache gMemCacheData, fileIdx
+    gMemCacheData[fileIdx].needsRecompile = Probing
     result = newModule(fileIdx)
     var rd = handleSymbolFile(result)
     result.flags = result.flags + flags
@@ -109,12 +167,12 @@ proc compileModule(fileIdx: int32, flags: TSymFlags): PSym =
     else:
       result.id = getID()
     processModule(result, nil, rd)
+    if optCaasEnabled in gGlobalOptions:
       gMemCacheData[fileIdx].compiledAt = gLastCmdTime
-    gMemCacheData[fileIdx].needsRecompile = false
+      gMemCacheData[fileIdx].needsRecompile = Recompiled
+      doCRC fileIdx
   else:
-    InternalAssert optCaasEnabled in gGlobalOptions
+    if checkDepMem(fileIdx) == Yes:
-    if checkDepMem(fileIdx):
-      gCompiledModules[fileIdx] = nil
       result = CompileModule(fileIdx, flags)
     else:
       result = gCompiledModules[fileIdx]
@@ -124,17 +182,17 @@ proc compileModule(filename: string, flags: TSymFlags): PSym =
 
 proc importModule(s: PSym, fileIdx: int32): PSym =
   # this is called by the semantic checking phase
-  result = getModule(fileIdx)
-  if result == nil: 
-    # compile the module
   result = compileModule(fileIdx, {})
   if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx)
-  elif sfSystemModule in result.flags: 
+  if sfSystemModule in result.flags:
     LocalError(result.info, errAttemptToRedefine, result.Name.s)
 
 proc includeModule(s: PSym, fileIdx: int32): PNode =
   result = syntaxes.parseFile(fileIdx)
-  if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx)
+  if optCaasEnabled in gGlobalOptions:
+    growCache gMemCacheData, fileIdx
+    addDep(s, fileIdx)
+    doCrc(fileIdx)
 
 proc `==^`(a, b: string): bool =
   try:
@@ -191,11 +249,55 @@ proc CommandCompileToC =
   registerPass(cgenPass)
   rodPass()
   #registerPass(cleanupPass())
+  if optCaasEnabled in gGlobalOptions:
+    # echo "BEFORE CHECK DEP"
+    # discard checkDepMem(gProjectMainIdx)
+    # echo "CHECK DEP COMPLETE"
+
   compileProject()
+
+  if optCaasEnabled in gGlobalOptions:
+    cgenCaasUpdate()
+
   if gCmd != cmdRun:
     extccomp.CallCCompiler(changeFileExt(gProjectFull, ""))
+
+  if optCaasEnabled in gGlobalOptions:
     # caas will keep track only of the compilation commands
     lastCaasCmd = curCaasCmd
+    resetCgenModules()
+    for i in 0 .. <gMemCacheData.len:
+      gMemCacheData[i].crcStatus = crcCached
+      gMemCacheData[i].needsRecompile = Maybe
+
+      # XXX: clean these global vars
+      # ccgstmts.gBreakpoints
+      # ccgthreadvars.nimtv
+      # ccgthreadvars.nimtVDeps
+      # ccgthreadvars.nimtvDeclared
+      # cgendata
+      # cgmeth?
+      # condsyms?
+      # depends?
+      # lexer.gLinesCompiled
+      # msgs - error counts
+      # magicsys, when system.nim changes
+      # rodread.rodcompilerProcs
+      # rodread.gTypeTable
+      # rodread.gMods
+      
+      # !! ropes.cache
+      # !! semdata.gGenericsCache
+      # semthreads.computed?
+      #
+      # suggest.usageSym
+      #
+      # XXX: can we run out of IDs?
+      # XXX: detect config reloading (implement as error/require restart)
+      # XXX: options are appended (they will accumulate over time)
+    resetCompilationLists()
+    ccgutils.resetCaches()
+    GC_fullCollect()
 
 when has_LLVM_Backend:
   proc CommandCompileToLLVM =

compiler/rodread.nim

@@ -795,7 +795,7 @@ proc GetCRC*(fileIdx: int32): TCrc32 =
   gMods[fileIdx].crc = result
 
 template growCache*(cache, pos) =
-  if cache.len <= fileIdx: cache.setLen(pos+1)
+  if cache.len <= pos: cache.setLen(pos+1)
 
 proc checkDep(fileIdx: int32): TReasonForRecompile =
   assert fileIdx != InvalidFileIDX

compiler/service.nim

@@ -12,7 +12,7 @@
 import 
   sockets,
   times, commands, options, msgs, nimconf,
-  extccomp, strutils, os, platform, main, parseopt
+  extccomp, strutils, os, platform, parseopt
 
 # We cache modules and the dependency graph. However, we don't check for
 # file changes but expect the client to tell us about them, otherwise the