trivial change scenario works with symbol files

compiler/ccgmerge.nim

@@ -43,30 +43,31 @@ const
     cpsInit: "NIM_merge_PROC_INIT",
     cpsStmts: "NIM_merge_PROC_BODY"
   ]
+  NimMergeEndMark = "/*\tNIM_merge_END:*/"
   
 proc genSectionStart*(fs: TCFileSection): PRope =
   if optSymbolFiles in gGlobalOptions:
     result = toRope(tnl)
     app(result, "/*\t")
     app(result, CFileSectionNames[fs])
-    app(result, "*/")
+    app(result, ":*/")
     app(result, tnl)
 
 proc genSectionEnd*(fs: TCFileSection): PRope =
   if optSymbolFiles in gGlobalOptions:
-    result = toRope("/*\tNIM_merge_END*/" & tnl)
+    result = toRope(NimMergeEndMark & tnl)
 
 proc genSectionStart*(ps: TCProcSection): PRope =
   if optSymbolFiles in gGlobalOptions:
     result = toRope(tnl)
     app(result, "/*\t")
     app(result, CProcSectionNames[ps])
-    app(result, "*/")
+    app(result, ":*/")
     app(result, tnl)
 
 proc genSectionEnd*(ps: TCProcSection): PRope =
   if optSymbolFiles in gGlobalOptions:
-    result = toRope("/*\tNIM_merge_END*/" & tnl)
+    result = toRope(NimMergeEndMark & tnl)
 
 proc writeTypeCache(a: TIdTable, s: var string) =
   var i = 0
@@ -137,8 +138,12 @@ proc skipUntilCmd(L: var TBaseLexer) =
     else: inc pos
   L.bufpos = pos
 
+proc atEndMark(buf: cstring, pos: int): bool =
+  var s = 0
+  while s < NimMergeEndMark.len and buf[pos+s] == NimMergeEndMark[s]: inc s
+  result = s == NimMergeEndMark.len
+
 proc readVerbatimSection(L: var TBaseLexer): PRope = 
-  const section = "/*\tNIM_merge_END*/"
   var pos = L.bufpos
   var buf = L.buf
   result = newMutableRope(30_000)
@@ -152,26 +157,22 @@ proc readVerbatimSection(L: var TBaseLexer): PRope =
       pos = lexbase.HandleLF(L, pos)
       buf = L.buf
       result.data.add(tnl)
-    of '\0': break
+    of '\0':
-    else: nil
+      InternalError("ccgmerge: expected: " & NimMergeEndMark)
-    if buf[pos] == section[0]:
+      break
-      var s = 0
+    else: 
-      while buf[pos+1] == section[s+1]:
+      if atEndMark(buf, pos):
-        inc s
+        inc pos, NimMergeEndMark.len
-        inc pos
-      if section[s] != '\0':
-        # reset:
-        dec pos, s
-      else:
         break
-    result.data.add(buf[pos])
+      result.data.add(buf[pos])
-    inc pos
+      inc pos
   L.bufpos = pos
   result.length = result.data.len
 
-proc readKey(L: var TBaseLexer): string =
+proc readKey(L: var TBaseLexer, result: var string) =
   var pos = L.bufpos
   var buf = L.buf
+  setLen(result, 0)
   while buf[pos] in IdentChars:
     result.add(buf[pos])
     inc pos
@@ -207,12 +208,13 @@ proc readIntSet(L: var TBaseLexer, result: var TIntSet) =
   inc L.bufpos
 
 proc processMergeInfo(L: var TBaseLexer, m: BModule) =
+  var k = newStringOfCap("typeCache".len)
   while true:
     skipWhite(L)
     if ^L.bufpos == '*' and ^(L.bufpos+1) == '/':
       inc(L.bufpos, 2)
       break
-    var k = readKey(L)
+    readKey(L, k)
     case k
     of "typeCache": readTypeCache(L, m.typeCache)
     of "declared":  readIntSet(L, m.declaredThings)
@@ -225,6 +227,7 @@ template withCFile(cfilename: string, body: stmt) =
   if s == nil: return
   var L: TBaseLexer
   openBaseLexer(L, s)
+  var k = newStringOfCap("NIM_merge_FORWARD_TYPES".len)
   while true:
     skipUntilCmd(L)
     if ^L.bufpos == '\0': break
@@ -234,7 +237,7 @@ template withCFile(cfilename: string, body: stmt) =
 proc readMergeInfo*(cfilename: string, m: BModule) =
   ## reads the merge meta information into `m`.
   withCFile(cfilename):
-    var k = readKey(L)
+    readKey(L, k)
     if k == "NIM_merge_INFO":
       processMergeInfo(L, m)
       break
@@ -247,7 +250,7 @@ type
 proc readMergeSections(cfilename: string, m: var TMergeSections) =
   ## reads the merge sections into `m`.
   withCFile(cfilename):
-    var k = readKey(L)
+    readKey(L, k)
     if k == "NIM_merge_INFO":   
       nil
     elif ^L.bufpos == '*' and ^(L.bufpos+1) == '/':
@@ -269,10 +272,14 @@ proc readMergeSections(cfilename: string, m: var TMergeSections) =
       InternalError("ccgmerge: '*/' expected")
 
 proc mergeRequired*(m: BModule): bool =
-  for i in low(TCFileSection)..high(TCFileSection):
+  for i in cfsHeaders..cfsProcs:
-    if m.s[i] != nil: return true
+    if m.s[i] != nil:
+      #echo "not empty: ", i, " ", ropeToStr(m.s[i])
+      return true
   for i in low(TCProcSection)..high(TCProcSection):
-    if m.initProc.s[i] != nil: return true
+    if m.initProc.s[i] != nil: 
+      #echo "not empty: ", i, " ", ropeToStr(m.initProc.s[i])
+      return true
 
 proc mergeFiles*(cfilename: string, m: BModule) =
   ## merges the C file with the old version on hard disc.

compiler/cgen.nim

@@ -808,12 +808,10 @@ proc genModule(m: BModule, cfilenoext: string): PRope =
   result = getFileHeader(cfilenoext)
   result.app(genMergeInfo(m))
   
-  app(m.s[cfsHeaders], genSectionStart(cfsHeaders))
   generateHeaders(m)
-  app(m.s[cfsHeaders], genSectionEnd(cfsHeaders))
 
   generateThreadLocalStorage(m)
-  for i in countup(low(TCFileSection), cfsProcs): 
+  for i in countup(cfsHeaders, cfsProcs): 
     app(result, genSectionStart(i))
     app(result, m.s[i])
     app(result, genSectionEnd(i))
@@ -899,29 +897,38 @@ proc shouldRecompile(code: PRope, cfile, cfilenoext: string): bool =
     if ExistsFile(objFile) and os.FileNewer(objFile, cfile): result = false
   else: 
     writeRope(code, cfile)
-  
+
-proc writeModule(m: BModule) = 
+# We need 2 different logics here: pending modules (including
+# 'nim__dat') may require file merging for the combination of dead code
+# elimination and incremental compilation! Non pending modules need no
+# such logic and in fact the logic hurts for the main module at least;
+# it would generate multiple 'main' procs, for instance.
+
+proc writeModule(m: BModule, pending: bool) =
   # generate code for the init statements of the module:
-  var cfile = completeCFilePath(m.cfilename)
+  var cfile = changeFileExt(completeCFilePath(m.cfilename), cExt)
   var cfilenoext = changeFileExt(cfile, "")
   
-  genInitCode(m)
-  finishTypeDescriptions(m)
-  if sfMainModule in m.module.flags: 
-    # generate main file:
-    app(m.s[cfsProcHeaders], mainModProcs)
-    GenerateThreadVarsSize(m)
-    
   if not m.fromCache or optForceFullMake in gGlobalOptions:
+    # XXX Bug: what if `m` is unchanged, but re-opened because of dead code
+    # elim and now depends on a previously unnecessary module that needs to
+    # be initialized here?
+    genInitCode(m)
+    finishTypeDescriptions(m)
+    if sfMainModule in m.module.flags: 
+      # generate main file:
+      app(m.s[cfsProcHeaders], mainModProcs)
+      GenerateThreadVarsSize(m)
+    
     var code = genModule(m, cfilenoext)
     when hasTinyCBackend:
       if gCmd == cmdRun:
         tccgen.compileCCode(ropeToStr(code))
         return
 
-    if shouldRecompile(code, changeFileExt(cfile, cExt), cfilenoext): 
+    if shouldRecompile(code, cfile, cfilenoext):
       addFileToCompile(cfilenoext)
-  elif mergeRequired(m):
+  elif pending and mergeRequired(m) and sfMainModule notin m.module.flags:
     mergeFiles(cfile, m)
     var code = genModule(m, cfilenoext)
     writeRope(code, cfile)
@@ -938,8 +945,8 @@ proc myClose(b: PPassContext, n: PNode): PNode =
   # cached modules need to registered too: 
   registerModuleToMain(m.module)
   
-  if not (optDeadCodeElim in gGlobalOptions) and
+  if optDeadCodeElim notin gGlobalOptions and
-      not (sfDeadCodeElim in m.module.flags): 
+      sfDeadCodeElim notin m.module.flags: 
     finishModule(m)
   if sfMainModule in m.module.flags: 
     var disp = generateMethodDispatchers()
@@ -952,11 +959,11 @@ proc myClose(b: PPassContext, n: PNode): PNode =
       for i in countup(0, high(gPendingModules)): 
         finishModule(gPendingModules[i])
     for i in countup(0, high(gPendingModules)): 
-      writeModule(gPendingModules[i])
+      writeModule(gPendingModules[i], pending=true)
     setlen(gPendingModules, 0)
-  if not (optDeadCodeElim in gGlobalOptions) and
+  if optDeadCodeElim notin gGlobalOptions and
-      not (sfDeadCodeElim in m.module.flags): 
+      sfDeadCodeElim notin m.module.flags:
-    writeModule(m)
+    writeModule(m, pending=false)
   if sfMainModule in m.module.flags: writeMapping(gMapping)
       
 proc cgenPass(): TPass = 

compiler/crc.nim

@@ -79,11 +79,11 @@ proc updateCrc32(val: int8, crc: TCrc32): TCrc32 =
       0x000000FF]) xor (crc shr TCrc32(8))
 
 proc updateCrc32(val: Char, crc: TCrc32): TCrc32 = 
-  result = updateCrc32(int8(ord(val)), crc)
+  result = updateCrc32(toU8(ord(val)), crc)
 
 proc strCrc32(s: string): TCrc32 = 
   result = InitCrc32
-  for i in countup(0, len(s) + 0 - 1): result = updateCrc32(s[i], result)
+  for i in countup(0, len(s) - 1): result = updateCrc32(s[i], result)
   
 proc `><`*(c: TCrc32, s: string): TCrc32 = 
   result = c

compiler/rodread.nim

@@ -713,8 +713,8 @@ proc checkDep(filename: string): TReasonForRecompile =
             # we cannot break here, because of side-effects of `checkDep`
   else: 
     result = rrRodDoesNotExist
-  if result != rrNone and gVerbosity > 0: 
+  if result != rrNone and gVerbosity > 0:
-    MsgWriteln(`%`(reasonToFrmt[result], [filename]))
+    rawMessage(hintProcessing, reasonToFrmt[result] % filename)
   if result != rrNone or optForceFullMake in gGlobalOptions: 
     # recompilation is necessary:
     r = nil

todo.txt

@@ -19,6 +19,7 @@ incremental compilation
 - adapt thread var implementation to care about the new merge operation
 - write test cases: needs test script support
   - test thread var
+  - test method generation
   - test DLL interfacing!
 - fix remaining bugs
 - write documentation