bugfixes for ROD file generation; nimcache dir is now flat

compiler/ccgtypes.nim

@@ -46,6 +46,9 @@ proc mangleName(s: PSym): PRope =
     app(result, toRope(mangle(s.name.s)))
     app(result, "_")
     app(result, toRope(s.id))
+    when false:
+      # deactivated to make mapping file smaller which is currently only used
+      # for the list of generated C files
       if optGenMapping in gGlobalOptions: 
         if s.owner != nil: 
           appf(gMapping, "r\"$1.$2\": $3$n", 

compiler/cgen.nim

@@ -920,7 +920,7 @@ proc newModule(module: PSym, filename: string): BModule =
 proc registerTypeInfoModule() = 
   const moduleName = "nim__dat"
   var s = NewSym(skModule, getIdent(moduleName), nil)
-  gNimDat = rawNewModule(s, joinPath(options.projectPath, moduleName) & ".nim")
+  gNimDat = rawNewModule(s, (options.projectPath / moduleName) & ".nim")
   gNimDat.PreventStackTrace = true
   addPendingModule(gNimDat)
   appff(mainModProcs, "N_NOINLINE(void, $1)(void);$n", 
@@ -934,13 +934,17 @@ proc myOpenCached(module: PSym, filename: string,
                   rd: PRodReader): PPassContext = 
   if gNimDat == nil: 
     registerTypeInfoModule()
-  var cfile = changeFileExt(completeCFilePath(filename), cExt)
+    # XXX load old nimcache data here!
-  var cfilenoext = changeFileExt(cfile, "")
+    gNimDat.fromCache = true
-  addFileToLink(cfilenoext)
+  result = newModule(module, filename)
-  registerModuleToMain(module)
+  #if gNimDat == nil: registerTypeInfoModule()
+  #var cfile = changeFileExt(completeCFilePath(filename), cExt)
+  #var cfilenoext = changeFileExt(cfile, "")
+  #addFileToLink(cfilenoext)
+  #registerModuleToMain(module)
   # XXX: this cannot be right here, initalization has to be appended during
   # the ``myClose`` call
-  result = nil
+  #result = nil
 
 proc shouldRecompile(code: PRope, cfile, cfilenoext: string): bool = 
   result = true
@@ -989,6 +993,7 @@ proc writeModule(m: BModule) =
       tccgen.compileCCode(ropeToStr(code))
       return
   
+  if not m.fromCache:
     if shouldRecompile(code, changeFileExt(cfile, cExt), cfilenoext): 
       addFileToCompile(cfilenoext)
   addFileToLink(cfilenoext)
@@ -1000,7 +1005,9 @@ proc myClose(b: PPassContext, n: PNode): PNode =
   if n != nil: 
     m.initProc.options = gOptions
     genStmts(m.initProc, n)
+  # cached modules need to registered too: 
   registerModuleToMain(m.module)
+  
   if not (optDeadCodeElim in gGlobalOptions) and
       not (sfDeadCodeElim in m.module.flags): 
     finishModule(m)
@@ -1014,7 +1021,8 @@ proc myClose(b: PPassContext, n: PNode): PNode =
     while gForwardedProcsCounter > 0: 
       for i in countup(0, high(gPendingModules)): 
         finishModule(gPendingModules[i])
-    for i in countup(0, high(gPendingModules)): writeModule(gPendingModules[i])
+    for i in countup(0, high(gPendingModules)): 
+      writeModule(gPendingModules[i])
     setlen(gPendingModules, 0)
   if not (optDeadCodeElim in gGlobalOptions) and
       not (sfDeadCodeElim in m.module.flags): 

compiler/extccomp.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2010 Andreas Rumpf
+#        (c) Copyright 2011 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.

compiler/msgs.nim

@@ -572,7 +572,7 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
     frmt = posErrorFormat
     # we try to filter error messages so that not two error message
     # in the same file and line are produced:
-    ignoreMsg = lastError == info
+    ignoreMsg = lastError == info and eh != doAbort
     lastError = info
   of warnMin..warnMax: 
     ignoreMsg = optWarns notin gOptions or msg notin gNotes

compiler/options.nim

@@ -82,8 +82,6 @@ var
   gVerbosity*: int            # how verbose the compiler is
   gNumberOfProcessors*: int   # number of processors
 
-proc FindFile*(f: string): string
-
 const 
   genSubDir* = "nimcache"
   NimExt* = "nim"
@@ -94,12 +92,6 @@ const
   DocConfig* = "nimdoc.cfg"
   DocTexConfig* = "nimdoc.tex.cfg"
 
-proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string
-proc toGeneratedFile*(path, ext: string): string
-  # converts "/home/a/mymodule.nim", "rod" to "/home/a/nimcache/mymodule.rod"
-proc getPrefixDir*(): string
-  # gets the application directory
-
 # additional configuration variables:
 var 
   gConfigVars*: PStringTable
@@ -110,36 +102,30 @@ var
   gKeepComments*: bool = true # whether the parser needs to keep comments
   gImplicitMods*: TStringSeq = @[] # modules that are to be implicitly imported
 
-proc existsConfigVar*(key: string): bool
+proc existsConfigVar*(key: string): bool = 
-proc getConfigVar*(key: string): string
-proc setConfigVar*(key, val: string)
-proc addImplicitMod*(filename: string)
-proc binaryStrSearch*(x: openarray[string], y: string): int
-# implementation
-
-proc existsConfigVar(key: string): bool = 
   result = hasKey(gConfigVars, key)
 
-proc getConfigVar(key: string): string = 
+proc getConfigVar*(key: string): string = 
   result = gConfigVars[key]
 
-proc setConfigVar(key, val: string) = 
+proc setConfigVar*(key, val: string) = 
   gConfigVars[key] = val
 
 proc getOutFile*(filename, ext: string): string = 
   if options.outFile != "": result = options.outFile
   else: result = changeFileExt(filename, ext)
   
-proc addImplicitMod(filename: string) = 
+proc addImplicitMod*(filename: string) = 
   var length = len(gImplicitMods)
   setlen(gImplicitMods, length + 1)
   gImplicitMods[length] = filename
 
-proc getPrefixDir(): string = 
+proc getPrefixDir*(): string = 
+  ## gets the application directory
   result = SplitPath(getAppDir()).head
 
 proc shortenDir(dir: string): string = 
-  # returns the interesting part of a dir
+  ## returns the interesting part of a dir
   var prefix = getPrefixDir() & dirSep
   if startsWith(dir, prefix): 
     return substr(dir, len(prefix))
@@ -161,15 +147,17 @@ proc removeTrailingDirSep*(path: string): string =
 proc getGeneratedPath: string =
   result = if nimcacheDir.len > 0: nimcacheDir else: projectPath / genSubDir
   
-proc toGeneratedFile(path, ext: string): string = 
+proc toGeneratedFile*(path, ext: string): string = 
+  ## converts "/home/a/mymodule.nim", "rod" to "/home/a/nimcache/mymodule.rod"
   var (head, tail) = splitPath(path)
-  if len(head) > 0: head = shortenDir(head & dirSep)
+  #if len(head) > 0: head = shortenDir(head & dirSep)
-  result = joinPath([getGeneratedPath(), head, changeFileExt(tail, ext)])
+  result = joinPath([getGeneratedPath(), changeFileExt(tail, ext)])
+  #echo "toGeneratedFile(", path, ", ", ext, ") = ", result
 
-proc completeGeneratedFilePath(f: string, createSubDir: bool = true): string = 
+proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string = 
   var (head, tail) = splitPath(f)
-  if len(head) > 0: head = removeTrailingDirSep(shortenDir(head & dirSep))
+  #if len(head) > 0: head = removeTrailingDirSep(shortenDir(head & dirSep))
-  var subdir = getGeneratedPath() / head
+  var subdir = getGeneratedPath() # / head
   if createSubDir:
     try: 
       createDir(subdir)
@@ -177,6 +165,7 @@ proc completeGeneratedFilePath(f: string, createSubDir: bool = true): string =
       writeln(stdout, "cannot create directory: " & subdir)
       quit(1)
   result = joinPath(subdir, tail)
+  #echo "completeGeneratedFilePath(", f, ") = ", result
 
 iterator iterSearchPath*(): string = 
   var it = PStrEntry(SearchPaths.head)
@@ -193,11 +182,11 @@ proc rawFindFile(f: string): string =
       if ExistsFile(result): return
     result = ""
 
-proc FindFile(f: string): string = 
+proc FindFile*(f: string): string = 
   result = rawFindFile(f)
   if len(result) == 0: result = rawFindFile(toLower(f))
   
-proc binaryStrSearch(x: openarray[string], y: string): int = 
+proc binaryStrSearch*(x: openarray[string], y: string): int = 
   var a = 0
   var b = len(x) - 1
   while a <= b: 

compiler/passes.nim

@@ -16,7 +16,14 @@ import
   nimsets, syntaxes, times, rodread, semthreads, idgen
 
 type
+  TPassComm* = object {.pure.} ## communication object between passes
+    optimizers*: TSymSeq       ## filled by semantic pass; used in HLO
+  PPassComm* = ref TPassComm
+  
   TPassContext* = object of TObject # the pass's context
+    comm*: PPassComm
+    fromCache*: bool  # true if created by "openCached"
+    
   PPassContext* = ref TPassContext
   TPass* = tuple[
     open: proc (module: PSym, filename: string): PPassContext, 
@@ -78,16 +85,27 @@ proc registerPass(p: TPass) =
   gPasses[gPassesLen] = p
   inc(gPassesLen)
 
+proc newPassComm(): PPassComm =
+  new(result)
+  result.optimizers = @[]
+
 proc openPasses(a: var TPassContextArray, module: PSym, filename: string) = 
+  var comm = newPassComm()
   for i in countup(0, gPassesLen - 1): 
-    if not isNil(gPasses[i].open): a[i] = gPasses[i].open(module, filename)
+    if not isNil(gPasses[i].open): 
+      a[i] = gPasses[i].open(module, filename)
+      if a[i] != nil: a[i].comm = comm
     else: a[i] = nil
   
 proc openPassesCached(a: var TPassContextArray, module: PSym, filename: string, 
                       rd: PRodReader) = 
+  var comm = newPassComm()
   for i in countup(0, gPassesLen - 1): 
     if not isNil(gPasses[i].openCached): 
       a[i] = gPasses[i].openCached(module, filename, rd)
+      if a[i] != nil: 
+        a[i].comm = comm
+        a[i].fromCache = true
     else:
       a[i] = nil
   
@@ -110,7 +128,7 @@ proc processTopLevelStmtCached(n: PNode, a: var TPassContextArray) =
     if not isNil(gPasses[i].openCached): m = gPasses[i].process(a[i], m)
   
 proc closePassesCached(a: var TPassContextArray) = 
-  var m = ast.emptyNode
+  var m: PNode = nil
   for i in countup(0, gPassesLen - 1): 
     if not isNil(gPasses[i].openCached) and not isNil(gPasses[i].close): 
       m = gPasses[i].close(a[i], m)
@@ -144,7 +162,7 @@ proc processModule(module: PSym, filename: string, stream: PLLStream,
     IDsynchronizationPoint(1000)
   else: 
     openPassesCached(a, module, filename, rd)
-    var n = loadInitSection(rd)   #MessageOut('init section' + renderTree(n));
+    var n = loadInitSection(rd)
     for i in countup(0, sonsLen(n) - 1): processTopLevelStmtCached(n.sons[i], a)
     closePassesCached(a)
 

compiler/rodread.nim

@@ -123,13 +123,12 @@ type
   PRodReader* = ref TRodReader
 
 const 
-  FileVersion* = "1012"       # modify this if the rod-format changes!
+  FileVersion* = "1018"       # modify this if the rod-format changes!
 
 var rodCompilerprocs*: TStrTable
 
 proc handleSymbolFile*(module: PSym, filename: string): PRodReader
 # global because this is needed by magicsys
-proc GetCRC*(filename: string): TCrc32
 proc loadInitSection*(r: PRodReader): PNode
 proc loadStub*(s: PSym)
 
@@ -144,11 +143,11 @@ proc rrGetType(r: PRodReader, id: int, info: TLineInfo): PType
 proc decodeLineInfo(r: PRodReader, info: var TLineInfo) = 
   if r.s[r.pos] == '?': 
     inc(r.pos)
-    if r.s[r.pos] == ',': info.col = int16(- 1)
+    if r.s[r.pos] == ',': info.col = -1'i16
     else: info.col = int16(decodeVInt(r.s, r.pos))
     if r.s[r.pos] == ',': 
       inc(r.pos)
-      if r.s[r.pos] == ',': info.line = int16(- 1)
+      if r.s[r.pos] == ',': info.line = -1'i16
       else: info.line = int16(decodeVInt(r.s, r.pos))
       if r.s[r.pos] == ',': 
         inc(r.pos)
@@ -671,7 +670,7 @@ proc loadConverters(r: PRodReader) =
   if r.convertersIdx == 0 or r.dataIdx == 0: 
     InternalError("importConverters")
   r.pos = r.convertersIdx
-  while (r.s[r.pos] > '\x0A'): 
+  while r.s[r.pos] > '\x0A': 
     var d = decodeVInt(r.s, r.pos)
     discard rrGetSym(r, d, UnknownLineInfo())
     if r.s[r.pos] == ' ': inc(r.pos)
@@ -741,13 +740,12 @@ proc handleSymbolFile(module: PSym, filename: string): PRodReader =
   else: 
     module.id = getID()
   
-proc GetCRC(filename: string): TCrc32 = 
+proc GetCRC*(filename: string): TCrc32 = 
   var idx = getModuleIdx(filename)
   result = gMods[idx].crc
 
 proc loadStub(s: PSym) =
-  if s.kind != skStub: 
+  if s.kind != skStub: InternalError("loadStub")
-    InternalError("loadStub") #MessageOut('loading stub: ' + s.name.s);
   var rd = gMods[s.position].rd
   var theId = s.id                # used for later check
   var d = IITableGet(rd.index.tab, s.id)

compiler/rodutils.nim

@@ -58,6 +58,12 @@ proc decodeStr*(s: cstring, pos: var int): string =
 const
   chars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 
+# since negative numbers require a leading '-' they use up 1 byte. Thus we
+# subtract/add vintDelta here to save space for little negative numbers
+# which are common in ROD files: 
+const
+  vintDelta = 5
+
 template encodeIntImpl(self: expr) =
   var d: char
   var v = x
@@ -74,13 +80,22 @@ template encodeIntImpl(self: expr) =
   if v != 0: self(v, result)
   add(result, d)
 
+proc encodeVBiggestIntAux(x: BiggestInt, result: var string) =
+  ## encode a biggest int as a variable length base 190 int.
+  encodeIntImpl(encodeVBiggestIntAux)
+
 proc encodeVBiggestInt*(x: BiggestInt, result: var string) =
   ## encode a biggest int as a variable length base 190 int.
-  encodeIntImpl(encodeVBiggestInt)
+  encodeVBiggestIntAux(x +% vintDelta, result)
+  #  encodeIntImpl(encodeVBiggestInt)
+
+proc encodeVIntAux(x: int, result: var string) = 
+  ## encode an int as a variable length base 190 int.
+  encodeIntImpl(encodeVIntAux)
   
 proc encodeVInt*(x: int, result: var string) = 
   ## encode an int as a variable length base 190 int.
-  encodeIntImpl(encodeVInt)
+  encodeVIntAux(x +% vintDelta, result)
 
 template decodeIntImpl() =
   var i = pos
@@ -98,7 +113,7 @@ template decodeIntImpl() =
     of '\x80'..'\xFF': result = result * 190 - (ord(s[i]) - 128 + 62)
     else: break
     inc(i)
-  result = result * sign
+  result = result * sign -% vintDelta
   pos = i
 
 proc decodeVInt*(s: cstring, pos: var int): int = 

compiler/rodwrite.nim

@@ -459,11 +459,11 @@ proc writeRod(w: PRodWriter) =
   f.write(w.data)
   f.write(')' & rodNL)
   f.close()
-  #MessageOut('interf ' + ToString(ropeLen(w.interf)))
+  
-  #MessageOut('index ' + ToString(ropeLen(w.indexRope)))
+  #echo "interf: ", w.interf.len
-  #MessageOut('init ' + ToString(ropeLen(w.init)))
+  #echo "index:  ", w.index.r.len
-  #MessageOut('data ' + ToString(ropeLen(w.data)))
+  #echo "init:   ", w.init.len
-  return 22 + 33
+  #echo "data:   ", w.data.len
 
 proc process(c: PPassContext, n: PNode): PNode = 
   result = n

compiler/sem.nim

@@ -157,9 +157,7 @@ proc myOpen(module: PSym, filename: string): PPassContext =
 
 proc myOpenCached(module: PSym, filename: string, 
                   rd: PRodReader): PPassContext = 
-  var c = PContext(myOpen(module, filename))
+  result = myOpen(module, filename)
-  c.fromCache = true
-  result = c
 
 proc SemStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = 
   result = semStmt(c, n)

compiler/semdata.nim

@@ -50,7 +50,6 @@ type
     converters*: TSymSeq      # sequence of converters
     optionStack*: TLinkedList
     libs*: TLinkedList        # all libs used by this module
-    fromCache*: bool          # is the module read from a cache?
     semConstExpr*: proc (c: PContext, n: PNode): PNode # for the pragmas
     semExpr*: proc (c: PContext, n: PNode): PNode      # for the pragmas
     includedFiles*: TIntSet   # used to detect recursive include files

doc/advopt.txt

@@ -41,6 +41,7 @@ Advanced options:
   --lineDir:on|off          generation of #line directive on|off
   --threadanalysis:on|off   turn thread analysis on|off
   --taintMode:on|off        turn taint mode on|off
+  --symbolFiles:on|off      turn symbol files on|off (experimental)
   --skipCfg                 do not read the general configuration file
   --skipProjCfg             do not read the project's configuration file
   --gc:refc|boehm|none      use Nimrod's native GC|Boehm GC|no GC