rod file viewer for easier debugging of rod files

compiler/nversion.nim

@@ -18,5 +18,5 @@ const
   VersionPatch* = 1
   VersionAsString* = $VersionMajor & "." & $VersionMinor & "." & $VersionPatch
 
-  RodFileVersion* = "1210"       # modify this if the rod-format changes!
+  RodFileVersion* = "1211"       # modify this if the rod-format changes!
 

compiler/rodread.nim

@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2013 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -126,7 +126,7 @@ type
     s: cstring               # mmap'ed file contents
     options: TOptions
     reason: TReasonForRecompile
-    modDeps: seq[int32]
+    modDeps: TStringSeq
     files: TStringSeq
     dataIdx: int             # offset of start of data section
     convertersIdx: int       # offset of start of converters section
@@ -370,12 +370,12 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym =
     inc(r.pos)
     id = decodeVInt(r.s, r.pos)
     setId(id)
-  else: 
+  else:
     InternalError(info, "decodeSym: no id")
   if r.s[r.pos] == '&': 
     inc(r.pos)
     ident = getIdent(decodeStr(r.s, r.pos))
-  else: 
+  else:
     InternalError(info, "decodeSym: no ident")
   #echo "decoding: {", ident.s
   result = PSym(IdTableGet(r.syms, id))
@@ -384,8 +384,14 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym =
     result.id = id
     IdTablePut(r.syms, result, result)
     if debugIds: registerID(result)
-  elif (result.id != id): 
+  elif result.id != id:
     InternalError(info, "decodeSym: wrong id")
+  elif result.kind != skStub:
+    # we already loaded the symbol
+    return
+  else:
+    reset(result[])
+    result.id = id
   result.kind = k
   result.name = ident         # read the rest of the symbol description:
   if r.s[r.pos] == '^': 
@@ -498,7 +504,7 @@ proc processCompilerProcs(r: PRodReader, module: PSym) =
       IdTablePut(r.syms, s, s)
     StrTableAdd(rodCompilerProcs, s)
 
-proc processIndex(r: PRodReader, idx: var TIndex) = 
+proc processIndex(r: PRodReader; idx: var TIndex; outf: TFile = nil) = 
   var key, val, tmp: int
   inc(r.pos, 2)               # skip "(\10"
   inc(r.line)
@@ -508,10 +514,11 @@ proc processIndex(r: PRodReader, idx: var TIndex) =
       inc(r.pos)
       key = idx.lastIdxKey + tmp
       val = decodeVInt(r.s, r.pos) + idx.lastIdxVal
-    else: 
+    else:
       key = idx.lastIdxKey + 1
       val = tmp + idx.lastIdxVal
     IITablePut(idx.tab, key, val)
+    if not outf.isNil: outf.write(key, " ", val, "\n")
     idx.lastIdxKey = key
     idx.lastIdxVal = val
     setID(key)                # ensure that this id will not be used
@@ -601,8 +608,8 @@ proc processRodFile(r: PRodReader, crc: TCrc32) =
       if r.s[r.pos] == ')': inc(r.pos)
     of "DEPS":
       inc(r.pos)              # skip ':'
-      while r.s[r.pos] > '\x0A': 
-        r.modDeps.add int32(decodeVInt(r.s, r.pos))
+      while r.s[r.pos] > '\x0A':
+        r.modDeps.add(r.files[int32(decodeVInt(r.s, r.pos))])
         if r.s[r.pos] == ' ': inc(r.pos)
     of "INTERF": 
       r.interfIdx = r.pos + 2
@@ -673,11 +680,9 @@ proc newRodReader(modfilename: string, crc: TCrc32,
       add(version, r.s[r.pos])
       inc(r.pos)
     if r.s[r.pos] == '\x0A': inc(r.pos)
-    if version == RodFileVersion: 
+    if version != RodFileVersion: 
       # since ROD files are only for caching, no backwards compatibility is
       # needed
-      processRodFile(r, crc)
-    else: 
       result = nil
   else:
     result = nil
@@ -813,6 +818,7 @@ proc checkDep(fileIdx: int32): TReasonForRecompile =
   if r == nil: 
     result = (if ExistsFile(rodfile): rrRodInvalid else: rrRodDoesNotExist)
   else:
+    processRodFile(r, crc)
     result = r.reason
     if result == rrNone: 
       # check modules it depends on
@@ -822,9 +828,9 @@ proc checkDep(fileIdx: int32): TReasonForRecompile =
       var res = checkDep(SystemFileIdx)
       if res != rrNone: result = rrModDeps
       for i in countup(0, high(r.modDeps)): 
-        res = checkDep(r.modDeps[i])
-        if res != rrNone: 
-          result = rrModDeps 
+        res = checkDep(r.modDeps[i].fileInfoIdx)
+        if res != rrNone:
+          result = rrModDeps
           # we cannot break here, because of side-effects of `checkDep`
   if result != rrNone and gVerbosity > 0:
     rawMessage(hintProcessing, reasonToFrmt[result] % filename)
@@ -862,7 +868,9 @@ proc rawLoadStub(s: PSym) =
   var d = IITableGet(rd.index.tab, s.id)
   if d == invalidKey: InternalError("loadStub: invalid key")
   var rs = decodeSymSafePos(rd, d, UnknownLineInfo())
-  if rs != s: 
+  if rs != s:
+    #echo "rs: ", toHex(cast[int](rs.position), int.sizeof * 2),
+    #     "\ns:  ", toHex(cast[int](s.position), int.sizeof * 2)
     InternalError(rs.info, "loadStub: wrong symbol")
   elif rs.id != theId: 
     InternalError(rs.info, "loadStub: wrong ID") 
@@ -897,3 +905,263 @@ proc getBody*(s: PSym): PNode =
   
 InitIdTable(gTypeTable)
 InitStrTable(rodCompilerProcs)
+
+# viewer:
+proc writeNode(f: TFile; n: PNode) =
+  f.write("(")
+  if n != nil:
+    f.write($n.kind)
+    if n.typ != nil:
+      f.write('^')
+      f.write(n.typ.id)
+    case n.kind
+    of nkCharLit..nkInt64Lit: 
+      if n.intVal != 0:
+        f.write('!')
+        f.write(n.intVal)
+    of nkFloatLit..nkFloat64Lit: 
+      if n.floatVal != 0.0: 
+        f.write('!')
+        f.write($n.floatVal)
+    of nkStrLit..nkTripleStrLit:
+      if n.strVal != "": 
+        f.write('!')
+        f.write(n.strVal.escape)
+    of nkIdent:
+      f.write('!')
+      f.write(n.ident.s)
+    of nkSym:
+      f.write('!')
+      f.write(n.sym.id)
+    else:
+      for i in countup(0, sonsLen(n) - 1): 
+        writeNode(f, n.sons[i])
+  f.write(")")
+
+proc writeSym(f: TFile; s: PSym) =
+  if s == nil:
+    f.write("{}\n")
+    return
+  f.write("{")
+  f.write($s.kind)
+  f.write('+')
+  f.write(s.id)
+  f.write('&')
+  f.write(s.name.s)
+  if s.typ != nil:
+    f.write('^')
+    f.write(s.typ.id)
+  if s.owner != nil:
+    f.write('*')
+    f.write(s.owner.id)
+  if s.flags != {}:
+    f.write('$')
+    f.write($s.flags)
+  if s.magic != mNone:
+    f.write('@')
+    f.write($s.magic)
+  if s.options != gOptions: 
+    f.write('!')
+    f.write($s.options)
+  if s.position != 0: 
+    f.write('%')
+    f.write($s.position)
+  if s.offset != -1:
+    f.write('`')
+    f.write($s.offset)
+  if s.constraint != nil:
+    f.write('#')
+    f.writeNode(s.constraint)
+  if s.ast != nil:
+    f.writeNode(s.ast)
+  f.write("}\n")
+
+proc writeType(f: TFile; t: PType) =
+  if t == nil:
+    f.write("[]\n")
+    return
+  f.write('[')
+  f.write($t.kind)
+  f.write('+')
+  f.write($t.id)
+  if t.n != nil: 
+    f.writeNode(t.n)
+  if t.flags != {}:
+    f.write('$')
+    f.write($t.flags)
+  if t.callConv != low(t.callConv): 
+    f.write('?')
+    f.write($t.callConv)
+  if t.owner != nil:
+    f.write('*')
+    f.write($t.owner.id)
+  if t.sym != nil:
+    f.write('&')
+    f.write(t.sym.id)
+  if t.size != -1:
+    f.write('/')
+    f.write($t.size)
+  if t.align != 2:
+    f.write('=')
+    f.write($t.align)
+  if t.containerID != 0: 
+    f.write('@')
+    f.write($t.containerID)
+  for i in countup(0, sonsLen(t) - 1): 
+    if t.sons[i] == nil: 
+      f.write("^()")
+    else:
+      f.write('^') 
+      f.write($t.sons[i].id)
+  f.write("]\n")
+
+proc viewFile(rodfile: string) =
+  var r = newRodReader(rodfile, 0, 0)
+  if r == nil:
+    rawMessage(errGenerated, "cannot open file (or maybe wrong version):" &
+       rodfile)
+    return
+  var outf = system.open(rodfile.changeFileExt(".rod.txt"), fmWrite)
+  while r.s[r.pos] != '\0':
+    let section = rdWord(r)
+    case section
+    of "CRC":
+      inc(r.pos)              # skip ':'
+      outf.writeln("CRC:", $decodeVInt(r.s, r.pos))
+    of "ID": 
+      inc(r.pos)              # skip ':'
+      r.moduleID = decodeVInt(r.s, r.pos)
+      setID(r.moduleID)
+      outf.writeln("ID:", $r.moduleID)
+    of "OPTIONS":
+      inc(r.pos)              # skip ':'
+      r.options = cast[TOptions](int32(decodeVInt(r.s, r.pos)))
+      outf.writeln("OPTIONS:", $r.options)
+    of "GOPTIONS":
+      inc(r.pos)              # skip ':'
+      let dep = cast[TGlobalOptions](int32(decodeVInt(r.s, r.pos)))
+      outf.writeln("GOPTIONS:", $dep)
+    of "CMD":
+      inc(r.pos)              # skip ':'
+      let dep = cast[TCommands](int32(decodeVInt(r.s, r.pos)))
+      outf.writeln("CMD:", $dep)
+    of "DEFINES":
+      inc(r.pos)              # skip ':'
+      var d = 0
+      outf.write("DEFINES:")
+      while r.s[r.pos] > '\x0A':
+        let w = decodeStr(r.s, r.pos)
+        inc(d)
+        outf.write(" ", w)
+        if r.s[r.pos] == ' ': inc(r.pos)
+      outf.write("\n")
+    of "FILES": 
+      inc(r.pos, 2)           # skip "(\10"
+      inc(r.line)
+      outf.write("FILES(\n")
+      while r.s[r.pos] != ')':
+        let relativePath = decodeStr(r.s, r.pos)
+        let resolvedPath = relativePath.findModule
+        let rr = if resolvedPath.len > 0: resolvedPath else: relativePath
+        r.files.add(rr)
+        inc(r.pos)            # skip #10
+        inc(r.line)
+        outf.writeln(rr)
+      if r.s[r.pos] == ')': inc(r.pos)
+      outf.write(")\n")
+    of "INCLUDES": 
+      inc(r.pos, 2)           # skip "(\10"
+      inc(r.line)
+      outf.write("INCLUDES(\n")
+      while r.s[r.pos] != ')': 
+        let w = r.files[decodeVInt(r.s, r.pos)]
+        inc(r.pos)            # skip ' '
+        let inclCrc = decodeVInt(r.s, r.pos)
+        if r.s[r.pos] == '\x0A': 
+          inc(r.pos)
+          inc(r.line)
+        outf.write(w, " ", inclCrc, "\n")
+      if r.s[r.pos] == ')': inc(r.pos)
+      outf.write(")\n")
+    of "DEPS":
+      inc(r.pos)              # skip ':'
+      outf.write("DEPS:")
+      while r.s[r.pos] > '\x0A': 
+        let v = int32(decodeVInt(r.s, r.pos))
+        r.modDeps.add(r.files[v])
+        if r.s[r.pos] == ' ': inc(r.pos)
+        outf.write(" ", r.files[v])
+      outf.write("\n")
+    of "INTERF",  "COMPILERPROCS":
+      inc r.pos, 2
+      if section == "INTERF": r.interfIdx = r.pos
+      else: r.compilerProcsIdx = r.pos
+      outf.write(section, "(\n")
+      while (r.s[r.pos] > '\x0A') and (r.s[r.pos] != ')'):
+        let w = decodeStr(r.s, r.pos)
+        inc(r.pos)
+        let key = decodeVInt(r.s, r.pos)
+        inc(r.pos)                # #10
+        outf.write(w, " ", key, "\n")
+      if r.s[r.pos] == ')': inc r.pos
+      outf.write(")\n")
+    of "INDEX":
+      outf.write(section, "(\n")
+      processIndex(r, r.index, outf)
+      outf.write(")\n")
+    of "IMPORTS":
+      outf.write(section, "(\n")
+      processIndex(r, r.imports, outf)
+      outf.write(")\n")
+    of "CONVERTERS",  "METHODS":
+      inc r.pos
+      if section == "METHODS": r.methodsIdx = r.pos
+      else: r.convertersIdx = r.pos
+      outf.write(section, ":")
+      while r.s[r.pos] > '\x0A': 
+        let d = decodeVInt(r.s, r.pos)
+        outf.write(" ", $d)
+        if r.s[r.pos] == ' ': inc(r.pos)
+      outf.write("\n")
+    of "DATA":
+      inc(r.pos, 2)
+      r.dataIdx = r.pos
+      outf.write("DATA(\n")
+      while r.s[r.pos] != ')':
+        if r.s[r.pos] == '(':
+          outf.writeNode decodeNode(r, UnknownLineInfo())
+          outf.write("\n")
+        elif r.s[r.pos] == '[':
+          outf.writeType decodeType(r, UnknownLineInfo())
+        else:
+          outf.writeSym decodeSym(r, UnknownLineInfo())
+        if r.s[r.pos] == '\x0A':
+          inc(r.pos)
+          inc(r.line)
+      if r.s[r.pos] == ')': inc r.pos
+      outf.write(")\n")
+    of "INIT":
+      outf.write("INIT(\n")
+      inc r.pos, 2
+      r.initIdx = r.pos
+      while r.s[r.pos] > '\x0A' and r.s[r.pos] != ')': 
+        let d = decodeVInt(r.s, r.pos)
+        inc(r.pos)                # #10
+        #let p = r.pos
+        #r.pos = d + r.dataIdx
+        #outf.writeNode decodeNode(r, UnknownLineInfo())
+        #outf.write("\n")
+        #r.pos = p
+      if r.s[r.pos] == ')': inc r.pos
+      outf.write("\n")
+    else:
+      InternalError("invalid section: '" & section &
+                    "' at " & $r.line & " in " & r.filename)
+      skipSection(r)
+    if r.s[r.pos] == '\x0A':
+      inc(r.pos)
+      inc(r.line)
+  outf.close
+
+when isMainModule:
+  viewFile(paramStr(1).addFileExt(rodExt))

compiler/rodwrite.nim

@@ -205,6 +205,9 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
     return
   # we need no surrounding [] here because the type is in a line of its own
   if t.kind == tyForward: InternalError("encodeType: tyForward")
+  # for the new rodfile viewer we use a preceeding [ so that the data section
+  # can easily be disambiguated:
+  add(result, '[')
   encodeVInt(ord(t.kind), result)
   add(result, '+')
   encodeVInt(t.id, result)

compiler/service.nim

@@ -83,7 +83,7 @@ proc serve*(action: proc (){.nimcall.}) =
     new(stdoutSocket)
     while true:
       accept(server, stdoutSocket)
-      discard stdoutSocket.recvLine(inp)
+      stdoutSocket.readLine(inp)
       execute inp.string
       stdoutSocket.send("\c\L")
       stdoutSocket.close()

lib/system.nim

@@ -1597,6 +1597,18 @@ proc `$`*[T: tuple|object](x: T): string =
     result.add($value)
   result.add(")")
 
+proc `$`*[T: set](x: T): string = 
+  ## generic ``$`` operator for sets that is lifted from the components
+  ## of `x`. Example:
+  ##
+  ## .. code-block:: nimrod
+  ##   ${23, 45} == "{23, 45}"
+  result = "{"
+  for value in items(x):
+    if result.len > 1: result.add(", ")
+    result.add($value)
+  result.add("}")
+
 when false:
   proc `$`*[T](a: openArray[T]): string = 
     ## generic ``$`` operator for open arrays that is lifted from the elements

tests/gc/closureleak.nim

@@ -30,4 +30,4 @@ for i in 0 .. <10:
     echo f.id
 
 gc_fullcollect()
-echo alive_foos.len <= 2
+echo alive_foos.len <= 3

tests/reject/teffects1.nim

@@ -1,5 +1,5 @@
 discard """
-  line: 49
+  line: 51
   file: "sysio.nim"
   errormsg: "can raise an unlisted exception: ref EIO"
 """

todo.txt

@@ -30,7 +30,6 @@ Bugs
 - osproc execProcesses can deadlock if all processes fail (as experienced
   in c++ mode)
 - bootstrapping does not work in C++ mode
-- the new m&s GC is still buggy
 
 
 version 0.9.4