temporary debugging code for the memory leak investigation

compiler/ast.nim

@@ -517,7 +517,7 @@ type
   TNodeSeq* = seq[PNode]
   PType* = ref TType
   PSym* = ref TSym
-  TNode*{.acyclic, final.} = object # on a 32bit machine, this takes 32 bytes
+  TNode*{.final.} = object # on a 32bit machine, this takes 32 bytes
     typ*: PType
     comment*: string
     info*: TLineInfo

compiler/magicsys.nim

@@ -77,6 +77,15 @@ proc getSysType(kind: TTypeKind): PType =
 var
   intTypeCache: array[-5..64, PType]
 
+proc resetSysTypes* =
+  systemModule = nil
+  initStrTable(compilerprocs)
+  for i in low(gSysTypes)..high(gSysTypes):
+    gSysTypes[i] = nil
+
+  for i in low(intTypeCache)..high(intTypeCache):
+    intTypeCache[i] = nil
+
 proc getIntLitType*(literal: PNode): PType =
   # we cache some common integer literal types for performance:
   let value = literal.intVal

compiler/main.nim

@@ -89,13 +89,23 @@ proc addDep(x: Psym, dep: int32) =
   growCache gMemCacheData, dep
   gMemCacheData[x.position].deps.safeAdd(dep)
 
+proc ResetModule(fileIdx: int32) =
+  echo "HARD RESETTING ", fileIdx.toFilename
+  gMemCacheData[fileIdx].needsRecompile = Yes
+  gCompiledModules[fileIdx] = nil
+  cgendata.gModules[fileIdx] = nil
+
+proc ResetAllModules =
+  for i in 0..gCompiledModules.high:
+    if gCompiledModules[i] != nil:
+      ResetModule(i.int32)
+
+  for m in cgenModules():
+    echo "CGEN MODULE FOUND"
+
 proc checkDepMem(fileIdx: int32): TNeedRecompile  =
   template markDirty =
-    echo "HARD RESETTING ", fileIdx.toFilename
+    ResetModule(fileIdx)
-    gMemCacheData[fileIdx].needsRecompile = Yes
-    gCompiledModules[fileIdx] = nil
-    cgendata.gModules[fileIdx] = nil
-      
     return Yes
 
   if gMemCacheData[fileIdx].needsRecompile != Maybe:
@@ -377,7 +387,13 @@ proc wantMainModule =
   if gProjectFull.len == 0:
     Fatal(gCmdLineInfo, errCommandExpectsFilename)
   gProjectMainIdx = addFileExt(gProjectFull, nimExt).fileInfoIdx
-  
+
+var oss: PGenericSeq
+proc dbgseqimp(x: PGenericSeq) {.cdecl.} =
+  oss = x
+
+seqdbg = dbgseqimp
+
 proc MainCommand =
   # In "nimrod serve" scenario, each command must reset the registered passes
   clearPasses()
@@ -482,6 +498,48 @@ proc MainCommand =
   of "e":
     # XXX: temporary command for easier testing
     commandEval(mainCommandArg())
+  of "reset":
+    ResetModule(gProjectMainIdx)
+    gcDebugging = true
+    GC_fullCollect()
+
+    resetCompilationLists()
+    ccgutils.resetCaches()
+    ResetAllModules()
+    resetRopeCache()
+    resetSysTypes()
+    gGenericsCache = nil
+    gOwners = @[]
+    rangeDestructorProc = nil
+    
+    # XXX: clean these global vars
+    # ccgstmts.gBreakpoints
+    # ccgthreadvars.nimtv
+    # ccgthreadvars.nimtVDeps
+    # ccgthreadvars.nimtvDeclared
+    # cgendata
+    # cgmeth?
+    # condsyms?
+    # depends?
+    # lexer.gLinesCompiled
+    # msgs - error counts
+    # magicsys, when system.nim changes
+    # rodread.rodcompilerProcs
+    # rodread.gTypeTable
+    # rodread.gMods
+    
+    # !! ropes.cache
+    # !! semdata.gGenericsCache
+    # semthreads.computed?
+    #
+    # suggest.usageSym
+    #
+    # XXX: can we run out of IDs?
+    # XXX: detect config reloading (implement as error/require restart)
+    # XXX: options are appended (they will accumulate over time)
+    # vis = visimpl
+    GC_fullCollect()
+    echo GC_getStatistics()
   of "idetools":
     gCmd = cmdIdeTools
     if gEvalExpr != "":

compiler/ropes.nim

@@ -114,6 +114,10 @@ proc freezeMutableRope*(r: PRope) {.inline.} =
 var 
   cache: array[0..2048*2 -1, PRope]
 
+proc resetRopeCache* =
+  for i in low(cache)..high(cache):
+    cache[i] = nil
+
 proc RopeInvariant(r: PRope): bool = 
   if r == nil: 
     result = true

compiler/semdata.nim

@@ -86,7 +86,7 @@ type
                                # naming it multiple times
 
 var
-  gGenericsCache: PGenericsCache # save for modularity
+  gGenericsCache*: PGenericsCache # save for modularity
 
 proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair =
   result.genericSym = s

compiler/semstmts.nim

@@ -933,7 +933,7 @@ var
   destructorName = getIdent"destroy_"
   destructorParam = getIdent"this_"
   destructorPragma = newIdentNode(getIdent"destructor", UnknownLineInfo())
-  rangeDestructorProc: PSym
+  rangeDestructorProc*: PSym
 
 proc destroyField(c: PContext, field: PSym, holder: PNode): PNode =
   if instantiateDestructor(c, field.typ):

lib/system.nim

@@ -179,9 +179,9 @@ when not defined(niminheritable):
 
 when not defined(EcmaScript) and not defined(NimrodVM):
   type
-    TGenericSeq {.compilerproc, pure, inheritable.} = object
+    TGenericSeq* {.compilerproc, pure, inheritable.} = object
       len, reserved: int
-    PGenericSeq {.exportc.} = ptr TGenericSeq
+    PGenericSeq* {.exportc.} = ptr TGenericSeq
     # len and space without counting the terminating zero:
     NimStringDesc {.compilerproc, final.} = object of TGenericSeq
       data: array[0..100_000_000, char]
@@ -558,7 +558,7 @@ proc abs*(x: int64): int64 {.magic: "AbsI64", noSideEffect.}
   ## checking is turned on).
 
 type
-  IntMax32 = int|int8|int16|int32
+  IntMax32 = bool|int|int8|int16|int32
 
 proc `+%` *(x, y: IntMax32): IntMax32 {.magic: "AddU", noSideEffect.}
 proc `+%` *(x, y: Int64): Int64 {.magic: "AddU", noSideEffect.}

lib/system/cellsets.nim

@@ -13,7 +13,7 @@ type
   TCell {.pure.} = object
     refcount: int  # the refcount and some flags
     typ: PNimType
-    when leakDetector:
+    when trackAllocationSource:
       filename: cstring
       line: int
 

lib/system/gc.nim

@@ -355,9 +355,10 @@ proc forAllChildren(cell: PCell, op: TWalkOp) =
       var d = cast[TAddress](cellToUsr(cell))
       var s = cast[PGenericSeq](d)
       if s != nil:
+        let baseAddr = d +% GenericSeqSize
         for i in 0..s.len-1:
-          forAllChildrenAux(cast[pointer](d +% i *% cell.typ.base.size +%
+          forAllChildrenAux(cast[pointer](baseAddr +% i *% cell.typ.base.size),
-            GenericSeqSize), cell.typ.base, op)
+                            cell.typ.base, op)
     else: nil
 
 proc addNewObjToZCT(res: PCell, gch: var TGcHeap) {.inline.} =
@@ -414,10 +415,12 @@ proc rawNewObj(typ: PNimType, size: int, gch: var TGcHeap): pointer =
   sysAssert((cast[TAddress](res) and (MemAlign-1)) == 0, "newObj: 2")
   # now it is buffered in the ZCT
   res.typ = typ
-  when leakDetector and not hasThreadSupport:
+  when trackAllocationSource and not hasThreadSupport:
-    if framePtr != nil and framePtr.prev != nil:
+    if framePtr != nil and framePtr.prev != nil and framePtr.prev.prev != nil:
-      res.filename = framePtr.prev.filename
+      res.filename = framePtr.prev.prev.filename
-      res.line = framePtr.prev.line
+      res.line = framePtr.prev.prev.line
+    else:
+      res.filename = "nofile"
   res.refcount = rcZct # refcount is zero, but mark it to be in the ZCT  
   sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3")
   # its refcount is zero, so add it to the ZCT:
@@ -456,10 +459,12 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} =
   sysAssert((cast[TAddress](res) and (MemAlign-1)) == 0, "newObj: 2")
   # now it is buffered in the ZCT
   res.typ = typ
-  when leakDetector and not hasThreadSupport:
+  when trackAllocationSource and not hasThreadSupport:
-    if framePtr != nil and framePtr.prev != nil:
+    if framePtr != nil and framePtr.prev != nil and framePtr.prev.prev != nil:
-      res.filename = framePtr.prev.filename
+      res.filename = framePtr.prev.prev.filename
-      res.line = framePtr.prev.line
+      res.line = framePtr.prev.prev.line
+    else:
+      res.filename = "nofile"
   res.refcount = rcIncrement # refcount is 1
   sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3")
   when logGC: writeCell("new cell", res)
@@ -597,6 +602,21 @@ proc collectCycles(gch: var TGcHeap) =
   Deinit(gch.cycleRoots)
   Init(gch.cycleRoots)
 
+var gcDebugging* = false
+var vis*: proc (a: pointer, b: PNimType)
+
+proc debugNode(n: ptr TNimNode) =
+  c_fprintf(c_stdout, "node %s\n", n.name)
+  for i in 0..n.len-1:
+    debugNode(n.sons[i])
+
+proc debugTyp(x: PNimType) =
+  c_fprintf(c_stdout, "type %d\n", x.kind)
+  if x.node != nil:
+    debugNode(x.node)
+
+var seqdbg* : proc (s: PGenericSeq) {.cdecl.}
+
 proc gcMark(gch: var TGcHeap, p: pointer) {.inline.} =
   # the addresses are not as cells on the stack, so turn them to cells:
   sysAssert(allocInv(gch.region), "gcMark begin")
@@ -607,8 +627,16 @@ proc gcMark(gch: var TGcHeap, p: pointer) {.inline.} =
     var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell))
     if objStart != nil:
       # mark the cell:
-      objStart.refcount = objStart.refcount +% rcIncrement
+      if gcDebugging:
-      add(gch.decStack, objStart)
+        c_fprintf(c_stdout, "object root found %d\nfile: %s\nline: %d\n", objStart.typ.kind, objStart.filename, objStart.line)
+        if objStart.typ.kind == tySequence:
+          let sq = cast[PGenericSeq](cellToUsr(objStart))
+          c_fprintf(c_stdout, "seq len: %d\noffset: %ld\n", sq.len, cast[TAddress](p) - cast[TAddress](sq))
+          seqdbg(sq)
+        
+      if not gcDebugging:
+        objStart.refcount = objStart.refcount +% rcIncrement
+        add(gch.decStack, objStart)
     when false:
       if isAllocatedPtr(gch.region, cell):
         sysAssert false, "allocated pointer but not interior?"

lib/system/mmdisp.nim

@@ -21,6 +21,7 @@ const
   alwaysGC = false # collect after every memory allocation (for debugging)
   leakDetector = false
   overwriteFree = false
+  trackAllocationSource = true or leakDetector
   
   cycleGC = true # (de)activate the cycle GC
   reallyDealloc = true # for debugging purposes this can be set to false