fixes the recently discovered GC memory leaks

This revision is intended as comparison point between the old and the new GC
The used GC can be switched in mmdisp and various statistics will be gathered during
execution (these will be removed/disabled in later revisions)

compiler/ccgtypes.nim

@@ -285,6 +285,12 @@ proc getParamTypeDesc(m: BModule, t: PType, check: var TIntSet): PRope =
         return getTypeDescAux(m, b, check)
   result = getTypeDescAux(m, t, check)
 
+proc paramStorageLoc(param: PSym): TStorageLoc =
+  if param.typ.skipTypes({tyVar}).kind notin {tyArray, tyOpenArray}:
+    result = OnStack
+  else:
+    result = OnUnknown
+
 proc genProcParams(m: BModule, t: PType, rettype, params: var PRope, 
                    check: var TIntSet, declareEnvironment=true) = 
   params = nil
@@ -297,7 +303,8 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var PRope,
     var param = t.n.sons[i].sym
     if isCompileTimeOnly(param.typ): continue
     if params != nil: app(params, ~", ")
-    fillLoc(param.loc, locParam, param.typ, mangleName(param), OnStack)
+    fillLoc(param.loc, locParam, param.typ, mangleName(param),
+            param.paramStorageLoc)
     app(params, getParamTypeDesc(m, param.typ, check))
     if ccgIntroducedPtr(param): 
       app(params, ~"*")

compiler/idents.nim

@@ -25,7 +25,7 @@ type
     next*: PIdent             # for hash-table chaining
     h*: THash                 # hash value of s
   
-var buckets: array[0..4096 * 2 - 1, PIdent]
+var buckets*: array[0..4096 * 2 - 1, PIdent]
 
 proc cmpIgnoreStyle(a, b: cstring, blen: int): int =
   var i = 0
@@ -102,5 +102,5 @@ proc getIdent*(identifier: string, h: THash): PIdent =
 proc IdentEq*(id: PIdent, name: string): bool = 
   result = id.id == getIdent(name).id
 
-let idAnon* = getIdent":anonymous"
+var idAnon* = getIdent":anonymous"
 

compiler/main.nim

@@ -394,7 +394,57 @@ proc dbgseqimp(x: PGenericSeq) {.cdecl.} =
 
 seqdbg = dbgseqimp
 
+proc resetMemory =
+  resetCompilationLists()
+  ccgutils.resetCaches()
+  ResetAllModules()
+  resetRopeCache()
+  resetSysTypes()
+  gGenericsCache = nil
+  gOwners = @[]
+  rangeDestructorProc = nil
+  for i in low(buckets)..high(buckets):
+    buckets[i] = nil
+  idAnon = nil
+  
+  # XXX: clean these global vars
+  # ccgstmts.gBreakpoints
+  # ccgthreadvars.nimtv
+  # ccgthreadvars.nimtVDeps
+  # ccgthreadvars.nimtvDeclared
+  # cgendata
+  # cgmeth?
+  # condsyms?
+  # depends?
+  # lexer.gLinesCompiled
+  # msgs - error counts
+  # magicsys, when system.nim changes
+  # rodread.rodcompilerProcs
+  # rodread.gTypeTable
+  # rodread.gMods
+  
+  # !! ropes.cache
+  # !! semdata.gGenericsCache
+  # semthreads.computed?
+  #
+  # suggest.usageSym
+  #
+  # XXX: can we run out of IDs?
+  # XXX: detect config reloading (implement as error/require restart)
+  # XXX: options are appended (they will accumulate over time)
+  # vis = visimpl
+  gcDebugging = true
+  echo "COLLECT 1"
+  GC_fullCollect()
+  echo "COLLECT 2"
+  GC_fullCollect()
+  echo "COLLECT 3"
+  GC_fullCollect()
+  echo GC_getStatistics()
+
 proc MainCommand =
+  gGlobalOptions.incl(optCaasEnabled)
+    
   # In "nimrod serve" scenario, each command must reset the registered passes
   clearPasses()
   gLastCmdTime = epochTime()
@@ -499,47 +549,7 @@ proc MainCommand =
     # XXX: temporary command for easier testing
     commandEval(mainCommandArg())
   of "reset":
-    ResetModule(gProjectMainIdx)
+    resetMemory()
-    gcDebugging = true
-    GC_fullCollect()
-
-    resetCompilationLists()
-    ccgutils.resetCaches()
-    ResetAllModules()
-    resetRopeCache()
-    resetSysTypes()
-    gGenericsCache = nil
-    gOwners = @[]
-    rangeDestructorProc = nil
-    
-    # XXX: clean these global vars
-    # ccgstmts.gBreakpoints
-    # ccgthreadvars.nimtv
-    # ccgthreadvars.nimtVDeps
-    # ccgthreadvars.nimtvDeclared
-    # cgendata
-    # cgmeth?
-    # condsyms?
-    # depends?
-    # lexer.gLinesCompiled
-    # msgs - error counts
-    # magicsys, when system.nim changes
-    # rodread.rodcompilerProcs
-    # rodread.gTypeTable
-    # rodread.gMods
-    
-    # !! ropes.cache
-    # !! semdata.gGenericsCache
-    # semthreads.computed?
-    #
-    # suggest.usageSym
-    #
-    # XXX: can we run out of IDs?
-    # XXX: detect config reloading (implement as error/require restart)
-    # XXX: options are appended (they will accumulate over time)
-    # vis = visimpl
-    GC_fullCollect()
-    echo GC_getStatistics()
   of "idetools":
     gCmd = cmdIdeTools
     if gEvalExpr != "":
@@ -565,3 +575,5 @@ proc MainCommand =
   echo "  int tries: ", gCacheIntTries
   echo "  efficiency: ", formatFloat(1-(gCacheMisses.float/gCacheTries.float), ffDecimal, 3)
 
+  # resetMemory()
+

lib/system/cellsets.nim

@@ -10,8 +10,10 @@
 # Efficient set of pointers for the GC (and repr)
 
 type
+  TRefCount = int
+
   TCell {.pure.} = object
-    refcount: int  # the refcount and some flags
+    refcount: TRefCount  # the refcount and some flags
     typ: PNimType
     when trackAllocationSource:
       filename: cstring

lib/system/hti.nim

@@ -13,11 +13,19 @@ when defined(NimString):
 else:
   {.pragma: codegenType.}
 
-type # This should be he same as ast.TTypeKind
+type 
+  # This should be he same as ast.TTypeKind
   # many enum fields are not used at runtime
   TNimKind = enum
-    tyNone, tyBool, tyChar,
+    tyNone,
-    tyEmpty, tyArrayConstr, tyNil, tyExpr, tyStmt, tyTypeDesc,
+    tyBool,
+    tyChar,
+    tyEmpty,
+    tyArrayConstr,
+    tyNil,
+    tyExpr,
+    tyStmt,
+    tyTypeDesc,
     tyGenericInvokation, # ``T[a, b]`` for types to invoke
     tyGenericBody,       # ``T[a, b, body]`` last parameter is the body
     tyGenericInst,       # ``T[a, b, realInstance]`` instantiated generic type
@@ -30,15 +38,30 @@ type # This should be he same as ast.TTypeKind
     tyTuple,             # WARNING: The compiler uses tyTuple for pure objects!
     tySet,
     tyRange,
-    tyPtr, tyRef,
+    tyPtr,
+    tyRef,
     tyVar,
     tySequence,
     tyProc,
-    tyPointer, tyOpenArray,
+    tyPointer,
-    tyString, tyCString, tyForward,
+    tyOpenArray,
-    tyInt, tyInt8, tyInt16, tyInt32, tyInt64,
+    tyString,
-    tyFloat, tyFloat32, tyFloat64, tyFloat128,
+    tyCString,
-    tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64,
+    tyForward,
+    tyInt,
+    tyInt8,
+    tyInt16,
+    tyInt32,
+    tyInt64,
+    tyFloat,
+    tyFloat32,
+    tyFloat64,
+    tyFloat128,
+    tyUInt,
+    tyUInt8,
+    tyUInt16,
+    tyUInt32,
+    tyUInt64,
     tyBigNum,
 
   TNimNodeKind = enum nkNone, nkSlot, nkList, nkCase

lib/system/mmdisp.nim

@@ -14,9 +14,9 @@
 {.push checks:off.}
 
 const
-  debugGC = false # we wish to debug the GC...
+  debugGC = true # we wish to debug the GC...
   logGC = false
-  traceGC = false # extensive debugging
+  traceGC = true # extensive debugging
   alwaysCycleGC = false
   alwaysGC = false # collect after every memory allocation (for debugging)
   leakDetector = false
@@ -307,7 +307,10 @@ else:
   include "system/cellsets"
   when not leakDetector:
     sysAssert(sizeof(TCell) == sizeof(TFreeCell), "sizeof TFreeCell")
+  when true:
     include "system/gc"
+  else:
+    include "system/oldgc"
   
 {.pop.}
 

lib/system/sysstr.nim

@@ -204,9 +204,16 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {.
     # we need to decref here, otherwise the GC leaks!
     when not defined(boehmGC) and not defined(nogc):
       for i in newLen..result.len-1:
+        let len0 = gch.tempStack.len
         forAllChildrenAux(cast[pointer](cast[TAddress](result) +%
                           GenericSeqSize +% (i*%elemSize)),
-                          extGetCellType(result).base, waZctDecRef)
+                          extGetCellType(result).base, waPush)
+        let len1 = gch.tempStack.len
+        for i in len0 .. <len1:
+          doDecRef(gch.tempStack.d[i], LocalHeap, MaybeCyclic)
+        gch.tempStack.len = len0
+                          
+        # XXX add a proper addCycleRoot barrier here!
     # and set the memory to nil:
     zeroMem(cast[pointer](cast[TAddress](result) +% GenericSeqSize +%
            (newLen*%elemSize)), (result.len-%newLen) *% elemSize)

lib/system/timers.nim

@@ -45,9 +45,10 @@ elif defined(macosx):
   proc getTicks(): TTicks {.inline.} =
     result = TTicks(mach_absolute_time())
   
-  proc `-`(a, b: TTicks): TNanos =
   var timeBaseInfo: TMachTimebaseInfoData
   mach_timebase_info(timeBaseInfo)
+    
+  proc `-`(a, b: TTicks): TNanos =
     result = (a.int64 - b.int64)  * timeBaseInfo.numer div timeBaseInfo.denom
 
 elif defined(posixRealtime):

tests/gc/refarrayleak.nim

@@ -0,0 +1,39 @@
+discard """
+  outputsub: "no leak: "
+"""
+
+type
+  TNode = object
+    data: array[0..300, char]
+
+  PNode = ref TNode
+
+  TNodeArray = array[0..10, PNode]
+
+  TArrayHolder = object
+    sons: TNodeArray
+
+proc nullify(a: var TNodeArray) =
+  for i in 0..high(a):
+    a[i] = nil
+
+proc newArrayHolder: ref TArrayHolder =
+  new result
+
+  for i in 0..high(result.sons):
+    new result.sons[i]
+
+  nullify result.sons
+
+proc loop =
+  for i in 0..10000:
+    discard newArrayHolder()
+    
+  if getOccupiedMem() > 300_000:
+    echo "still a leak! ", getOccupiedMem()
+    quit 1
+  else:
+    echo "no leak: ", getOccupiedMem()
+
+loop()
+

tests/gc/stackrefleak.nim

@@ -29,5 +29,3 @@ proc loop =
 
 loop()
 
-
-