GC v2 works sometimes

compiler/nim.nim

@@ -119,4 +119,6 @@ condsyms.initDefines()
 
 when not defined(selftest):
   handleCmdLine(newIdentCache(), newConfigRef())
+  when declared(GC_setMaxPause):
+    echo GC_getStatistics()
   msgQuit(int8(msgs.gErrorCounter > 0))

lib/system/gc2.nim

@@ -1,7 +1,7 @@
 #
 #
 #            Nim's Runtime Library
-#        (c) Copyright 2015 Andreas Rumpf
+#        (c) Copyright 2017 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -9,17 +9,19 @@
 
 #            Garbage Collector
 #
-# The basic algorithm is *Deferred Reference Counting* with an incremental mark
+# The basic algorithm is an incremental mark
 # and sweep GC to free cycles. It is hard realtime in that if you play
 # according to its rules, no deadline will ever be missed.
-
+# Since this kind of collector is very bad at recycling dead objects
-# XXX Ensure by smart color masking that the object is not in the ZCT.
+# early, Nim's codegen emits ``nimEscape`` calls at strategic
+# places. For this to work even 'unsureAsgnRef' needs to mark things
+# so that only return values need to be considered in ``nimEscape``.
 
 {.push profiler:off.}
 
 const
   CycleIncrease = 2 # is a multiplicative increase
-  InitialCycleThreshold = 4*1024*1024 # X MB because cycle checking is slow
+  InitialCycleThreshold = 512*1024 # start collecting after 500KB
   ZctThreshold = 500  # we collect garbage if the ZCT's size
                       # reaches this threshold
                       # this seems to be a good value
@@ -40,13 +42,11 @@ type
 iterToProc(allObjects, ptr ObjectSpaceIter, allObjectsAsProc)
 
 const
-  rcIncrement = 0b1000 # so that lowest 3 bits are not touched
+  escapedBit = 0b1000 # so that lowest 3 bits are not touched
   rcBlackOrig = 0b000
   rcWhiteOrig = 0b001
   rcGrey = 0b010   # traditional color for incremental mark&sweep
   rcUnused = 0b011
-  ZctFlag = 0b100  # in ZCT
-  rcShift = 3      # shift by rcShift to get the reference counter
   colorMask = 0b011
 type
   WalkOp = enum
@@ -63,7 +63,7 @@ type
 
   GcStat = object
     stackScans: int          # number of performed stack scans (for statistics)
-    cycleCollections: int    # number of performed full collections
+    completedCollections: int    # number of performed full collections
     maxThreshold: int        # max threshold that has been set
     maxStackSize: int        # max stack size
     maxStackCells: int       # max stack cells in ``decStack``
@@ -93,15 +93,13 @@ type
     cycleThreshold: int
     when useCellIds:
       idGenerator: int
-    zct: CellSeq             # the zero count table
-    decStack: CellSeq        # cells in the stack that are to decref again
     greyStack: CellSeq
     recGcLock: int           # prevent recursion via finalizers; no thread lock
     when withRealTime:
       maxPause: Nanos        # max allowed pause in nanoseconds; active if > 0
     region: MemRegion        # garbage collected region
     stat: GcStat
-    additionalRoots: CellSeq # dummy roots for GC_ref/unref
+    additionalRoots: CellSeq # explicit roots for GC_ref/unref
     spaceIter: ObjectSpaceIter
     pDumpHeapFile: pointer # File that is used for GC_dumpHeap
     when hasThreadSupport:
@@ -113,19 +111,25 @@ var
 when not defined(useNimRtl):
   instantiateForRegion(gch.region)
 
+template acquire(gch: GcHeap) =
+  when hasThreadSupport and hasSharedHeap:
+    acquireSys(HeapLock)
+
+template release(gch: GcHeap) =
+  when hasThreadSupport and hasSharedHeap:
+    releaseSys(HeapLock)
+
 proc initGC() =
   when not defined(useNimRtl):
     gch.red = (1-gch.black)
     gch.cycleThreshold = InitialCycleThreshold
     gch.stat.stackScans = 0
-    gch.stat.cycleCollections = 0
+    gch.stat.completedCollections = 0
     gch.stat.maxThreshold = 0
     gch.stat.maxStackSize = 0
     gch.stat.maxStackCells = 0
     gch.stat.cycleTableSize = 0
     # init the rt
-    init(gch.zct)
-    init(gch.decStack)
     init(gch.additionalRoots)
     init(gch.greyStack)
     when hasThreadSupport:
@@ -147,11 +151,6 @@ template gcAssert(cond: bool, msg: string) =
       writeStackTrace()
       quit 1
 
-proc addZCT(s: var CellSeq, c: PCell) {.noinline.} =
-  if (c.refcount and ZctFlag) == 0:
-    c.refcount = c.refcount or ZctFlag
-    add(s, c)
-
 proc cellToUsr(cell: PCell): pointer {.inline.} =
   # convert object (=pointer to refcount) to pointer to userdata
   result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell)))
@@ -168,7 +167,7 @@ proc extGetCellType(c: pointer): PNimType {.compilerproc.} =
   result = usrToCell(c).typ
 
 proc internRefcount(p: pointer): int {.exportc: "getRefcount".} =
-  result = int(usrToCell(p).refcount) shr rcShift
+  result = 0
 
 # this that has to equals zero, otherwise we have to round up UnitsPerPage:
 when BitsPerPage mod (sizeof(int)*8) != 0:
@@ -178,6 +177,12 @@ template color(c): expr = c.refCount and colorMask
 template setColor(c, col) =
   c.refcount = c.refcount and not colorMask or col
 
+template markAsEscaped(c: PCell) =
+  c.refcount = c.refcount or escapedBit
+
+template didEscape(c: PCell): bool =
+  (c.refCount and escapedBit) != 0
+
 proc writeCell(file: File; msg: cstring, c: PCell) =
   var kind = -1
   if c.typ != nil: kind = ord(c.typ.kind)
@@ -189,18 +194,18 @@ proc writeCell(file: File; msg: cstring, c: PCell) =
   else:
     let id = c
   when leakDetector:
-    c_fprintf(file, "%s %p %d rc=%ld color=%c from %s(%ld)\n",
+    c_fprintf(file, "%s %p %d escaped=%ld color=%c from %s(%ld)\n",
-              msg, id, kind, c.refcount shr rcShift, col, c.filename, c.line)
+              msg, id, kind, didEscape(c), col, c.filename, c.line)
   else:
-    c_fprintf(file, "%s %p %d rc=%ld color=%c\n",
+    c_fprintf(file, "%s %p %d escaped=%ld color=%c\n",
-              msg, id, kind, c.refcount shr rcShift, col)
+              msg, id, kind, didEscape(c), col)
 
 proc writeCell(msg: cstring, c: PCell) =
   stdout.writeCell(msg, c)
 
 proc myastToStr[T](x: T): string {.magic: "AstToStr", noSideEffect.}
 
-template gcTrace(cell, state: expr): stmt {.immediate.} =
+template gcTrace(cell, state: untyped) =
   when traceGC: writeCell(myastToStr(state), cell)
 
 # forward declarations:
@@ -211,44 +216,17 @@ proc doOperation(p: pointer, op: WalkOp) {.benign.}
 proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.}
 # we need the prototype here for debugging purposes
 
-when hasThreadSupport and hasSharedHeap:
-  template `--`(x: expr): expr = atomicDec(x, rcIncrement) <% rcIncrement
-  template `++`(x: expr): stmt = discard atomicInc(x, rcIncrement)
-else:
-  template `--`(x: expr): expr =
-    dec(x, rcIncrement)
-    x <% rcIncrement
-  template `++`(x: expr): stmt = inc(x, rcIncrement)
-
 proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} =
   # we MUST access gch as a global here, because this crosses DLL boundaries!
   discard
 
-proc rtlAddZCT(c: PCell) {.rtl, inl.} =
-  # we MUST access gch as a global here, because this crosses DLL boundaries!
-  addZCT(gch.zct, c)
-
-proc decRef(c: PCell) {.inline.} =
-  gcAssert(isAllocatedPtr(gch.region, c), "decRef: interiorPtr")
-  gcAssert(c.refcount >=% rcIncrement, "decRef")
-  if --c.refcount:
-    rtlAddZCT(c)
-
-proc incRef(c: PCell) {.inline.} =
-  gcAssert(isAllocatedPtr(gch.region, c), "incRef: interiorPtr")
-  c.refcount = c.refcount +% rcIncrement
-
-proc nimGCunrefRC1(p: pointer) {.compilerProc, inline.} =
-  decRef(usrToCell(p))
-
 proc nimGCref(p: pointer) {.compilerProc.} =
   let cell = usrToCell(p)
-  incRef(cell)
+  markAsEscaped(cell)
   add(gch.additionalRoots, cell)
 
 proc nimGCunref(p: pointer) {.compilerProc.} =
   let cell = usrToCell(p)
-  decRef(cell)
   var L = gch.additionalRoots.len-1
   var i = L
   let d = gch.additionalRoots.d
@@ -259,6 +237,12 @@ proc nimGCunref(p: pointer) {.compilerProc.} =
       break
     dec(i)
 
+proc nimGCunrefNoCycle(p: pointer) {.compilerProc, inline.} =
+  discard "can we do some freeing here?"
+
+proc nimGCunrefRC1(p: pointer) {.compilerProc, inline.} =
+  discard "can we do some freeing here?"
+
 template markGrey(x: PCell) =
   if x.color != 1-gch.black and gch.phase == Phase.Marking:
     if not isAllocatedPtr(gch.region, x):
@@ -272,59 +256,32 @@ proc GC_addCycleRoot*[T](p: ref T) {.inline.} =
   ## adds 'p' to the cycle candidate set for the cycle collector. It is
   ## necessary if you used the 'acyclic' pragma for optimization
   ## purposes and need to break cycles manually.
-  rtlAddCycleRoot(usrToCell(cast[pointer](p)))
+  discard
 
-proc nimGCunrefNoCycle(p: pointer) {.compilerProc, inline.} =
+template asgnRefImpl =
-  sysAssert(allocInv(gch.region), "begin nimGCunrefNoCycle")
-  var c = usrToCell(p)
-  gcAssert(isAllocatedPtr(gch.region, c), "nimGCunrefNoCycle: isAllocatedPtr")
-  if --c.refcount:
-    rtlAddZCT(c)
-    sysAssert(allocInv(gch.region), "end nimGCunrefNoCycle 2")
-  sysAssert(allocInv(gch.region), "end nimGCunrefNoCycle 5")
-
-proc asgnRef(dest: PPointer, src: pointer) {.compilerProc, inline.} =
-  # the code generator calls this proc!
   gcAssert(not isOnStack(dest), "asgnRef")
   # BUGFIX: first incRef then decRef!
   if src != nil:
     let s = usrToCell(src)
-    incRef(s)
+    markAsEscaped(s)
     markGrey(s)
-  if dest[] != nil: decRef(usrToCell(dest[]))
   dest[] = src
 
+proc asgnRef(dest: PPointer, src: pointer) {.compilerProc, inline.} =
+  # the code generator calls this proc!
+  asgnRefImpl()
+
 proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerProc, inline.} =
-  # the code generator calls this proc if it is known at compile time that no
+  asgnRefImpl()
-  # cycle is possible.
-  gcAssert(not isOnStack(dest), "asgnRefNoCycle")
-  if src != nil:
-    var c = usrToCell(src)
-    ++c.refcount
-    markGrey(c)
-  if dest[] != nil:
-    var c = usrToCell(dest[])
-    if --c.refcount:
-      rtlAddZCT(c)
-  dest[] = src
 
 proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerProc.} =
-  # unsureAsgnRef updates the reference counters only if dest is not on the
+  # unsureAsgnRef marks 'src' as grey only if dest is not on the
   # stack. It is used by the code generator if it cannot decide wether a
   # reference is in the stack or not (this can happen for var parameters).
-  if not isOnStack(dest):
   if src != nil:
     let s = usrToCell(src)
-      incRef(s)
+    markAsEscaped(s)
-      markGrey(s)
+    if not isOnStack(dest): markGrey(s)
-    # XXX finally use assembler for the stack checking instead!
-    # the test for '!= nil' is correct, but I got tired of the segfaults
-    # resulting from the crappy stack checking:
-    if cast[int](dest[]) >=% PageSize: decRef(usrToCell(dest[]))
-  else:
-    # can't be an interior pointer if it's a stack location!
-    gcAssert(interiorAllocatedPtr(gch.region, dest) == nil,
-             "stack loc AND interior pointer")
   dest[] = src
 
 type
@@ -388,50 +345,6 @@ proc forAllChildren(cell: PCell, op: WalkOp) =
             GenericSeqSize), cell.typ.base, op)
     else: discard
 
-proc addNewObjToZCT(res: PCell, gch: var GcHeap) {.inline.} =
-  # we check the last 8 entries (cache line) for a slot that could be reused.
-  # In 63% of all cases we succeed here! But we have to optimize the heck
-  # out of this small linear search so that ``newObj`` is not slowed down.
-  #
-  # Slots to try          cache hit
-  # 1                     32%
-  # 4                     59%
-  # 8                     63%
-  # 16                    66%
-  # all slots             68%
-  var L = gch.zct.len
-  var d = gch.zct.d
-  when true:
-    # loop unrolled for performance:
-    template replaceZctEntry(i: expr) =
-      c = d[i]
-      if c.refcount >=% rcIncrement:
-        c.refcount = c.refcount and not ZctFlag
-        d[i] = res
-        return
-    if L > 8:
-      var c: PCell
-      replaceZctEntry(L-1)
-      replaceZctEntry(L-2)
-      replaceZctEntry(L-3)
-      replaceZctEntry(L-4)
-      replaceZctEntry(L-5)
-      replaceZctEntry(L-6)
-      replaceZctEntry(L-7)
-      replaceZctEntry(L-8)
-      add(gch.zct, res)
-    else:
-      d[L] = res
-      inc(gch.zct.len)
-  else:
-    for i in countdown(L-1, max(0, L-8)):
-      var c = d[i]
-      if c.refcount >=% rcIncrement:
-        c.refcount = c.refcount and not ZctFlag
-        d[i] = res
-        return
-    add(gch.zct, res)
-
 {.push stackTrace: off, profiler:off.}
 proc gcInvariant*() =
   sysAssert(allocInv(gch.region), "injected")
@@ -439,6 +352,8 @@ proc gcInvariant*() =
     markForDebug(gch)
 {.pop.}
 
+include gc_common
+
 proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer =
   # generates a new object and sets its reference counter to 0
   sysAssert(allocInv(gch.region), "rawNewObj begin")
@@ -453,10 +368,8 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer =
       res.filename = framePtr.prev.filename
       res.line = framePtr.prev.line
   # refcount is zero, color is black, but mark it to be in the ZCT
-  res.refcount = ZctFlag or allocColor()
+  res.refcount = allocColor()
   sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3")
-  # its refcount is zero, so add it to the ZCT:
-  addNewObjToZCT(res, gch)
   when logGC: writeCell("new cell", res)
   gcTrace(res, csAllocated)
   when useCellIds:
@@ -485,95 +398,38 @@ proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} =
   when defined(memProfiler): nimProfile(size)
 
 proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} =
-  # generates a new object and sets its reference counter to 1
+  result = newObj(typ, size)
-  sysAssert(allocInv(gch.region), "newObjRC1 begin")
-  gcAssert(typ.kind in {tyRef, tyOptAsRef, tyString, tySequence}, "newObj: 1")
-  collectCT(gch)
-  sysAssert(allocInv(gch.region), "newObjRC1 after collectCT")
-
-  var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell)))
-  sysAssert(allocInv(gch.region), "newObjRC1 after rawAlloc")
-  sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2")
-  # now it is buffered in the ZCT
-  res.typ = typ
-  when leakDetector:
-    if framePtr != nil and framePtr.prev != nil:
-      res.filename = framePtr.prev.filename
-      res.line = framePtr.prev.line
-  res.refcount = rcIncrement or allocColor() # refcount is 1
-  sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3")
-  when logGC: writeCell("new cell", res)
-  gcTrace(res, csAllocated)
-  when useCellIds:
-    inc gch.idGenerator
-    res.id = gch.idGenerator
-  result = cellToUsr(res)
-  zeroMem(result, size)
-  sysAssert(allocInv(gch.region), "newObjRC1 end")
-  when defined(memProfiler): nimProfile(size)
 
 proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} =
-  let size = addInt(mulInt(len, typ.base.size), GenericSeqSize)
+  result = newSeq(typ, len)
-  result = newObjRC1(typ, size)
-  cast[PGenericSeq](result).len = len
-  cast[PGenericSeq](result).reserved = len
-  when defined(memProfiler): nimProfile(size)
 
 proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer =
+  acquire(gch)
   collectCT(gch)
   var ol = usrToCell(old)
-  gcAssert(isAllocatedPtr(gch.region, ol), "growObj: freed pointer?")
-
   sysAssert(ol.typ != nil, "growObj: 1")
   gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2")
-  sysAssert(allocInv(gch.region), "growObj begin")
 
   var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell)))
   var elemSize = 1
   if ol.typ.kind != tyString: elemSize = ol.typ.base.size
+  incTypeSize ol.typ, newsize
 
-  let oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize
+  var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize
   copyMem(res, ol, oldsize + sizeof(Cell))
   zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)),
           newsize-oldsize)
   sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3")
-  # This can be wrong for intermediate temps that are nevertheless on the
+  when false:
-  # heap because of lambda lifting:
+    # this is wrong since seqs can be shared via 'shallow':
-  #gcAssert(res.refcount shr rcShift <=% 1, "growObj: 4")
+    when reallyDealloc: rawDealloc(gch.region, ol)
-  when logGC:
-    writeCell("growObj old cell", ol)
-    writeCell("growObj new cell", res)
-  gcTrace(ol, csZctFreed)
-  gcTrace(res, csAllocated)
-  when reallyDealloc:
-    sysAssert(allocInv(gch.region), "growObj before dealloc")
-    if ol.refcount shr rcShift <=% 1:
-      # free immediately to save space:
-      if (ol.refcount and ZctFlag) != 0:
-        var j = gch.zct.len-1
-        var d = gch.zct.d
-        while j >= 0:
-          if d[j] == ol:
-            d[j] = res
-            break
-          dec(j)
-      rawDealloc(gch.region, ol)
     else:
-      # we split the old refcount in 2 parts. XXX This is still not entirely
-      # correct if the pointer that receives growObj's result is on the stack.
-      # A better fix would be to emit the location specific write barrier for
-      # 'growObj', but this is lots of more work and who knows what new problems
-      # this would create.
-      res.refcount = rcIncrement or allocColor()
-      decRef(ol)
-  else:
-    sysAssert(ol.typ != nil, "growObj: 5")
       zeroMem(ol, sizeof(Cell))
   when useCellIds:
     inc gch.idGenerator
     res.id = gch.idGenerator
+  release(gch)
   result = cellToUsr(res)
-  sysAssert(allocInv(gch.region), "growObj end")
   when defined(memProfiler): nimProfile(newsize-oldsize)
 
 proc growObj(old: pointer, newsize: int): pointer {.rtl.} =
@@ -629,6 +485,7 @@ proc GC_dumpHeap*(file: File) =
   ## can be translated into "dot" syntax via the "heapdump2dot" tool.
   gch.pDumpHeapFile = file
   var spaceIter: ObjectSpaceIter
+  when false:
     var d = gch.decStack.d
     for i in 0 .. < gch.decStack.len:
       if isAllocatedPtr(gch.region, d[i]):
@@ -657,18 +514,8 @@ proc GC_dumpHeap() =
 
 # ---------------- cycle collector -------------------------------------------
 
-include gc_common
-
 proc freeCyclicCell(gch: var GcHeap, c: PCell) =
   gcAssert(isAllocatedPtr(gch.region, c), "freeCyclicCell: freed pointer?")
-
-  var d = gch.decStack.d
-  for i in 0..gch.decStack.len-1:
-    if d[i] == c:
-      writeCell("freeing ", c)
-      GC_dumpHeap()
-    gcAssert d[i] != c, "wtf man, freeing obviously alive stuff?!!"
-
   prepareDealloc(c)
   gcTrace(c, csCycFreed)
   when logGC: writeCell("cycle collector dealloc cell", c)
@@ -707,15 +554,6 @@ proc markRoot(gch: var GcHeap, c: PCell) {.inline.} =
   if c.color == 1-gch.black:
     c.setColor(rcGrey)
     add(gch.greyStack, c)
-  elif c.color == rcGrey:
-    var isGrey = false
-    var d = gch.decStack.d
-    for i in 0..gch.decStack.len-1:
-      if d[i] == c:
-        isGrey = true
-        break
-    if not isGrey:
-      gcAssert false, "markRoot: root is already grey?!"
 
 proc markIncremental(gch: var GcHeap): bool =
   var L = addr(gch.greyStack.len)
@@ -735,30 +573,14 @@ proc markIncremental(gch: var GcHeap): bool =
       c.setColor(gch.black)
       forAllChildren(c, waMarkGrey)
     elif c.color == (1-gch.black):
-      gcAssert false, "wtf why are there white object in the greystack?"
+      gcAssert false, "wtf why are there white objects in the greystack?"
     checkTime()
   gcAssert gch.greyStack.len == 0, "markIncremental: greystack not empty "
-
-  # assert that all local roots are black by now:
-  var d = gch.decStack.d
-  var errors = false
-  for i in 0..gch.decStack.len-1:
-    gcAssert(isAllocatedPtr(gch.region, d[i]), "markIncremental: isAllocatedPtr 2")
-    if d[i].color != gch.black:
-      writeCell("not black ", d[i])
-      errors = true
-  gcAssert(not errors, "wtf something wrong hre")
   result = true
 
 proc markGlobals(gch: var GcHeap) =
   for i in 0 .. < globalMarkersLen: globalMarkers[i]()
 
-proc markLocals(gch: var GcHeap) =
-  var d = gch.decStack.d
-  for i in 0 .. < gch.decStack.len:
-    sysAssert isAllocatedPtr(gch.region, d[i]), "markLocals"
-    markRoot(gch, d[i])
-
 proc doOperation(p: pointer, op: WalkOp) =
   if p == nil: return
   var c: PCell = usrToCell(p)
@@ -770,11 +592,7 @@ proc doOperation(p: pointer, op: WalkOp) =
     #if not isAllocatedPtr(gch.region, c):
     #  c_fprintf(stdout, "[GC] decref bug: %p", c)
     gcAssert(isAllocatedPtr(gch.region, c), "decRef: waZctDecRef")
-    gcAssert(c.refcount >=% rcIncrement, "doOperation 2")
+    discard "use me for nimEscape?"
-    #c.refcount = c.refcount -% rcIncrement
-    when logGC: writeCell("decref (from doOperation)", c)
-    decRef(c)
-    #if c.refcount <% rcIncrement: addZCT(gch.zct, c)
   of waMarkGlobal:
     template handleRoot =
       if gch.dumpHeapFile.isNil:
@@ -805,40 +623,6 @@ proc doOperation(p: pointer, op: WalkOp) =
 proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} =
   doOperation(d, WalkOp(op))
 
-proc collectZCT(gch: var GcHeap): bool {.benign.}
-
-proc collectCycles(gch: var GcHeap): bool =
-  when hasThreadSupport:
-    for c in gch.toDispose:
-      nimGCunref(c)
-
-  # ensure the ZCT 'color' is not used:
-  while gch.zct.len > 0: discard collectZCT(gch)
-
-  case gch.phase
-  of Phase.None:
-    gch.phase = Phase.Marking
-    markGlobals(gch)
-
-    c_fprintf(stdout, "collectCycles: introduced bug E %ld\n", gch.phase)
-    discard allocInv(gch.region)
-  of Phase.Marking:
-    # since locals do not have a write barrier, we need
-    # to keep re-scanning them :-( but there is really nothing we can
-    # do about that.
-    markLocals(gch)
-    if markIncremental(gch):
-      gch.phase = Phase.Sweeping
-      gch.red = 1 - gch.red
-  of Phase.Sweeping:
-    gcAssert gch.greyStack.len == 0, "greystack not empty"
-    if sweep(gch):
-      gch.phase = Phase.None
-      # flip black/white meanings:
-      gch.black = 1 - gch.black
-      gcAssert gch.red == 1 - gch.black, "red color is wrong"
-      result = true
-
 proc gcMark(gch: var GcHeap, p: pointer) {.inline.} =
   # the addresses are not as cells on the stack, so turn them to cells:
   sysAssert(allocInv(gch.region), "gcMark begin")
@@ -849,61 +633,44 @@ proc gcMark(gch: var GcHeap, p: pointer) {.inline.} =
     var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell))
     if objStart != nil:
       # mark the cell:
-      objStart.refcount = objStart.refcount +% rcIncrement
+      markRoot(gch, objStart)
-      add(gch.decStack, objStart)
   sysAssert(allocInv(gch.region), "gcMark end")
 
 proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} =
   forEachStackSlot(gch, gcMark)
 
-proc collectZCT(gch: var GcHeap): bool =
+proc collectALittle(gch: var GcHeap): bool =
-  # Note: Freeing may add child objects to the ZCT! So essentially we do
+  case gch.phase
-  # deep freeing, which is bad for incremental operation. In order to
+  of Phase.None:
-  # avoid a deep stack, we move objects to keep the ZCT small.
+    if getOccupiedMem(gch.region) >= gch.cycleThreshold:
-  # This is performance critical!
+      gch.phase = Phase.Marking
-  var L = addr(gch.zct.len)
+      markGlobals(gch)
-  takeStartTime(100)
+      result = collectALittle(gch)
-
+      #when false: c_fprintf(stdout, "collectALittle: introduced bug E %ld\n", gch.phase)
-  while L[] > 0:
+      #discard allocInv(gch.region)
-    var c = gch.zct.d[0]
+  of Phase.Marking:
-    sysAssert(isAllocatedPtr(gch.region, c), "CollectZCT: isAllocatedPtr")
+    when hasThreadSupport:
-    # remove from ZCT:
+      for c in gch.toDispose:
-    gcAssert((c.refcount and ZctFlag) == ZctFlag, "collectZCT")
+        nimGCunref(c)
-
+    prepareForInteriorPointerChecking(gch.region)
-    c.refcount = c.refcount and not ZctFlag
+    markStackAndRegisters(gch)
-    gch.zct.d[0] = gch.zct.d[L[] - 1]
+    inc(gch.stat.stackScans)
-    dec(L[])
+    if markIncremental(gch):
-    takeTime()
+      gch.phase = Phase.Sweeping
-    if c.refcount <% rcIncrement and c.color != rcGrey:
+      gch.red = 1 - gch.red
-      # It may have a RC > 0, if it is in the hardware stack or
+  of Phase.Sweeping:
-      # it has not been removed yet from the ZCT. This is because
+    gcAssert gch.greyStack.len == 0, "greystack not empty"
-      # ``incref`` does not bother to remove the cell from the ZCT
+    when hasThreadSupport:
-      # as this might be too slow.
+      for c in gch.toDispose:
-      # In any case, it should be removed from the ZCT. But not
+        nimGCunref(c)
-      # freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!**
+    if sweep(gch):
-      when logGC: writeCell("zct dealloc cell", c)
+      gch.phase = Phase.None
-      gcTrace(c, csZctFreed)
+      # flip black/white meanings:
-      # We are about to free the object, call the finalizer BEFORE its
+      gch.black = 1 - gch.black
-      # children are deleted as well, because otherwise the finalizer may
+      gcAssert gch.red == 1 - gch.black, "red color is wrong"
-      # access invalid memory. This is done by prepareDealloc():
+      inc(gch.stat.completedCollections)
-      prepareDealloc(c)
-      forAllChildren(c, waZctDecRef)
-      when reallyDealloc:
-        sysAssert(allocInv(gch.region), "collectZCT: rawDealloc")
-        rawDealloc(gch.region, c)
-      else:
-        sysAssert(c.typ != nil, "collectZCT 2")
-        zeroMem(c, sizeof(Cell))
-    checkTime()
       result = true
 
-proc unmarkStackAndRegisters(gch: var GcHeap) =
-  var d = gch.decStack.d
-  for i in 0..gch.decStack.len-1:
-    sysAssert isAllocatedPtr(gch.region, d[i]), "unmarkStackAndRegisters"
-    decRef(d[i])
-  gch.decStack.len = 0
-
 proc collectCTBody(gch: var GcHeap) =
   when withRealTime:
     let t0 = getticks()
@@ -911,22 +678,12 @@ proc collectCTBody(gch: var GcHeap) =
 
   when not nimCoroutines:
     gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize())
-  sysAssert(gch.decStack.len == 0, "collectCT")
+  #gch.stat.maxStackCells = max(gch.stat.maxStackCells, gch.decStack.len)
-  prepareForInteriorPointerChecking(gch.region)
+  if collectALittle(gch):
-  markStackAndRegisters(gch)
-  gch.stat.maxStackCells = max(gch.stat.maxStackCells, gch.decStack.len)
-  inc(gch.stat.stackScans)
-  if collectZCT(gch):
-    when cycleGC:
-      if getOccupiedMem(gch.region) >= gch.cycleThreshold or alwaysCycleGC:
-        if collectCycles(gch):
-          inc(gch.stat.cycleCollections)
     gch.cycleThreshold = max(InitialCycleThreshold, getOccupiedMem() *
                               CycleIncrease)
     gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold)
-  unmarkStackAndRegisters(gch)
   sysAssert(allocInv(gch.region), "collectCT: end")
-
   when withRealTime:
     let duration = getticks() - t0
     gch.stat.maxPause = max(gch.stat.maxPause, duration)
@@ -947,7 +704,7 @@ proc collectCT(gch: var GcHeap) =
     let stackMarkCosts = max(currentStackSizes() div (16*sizeof(int)), ZctThreshold)
   else:
     let stackMarkCosts = max(stackSize() div (16*sizeof(int)), ZctThreshold)
-  if (gch.zct.len >= stackMarkCosts or (cycleGC and
+  if (gch.greyStack.len >= stackMarkCosts or (cycleGC and
       getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and
       gch.recGcLock == 0:
     collectCTBody(gch)
@@ -961,9 +718,8 @@ when withRealTime:
 
   proc GC_step(gch: var GcHeap, us: int, strongAdvice: bool) =
     gch.maxPause = us.toNano
-    if (gch.zct.len >= ZctThreshold or (cycleGC and
+    #if (getOccupiedMem(gch.region)>=gch.cycleThreshold) or
-        getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) or
+    #    alwaysGC or strongAdvice:
-        strongAdvice:
     collectCTBody(gch)
 
   proc GC_step*(us: int, strongAdvice = false, stackSize = -1) {.noinline.} =
@@ -1002,12 +758,8 @@ when not defined(useNimRtl):
   proc GC_setStrategy(strategy: GC_Strategy) =
     discard
 
-  proc GC_enableMarkAndSweep() =
+  proc GC_enableMarkAndSweep() = discard
-    gch.cycleThreshold = InitialCycleThreshold
+  proc GC_disableMarkAndSweep() = discard
-
-  proc GC_disableMarkAndSweep() =
-    gch.cycleThreshold = high(gch.cycleThreshold)-1
-    # set to the max value to suppress the cycle detector
 
   proc GC_fullCollect() =
     var oldThreshold = gch.cycleThreshold
@@ -1021,17 +773,17 @@ when not defined(useNimRtl):
              "[GC] occupied memory: " & $(getOccupiedMem()) & "\n" &
              "[GC] stack scans: " & $gch.stat.stackScans & "\n" &
              "[GC] stack cells: " & $gch.stat.maxStackCells & "\n" &
-             "[GC] cycle collections: " & $gch.stat.cycleCollections & "\n" &
+             "[GC] completed collections: " & $gch.stat.completedCollections & "\n" &
              "[GC] max threshold: " & $gch.stat.maxThreshold & "\n" &
-             "[GC] zct capacity: " & $gch.zct.cap & "\n" &
+             "[GC] grey stack capacity: " & $gch.greyStack.cap & "\n" &
              "[GC] max cycle table size: " & $gch.stat.cycleTableSize & "\n" &
-             "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000)
+             "[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) & "\n"
     when nimCoroutines:
-      result = result & "[GC] number of stacks: " & $gch.stack.len & "\n"
+      result.add "[GC] number of stacks: " & $gch.stack.len & "\n"
       for stack in items(gch.stack):
-        result = result & "[GC]   stack " & stack.bottom.repr & "[GC]     max stack size " & $stack.maxStackSize & "\n"
+        result.add "[GC]   stack " & stack.bottom.repr & "[GC]     max stack size " & $stack.maxStackSize & "\n"
     else:
-      result = result & "[GC] max stack size: " & $gch.stat.maxStackSize & "\n"
+      result.add "[GC] max stack size: " & $gch.stat.maxStackSize & "\n"
     GC_enable()
 
 {.pop.}