GC: get rid of pathological behaviour for stack marking
This commit is contained in:
Araq
parent
e390d8ec4a
commit
c40aac8e20
5 changed files with 326 additions and 323 deletions
|
|
@ -1,7 +1,7 @@
|
|||
#
|
||||
#
|
||||
# Nim's Runtime Library
|
||||
# (c) Copyright 2013 Andreas Rumpf
|
||||
# (c) Copyright 2015 Andreas Rumpf
|
||||
#
|
||||
# See the file "copying.txt", included in this
|
||||
# distribution, for details about the copyright.
|
||||
|
|
@ -48,7 +48,7 @@ type
|
|||
TWalkOp = enum
|
||||
waMarkGlobal, # part of the backup/debug mark&sweep
|
||||
waMarkPrecise, # part of the backup/debug mark&sweep
|
||||
waZctDecRef, waPush, waCycleDecRef, waMarkGray, waScan, waScanBlack,
|
||||
waZctDecRef, waPush, waCycleDecRef, waMarkGray, waScan, waScanBlack,
|
||||
waCollectWhite #, waDebug
|
||||
|
||||
TFinalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.}
|
||||
|
|
@ -61,9 +61,9 @@ type
|
|||
maxThreshold: int # max threshold that has been set
|
||||
maxStackSize: int # max stack size
|
||||
maxStackCells: int # max stack cells in ``decStack``
|
||||
cycleTableSize: int # max entries in cycle table
|
||||
cycleTableSize: int # max entries in cycle table
|
||||
maxPause: int64 # max measured GC pause in nanoseconds
|
||||
|
||||
|
||||
TGcHeap {.final, pure.} = object # this contains the zero count and
|
||||
# non-zero count table
|
||||
stackBottom: pointer
|
||||
|
|
@ -88,11 +88,11 @@ var
|
|||
when not defined(useNimRtl):
|
||||
instantiateForRegion(gch.region)
|
||||
|
||||
template acquire(gch: TGcHeap) =
|
||||
template acquire(gch: TGcHeap) =
|
||||
when hasThreadSupport and hasSharedHeap:
|
||||
acquireSys(HeapLock)
|
||||
|
||||
template release(gch: TGcHeap) =
|
||||
template release(gch: TGcHeap) =
|
||||
when hasThreadSupport and hasSharedHeap:
|
||||
releaseSys(HeapLock)
|
||||
|
||||
|
|
@ -163,7 +163,7 @@ when hasThreadSupport and hasSharedHeap:
|
|||
template `--`(x: expr): expr = atomicDec(x, rcIncrement) <% rcIncrement
|
||||
template `++`(x: expr): stmt = discard atomicInc(x, rcIncrement)
|
||||
else:
|
||||
template `--`(x: expr): expr =
|
||||
template `--`(x: expr): expr =
|
||||
dec(x, rcIncrement)
|
||||
x <% rcIncrement
|
||||
template `++`(x: expr): stmt = inc(x, rcIncrement)
|
||||
|
|
@ -181,7 +181,7 @@ proc prepareDealloc(cell: PCell) =
|
|||
(cast[TFinalizer](cell.typ.finalizer))(cellToUsr(cell))
|
||||
dec(gch.recGcLock)
|
||||
|
||||
proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} =
|
||||
proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} =
|
||||
# we MUST access gch as a global here, because this crosses DLL boundaries!
|
||||
when hasThreadSupport and hasSharedHeap:
|
||||
acquireSys(HeapLock)
|
||||
|
|
@ -211,7 +211,7 @@ proc decRef(c: PCell) {.inline.} =
|
|||
rtlAddCycleRoot(c)
|
||||
#writeCell("decRef", c)
|
||||
|
||||
proc incRef(c: PCell) {.inline.} =
|
||||
proc incRef(c: PCell) {.inline.} =
|
||||
gcAssert(isAllocatedPtr(gch.region, c), "incRef: interiorPtr")
|
||||
c.refcount = c.refcount +% rcIncrement
|
||||
# and not colorMask
|
||||
|
|
@ -246,12 +246,12 @@ proc asgnRef(dest: PPointer, src: pointer) {.compilerProc, inline.} =
|
|||
dest[] = src
|
||||
|
||||
proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerProc, inline.} =
|
||||
# the code generator calls this proc if it is known at compile time that no
|
||||
# the code generator calls this proc if it is known at compile time that no
|
||||
# cycle is possible.
|
||||
if src != nil:
|
||||
var c = usrToCell(src)
|
||||
++c.refcount
|
||||
if dest[] != nil:
|
||||
if dest[] != nil:
|
||||
var c = usrToCell(dest[])
|
||||
if --c.refcount:
|
||||
rtlAddZCT(c)
|
||||
|
|
@ -269,7 +269,7 @@ proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerProc.} =
|
|||
if cast[int](dest[]) >=% PageSize: decRef(usrToCell(dest[]))
|
||||
else:
|
||||
# can't be an interior pointer if it's a stack location!
|
||||
gcAssert(interiorAllocatedPtr(gch.region, dest) == nil,
|
||||
gcAssert(interiorAllocatedPtr(gch.region, dest) == nil,
|
||||
"stack loc AND interior pointer")
|
||||
dest[] = src
|
||||
|
||||
|
|
@ -321,7 +321,7 @@ when useMarkForDebug or useBackupGc:
|
|||
echo "[GC] cannot register global variable; too many global variables"
|
||||
quit 1
|
||||
|
||||
proc cellsetReset(s: var TCellSet) =
|
||||
proc cellsetReset(s: var TCellSet) =
|
||||
deinit(s)
|
||||
init(s)
|
||||
|
||||
|
|
@ -336,7 +336,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: TWalkOp) {.benign.} =
|
|||
if n.sons[i].typ.kind in {tyRef, tyString, tySequence}:
|
||||
doOperation(cast[PPointer](d +% n.sons[i].offset)[], op)
|
||||
else:
|
||||
forAllChildrenAux(cast[pointer](d +% n.sons[i].offset),
|
||||
forAllChildrenAux(cast[pointer](d +% n.sons[i].offset),
|
||||
n.sons[i].typ, op)
|
||||
else:
|
||||
forAllSlotsAux(dest, n.sons[i], op)
|
||||
|
|
@ -384,7 +384,7 @@ proc addNewObjToZCT(res: PCell, gch: var TGcHeap) {.inline.} =
|
|||
# we check the last 8 entries (cache line) for a slot that could be reused.
|
||||
# In 63% of all cases we succeed here! But we have to optimize the heck
|
||||
# out of this small linear search so that ``newObj`` is not slowed down.
|
||||
#
|
||||
#
|
||||
# Slots to try cache hit
|
||||
# 1 32%
|
||||
# 4 59%
|
||||
|
|
@ -481,7 +481,7 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} =
|
|||
gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1")
|
||||
collectCT(gch)
|
||||
sysAssert(allocInv(gch.region), "newObjRC1 after collectCT")
|
||||
|
||||
|
||||
var res = cast[PCell](rawAlloc(gch.region, size + sizeof(TCell)))
|
||||
sysAssert(allocInv(gch.region), "newObjRC1 after rawAlloc")
|
||||
sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2")
|
||||
|
|
@ -510,7 +510,7 @@ proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} =
|
|||
cast[PGenericSeq](result).len = len
|
||||
cast[PGenericSeq](result).reserved = len
|
||||
when defined(memProfiler): nimProfile(size)
|
||||
|
||||
|
||||
proc growObj(old: pointer, newsize: int, gch: var TGcHeap): pointer =
|
||||
acquire(gch)
|
||||
collectCT(gch)
|
||||
|
|
@ -522,7 +522,7 @@ proc growObj(old: pointer, newsize: int, gch: var TGcHeap): pointer =
|
|||
var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(TCell)))
|
||||
var elemSize = 1
|
||||
if ol.typ.kind != tyString: elemSize = ol.typ.base.size
|
||||
|
||||
|
||||
var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize
|
||||
copyMem(res, ol, oldsize + sizeof(TCell))
|
||||
zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(TCell)),
|
||||
|
|
@ -536,7 +536,7 @@ proc growObj(old: pointer, newsize: int, gch: var TGcHeap): pointer =
|
|||
writeCell("growObj new cell", res)
|
||||
gcTrace(ol, csZctFreed)
|
||||
gcTrace(res, csAllocated)
|
||||
when reallyDealloc:
|
||||
when reallyDealloc:
|
||||
sysAssert(allocInv(gch.region), "growObj before dealloc")
|
||||
if ol.refcount shr rcShift <=% 1:
|
||||
# free immediately to save space:
|
||||
|
|
@ -580,7 +580,7 @@ proc freeCyclicCell(gch: var TGcHeap, c: PCell) =
|
|||
prepareDealloc(c)
|
||||
gcTrace(c, csCycFreed)
|
||||
when logGC: writeCell("cycle collector dealloc cell", c)
|
||||
when reallyDealloc:
|
||||
when reallyDealloc:
|
||||
sysAssert(allocInv(gch.region), "free cyclic cell")
|
||||
rawDealloc(gch.region, c)
|
||||
else:
|
||||
|
|
@ -767,7 +767,7 @@ proc collectCycles(gch: var TGcHeap) =
|
|||
gcAssert isAllocatedPtr(gch.region, c), "addBackStackRoots"
|
||||
gcAssert c.refcount >=% rcIncrement, "addBackStackRoots: dead cell"
|
||||
if canBeCycleRoot(c):
|
||||
#if c notin gch.cycleRoots:
|
||||
#if c notin gch.cycleRoots:
|
||||
inc cycleRootsLen
|
||||
incl(gch.cycleRoots, c)
|
||||
gcAssert c.typ != nil, "addBackStackRoots 2"
|
||||
|
|
@ -794,12 +794,12 @@ proc gcMark(gch: var TGcHeap, p: pointer) {.inline.} =
|
|||
add(gch.decStack, cell)
|
||||
sysAssert(allocInv(gch.region), "gcMark end")
|
||||
|
||||
proc markThreadStacks(gch: var TGcHeap) =
|
||||
proc markThreadStacks(gch: var TGcHeap) =
|
||||
when hasThreadSupport and hasSharedHeap:
|
||||
{.error: "not fully implemented".}
|
||||
var it = threadList
|
||||
while it != nil:
|
||||
# mark registers:
|
||||
# mark registers:
|
||||
for i in 0 .. high(it.registers): gcMark(gch, it.registers[i])
|
||||
var sp = cast[TAddress](it.stackBottom)
|
||||
var max = cast[TAddress](it.stackTop)
|
||||
|
|
@ -933,7 +933,7 @@ else:
|
|||
while sp <=% max:
|
||||
gcMark(gch, cast[PPointer](sp)[])
|
||||
sp = sp +% sizeof(pointer)
|
||||
|
||||
|
||||
proc markStackAndRegisters(gch: var TGcHeap) {.noinline, cdecl.} =
|
||||
forEachStackSlot(gch, gcMark)
|
||||
|
||||
|
|
@ -946,13 +946,13 @@ when useMarkForDebug or useBackupGc:
|
|||
# ----------------------------------------------------------------------------
|
||||
|
||||
proc collectZCT(gch: var TGcHeap): bool =
|
||||
# Note: Freeing may add child objects to the ZCT! So essentially we do
|
||||
# deep freeing, which is bad for incremental operation. In order to
|
||||
# Note: Freeing may add child objects to the ZCT! So essentially we do
|
||||
# deep freeing, which is bad for incremental operation. In order to
|
||||
# avoid a deep stack, we move objects to keep the ZCT small.
|
||||
# This is performance critical!
|
||||
const workPackage = 100
|
||||
var L = addr(gch.zct.len)
|
||||
|
||||
|
||||
when withRealTime:
|
||||
var steps = workPackage
|
||||
var t0: TTicks
|
||||
|
|
@ -962,15 +962,15 @@ proc collectZCT(gch: var TGcHeap): bool =
|
|||
sysAssert(isAllocatedPtr(gch.region, c), "CollectZCT: isAllocatedPtr")
|
||||
# remove from ZCT:
|
||||
gcAssert((c.refcount and ZctFlag) == ZctFlag, "collectZCT")
|
||||
|
||||
|
||||
c.refcount = c.refcount and not ZctFlag
|
||||
gch.zct.d[0] = gch.zct.d[L[] - 1]
|
||||
dec(L[])
|
||||
when withRealTime: dec steps
|
||||
if c.refcount <% rcIncrement:
|
||||
if c.refcount <% rcIncrement:
|
||||
# It may have a RC > 0, if it is in the hardware stack or
|
||||
# it has not been removed yet from the ZCT. This is because
|
||||
# ``incref`` does not bother to remove the cell from the ZCT
|
||||
# ``incref`` does not bother to remove the cell from the ZCT
|
||||
# as this might be too slow.
|
||||
# In any case, it should be removed from the ZCT. But not
|
||||
# freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!**
|
||||
|
|
@ -983,7 +983,7 @@ proc collectZCT(gch: var TGcHeap): bool =
|
|||
# access invalid memory. This is done by prepareDealloc():
|
||||
prepareDealloc(c)
|
||||
forAllChildren(c, waZctDecRef)
|
||||
when reallyDealloc:
|
||||
when reallyDealloc:
|
||||
sysAssert(allocInv(gch.region), "collectZCT: rawDealloc")
|
||||
rawDealloc(gch.region, c)
|
||||
else:
|
||||
|
|
@ -994,7 +994,7 @@ proc collectZCT(gch: var TGcHeap): bool =
|
|||
steps = workPackage
|
||||
if gch.maxPause > 0:
|
||||
let duration = getticks() - t0
|
||||
# the GC's measuring is not accurate and needs some cleanup actions
|
||||
# the GC's measuring is not accurate and needs some cleanup actions
|
||||
# (stack unmarking), so subtract some short amount of time in
|
||||
# order to miss deadlines less often:
|
||||
if duration >= gch.maxPause - 50_000:
|
||||
|
|
@ -1017,7 +1017,7 @@ proc collectCTBody(gch: var TGcHeap) =
|
|||
when withRealTime:
|
||||
let t0 = getticks()
|
||||
sysAssert(allocInv(gch.region), "collectCT: begin")
|
||||
|
||||
|
||||
gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize())
|
||||
sysAssert(gch.decStack.len == 0, "collectCT")
|
||||
prepareForInteriorPointerChecking(gch.region)
|
||||
|
|
@ -1036,7 +1036,7 @@ proc collectCTBody(gch: var TGcHeap) =
|
|||
gch.stat.maxThreshold = max(gch.stat.maxThreshold, gch.cycleThreshold)
|
||||
unmarkStackAndRegisters(gch)
|
||||
sysAssert(allocInv(gch.region), "collectCT: end")
|
||||
|
||||
|
||||
when withRealTime:
|
||||
let duration = getticks() - t0
|
||||
gch.stat.maxPause = max(gch.stat.maxPause, duration)
|
||||
|
|
@ -1050,8 +1050,12 @@ when useMarkForDebug or useBackupGc:
|
|||
markGlobals(gch)
|
||||
|
||||
proc collectCT(gch: var TGcHeap) =
|
||||
if (gch.zct.len >= ZctThreshold or (cycleGC and
|
||||
getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and
|
||||
# stackMarkCosts prevents some pathological behaviour: Stack marking
|
||||
# becomes more expensive with large stacks and large stacks mean that
|
||||
# cells with RC=0 are more likely to be kept alive by the stack.
|
||||
let stackMarkCosts = max(stackSize() div (16*sizeof(int)), ZctThreshold)
|
||||
if (gch.zct.len >= stackMarkCosts or (cycleGC and
|
||||
getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and
|
||||
gch.recGcLock == 0:
|
||||
when useMarkForDebug:
|
||||
prepareForInteriorPointerChecking(gch.region)
|
||||
|
|
@ -1070,7 +1074,7 @@ when withRealTime:
|
|||
acquire(gch)
|
||||
gch.maxPause = us.toNano
|
||||
if (gch.zct.len >= ZctThreshold or (cycleGC and
|
||||
getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) or
|
||||
getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) or
|
||||
strongAdvice:
|
||||
collectCTBody(gch)
|
||||
release(gch)
|
||||
|
|
@ -1078,13 +1082,13 @@ when withRealTime:
|
|||
proc GC_step*(us: int, strongAdvice = false) = GC_step(gch, us, strongAdvice)
|
||||
|
||||
when not defined(useNimRtl):
|
||||
proc GC_disable() =
|
||||
proc GC_disable() =
|
||||
when hasThreadSupport and hasSharedHeap:
|
||||
discard atomicInc(gch.recGcLock, 1)
|
||||
else:
|
||||
inc(gch.recGcLock)
|
||||
proc GC_enable() =
|
||||
if gch.recGcLock > 0:
|
||||
if gch.recGcLock > 0:
|
||||
when hasThreadSupport and hasSharedHeap:
|
||||
discard atomicDec(gch.recGcLock, 1)
|
||||
else:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue