Merge branch 'prim-gc' into devel

This commit is contained in:
Andreas Rumpf 2016-05-22 15:01:36 +02:00
commit 2ecf4633f4
6 changed files with 549 additions and 186 deletions

View file

@ -27,15 +27,14 @@ const
type
PTrunk = ptr Trunk
Trunk {.final.} = object
Trunk = object
next: PTrunk # all nodes are connected with this pointer
key: int # start address at bit 0
bits: array[0..IntsPerTrunk-1, int] # a bit vector
TrunkBuckets = array[0..255, PTrunk]
IntSet {.final.} = object
IntSet = object
data: TrunkBuckets
{.deprecated: [TIntSet: IntSet, TTrunk: Trunk, TTrunkBuckets: TrunkBuckets].}
type
AlignType = BiggestFloat
@ -64,8 +63,6 @@ type
next, prev: PBigChunk # chunks of the same (or bigger) size
align: int
data: AlignType # start of usable memory
{.deprecated: [TAlignType: AlignType, TFreeCell: FreeCell, TBaseChunk: BaseChunk,
TBigChunk: BigChunk, TSmallChunk: SmallChunk].}
template smallChunkOverhead(): expr = sizeof(SmallChunk)-sizeof(AlignType)
template bigChunkOverhead(): expr = sizeof(BigChunk)-sizeof(AlignType)
@ -79,18 +76,18 @@ template bigChunkOverhead(): expr = sizeof(BigChunk)-sizeof(AlignType)
type
PLLChunk = ptr LLChunk
LLChunk {.pure.} = object ## *low-level* chunk
LLChunk = object ## *low-level* chunk
size: int # remaining size
acc: int # accumulator
next: PLLChunk # next low-level chunk; only needed for dealloc
PAvlNode = ptr AvlNode
AvlNode {.pure, final.} = object
AvlNode = object
link: array[0..1, PAvlNode] # Left (0) and right (1) links
key, upperBound: int
level: int
MemRegion {.final, pure.} = object
MemRegion = object
minLargeObj, maxLargeObj: int
freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk]
llmem: PLLChunk
@ -99,6 +96,7 @@ type
freeChunksList: PBigChunk # XXX make this a datastructure with O(1) access
chunkStarts: IntSet
root, deleted, last, freeAvlNodes: PAvlNode
locked: bool # if locked, we cannot free pages.
{.deprecated: [TLLChunk: LLChunk, TAvlNode: AvlNode, TMemRegion: MemRegion].}
# shared:
@ -234,7 +232,8 @@ proc isSmallChunk(c: PChunk): bool {.inline.} =
proc chunkUnused(c: PChunk): bool {.inline.} =
result = not c.used
iterator allObjects(m: MemRegion): pointer {.inline.} =
iterator allObjects(m: var MemRegion): pointer {.inline.} =
m.locked = true
for s in elements(m.chunkStarts):
# we need to check here again as it could have been modified:
if s in m.chunkStarts:
@ -252,6 +251,7 @@ iterator allObjects(m: MemRegion): pointer {.inline.} =
else:
let c = cast[PBigChunk](c)
yield addr(c.data)
m.locked = false
proc iterToProc*(iter: typed, envType: typedesc; procName: untyped) {.
magic: "Plugin", compileTime.}
@ -385,7 +385,7 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) =
excl(a.chunkStarts, pageIndex(c))
c = cast[PBigChunk](le)
if c.size < ChunkOsReturn or doNotUnmap:
if c.size < ChunkOsReturn or doNotUnmap or a.locked:
incl(a, a.chunkStarts, pageIndex(c))
updatePrevSize(a, c, c.size)
listAdd(a.freeChunksList, c)
@ -442,26 +442,29 @@ proc getSmallChunk(a: var MemRegion): PSmallChunk =
# -----------------------------------------------------------------------------
proc isAllocatedPtr(a: MemRegion, p: pointer): bool {.benign.}
proc allocInv(a: MemRegion): bool =
## checks some (not all yet) invariants of the allocator's data structures.
for s in low(a.freeSmallChunks)..high(a.freeSmallChunks):
var c = a.freeSmallChunks[s]
while not (c == nil):
if c.next == c:
echo "[SYSASSERT] c.next == c"
return false
if not (c.size == s * MemAlign):
echo "[SYSASSERT] c.size != s * MemAlign"
return false
var it = c.freeList
while not (it == nil):
if not (it.zeroField == 0):
echo "[SYSASSERT] it.zeroField != 0"
c_printf("%ld %p\n", it.zeroField, it)
when true:
template allocInv(a: MemRegion): bool = true
else:
proc allocInv(a: MemRegion): bool =
## checks some (not all yet) invariants of the allocator's data structures.
for s in low(a.freeSmallChunks)..high(a.freeSmallChunks):
var c = a.freeSmallChunks[s]
while not (c == nil):
if c.next == c:
echo "[SYSASSERT] c.next == c"
return false
it = it.next
c = c.next
result = true
if not (c.size == s * MemAlign):
echo "[SYSASSERT] c.size != s * MemAlign"
return false
var it = c.freeList
while not (it == nil):
if not (it.zeroField == 0):
echo "[SYSASSERT] it.zeroField != 0"
c_printf("%ld %p\n", it.zeroField, it)
return false
it = it.next
c = c.next
result = true
proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
sysAssert(allocInv(a), "rawAlloc: begin")

View file

@ -1,7 +1,7 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2015 Andreas Rumpf
# (c) Copyright 2016 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
@ -9,13 +9,8 @@
# Garbage Collector
#
# The basic algorithm is *Deferred Reference Counting* with cycle detection.
# This is achieved by combining a Deutsch-Bobrow garbage collector
# together with Christoper's partial mark-sweep garbage collector.
#
# Special care has been taken to avoid recursion as far as possible to avoid
# stack overflows when traversing deep datastructures. It is well-suited
# for soft real time applications (like games).
# Refcounting + Mark&Sweep. Complex algorithms avoided.
# Been there, done that, didn't work.
when defined(nimCoroutines):
import arch
@ -30,7 +25,7 @@ const
# this seems to be a good value
withRealTime = defined(useRealtimeGC)
useMarkForDebug = defined(gcGenerational)
useBackupGc = false # use a simple M&S GC to collect
useBackupGc = true # use a simple M&S GC to collect
# cycles instead of the complex
# algorithm
@ -55,8 +50,7 @@ type
WalkOp = enum
waMarkGlobal, # part of the backup/debug mark&sweep
waMarkPrecise, # part of the backup/debug mark&sweep
waZctDecRef, waPush, waCycleDecRef, waMarkGray, waScan, waScanBlack,
waCollectWhite #, waDebug
waZctDecRef, waPush
Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.}
# A ref type can have a finalizer that is called before the object's
@ -87,7 +81,6 @@ type
idGenerator: int
zct: CellSeq # the zero count table
decStack: CellSeq # cells in the stack that are to decref again
cycleRoots: CellSet
tempStack: CellSeq # temporary stack for recursion elimination
recGcLock: int # prevent recursion via finalizers; no thread lock
when withRealTime:
@ -136,9 +129,6 @@ proc usrToCell(usr: pointer): PCell {.inline.} =
# convert pointer to userdata to object (=pointer to refcount)
result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell)))
proc canBeCycleRoot(c: PCell): bool {.inline.} =
result = ntfAcyclic notin c.typ.flags
proc extGetCellType(c: pointer): PNimType {.compilerproc.} =
# used for code generation concerning debugging
result = usrToCell(c).typ
@ -200,14 +190,16 @@ proc prepareDealloc(cell: PCell) =
(cast[Finalizer](cell.typ.finalizer))(cellToUsr(cell))
dec(gch.recGcLock)
template beforeDealloc(gch: var GcHeap; c: PCell; msg: typed) =
when false:
for i in 0..gch.decStack.len-1:
if gch.decStack.d[i] == c:
sysAssert(false, msg)
proc rtlAddCycleRoot(c: PCell) {.rtl, inl.} =
# we MUST access gch as a global here, because this crosses DLL boundaries!
when hasThreadSupport and hasSharedHeap:
acquireSys(HeapLock)
when cycleGC:
if c.color != rcPurple:
c.setColor(rcPurple)
incl(gch.cycleRoots, c)
when hasThreadSupport and hasSharedHeap:
releaseSys(HeapLock)
@ -224,19 +216,12 @@ proc decRef(c: PCell) {.inline.} =
gcAssert(c.refcount >=% rcIncrement, "decRef")
if --c.refcount:
rtlAddZCT(c)
elif canbeCycleRoot(c):
# unfortunately this is necessary here too, because a cycle might just
# have been broken up and we could recycle it.
rtlAddCycleRoot(c)
#writeCell("decRef", c)
proc incRef(c: PCell) {.inline.} =
gcAssert(isAllocatedPtr(gch.region, c), "incRef: interiorPtr")
c.refcount = c.refcount +% rcIncrement
# and not colorMask
#writeCell("incRef", c)
if canbeCycleRoot(c):
rtlAddCycleRoot(c)
proc nimGCref(p: pointer) {.compilerProc, inline.} = incRef(usrToCell(p))
proc nimGCunref(p: pointer) {.compilerProc, inline.} = decRef(usrToCell(p))
@ -306,7 +291,6 @@ proc initGC() =
# init the rt
init(gch.zct)
init(gch.tempStack)
init(gch.cycleRoots)
init(gch.decStack)
when useMarkForDebug or useBackupGc:
init(gch.marked)
@ -563,7 +547,7 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer =
d[j] = res
break
dec(j)
if canbeCycleRoot(ol): excl(gch.cycleRoots, ol)
beforeDealloc(gch, ol, "growObj stack trash")
rawDealloc(gch.region, ol)
else:
# we split the old refcount in 2 parts. XXX This is still not entirely
@ -597,54 +581,12 @@ proc freeCyclicCell(gch: var GcHeap, c: PCell) =
when logGC: writeCell("cycle collector dealloc cell", c)
when reallyDealloc:
sysAssert(allocInv(gch.region), "free cyclic cell")
beforeDealloc(gch, c, "freeCyclicCell: stack trash")
rawDealloc(gch.region, c)
else:
gcAssert(c.typ != nil, "freeCyclicCell")
zeroMem(c, sizeof(Cell))
proc markGray(s: PCell) =
if s.color != rcGray:
setColor(s, rcGray)
forAllChildren(s, waMarkGray)
proc scanBlack(s: PCell) =
s.setColor(rcBlack)
forAllChildren(s, waScanBlack)
proc scan(s: PCell) =
if s.color == rcGray:
if s.refcount >=% rcIncrement:
scanBlack(s)
else:
s.setColor(rcWhite)
forAllChildren(s, waScan)
proc collectWhite(s: PCell) =
# This is a hacky way to deal with the following problem (bug #1796)
# Consider this content in cycleRoots:
# x -> a; y -> a where 'a' is an acyclic object so not included in
# cycleRoots itself. Then 'collectWhite' used to free 'a' twice. The
# 'isAllocatedPtr' check prevents this. This also means we do not need
# to query 's notin gch.cycleRoots' at all.
if isAllocatedPtr(gch.region, s) and s.color == rcWhite:
s.setColor(rcBlack)
forAllChildren(s, waCollectWhite)
freeCyclicCell(gch, s)
proc markRoots(gch: var GcHeap) =
var tabSize = 0
for s in elements(gch.cycleRoots):
#writeCell("markRoot", s)
inc tabSize
if s.color == rcPurple and s.refcount >=% rcIncrement:
markGray(s)
else:
excl(gch.cycleRoots, s)
# (s.color == rcBlack and rc == 0) as 1 condition:
if s.refcount == 0:
freeCyclicCell(gch, s)
gch.stat.cycleTableSize = max(gch.stat.cycleTableSize, tabSize)
when useBackupGc:
proc sweep(gch: var GcHeap) =
for x in allObjects(gch.region):
@ -667,16 +609,6 @@ when useMarkForDebug or useBackupGc:
proc markGlobals(gch: var GcHeap) =
for i in 0 .. < globalMarkersLen: globalMarkers[i]()
proc stackMarkS(gch: var GcHeap, p: pointer) {.inline.} =
# the addresses are not as cells on the stack, so turn them to cells:
var cell = usrToCell(p)
var c = cast[ByteAddress](cell)
if c >% PageSize:
# fast check: does it look like a cell?
var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell))
if objStart != nil:
markS(gch, objStart)
when logGC:
var
cycleCheckA: array[100, PCell]
@ -717,19 +649,6 @@ proc doOperation(p: pointer, op: WalkOp) =
#if c.refcount <% rcIncrement: addZCT(gch.zct, c)
of waPush:
add(gch.tempStack, c)
of waCycleDecRef:
gcAssert(c.refcount >=% rcIncrement, "doOperation 3")
c.refcount = c.refcount -% rcIncrement
of waMarkGray:
gcAssert(c.refcount >=% rcIncrement, "waMarkGray")
c.refcount = c.refcount -% rcIncrement
markGray(c)
of waScan: scan(c)
of waScanBlack:
c.refcount = c.refcount +% rcIncrement
if c.color != rcBlack:
scanBlack(c)
of waCollectWhite: collectWhite(c)
of waMarkGlobal:
when useMarkForDebug or useBackupGc:
when hasThreadSupport:
@ -748,14 +667,6 @@ proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} =
proc collectZCT(gch: var GcHeap): bool {.benign.}
when useMarkForDebug or useBackupGc:
proc markStackAndRegistersForSweep(gch: var GcHeap) {.noinline, cdecl,
benign.}
proc collectRoots(gch: var GcHeap) =
for s in elements(gch.cycleRoots):
collectWhite(s)
proc collectCycles(gch: var GcHeap) =
when hasThreadSupport:
for c in gch.toDispose:
@ -764,33 +675,12 @@ proc collectCycles(gch: var GcHeap) =
while gch.zct.len > 0: discard collectZCT(gch)
when useBackupGc:
cellsetReset(gch.marked)
markStackAndRegistersForSweep(gch)
var d = gch.decStack.d
for i in 0..gch.decStack.len-1:
sysAssert isAllocatedPtr(gch.region, d[i]), "collectCycles"
markS(gch, d[i])
markGlobals(gch)
sweep(gch)
else:
markRoots(gch)
# scanRoots:
for s in elements(gch.cycleRoots): scan(s)
collectRoots(gch)
cellsetReset(gch.cycleRoots)
# alive cycles need to be kept in 'cycleRoots' if they are referenced
# from the stack; otherwise the write barrier will add the cycle root again
# anyway:
when false:
var d = gch.decStack.d
var cycleRootsLen = 0
for i in 0..gch.decStack.len-1:
var c = d[i]
gcAssert isAllocatedPtr(gch.region, c), "addBackStackRoots"
gcAssert c.refcount >=% rcIncrement, "addBackStackRoots: dead cell"
if canBeCycleRoot(c):
#if c notin gch.cycleRoots:
inc cycleRootsLen
incl(gch.cycleRoots, c)
gcAssert c.typ != nil, "addBackStackRoots 2"
if cycleRootsLen != 0:
cfprintf(cstdout, "cycle roots: %ld\n", cycleRootsLen)
proc gcMark(gch: var GcHeap, p: pointer) {.inline.} =
# the addresses are not as cells on the stack, so turn them to cells:
@ -812,31 +702,11 @@ proc gcMark(gch: var GcHeap, p: pointer) {.inline.} =
add(gch.decStack, cell)
sysAssert(allocInv(gch.region), "gcMark end")
proc markThreadStacks(gch: var GcHeap) =
when hasThreadSupport and hasSharedHeap:
{.error: "not fully implemented".}
var it = threadList
while it != nil:
# mark registers:
for i in 0 .. high(it.registers): gcMark(gch, it.registers[i])
var sp = cast[ByteAddress](it.stackBottom)
var max = cast[ByteAddress](it.stackTop)
# XXX stack direction?
# XXX unroll this loop:
while sp <=% max:
gcMark(gch, cast[ppointer](sp)[])
sp = sp +% sizeof(pointer)
it = it.next
include gc_common
proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl.} =
forEachStackSlot(gch, gcMark)
when useMarkForDebug or useBackupGc:
proc markStackAndRegistersForSweep(gch: var GcHeap) =
forEachStackSlot(gch, stackMarkS)
proc collectZCT(gch: var GcHeap): bool =
# Note: Freeing may add child objects to the ZCT! So essentially we do
# deep freeing, which is bad for incremental operation. In order to
@ -866,8 +736,6 @@ proc collectZCT(gch: var GcHeap): bool =
# as this might be too slow.
# In any case, it should be removed from the ZCT. But not
# freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!**
when cycleGC:
if canbeCycleRoot(c): excl(gch.cycleRoots, c)
when logGC: writeCell("zct dealloc cell", c)
gcTrace(c, csZctFreed)
# We are about to free the object, call the finalizer BEFORE its
@ -877,6 +745,7 @@ proc collectZCT(gch: var GcHeap): bool =
forAllChildren(c, waZctDecRef)
when reallyDealloc:
sysAssert(allocInv(gch.region), "collectZCT: rawDealloc")
beforeDealloc(gch, c, "collectZCT: stack trash")
rawDealloc(gch.region, c)
else:
sysAssert(c.typ != nil, "collectZCT 2")
@ -915,7 +784,6 @@ proc collectCTBody(gch: var GcHeap) =
sysAssert(gch.decStack.len == 0, "collectCT")
prepareForInteriorPointerChecking(gch.region)
markStackAndRegisters(gch)
markThreadStacks(gch)
gch.stat.maxStackCells = max(gch.stat.maxStackCells, gch.decStack.len)
inc(gch.stat.stackScans)
if collectZCT(gch):
@ -937,11 +805,6 @@ proc collectCTBody(gch: var GcHeap) =
if gch.maxPause > 0 and duration > gch.maxPause:
c_fprintf(c_stdout, "[GC] missed deadline: %ld\n", duration)
when useMarkForDebug or useBackupGc:
proc markForDebug(gch: var GcHeap) =
markStackAndRegistersForSweep(gch)
markGlobals(gch)
when defined(nimCoroutines):
proc currentStackSizes(): int =
for stack in items(gch.stack):
@ -1035,7 +898,7 @@ when not defined(useNimRtl):
"[GC] max threshold: " & $gch.stat.maxThreshold & "\n" &
"[GC] zct capacity: " & $gch.zct.cap & "\n" &
"[GC] max cycle table size: " & $gch.stat.cycleTableSize & "\n" &
"[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000)
"[GC] max pause time [ms]: " & $(gch.stat.maxPause div 1000_000) & "\n"
when defined(nimCoroutines):
result = result & "[GC] number of stacks: " & $gch.stack.len & "\n"
for stack in items(gch.stack):