jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Reworked gc support for coroutines. Nim now bootstraps with -d:nimCoroutines

Browse source 
Added gc test to coro.nim
Lots of misc improvements and comments in coro.nim
This commit is contained in:
Rokas Kupstys 2017-02-13 19:42:47 +02:00
commit c3d1b732d6
3 changed files with 231 additions and 164 deletions
Download patch file Download diff file Expand all files Collapse all files

129
lib/pure/coro.nim
View file

@ -26,9 +26,9 @@ include system/timers
const defaultStackSize = 512 * 1024 const defaultStackSize = 512 * 1024
proc GC_addStack(starts: pointer) {.cdecl, importc.} proc GC_addStack(bottom: pointer) {.cdecl, importc.}
proc GC_removeStack(starts: pointer) {.cdecl, importc.} proc GC_removeStack(bottom: pointer) {.cdecl, importc.}
proc GC_setCurrentStack(starts, pos: pointer) {.cdecl, importc.} proc GC_setActiveStack(bottom: pointer) {.cdecl, importc.}
const const
CORO_BACKEND_UCONTEXT = 0 CORO_BACKEND_UCONTEXT = 0
@ -50,11 +50,6 @@ when coroBackend == CORO_BACKEND_FIBERS:
import windows.winlean import windows.winlean
type type
Context = pointer Context = pointer
Fiber {.final, pure.} = object
parameter: pointer
pad1: pointer
stackStart: pointer
stackEnd: pointer
elif coroBackend == CORO_BACKEND_UCONTEXT: elif coroBackend == CORO_BACKEND_UCONTEXT:
type type
@ -123,8 +118,8 @@ const
type type
Stack = object Stack = object
start: pointer top: pointer # Top of the stack. Pointer used for deallocating stack if we own it.
ends: pointer bottom: pointer # Very bottom of the stack, acts as unique stack identifier.
size: int size: int
Coroutine = ref object Coroutine = ref object
@ -164,44 +159,59 @@ proc runCurrentTask()
proc switchTo(current, to: Coroutine) = proc switchTo(current, to: Coroutine) =
## Switches execution from `current` into `to` context. ## Switches execution from `current` into `to` context.
to.lastRun = getTicks() to.lastRun = getTicks()
# Execution will switch to another fiber now. # Update position of current stack so gc invoked from another stack knows how much to scan.
when coroBackend == CORO_BACKEND_FIBERS: GC_setActiveStack(current.stack.bottom)
SwitchToFiber(to.execContext) var frame = getFrameState()
elif coroBackend == CORO_BACKEND_UCONTEXT: block:
discard swapcontext(current.execContext, to.execContext) # Execution will switch to another fiber now. We do not need to update current stack
elif coroBackend == CORO_BACKEND_SETJMP: when coroBackend == CORO_BACKEND_FIBERS:
var res = setjmp(current.execContext) SwitchToFiber(to.execContext)
if res == 0: elif coroBackend == CORO_BACKEND_UCONTEXT:
if to.state == CORO_EXECUTING: discard swapcontext(current.execContext, to.execContext)
# Coroutine is resumed. elif coroBackend == CORO_BACKEND_SETJMP:
longjmp(to.execContext, 1) var res = setjmp(current.execContext)
elif to.state == CORO_CREATED: if res == 0:
# Coroutine is started. if to.state == CORO_EXECUTING:
coroExecWithStack(runCurrentTask, to.stack.ends) # Coroutine is resumed.
doAssert false longjmp(to.execContext, 1)
else: elif to.state == CORO_CREATED:
{.error: "Invalid coroutine backend set.".} # Coroutine is started.
# Execution was just resumed. Set active stack to current one. coroExecWithStack(runCurrentTask, to.stack.bottom)
GC_setCurrentStack(current.stack.start) doAssert false
else:
{.error: "Invalid coroutine backend set.".}
# Execution was just resumed. Restore frame information and set active stack.
setFrameState(frame)
GC_setActiveStack(current.stack.bottom)
proc suspend*(sleepTime: float=0) = proc suspend*(sleepTime: float=0) =
## Stops coroutine execution and resumes no sooner than after ``sleeptime`` seconds. ## Stops coroutine execution and resumes no sooner than after ``sleeptime`` seconds.
## Until then other coroutines are executed. ## Until then other coroutines are executed.
var current = getCurrent() var current = getCurrent()
current.sleepTime = sleepTime current.sleepTime = sleepTime
var frame = getFrameState()
switchTo(current, ctx.loop) switchTo(current, ctx.loop)
setFrameState(frame)
proc runCurrentTask() = proc runCurrentTask() =
## Starts execution of current coroutine and updates it's state through coroutine's life. ## Starts execution of current coroutine and updates it's state through coroutine's life.
var current = getCurrent() var sp {.volatile.}: pointer
# Execution of new fiber just started. Since it was entered not through `switchTo` we sp = addr(sp)
# have to set active stack here as well. block:
GC_setCurrentStack(current.stack.start) var current = getCurrent()
current.state = CORO_EXECUTING current.stack.bottom = sp
current.fn() # Start coroutine execution # Execution of new fiber just started. Since it was entered not through `switchTo` we
current.state = CORO_FINISHED # have to set active stack here as well. GC_removeStack() has to be called in main loop
# because we still need stack available in final suspend(0) call from which we will not
# return.
GC_addStack(sp)
# Activate current stack because we are executing in a new coroutine.
GC_setActiveStack(sp)
current.state = CORO_EXECUTING
try:
current.fn() # Start coroutine execution
except:
echo "Unhandled exception in coroutine."
writeStackTrace()
current.state = CORO_FINISHED
suspend(0) # Exit coroutine without returning from coroExecWithStack() suspend(0) # Exit coroutine without returning from coroExecWithStack()
doAssert false doAssert false
@ -215,25 +225,20 @@ proc start*(c: proc(), stacksize: int=defaultStackSize) =
when coroBackend == CORO_BACKEND_FIBERS: when coroBackend == CORO_BACKEND_FIBERS:
coro.execContext = CreateFiberEx(stacksize, stacksize, coro.execContext = CreateFiberEx(stacksize, stacksize,
FIBER_FLAG_FLOAT_SWITCH, (proc(p: pointer): void {.stdcall.} = runCurrentTask()), nil) FIBER_FLAG_FLOAT_SWITCH, (proc(p: pointer): void {.stdcall.} = runCurrentTask()), nil)
var fiber = cast[ptr Fiber](coro.execContext)
coro.stack.start = fiber.stackStart
coro.stack.ends = fiber.stackEnd
coro.stack.size = stacksize coro.stack.size = stacksize
else: else:
var stack: pointer var stack: pointer
while stack == nil: while stack == nil:
stack = alloc0(stacksize) stack = alloc0(stacksize)
coro.stack.start = stack coro.stack.top = stack
coro.stack.ends = cast[pointer](cast[ByteAddress](stack) + stacksize)
when coroBackend == CORO_BACKEND_UCONTEXT: when coroBackend == CORO_BACKEND_UCONTEXT:
discard getcontext(coro.execContext) discard getcontext(coro.execContext)
coro.execContext.uc_stack.ss_sp = coro.stack.ends coro.execContext.uc_stack.ss_sp = cast[pointer](cast[ByteAddress](stack) + stacksize)
coro.execContext.uc_stack.ss_size = coro.stack.size coro.execContext.uc_stack.ss_size = coro.stack.size
coro.execContext.uc_link = addr ctx.loop.execContext coro.execContext.uc_link = addr ctx.loop.execContext
makecontext(coro.execContext, runCurrentTask, 0) makecontext(coro.execContext, runCurrentTask, 0)
coro.stack.size = stacksize coro.stack.size = stacksize
coro.state = CORO_CREATED coro.state = CORO_CREATED
GC_addStack(coro.stack.ends)
ctx.coroutines.append(coro) ctx.coroutines.append(coro)
proc run*() = proc run*() =
@ -248,9 +253,7 @@ proc run*() =
var remaining = current.sleepTime - (float(getTicks() - current.lastRun) / 1_000_000_000) var remaining = current.sleepTime - (float(getTicks() - current.lastRun) / 1_000_000_000)
if remaining <= 0: if remaining <= 0:
# Save main loop context. Suspending coroutine will resume after this statement with # Save main loop context. Suspending coroutine will resume after this statement with
var frame = getFrameState()
switchTo(ctx.loop, current) switchTo(ctx.loop, current)
setFrameState(frame)
else: else:
if minDelay > 0 and remaining > 0: if minDelay > 0 and remaining > 0:
minDelay = min(remaining, minDelay) minDelay = min(remaining, minDelay)
@ -258,19 +261,19 @@ proc run*() =
minDelay = remaining minDelay = remaining
if current.state == CORO_FINISHED: if current.state == CORO_FINISHED:
GC_removeStack(current.stack.start)
var next = ctx.current.prev var next = ctx.current.prev
if next == nil: if next == nil:
# If first coroutine ends then `prev` is nil even if more coroutines # If first coroutine ends then `prev` is nil even if more coroutines
# are to be scheduled. # are to be scheduled.
next = ctx.current.next next = ctx.current.next
ctx.coroutines.remove(ctx.current) ctx.coroutines.remove(ctx.current)
GC_removeStack(current.stack.bottom)
when coroBackend == CORO_BACKEND_FIBERS: when coroBackend == CORO_BACKEND_FIBERS:
DeleteFiber(coro.execContext) DeleteFiber(current.execContext)
else: else:
dealloc(current.stack.start) dealloc(current.stack.top)
current.stack.start = nil current.stack.top = nil
current.stack.ends = nil current.stack.bottom = nil
ctx.current = next ctx.current = next
elif ctx.current == nil or ctx.current.next == nil: elif ctx.current == nil or ctx.current.next == nil:
ctx.current = ctx.coroutines.head ctx.current = ctx.coroutines.head
@ -358,3 +361,25 @@ when isMainModule:
run() run()
doAssert order == @[0, 0, 1, 2, 1, 3, 4, 2, 3, 4] doAssert order == @[0, 0, 1, 2, 1, 3, 4, 2, 3, 4]
doAssert stackCheckValue == 1100220033 doAssert stackCheckValue == 1100220033
type Foo = ref object
number: int
GC_fullCollect()
var occupiedMemory = getOccupiedMem()
i = 0
var objects = newSeq[Foo](100)
proc terstGc(id: int, sleep: float) =
for n in 0..<50:
objects[i] = Foo(number: n)
i += 1
start(proc() = terstIterators(1, 0.01))
start(proc() = terstIterators(2, 0.021))
run()
doAssert occupiedMemory < getOccupiedMem()
objects = nil
GC_fullCollect()
doAssert occupiedMemory >= getOccupiedMem()

72
lib/system/gc.nim
View file

@ -63,18 +63,18 @@ type
cycleTableSize: int # max entries in cycle table cycleTableSize: int # max entries in cycle table
maxPause: int64 # max measured GC pause in nanoseconds maxPause: int64 # max measured GC pause in nanoseconds
GcStack {.final.} = object GcStack {.final, pure.} = object
prev: ptr GcStack prev: ptr GcStack
next: ptr GcStack next: ptr GcStack
starts: pointer bottom: pointer
pos: pointer pos: pointer
maxStackSize: int maxStackSize: int
GcHeap {.final, pure.} = object # this contains the zero count and GcHeap {.final, pure.} = object # this contains the zero count and
# non-zero count table # non-zero count table
when defined(nimCoroutines): when defined(nimCoroutines):
stack: ptr GcStack stack: GcStack
stackActive: ptr GcStack activeStack: ptr GcStack
else: else:
stackBottom: pointer stackBottom: pointer
cycleThreshold: int cycleThreshold: int
@ -120,6 +120,53 @@ template gcAssert(cond: bool, msg: string) =
#echo x[] #echo x[]
quit 1 quit 1
when defined(nimCoroutines):
iterator items(first: var GcStack): ptr GcStack =
var item = addr(first)
while true:
yield item
item = item.next
if item == addr(first):
break
proc append(first: var GcStack, stack: ptr GcStack) =
## Append stack to the ring of stacks.
first.prev.next = stack
stack.prev = first.prev
first.prev = stack
stack.next = addr(first)
proc append(first: var GcStack): ptr GcStack =
## Allocate new GcStack object, append it to the ring of stacks and return it.
result = cast[ptr GcStack](alloc0(sizeof(GcStack)))
first.append(result)
proc remove(first: var GcStack, stack: ptr GcStack) =
## Remove stack from ring of stacks.
gcAssert(addr(first) != stack, "Main application stack can not be removed")
if addr(first) == stack or stack == nil:
return
stack.prev.next = stack.next
stack.next.prev = stack.prev
dealloc(stack)
proc remove(stack: ptr GcStack) =
gch.stack.remove(stack)
proc find(first: var GcStack, bottom: pointer): ptr GcStack =
## Find stack struct based on bottom pointer. If `bottom` is nil then main
## thread stack is is returned.
if bottom == nil:
return addr(gch.stack)
for stack in first.items():
if stack.bottom == bottom:
return stack
proc len(stack: var GcStack): int =
for _ in stack.items():
result = result + 1
proc addZCT(s: var CellSeq, c: PCell) {.noinline.} = proc addZCT(s: var CellSeq, c: PCell) {.noinline.} =
if (c.refcount and ZctFlag) == 0: if (c.refcount and ZctFlag) == 0:
c.refcount = c.refcount or ZctFlag c.refcount = c.refcount or ZctFlag
@ -823,7 +870,10 @@ proc collectCTBody(gch: var GcHeap) =
let t0 = getticks() let t0 = getticks()
sysAssert(allocInv(gch.region), "collectCT: begin") sysAssert(allocInv(gch.region), "collectCT: begin")
when not defined(nimCoroutines): when defined(nimCoroutines):
for stack in gch.stack.items():
gch.stat.maxStackSize = max(gch.stat.maxStackSize, stack.stackSize())
else:
gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize()) gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize())
sysAssert(gch.decStack.len == 0, "collectCT") sysAssert(gch.decStack.len == 0, "collectCT")
prepareForInteriorPointerChecking(gch.region) prepareForInteriorPointerChecking(gch.region)
@ -849,19 +899,11 @@ proc collectCTBody(gch: var GcHeap) =
if gch.maxPause > 0 and duration > gch.maxPause: if gch.maxPause > 0 and duration > gch.maxPause:
c_fprintf(stdout, "[GC] missed deadline: %ld\n", duration) c_fprintf(stdout, "[GC] missed deadline: %ld\n", duration)
when defined(nimCoroutines):
proc currentStackSizes(): int =
for stack in items(gch.stack):
result = result + stackSize(stack.starts, stack.pos)
proc collectCT(gch: var GcHeap) = proc collectCT(gch: var GcHeap) =
# stackMarkCosts prevents some pathological behaviour: Stack marking # stackMarkCosts prevents some pathological behaviour: Stack marking
# becomes more expensive with large stacks and large stacks mean that # becomes more expensive with large stacks and large stacks mean that
# cells with RC=0 are more likely to be kept alive by the stack. # cells with RC=0 are more likely to be kept alive by the stack.
when defined(nimCoroutines): let stackMarkCosts = max(stackSize() div (16*sizeof(int)), ZctThreshold)
let stackMarkCosts = max(currentStackSizes() div (16*sizeof(int)), ZctThreshold)
else:
let stackMarkCosts = max(stackSize() div (16*sizeof(int)), ZctThreshold)
if (gch.zct.len >= stackMarkCosts or (cycleGC and if (gch.zct.len >= stackMarkCosts or (cycleGC and
getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and
gch.recGcLock == 0: gch.recGcLock == 0:
@ -946,7 +988,7 @@ when not defined(useNimRtl):
when defined(nimCoroutines): when defined(nimCoroutines):
result = result & "[GC] number of stacks: " & $gch.stack.len & "\n" result = result & "[GC] number of stacks: " & $gch.stack.len & "\n"
for stack in items(gch.stack): for stack in items(gch.stack):
result = result & "[GC] stack " & stack.starts.repr & "[GC] max stack size " & $stack.maxStackSize & "\n" result = result & "[GC] stack " & stack.bottom.repr & "[GC] max stack size " & cast[pointer](stack.maxStackSize).repr & "\n"
else: else:
result = result & "[GC] max stack size: " & $gch.stat.maxStackSize & "\n" result = result & "[GC] max stack size: " & $gch.stat.maxStackSize & "\n"
GC_enable() GC_enable()

194
lib/system/gc_common.nim
View file

@ -68,58 +68,25 @@ proc len(stack: ptr GcStack): int =
s = s.next s = s.next
when defined(nimCoroutines): when defined(nimCoroutines):
proc stackSize(stackBottom: pointer, pos: pointer=nil): int {.noinline.} = proc stackSize(stack: ptr GcStack): int {.noinline.} =
var sp: pointer if stack.pos != nil:
if pos == nil: when defined(stackIncreases):
var stackTop {.volatile.}: pointer result = cast[ByteAddress](stack.pos) -% cast[ByteAddress](stack.bottom)
sp = addr(stackTop)
else:
sp = pos
result = abs(cast[int](sp) - cast[int](stackBottom))
proc GC_addStack*(starts: pointer) {.cdecl, exportc.} =
var sp {.volatile.}: pointer
var stack = cast[ptr GcStack](alloc0(sizeof(GcStack)))
stack.starts = starts
stack.pos = addr sp
if gch.stack == nil:
gch.stack = stack
else:
stack.next = gch.stack
gch.stack.prev = stack
gch.stack = stack
# c_fprintf(stdout, "[GC] added stack 0x%016X\n", starts)
proc GC_removeStack*(starts: pointer) {.cdecl, exportc.} =
var stack = gch.stack
while stack != nil:
if stack.starts == starts:
if stack.prev == nil:
if stack.next != nil:
stack.next.prev = nil
gch.stack = stack.next
else:
stack.prev.next = stack.next
if stack.next != nil:
stack.next.prev = stack.prev
dealloc(stack)
# echo "[GC] removed stack ", starts.repr
break
else: else:
stack = stack.next result = cast[ByteAddress](stack.bottom) -% cast[ByteAddress](stack.pos)
else:
result = 0
proc GC_setCurrentStack*(starts: pointer) {.cdecl, exportc.} = proc setPosition(stack: ptr GcStack, position: pointer) =
var pos {.volatile.}: pointer stack.pos = position
pos = addr(pos) stack.maxStackSize = max(stack.maxStackSize, stack.stackSize())
var stack = gch.stack
while stack != nil: proc setPosition(stack: var GcStack, position: pointer) =
if stack.starts == starts: setPosition(addr(stack), position)
stack.pos = pos
stack.maxStackSize = max(stack.maxStackSize, stackSize(stack.starts, pos)) proc stackSize(): int {.noinline.} =
gch.stackActive = stack for stack in gch.stack.items():
return result = result + stack.stackSize()
stack = stack.next
gcAssert(false, "Current stack position does not belong to registered stack")
else: else:
proc stackSize(): int {.noinline.} = proc stackSize(): int {.noinline.} =
var stackTop {.volatile.}: pointer var stackTop {.volatile.}: pointer
@ -180,18 +147,50 @@ elif defined(hppa) or defined(hp9000) or defined(hp9000s300) or
else: else:
const stackIncreases = false const stackIncreases = false
{.push stack_trace: off.}
when defined(nimCoroutines):
proc GC_addStack(bottom: pointer) {.cdecl, exportc.} =
# c_fprintf(stdout, "GC_addStack: %p;\n", bottom)
var stack = gch.stack.append()
stack.bottom = bottom
stack.setPosition(bottom)
proc GC_removeStack(bottom: pointer) {.cdecl, exportc.} =
# c_fprintf(stdout, "GC_removeStack: %p;\n", bottom)
gch.stack.find(bottom).remove()
proc GC_setActiveStack(bottom: pointer) {.cdecl, exportc.} =
## Sets active stack and updates current stack position.
# c_fprintf(stdout, "GC_setActiveStack: %p;\n", bottom)
var sp {.volatile.}: pointer
gch.activeStack = gch.stack.find(bottom)
gch.activeStack.setPosition(addr(sp))
when not defined(useNimRtl): when not defined(useNimRtl):
{.push stack_trace: off.} when defined(nimCoroutines):
proc setStackBottom(theStackBottom: pointer) = proc setStackBottom(theStackBottom: pointer) =
#c_fprintf(stdout, "stack bottom: %p;\n", theStackBottom) # Initializes main stack of the thread.
# the first init must be the one that defines the stack bottom: if gch.stack.next == nil:
when defined(nimCoroutines): # Main stack was not initialized yet
if gch.stack == nil: gch.stack.next = addr(gch.stack)
# `setStackBottom()` gets called multiple times from main thread. gch.stack.prev = addr(gch.stack)
# Add it only once. gch.stack.bottom = theStackBottom
GC_addStack(theStackBottom) gch.stack.maxStackSize = 0
GC_setCurrentStack(theStackBottom) gch.activeStack = addr(gch.stack)
else: else:
var a = cast[ByteAddress](theStackBottom) # and not PageMask - PageSize*2
var b = cast[ByteAddress](gch.stack.bottom)
#c_fprintf(stdout, "old: %p new: %p;\n",gch.stackBottom,theStackBottom)
when stackIncreases:
gch.stack.bottom = cast[pointer](min(a, b))
else:
gch.stack.bottom = cast[pointer](max(a, b))
gch.stack.setPosition(theStackBottom)
else:
proc setStackBottom(theStackBottom: pointer) =
#c_fprintf(stdout, "stack bottom: %p;\n", theStackBottom)
# the first init must be the one that defines the stack bottom:
if gch.stackBottom == nil: gch.stackBottom = theStackBottom if gch.stackBottom == nil: gch.stackBottom = theStackBottom
else: else:
var a = cast[ByteAddress](theStackBottom) # and not PageMask - PageSize*2 var a = cast[ByteAddress](theStackBottom) # and not PageMask - PageSize*2
@ -201,7 +200,7 @@ when not defined(useNimRtl):
gch.stackBottom = cast[pointer](min(a, b)) gch.stackBottom = cast[pointer](min(a, b))
else: else:
gch.stackBottom = cast[pointer](max(a, b)) gch.stackBottom = cast[pointer](max(a, b))
{.pop.} {.pop.}
when defined(sparc): # For SPARC architecture. when defined(sparc): # For SPARC architecture.
when defined(nimCoroutines): when defined(nimCoroutines):
@ -272,12 +271,10 @@ else:
proc isOnStack(p: pointer): bool = proc isOnStack(p: pointer): bool =
var stackTop {.volatile.}: pointer var stackTop {.volatile.}: pointer
stackTop = addr(stackTop) stackTop = addr(stackTop)
for stack in items(gch.stack): var b = cast[ByteAddress](gch.activeStack.bottom)
var b = cast[ByteAddress](stack.starts) var a = cast[ByteAddress](stackTop)
var a = cast[ByteAddress](stack.starts) - stack.maxStackSize var x = cast[ByteAddress](p)
var x = cast[ByteAddress](p) result = a <=% x and x <=% b
if a <=% x and x <=% b:
return true
template forEachStackSlot(gch, gcMark: untyped) {.dirty.} = template forEachStackSlot(gch, gcMark: untyped) {.dirty.} =
# We use a jmp_buf buffer that is in the C stack. # We use a jmp_buf buffer that is in the C stack.
@ -285,35 +282,38 @@ else:
# that 'setjmp' will save registers in the C stack. # that 'setjmp' will save registers in the C stack.
type PStackSlice = ptr array[0..7, pointer] type PStackSlice = ptr array[0..7, pointer]
var registers {.noinit.}: C_JmpBuf var registers {.noinit.}: C_JmpBuf
discard c_setjmp(registers) # Update position of stack gc is executing in.
gch.stackActive.pos = addr(registers) gch.activeStack.setPosition(addr(registers))
for stack in items(gch.stack): if c_setjmp(registers) == 0'i32: # To fill the C stack with registers.
stack.maxStackSize = max(stack.maxStackSize, stackSize(stack.starts)) for stack in gch.stack.items():
var max = cast[ByteAddress](stack.starts) var max = cast[ByteAddress](stack.bottom)
var sp = cast[ByteAddress](stack.pos) var sp = cast[ByteAddress](addr(registers))
when defined(amd64): when defined(amd64):
if stack == gch.stackActive: if stack == gch.activeStack:
# words within the jmp_buf structure may not be properly aligned. # words within the jmp_buf structure may not be properly aligned.
let regEnd = sp +% sizeof(registers) let regEnd = sp +% sizeof(registers)
while sp <% regEnd: while sp <% regEnd:
gcMark(gch, cast[PPointer](sp)[]) gcMark(gch, cast[PPointer](sp)[])
gcMark(gch, cast[PPointer](sp +% sizeof(pointer) div 2)[]) gcMark(gch, cast[PPointer](sp +% sizeof(pointer) div 2)[])
sp = sp +% sizeof(pointer) sp = sp +% sizeof(pointer)
# loop unrolled: # Make sure sp is word-aligned
while sp <% max - 8*sizeof(pointer): sp = sp and not (sizeof(pointer) - 1)
gcMark(gch, cast[PStackSlice](sp)[0]) # loop unrolled:
gcMark(gch, cast[PStackSlice](sp)[1]) while sp <% max - 8*sizeof(pointer):
gcMark(gch, cast[PStackSlice](sp)[2]) gcMark(gch, cast[PStackSlice](sp)[0])
gcMark(gch, cast[PStackSlice](sp)[3]) gcMark(gch, cast[PStackSlice](sp)[1])
gcMark(gch, cast[PStackSlice](sp)[4]) gcMark(gch, cast[PStackSlice](sp)[2])
gcMark(gch, cast[PStackSlice](sp)[5]) gcMark(gch, cast[PStackSlice](sp)[3])
gcMark(gch, cast[PStackSlice](sp)[6]) gcMark(gch, cast[PStackSlice](sp)[4])
gcMark(gch, cast[PStackSlice](sp)[7]) gcMark(gch, cast[PStackSlice](sp)[5])
sp = sp +% sizeof(pointer)*8 gcMark(gch, cast[PStackSlice](sp)[6])
# last few entries: gcMark(gch, cast[PStackSlice](sp)[7])
while sp <=% max: sp = sp +% sizeof(pointer)*8
gcMark(gch, cast[PPointer](sp)[]) # last few entries:
sp = sp +% sizeof(pointer) while sp <=% max:
gcMark(gch, cast[PPointer](sp)[])
sp = sp +% sizeof(pointer)
else: else:
proc isOnStack(p: pointer): bool = proc isOnStack(p: pointer): bool =
var stackTop {.volatile.}: pointer var stackTop {.volatile.}: pointer