Nim/lib/system/mmdisp.nim
Reimer Behrends 06e3d3ab4d Disable interior pointer checking by default for the Boehm GC. (#10316)
The Boehm GC only needs interior pointer checking if pointers in global
variables or on the heap point to the interior of an object rather than
the beginning. If this does not happen, then checking for interior
pointers causes additional overhead, in particular because any objects
whose sizes are an exact multiple of two words gain another two words of
padding, wasting memory.

If checking of interior pointers is still desired, this can be achieved
by setting the environment variable GC_ALL_INTERIOR_POINTERS.

Pointers on the stack will always be treated as potential interior
pointers, as compiler optimizations may advance the only live reference
to a point past the beginning of an object.
2019-01-15 19:00:02 +01:00

527 lines
19 KiB
Nim

#
#
# Nim's Runtime Library
# (c) Copyright 2015 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
# Nim high-level memory manager: It supports Boehm's GC, Go's GC, no GC and the
# native Nim GC. The native Nim GC is the default.
#{.push checks:on, assertions:on.}
{.push checks:off.}
const
debugGC = false # we wish to debug the GC...
logGC = false
traceGC = false # extensive debugging
alwaysCycleGC = defined(smokeCycles)
alwaysGC = defined(fulldebug) # collect after every memory
# allocation (for debugging)
leakDetector = defined(leakDetector)
overwriteFree = defined(nimBurnFree) # overwrite memory with 0xFF before free
trackAllocationSource = leakDetector
cycleGC = true # (de)activate the cycle GC
reallyDealloc = true # for debugging purposes this can be set to false
reallyOsDealloc = true
coalescRight = true
coalescLeft = true
logAlloc = false
useCellIds = defined(corruption)
type
PPointer = ptr pointer
ByteArray = UncheckedArray[byte]
PByte = ptr ByteArray
PString = ptr string
# Page size of the system; in most cases 4096 bytes. For exotic OS or
# CPU this needs to be changed:
const
PageShift = when defined(cpu16): 8 else: 12 # \
# my tests showed no improvments for using larger page sizes.
PageSize = 1 shl PageShift
PageMask = PageSize-1
MemAlign = 8 # also minimal allocatable memory block
BitsPerPage = PageSize div MemAlign
UnitsPerPage = BitsPerPage div (sizeof(int)*8)
# how many ints do we need to describe a page:
# on 32 bit systems this is only 16 (!)
TrunkShift = 9
BitsPerTrunk = 1 shl TrunkShift # needs to be power of 2 and divisible by 64
TrunkMask = BitsPerTrunk - 1
IntsPerTrunk = BitsPerTrunk div (sizeof(int)*8)
IntShift = 5 + ord(sizeof(int) == 8) # 5 or 6, depending on int width
IntMask = 1 shl IntShift - 1
proc raiseOutOfMem() {.noinline.} =
if outOfMemHook != nil: outOfMemHook()
echo("out of memory")
quit(1)
when defined(boehmgc):
proc boehmGCinit {.importc: "GC_init", boehmGC.}
proc boehmGC_disable {.importc: "GC_disable", boehmGC.}
proc boehmGC_enable {.importc: "GC_enable", boehmGC.}
proc boehmGCincremental {.
importc: "GC_enable_incremental", boehmGC.}
proc boehmGCfullCollect {.importc: "GC_gcollect", boehmGC.}
proc boehmGC_set_all_interior_pointers(flag: cint) {.
importc: "GC_set_all_interior_pointers", boehmGC.}
proc boehmAlloc(size: int): pointer {.importc: "GC_malloc", boehmGC.}
proc boehmAllocAtomic(size: int): pointer {.
importc: "GC_malloc_atomic", boehmGC.}
proc boehmRealloc(p: pointer, size: int): pointer {.
importc: "GC_realloc", boehmGC.}
proc boehmDealloc(p: pointer) {.importc: "GC_free", boehmGC.}
when hasThreadSupport:
proc boehmGC_allow_register_threads {.
importc: "GC_allow_register_threads", boehmGC.}
proc boehmGetHeapSize: int {.importc: "GC_get_heap_size", boehmGC.}
## Return the number of bytes in the heap. Excludes collector private
## data structures. Includes empty blocks and fragmentation loss.
## Includes some pages that were allocated but never written.
proc boehmGetFreeBytes: int {.importc: "GC_get_free_bytes", boehmGC.}
## Return a lower bound on the number of free bytes in the heap.
proc boehmGetBytesSinceGC: int {.importc: "GC_get_bytes_since_gc", boehmGC.}
## Return the number of bytes allocated since the last collection.
proc boehmGetTotalBytes: int {.importc: "GC_get_total_bytes", boehmGC.}
## Return the total number of bytes allocated in this process.
## Never decreases.
proc allocAtomic(size: int): pointer =
result = boehmAllocAtomic(size)
zeroMem(result, size)
when not defined(useNimRtl):
proc alloc(size: Natural): pointer =
result = boehmAlloc(size)
if result == nil: raiseOutOfMem()
proc alloc0(size: Natural): pointer =
result = alloc(size)
proc realloc(p: pointer, newsize: Natural): pointer =
result = boehmRealloc(p, newsize)
if result == nil: raiseOutOfMem()
proc dealloc(p: pointer) = boehmDealloc(p)
proc allocShared(size: Natural): pointer =
result = boehmAlloc(size)
if result == nil: raiseOutOfMem()
proc allocShared0(size: Natural): pointer =
result = allocShared(size)
proc reallocShared(p: pointer, newsize: Natural): pointer =
result = boehmRealloc(p, newsize)
if result == nil: raiseOutOfMem()
proc deallocShared(p: pointer) = boehmDealloc(p)
when hasThreadSupport:
proc getFreeSharedMem(): int =
boehmGetFreeBytes()
proc getTotalSharedMem(): int =
boehmGetHeapSize()
proc getOccupiedSharedMem(): int =
getTotalSharedMem() - getFreeSharedMem()
#boehmGCincremental()
proc GC_disable() = boehmGC_disable()
proc GC_enable() = boehmGC_enable()
proc GC_fullCollect() = boehmGCfullCollect()
proc GC_setStrategy(strategy: GC_Strategy) = discard
proc GC_enableMarkAndSweep() = discard
proc GC_disableMarkAndSweep() = discard
proc GC_getStatistics(): string = return ""
proc getOccupiedMem(): int = return boehmGetHeapSize()-boehmGetFreeBytes()
proc getFreeMem(): int = return boehmGetFreeBytes()
proc getTotalMem(): int = return boehmGetHeapSize()
proc nimGC_setStackBottom(theStackBottom: pointer) = discard
proc initGC() =
boehmGC_set_all_interior_pointers(0)
boehmGCinit()
when hasThreadSupport:
boehmGC_allow_register_threads()
proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} =
if ntfNoRefs in typ.flags: result = allocAtomic(size)
else: result = alloc(size)
proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} =
result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize))
cast[PGenericSeq](result).len = len
cast[PGenericSeq](result).reserved = len
proc growObj(old: pointer, newsize: int): pointer =
result = realloc(old, newsize)
proc nimGCref(p: pointer) {.compilerproc, inline.} = discard
proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard
proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} =
dest[] = src
proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} =
dest[] = src
proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline,
deprecated: "old compiler compat".} = asgnRef(dest, src)
type
MemRegion = object
proc alloc(r: var MemRegion, size: int): pointer =
result = boehmAlloc(size)
if result == nil: raiseOutOfMem()
proc alloc0(r: var MemRegion, size: int): pointer =
result = alloc(size)
zeroMem(result, size)
proc dealloc(r: var MemRegion, p: pointer) = boehmDealloc(p)
proc deallocOsPages(r: var MemRegion) {.inline.} = discard
proc deallocOsPages() {.inline.} = discard
include "system/cellsets"
elif defined(gogc):
when defined(windows):
const goLib = "libgo.dll"
elif defined(macosx):
const goLib = "libgo.dylib"
else:
const goLib = "libgo.so"
proc initGC() = discard
proc GC_disable() = discard
proc GC_enable() = discard
proc go_gc() {.importc: "go_gc", dynlib: goLib.}
proc GC_fullCollect() = go_gc()
proc GC_setStrategy(strategy: GC_Strategy) = discard
proc GC_enableMarkAndSweep() = discard
proc GC_disableMarkAndSweep() = discard
const
goNumSizeClasses = 67
type
goMStats = object
alloc: uint64 # bytes allocated and still in use
total_alloc: uint64 # bytes allocated (even if freed)
sys: uint64 # bytes obtained from system
nlookup: uint64 # number of pointer lookups
nmalloc: uint64 # number of mallocs
nfree: uint64 # number of frees
heap_objects: uint64 # total number of allocated objects
pause_total_ns: uint64 # cumulative nanoseconds in GC stop-the-world pauses since the program started
numgc: uint32 # number of completed GC cycles
proc goMemStats(): goMStats {.importc: "go_mem_stats", dynlib: goLib.}
proc goMalloc(size: uint): pointer {.importc: "go_malloc", dynlib: goLib.}
proc goSetFinalizer(obj: pointer, f: pointer) {.importc: "set_finalizer", codegenDecl:"$1 $2$3 __asm__ (\"main.Set_finalizer\");\n$1 $2$3", dynlib: goLib.}
proc writebarrierptr(dest: PPointer, src: pointer) {.importc: "writebarrierptr", codegenDecl:"$1 $2$3 __asm__ (\"main.Atomic_store_pointer\");\n$1 $2$3", dynlib: goLib.}
proc GC_getStatistics(): string =
var mstats = goMemStats()
result = "[GC] total allocated memory: " & $(mstats.total_alloc) & "\n" &
"[GC] total memory obtained from system: " & $(mstats.sys) & "\n" &
"[GC] occupied memory: " & $(mstats.alloc) & "\n" &
"[GC] number of pointer lookups: " & $(mstats.nlookup) & "\n" &
"[GC] number of mallocs: " & $(mstats.nmalloc) & "\n" &
"[GC] number of frees: " & $(mstats.nfree) & "\n" &
"[GC] heap objects: " & $(mstats.heap_objects) & "\n" &
"[GC] number of completed GC cycles: " & $(mstats.numgc) & "\n" &
"[GC] total GC pause time [ms]: " & $(mstats.pause_total_ns div 1000_000)
proc getOccupiedMem(): int =
var mstats = goMemStats()
result = int(mstats.alloc)
proc getFreeMem(): int =
var mstats = goMemStats()
result = int(mstats.sys - mstats.alloc)
proc getTotalMem(): int =
var mstats = goMemStats()
result = int(mstats.sys)
proc nimGC_setStackBottom(theStackBottom: pointer) = discard
proc alloc(size: Natural): pointer =
result = goMalloc(size.uint)
proc alloc0(size: Natural): pointer =
result = goMalloc(size.uint)
proc realloc(p: pointer, newsize: Natural): pointer =
raise newException(Exception, "not implemented")
proc dealloc(p: pointer) =
discard
proc allocShared(size: Natural): pointer =
result = alloc(size)
proc allocShared0(size: Natural): pointer =
result = alloc0(size)
proc reallocShared(p: pointer, newsize: Natural): pointer =
result = realloc(p, newsize)
proc deallocShared(p: pointer) = dealloc(p)
when hasThreadSupport:
proc getFreeSharedMem(): int = discard
proc getTotalSharedMem(): int = discard
proc getOccupiedSharedMem(): int = discard
proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} =
writebarrierptr(addr(result), goMalloc(size.uint))
if typ.finalizer != nil:
goSetFinalizer(result, typ.finalizer)
proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} =
writebarrierptr(addr(result), newObj(typ, size))
proc newObjNoInit(typ: PNimType, size: int): pointer =
writebarrierptr(addr(result), newObj(typ, size))
proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} =
writebarrierptr(addr(result), newObj(typ, len * typ.base.size + GenericSeqSize))
cast[PGenericSeq](result).len = len
cast[PGenericSeq](result).reserved = len
cast[PGenericSeq](result).elemSize = typ.base.size
proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} =
writebarrierptr(addr(result), newSeq(typ, len))
proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} =
result = newObj(typ, cap * typ.base.size + GenericSeqSize)
cast[PGenericSeq](result).len = 0
cast[PGenericSeq](result).reserved = cap
cast[PGenericSeq](result).elemSize = typ.base.size
proc typedMemMove(dest: pointer, src: pointer, size: uint) {.importc: "typedmemmove", dynlib: goLib.}
proc growObj(old: pointer, newsize: int): pointer =
# the Go GC doesn't have a realloc
var metadataOld = cast[PGenericSeq](old)
if metadataOld.elemSize == 0:
metadataOld.elemSize = 1
let oldsize = cast[PGenericSeq](old).len * cast[PGenericSeq](old).elemSize + GenericSeqSize
writebarrierptr(addr(result), goMalloc(newsize.uint))
typedMemMove(result, old, oldsize.uint)
proc nimGCref(p: pointer) {.compilerproc, inline.} = discard
proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard
proc nimGCunrefNoCycle(p: pointer) {.compilerProc, inline.} = discard
proc nimGCunrefRC1(p: pointer) {.compilerProc, inline.} = discard
proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = discard
proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} =
writebarrierptr(dest, src)
proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} =
writebarrierptr(dest, src)
proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline,
deprecated: "old compiler compat".} = asgnRef(dest, src)
type
MemRegion = object
proc alloc(r: var MemRegion, size: int): pointer =
result = alloc(size)
proc alloc0(r: var MemRegion, size: int): pointer =
result = alloc0(size)
proc dealloc(r: var MemRegion, p: pointer) = dealloc(p)
proc deallocOsPages(r: var MemRegion) {.inline.} = discard
proc deallocOsPages() {.inline.} = discard
elif defined(nogc) and defined(useMalloc):
when not defined(useNimRtl):
proc alloc(size: Natural): pointer =
var x = c_malloc(size + sizeof(size))
if x == nil: raiseOutOfMem()
cast[ptr int](x)[] = size
result = cast[pointer](cast[int](x) + sizeof(size))
proc alloc0(size: Natural): pointer =
result = alloc(size)
zeroMem(result, size)
proc realloc(p: pointer, newsize: Natural): pointer =
var x = cast[pointer](cast[int](p) - sizeof(newsize))
let oldsize = cast[ptr int](x)[]
x = c_realloc(x, newsize + sizeof(newsize))
if x == nil: raiseOutOfMem()
cast[ptr int](x)[] = newsize
result = cast[pointer](cast[int](x) + sizeof(newsize))
if newsize > oldsize:
zeroMem(cast[pointer](cast[int](result) + oldsize), newsize - oldsize)
proc dealloc(p: pointer) = c_free(cast[pointer](cast[int](p) - sizeof(int)))
proc allocShared(size: Natural): pointer =
result = c_malloc(size)
if result == nil: raiseOutOfMem()
proc allocShared0(size: Natural): pointer =
result = alloc(size)
zeroMem(result, size)
proc reallocShared(p: pointer, newsize: Natural): pointer =
result = c_realloc(p, newsize)
if result == nil: raiseOutOfMem()
proc deallocShared(p: pointer) = c_free(p)
proc GC_disable() = discard
proc GC_enable() = discard
proc GC_fullCollect() = discard
proc GC_setStrategy(strategy: GC_Strategy) = discard
proc GC_enableMarkAndSweep() = discard
proc GC_disableMarkAndSweep() = discard
proc GC_getStatistics(): string = return ""
proc getOccupiedMem(): int = discard
proc getFreeMem(): int = discard
proc getTotalMem(): int = discard
proc nimGC_setStackBottom(theStackBottom: pointer) = discard
proc initGC() = discard
proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} =
result = alloc0(size)
proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} =
result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize))
cast[PGenericSeq](result).len = len
cast[PGenericSeq](result).reserved = len
proc newObjNoInit(typ: PNimType, size: int): pointer =
result = alloc(size)
proc growObj(old: pointer, newsize: int): pointer =
result = realloc(old, newsize)
proc nimGCref(p: pointer) {.compilerproc, inline.} = discard
proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard
proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} =
dest[] = src
proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} =
dest[] = src
proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline,
deprecated: "old compiler compat".} = asgnRef(dest, src)
type
MemRegion = object
proc alloc(r: var MemRegion, size: int): pointer =
result = alloc(size)
proc alloc0(r: var MemRegion, size: int): pointer =
result = alloc0(size)
proc dealloc(r: var MemRegion, p: pointer) = dealloc(p)
proc deallocOsPages(r: var MemRegion) {.inline.} = discard
proc deallocOsPages() {.inline.} = discard
elif defined(nogc):
# Even though we don't want the GC, we cannot simply use C's memory manager
# because Nim's runtime wants ``realloc`` to zero out the additional
# space which C's ``realloc`` does not. And we cannot get the old size of an
# object, because C does not support this operation... Even though every
# possible implementation has to have a way to determine the object's size.
# C just sucks.
when appType == "lib":
{.warning: "nogc in a library context may not work".}
include "system/alloc"
proc initGC() = discard
proc GC_disable() = discard
proc GC_enable() = discard
proc GC_fullCollect() = discard
proc GC_setStrategy(strategy: GC_Strategy) = discard
proc GC_enableMarkAndSweep() = discard
proc GC_disableMarkAndSweep() = discard
proc GC_getStatistics(): string = return ""
proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} =
result = alloc0(size)
proc newObjNoInit(typ: PNimType, size: int): pointer =
result = alloc(size)
proc newSeq(typ: PNimType, len: int): pointer {.compilerproc.} =
result = newObj(typ, addInt(mulInt(len, typ.base.size), GenericSeqSize))
cast[PGenericSeq](result).len = len
cast[PGenericSeq](result).reserved = len
proc growObj(old: pointer, newsize: int): pointer =
result = realloc(old, newsize)
proc nimGC_setStackBottom(theStackBottom: pointer) = discard
proc nimGCref(p: pointer) {.compilerproc, inline.} = discard
proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard
proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} =
dest[] = src
proc asgnRef(dest: PPointer, src: pointer) {.compilerproc, inline.} =
dest[] = src
proc asgnRefNoCycle(dest: PPointer, src: pointer) {.compilerproc, inline,
deprecated: "old compiler compat".} = asgnRef(dest, src)
var allocator {.rtlThreadVar.}: MemRegion
instantiateForRegion(allocator)
include "system/cellsets"
else:
when not defined(gcRegions):
include "system/alloc"
include "system/cellsets"
when not leakDetector and not useCellIds:
sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell")
when compileOption("gc", "v2"):
include "system/gc2"
elif defined(gcRegions):
# XXX due to bootstrapping reasons, we cannot use compileOption("gc", "stack") here
include "system/gc_regions"
elif defined(gcMarkAndSweep) or defined(gcDestructors):
# XXX use 'compileOption' here
include "system/gc_ms"
else:
include "system/gc"
when not declared(nimNewSeqOfCap) and not defined(gcDestructors):
proc nimNewSeqOfCap(typ: PNimType, cap: int): pointer {.compilerproc.} =
when defined(gcRegions):
let s = mulInt(cap, typ.base.size) # newStr already adds GenericSeqSize
result = newStr(typ, s, ntfNoRefs notin typ.base.flags)
else:
let s = addInt(mulInt(cap, typ.base.size), GenericSeqSize)
when declared(newObjNoInit):
result = if ntfNoRefs in typ.base.flags: newObjNoInit(typ, s) else: newObj(typ, s)
else:
result = newObj(typ, s)
cast[PGenericSeq](result).len = 0
cast[PGenericSeq](result).reserved = cap
{.pop.}
when not declared(ForeignCell):
type ForeignCell* = object
data*: pointer
proc protect*(x: pointer): ForeignCell = ForeignCell(data: x)
proc dispose*(x: ForeignCell) = discard
proc isNotForeign*(x: ForeignCell): bool = false