jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

TlSF Alloctor: use less memory for --gc:arc (#13280)

Browse source
This commit is contained in:
Andreas Rumpf 2020-01-28 17:09:48 +01:00 • committed by GitHub
commit be795bbf1a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 103 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

179
lib/system/alloc.nim
View file

@ -48,9 +48,12 @@ type
AlignType = BiggestFloat AlignType = BiggestFloat
FreeCell {.final, pure.} = object FreeCell {.final, pure.} = object
next: ptr FreeCell # next free cell in chunk (overlaid with refcount) next: ptr FreeCell # next free cell in chunk (overlaid with refcount)
zeroField: int # 0 means cell is not used (overlaid with typ field) when not defined(gcDestructors):
# 1 means cell is manually managed pointer zeroField: int # 0 means cell is not used (overlaid with typ field)
# otherwise a PNimType is stored in there # 1 means cell is manually managed pointer
# otherwise a PNimType is stored in there
else:
alignment: int
PChunk = ptr BaseChunk PChunk = ptr BaseChunk
PBigChunk = ptr BigChunk PBigChunk = ptr BigChunk
@ -396,8 +399,9 @@ iterator allObjects(m: var MemRegion): pointer {.inline.} =
proc iterToProc*(iter: typed, envType: typedesc; procName: untyped) {. proc iterToProc*(iter: typed, envType: typedesc; procName: untyped) {.
magic: "Plugin", compileTime.} magic: "Plugin", compileTime.}
proc isCell(p: pointer): bool {.inline.} = when not defined(gcDestructors):
result = cast[ptr FreeCell](p).zeroField >% 1 proc isCell(p: pointer): bool {.inline.} =
result = cast[ptr FreeCell](p).zeroField >% 1
# ------------- chunk management ---------------------------------------------- # ------------- chunk management ----------------------------------------------
proc pageIndex(c: PChunk): int {.inline.} = proc pageIndex(c: PChunk): int {.inline.} =
@ -630,7 +634,8 @@ proc getSmallChunk(a: var MemRegion): PSmallChunk =
result = cast[PSmallChunk](res) result = cast[PSmallChunk](res)
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
proc isAllocatedPtr(a: MemRegion, p: pointer): bool {.benign.} when not defined(gcDestructors):
proc isAllocatedPtr(a: MemRegion, p: pointer): bool {.benign.}
when true: when true:
template allocInv(a: MemRegion): bool = true template allocInv(a: MemRegion): bool = true
@ -773,7 +778,8 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
inc(c.acc, size) inc(c.acc, size)
else: else:
result = c.freeList result = c.freeList
sysAssert(c.freeList.zeroField == 0, "rawAlloc 8") when not defined(gcDestructors):
sysAssert(c.freeList.zeroField == 0, "rawAlloc 8")
c.freeList = c.freeList.next c.freeList = c.freeList.next
dec(c.free, size) dec(c.free, size)
sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9") sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9")
@ -826,9 +832,10 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %%
s == 0, "rawDealloc 3") s == 0, "rawDealloc 3")
var f = cast[ptr FreeCell](p) var f = cast[ptr FreeCell](p)
#echo("setting to nil: ", $cast[ByteAddress](addr(f.zeroField))) when not defined(gcDestructors):
sysAssert(f.zeroField != 0, "rawDealloc 1") #echo("setting to nil: ", $cast[ByteAddress](addr(f.zeroField)))
f.zeroField = 0 sysAssert(f.zeroField != 0, "rawDealloc 1")
f.zeroField = 0
f.next = c.freeList f.next = c.freeList
c.freeList = f c.freeList = f
when overwriteFree: when overwriteFree:
@ -863,88 +870,102 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
sysAssert(allocInv(a), "rawDealloc: end") sysAssert(allocInv(a), "rawDealloc: end")
when logAlloc: cprintf("dealloc(pointer_%p)\n", p) when logAlloc: cprintf("dealloc(pointer_%p)\n", p)
proc isAllocatedPtr(a: MemRegion, p: pointer): bool = when not defined(gcDestructors):
if isAccessible(a, p): proc isAllocatedPtr(a: MemRegion, p: pointer): bool =
var c = pageAddr(p) if isAccessible(a, p):
if not chunkUnused(c): var c = pageAddr(p)
if isSmallChunk(c): if not chunkUnused(c):
var c = cast[PSmallChunk](c) if isSmallChunk(c):
var offset = (cast[ByteAddress](p) and (PageSize-1)) -% var c = cast[PSmallChunk](c)
smallChunkOverhead() var offset = (cast[ByteAddress](p) and (PageSize-1)) -%
result = (c.acc >% offset) and (offset %% c.size == 0) and smallChunkOverhead()
(cast[ptr FreeCell](p).zeroField >% 1) result = (c.acc >% offset) and (offset %% c.size == 0) and
else: (cast[ptr FreeCell](p).zeroField >% 1)
var c = cast[PBigChunk](c) else:
result = p == addr(c.data) and cast[ptr FreeCell](p).zeroField >% 1 var c = cast[PBigChunk](c)
result = p == addr(c.data) and cast[ptr FreeCell](p).zeroField >% 1
proc prepareForInteriorPointerChecking(a: var MemRegion) {.inline.} = proc prepareForInteriorPointerChecking(a: var MemRegion) {.inline.} =
a.minLargeObj = lowGauge(a.root) a.minLargeObj = lowGauge(a.root)
a.maxLargeObj = highGauge(a.root) a.maxLargeObj = highGauge(a.root)
proc interiorAllocatedPtr(a: MemRegion, p: pointer): pointer = proc interiorAllocatedPtr(a: MemRegion, p: pointer): pointer =
if isAccessible(a, p): if isAccessible(a, p):
var c = pageAddr(p) var c = pageAddr(p)
if not chunkUnused(c): if not chunkUnused(c):
if isSmallChunk(c): if isSmallChunk(c):
var c = cast[PSmallChunk](c) var c = cast[PSmallChunk](c)
var offset = (cast[ByteAddress](p) and (PageSize-1)) -% var offset = (cast[ByteAddress](p) and (PageSize-1)) -%
smallChunkOverhead() smallChunkOverhead()
if c.acc >% offset: if c.acc >% offset:
sysAssert(cast[ByteAddress](addr(c.data)) +% offset == sysAssert(cast[ByteAddress](addr(c.data)) +% offset ==
cast[ByteAddress](p), "offset is not what you think it is") cast[ByteAddress](p), "offset is not what you think it is")
var d = cast[ptr FreeCell](cast[ByteAddress](addr(c.data)) +% var d = cast[ptr FreeCell](cast[ByteAddress](addr(c.data)) +%
offset -% (offset %% c.size)) offset -% (offset %% c.size))
if d.zeroField >% 1: if d.zeroField >% 1:
result = d
sysAssert isAllocatedPtr(a, result), " result wrong pointer!"
else:
var c = cast[PBigChunk](c)
var d = addr(c.data)
if p >= d and cast[ptr FreeCell](d).zeroField >% 1:
result = d result = d
sysAssert isAllocatedPtr(a, result), " result wrong pointer!" sysAssert isAllocatedPtr(a, result), " result wrong pointer!"
else: else:
var c = cast[PBigChunk](c) var q = cast[int](p)
var d = addr(c.data) if q >=% a.minLargeObj and q <=% a.maxLargeObj:
if p >= d and cast[ptr FreeCell](d).zeroField >% 1: # this check is highly effective! Test fails for 99,96% of all checks on
result = d # an x86-64.
sysAssert isAllocatedPtr(a, result), " result wrong pointer!" var avlNode = inRange(a.root, q)
else: if avlNode != nil:
var q = cast[int](p) var k = cast[pointer](avlNode.key)
if q >=% a.minLargeObj and q <=% a.maxLargeObj: var c = cast[PBigChunk](pageAddr(k))
# this check is highly effective! Test fails for 99,96% of all checks on sysAssert(addr(c.data) == k, " k is not the same as addr(c.data)!")
# an x86-64. if cast[ptr FreeCell](k).zeroField >% 1:
var avlNode = inRange(a.root, q) result = k
if avlNode != nil: sysAssert isAllocatedPtr(a, result), " result wrong pointer!"
var k = cast[pointer](avlNode.key)
var c = cast[PBigChunk](pageAddr(k))
sysAssert(addr(c.data) == k, " k is not the same as addr(c.data)!")
if cast[ptr FreeCell](k).zeroField >% 1:
result = k
sysAssert isAllocatedPtr(a, result), " result wrong pointer!"
proc ptrSize(p: pointer): int = proc ptrSize(p: pointer): int =
var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) when not defined(gcDestructors):
var c = pageAddr(p) var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell))
sysAssert(not chunkUnused(c), "ptrSize") var c = pageAddr(p)
result = c.size -% sizeof(FreeCell) sysAssert(not chunkUnused(c), "ptrSize")
if not isSmallChunk(c): result = c.size -% sizeof(FreeCell)
dec result, bigChunkOverhead() if not isSmallChunk(c):
dec result, bigChunkOverhead()
else:
var c = pageAddr(p)
sysAssert(not chunkUnused(c), "ptrSize")
result = c.size
if not isSmallChunk(c):
dec result, bigChunkOverhead()
proc alloc(allocator: var MemRegion, size: Natural): pointer {.gcsafe.} = proc alloc(allocator: var MemRegion, size: Natural): pointer {.gcsafe.} =
result = rawAlloc(allocator, size+sizeof(FreeCell)) when not defined(gcDestructors):
cast[ptr FreeCell](result).zeroField = 1 # mark it as used result = rawAlloc(allocator, size+sizeof(FreeCell))
sysAssert(not isAllocatedPtr(allocator, result), "alloc") cast[ptr FreeCell](result).zeroField = 1 # mark it as used
result = cast[pointer](cast[ByteAddress](result) +% sizeof(FreeCell)) sysAssert(not isAllocatedPtr(allocator, result), "alloc")
track("alloc", result, size) result = cast[pointer](cast[ByteAddress](result) +% sizeof(FreeCell))
track("alloc", result, size)
else:
result = rawAlloc(allocator, size)
proc alloc0(allocator: var MemRegion, size: Natural): pointer = proc alloc0(allocator: var MemRegion, size: Natural): pointer =
result = alloc(allocator, size) result = alloc(allocator, size)
zeroMem(result, size) zeroMem(result, size)
proc dealloc(allocator: var MemRegion, p: pointer) = proc dealloc(allocator: var MemRegion, p: pointer) =
sysAssert(p != nil, "dealloc: p is nil") when not defined(gcDestructors):
var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) sysAssert(p != nil, "dealloc: p is nil")
sysAssert(x != nil, "dealloc: x is nil") var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell))
sysAssert(isAccessible(allocator, x), "is not accessible") sysAssert(x != nil, "dealloc: x is nil")
sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc: object header corrupted") sysAssert(isAccessible(allocator, x), "is not accessible")
rawDealloc(allocator, x) sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc: object header corrupted")
sysAssert(not isAllocatedPtr(allocator, x), "dealloc: object still accessible") rawDealloc(allocator, x)
track("dealloc", p, 0) sysAssert(not isAllocatedPtr(allocator, x), "dealloc: object still accessible")
track("dealloc", p, 0)
else:
rawDealloc(allocator, p)
proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer = proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer =
if newsize > 0: if newsize > 0:
@ -958,7 +979,7 @@ proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer =
proc realloc0(allocator: var MemRegion, p: pointer, oldsize, newsize: Natural): pointer = proc realloc0(allocator: var MemRegion, p: pointer, oldsize, newsize: Natural): pointer =
result = realloc(allocator, p, newsize) result = realloc(allocator, p, newsize)
if newsize > oldsize: if newsize > oldsize:
zeroMem(cast[pointer](cast[int](result) + oldsize), newsize - oldsize) zeroMem(cast[pointer](cast[uint](result) + uint(oldsize)), newsize - oldsize)
proc deallocOsPages(a: var MemRegion) = proc deallocOsPages(a: var MemRegion) =
# we free every 'ordinarily' allocated page by iterating over the page bits: # we free every 'ordinarily' allocated page by iterating over the page bits:

2
lib/system/strs_v2.nim
View file

@ -122,7 +122,7 @@ proc nimAsgnStrV2(a: var NimStringV2, b: NimStringV2) {.compilerRtl.} =
a.len = b.len a.len = b.len
a.p = b.p a.p = b.p
else: else:
if isLiteral(a) or (a.p.cap and not strlitFlag) < b.len: if isLiteral(a) or (a.p.cap and not strlitFlag) < b.len:
# we have to allocate the 'cap' here, consider # we have to allocate the 'cap' here, consider
# 'let y = newStringOfCap(); var x = y' # 'let y = newStringOfCap(); var x = y'
# on the other hand... These get turned into moves now. # on the other hand... These get turned into moves now.

2
tests/gc/gcbench.nim
View file

@ -167,6 +167,8 @@ proc main() =
var elapsed = epochTime() - t var elapsed = epochTime() - t
printDiagnostics() printDiagnostics()
echo("Completed in " & $elapsed & "s. Success!") echo("Completed in " & $elapsed & "s. Success!")
when declared(getMaxMem):
echo "Max memory ", formatSize getMaxMem()
when defined(GC_setMaxPause): when defined(GC_setMaxPause):
GC_setMaxPause 2_000 GC_setMaxPause 2_000