GC stack scanning cares about interior pointers

lib/system/alloc.nim

@@ -151,14 +151,22 @@ type
     size: int                # remaining size
     acc: int                 # accumulator
     next: PLLChunk           # next low-level chunk; only needed for dealloc
+
+  PAvlNode = ptr TAvlNode
+  TAvlNode {.pure, final.} = object 
+    link: array[0..1, PAvlNode] # Left (0) and right (1) links 
+    key, upperBound: int
+    balance: int            # Balance factor 
     
   TMemRegion {.final, pure.} = object
+    minLargeObj, maxLargeObj: int
+    freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk]
     llmem: PLLChunk
     currMem, maxMem, freeMem: int # memory sizes (allocated from OS)
     lastSize: int # needed for the case that OS gives us pages linearly 
-    freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk]
     freeChunksList: PBigChunk # XXX make this a datastructure with O(1) access
     chunkStarts: TIntSet
+    root, freeAvlNodes: PAvlNode
   
 proc incCurrMem(a: var TMemRegion, bytes: int) {.inline.} = 
   inc(a.currMem, bytes)
@@ -191,7 +199,25 @@ proc llAlloc(a: var TMemRegion, size: int): pointer =
   dec(a.llmem.size, size)
   inc(a.llmem.acc, size)
   zeroMem(result, size)
-  
+
+proc allocAvlNode(a: var TMemRegion, key, upperBound: int): PAvlNode =
+  if a.freeAvlNodes != nil:
+    result = a.freeAvlNodes
+    a.freeAvlNodes = a.freeAvlNodes.link[0]
+    result.link[0] = nil
+    result.link[1] = nil
+    result.balance = 0
+  else:
+    result = cast[PAvlNode](llAlloc(a, sizeof(TAvlNode)))
+  result.key = key
+  result.upperBound = upperBound
+
+proc deallocAvlNode(a: var TMemRegion, n: PAvlNode) {.inline.} =
+  n.link[0] = a.freeAvlNodes
+  a.freeAvlNodes = n
+
+include "system/avltree"
+
 proc llDeallocAll(a: var TMemRegion) =
   var it = a.llmem
   while it != nil:
@@ -497,6 +523,7 @@ proc rawAlloc(a: var TMemRegion, requestedSize: int): pointer =
     sysAssert c.size == size, "rawAlloc 12"
     result = addr(c.data)
     sysAssert((cast[TAddress](result) and (MemAlign-1)) == 0, "rawAlloc 13")
+    add(a, cast[TAddress](result), cast[TAddress](result)+%size)
   sysAssert(isAccessible(a, result), "rawAlloc 14")
 
 proc rawAlloc0(a: var TMemRegion, requestedSize: int): pointer =
@@ -534,7 +561,9 @@ proc rawDealloc(a: var TMemRegion, p: pointer) =
     # set to 0xff to check for usage after free bugs:
     when overwriteFree: c_memset(p, -1'i32, c.size -% bigChunkOverhead())
     # free big chunk
-    freeBigChunk(a, cast[PBigChunk](c))
+    var c = cast[PBigChunk](c)
+    del(a, cast[int](addr(c.data)))
+    freeBigChunk(a, c)
 
 proc isAllocatedPtr(a: TMemRegion, p: pointer): bool = 
   if isAccessible(a, p):
@@ -550,6 +579,10 @@ proc isAllocatedPtr(a: TMemRegion, p: pointer): bool =
         var c = cast[PBigChunk](c)
         result = p == addr(c.data) and cast[ptr TFreeCell](p).zeroField >% 1
 
+proc prepareForInteriorPointerChecking(a: var TMemRegion) {.inline.} =
+  a.minLargeObj = lowGauge(a.root)
+  a.maxLargeObj = highGauge(a.root)
+
 proc interiorAllocatedPtr(a: TMemRegion, p: pointer): pointer =
   if isAccessible(a, p):
     var c = pageAddr(p)
@@ -566,6 +599,18 @@ proc interiorAllocatedPtr(a: TMemRegion, p: pointer): pointer =
         var c = cast[PBigChunk](c)
         var d = addr(c.data)
         if p >= d and cast[ptr TFreeCell](d).zeroField >% 1: result = d
+  else:
+    var q = cast[int](p)
+    if q >=% a.minLargeObj and q <=% a.maxLargeObj:
+      # this check is highly effective! Test fails for 99,96% of all checks on
+      # an x86-64.
+      var avlNode = inRange(a.root, q)
+      if avlNode != nil:
+        var k = cast[pointer](avlNode.key)
+        var c = cast[PBigChunk](pageAddr(k))
+        sysAssert(addr(c.data) == k, " k is not the same as addr(c.data)!")
+        if cast[ptr TFreeCell](k).zeroField >% 1:
+          result = k
 
 proc ptrSize(p: pointer): int =
   var x = cast[pointer](cast[TAddress](p) -% sizeof(TFreeCell))

lib/system/avltree.nim

@@ -0,0 +1,251 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2011 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+
+## AVL balanced tree based on a C implementation by Julienne Walker
+
+const 
+  HeightLimit = 128        # Tallest allowable tree
+  
+# Two way single rotation 
+
+template singleRot(root, dir: expr): stmt =
+  block:
+    var save = root.link[1-dir]
+    root.link[1-dir] = save.link[dir]
+    save.link[dir] = root
+    root = save
+
+# Two way double rotation 
+
+template doubleRot(root, dir: expr): stmt =
+  block:
+    var save = root.link[1-dir].link[dir]
+    root.link[1-dir].link[dir] = save.link[1-dir]
+    save.link[1-dir] = root.link[1-dir]
+    root.link[1-dir] = save
+    save = root.link[1-dir]
+    root.link[1-dir] = save.link[dir]
+    save.link[dir] = root
+    root = save
+
+# Adjust balance before double rotation 
+
+template adjustBalance(root, dir, bal: expr): stmt = 
+  block:
+    var n = root.link[dir]
+    var nn = n.link[1-dir]
+    if nn.balance == 0:
+      root.balance = 0
+      n.balance = 0
+    elif nn.balance == bal:
+      root.balance = -bal
+      n.balance = 0
+    else:
+      # nn->balance == -bal 
+      root.balance = 0
+      n.balance = bal
+    nn.balance = 0
+
+# Rebalance after insertion 
+
+template insertBalance(root, dir: expr): stmt = 
+  block:
+    var n = root.link[dir]
+    var bal = if dir == 0: -1 else: +1
+    if n.balance == bal:
+      root.balance = 0
+      n.balance = 0
+      singleRot(root, 1-dir)
+    else: 
+      # n->balance == -bal 
+      adjustBalance(root, dir, bal)
+      doubleRot(root, 1-dir)
+
+# Rebalance after deletion 
+
+template removeBalance(root, dir, done: expr): stmt =
+  block:
+    var n = root.link[1-dir]
+    var bal = if dir == 0: -1 else: + 1
+    if n.balance == - bal: 
+      root.balance = 0
+      n.balance = 0
+      singleRot(root, dir)
+    elif n.balance == bal: 
+      adjustBalance(root, 1-dir, - bal)
+      doubleRot(root, dir)
+    else: 
+      # n->balance == 0 
+      root.balance = -bal
+      n.balance = bal
+      singleRot(root, dir)
+      done = true
+
+proc find(root: PAvlNode, key: int): PAvlNode = 
+  var it = root
+  while it != nil:
+    if it.key == key: return it
+    it = it.link[ord(it.key < key)]
+
+proc inRange(root: PAvlNode, key: int): PAvlNode =
+  var it = root
+  while it != nil:
+    if it.key <= key and key <= it.upperBound: return it
+    it = it.link[ord(it.key < key)]
+
+proc contains(root: PAvlNode, key: int): bool {.inline.} =
+  result = find(root, key) != nil
+
+proc maxheight(n: PAvlNode): int =
+  if n != nil:
+    result = max(maxheight(n.link[0]), maxheight(n.link[1])) + 1
+
+proc minheight(n: PAvlNode): int =
+  if n != nil:
+    result = min(minheight(n.link[0]), minheight(n.link[1])) + 1
+
+proc lowGauge(n: PAvlNode): int =
+  var it = n
+  while it != nil:
+    result = it.key
+    it = it.link[0]
+  
+proc highGauge(n: PAvlNode): int =
+  result = -1
+  var it = n
+  while it != nil:
+    result = it.upperBound
+    it = it.link[1]
+
+proc add(a: var TMemRegion, key, upperBound: int) = 
+  # Empty tree case
+  if a.root == nil:
+    a.root = allocAvlNode(a, key, upperBound)
+  else:
+    var head: TAvlNode # Temporary tree root
+    var s, t, p, q: PAvlNode
+    # Iterator and save pointer 
+    var dir: int
+    # Set up false root to ease maintenance:
+    t = addr(head)
+    t.link[1] = a.root
+    # Search down the tree, saving rebalance points
+    s = t.link[1]
+    p = s
+    while true:
+      dir = ord(p.key < key)
+      q = p.link[dir]
+      if q == nil: break 
+      if q.balance != 0:
+        t = p
+        s = q
+      p = q
+    q = allocAvlNode(a, key, upperBound)
+    p.link[dir] = q
+    # Update balance factors
+    p = s
+    while p != q:
+      dir = ord(p.key < key)
+      if dir == 0: dec p.balance
+      else: inc p.balance
+      p = p.link[dir]
+    q = s
+    # Save rebalance point for parent fix 
+    # Rebalance if necessary 
+    if abs(s.balance) > 1:
+      dir = ord(s.key < key)
+      insertBalance(s, dir)
+    # Fix parent
+    if q == head.link[1]: a.root = s
+    else: t.link[ord(q == t.link[1])] = s
+
+proc del(a: var TMemRegion, key: int) =
+  if a.root == nil: return
+  var
+    upd: array[0..HeightLimit-1, int]
+    up: array[0..HeightLimit-1, PAvlNode]
+  var top = 0
+  var it = a.root
+  # Search down tree and save path
+  while true:
+    if it == nil: return
+    elif it.key == key: break 
+    # Push direction and node onto stack 
+    upd[top] = ord(it.key < key)
+    up[top] = it
+    it = it.link[upd[top]]
+    inc top
+  # Remove the node 
+  if it.link[0] == nil or it.link[1] == nil: 
+    # Which child is not null? 
+    var dir = ord(it.link[0] == nil)
+    # Fix parent 
+    if top != 0: up[top - 1].link[upd[top - 1]] = it.link[dir]
+    else: a.root = it.link[dir]
+    deallocAvlNode(a, it)
+  else:
+    # Find the inorder successor 
+    var heir = it.link[1]
+    # Save this path too 
+    upd[top] = 1
+    up[top] = it
+    inc top
+    while heir.link[0] != nil: 
+      upd[top] = 0
+      up[top] = heir
+      inc top
+      heir = heir.link[0]
+    swap(it.key, heir.key)
+    swap(it.upperBound, heir.upperBound)
+    
+    # Unlink successor and fix parent 
+    up[top - 1].link[ord(up[top - 1] == it)] = heir.link[1]
+    deallocAvlNode(a, heir)
+  # Walk back up the search path
+  dec top
+  var done = false
+  while top >= 0 and not done:
+    # Update balance factors
+    if upd[top] != 0: dec up[top].balance
+    else: inc up[top].balance
+    # Terminate or rebalance as necessary 
+    if abs(up[top].balance) == 1: 
+      break
+    elif abs(up[top].balance) > 1: 
+      removeBalance(up[top], upd[top], done)
+      # Fix parent
+      if top != 0: up[top-1].link[upd[top-1]] = up[top]
+      else: a.root = up[0]
+    dec top
+
+when isMainModule:
+  import math
+  var
+    r: PAvlNode
+    s: seq[int]
+  const N = 1000_000
+  newSeq s, N
+
+  for i in 0..N-1:
+    var key = i #random(10_000)
+    s[i] = key
+    r.add(key, 12_000_000)
+  for i in 0..N-1:
+    var key = s[i]
+    doAssert inRange(r, key+1000) != nil
+    doAssert key in r
+  echo "Min-Height: ", minheight(r), " max-height: ", maxheight(r)
+  for i in 0..N-1:
+    var key = s[i]
+    del r, key
+    doAssert key notin r
+    
+  doAssert r == nil
+

lib/system/gc.nim

@@ -561,18 +561,17 @@ proc gcMark(gch: var TGcHeap, p: pointer) {.inline.} =
   var c = cast[TAddress](cell)
   if c >% PageSize and (c and (MemAlign-1)) == 0:
     # fast check: does it look like a cell?
-    if isAllocatedPtr(gch.region, cell): 
+    var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell))
+    if objStart != nil:
       # mark the cell:
-      cell.refcount = cell.refcount +% rcIncrement
-      add(gch.decStack, cell)
+      objStart.refcount = objStart.refcount +% rcIncrement
+      add(gch.decStack, objStart)
     when false:
-      # Care for string->cstring and seq->openArray conversions.
-      # Now the codegen deals with it, it generated ``nimKeepAlive`` calls.
-      var b = cast[PCell](c -% sizeof(TGenericSeq))
-      if isAllocatedPtr(gch.region, b):
+      if isAllocatedPtr(gch.region, cell):
+        sysAssert false, "allocated pointer but not interior?"
         # mark the cell:
-        b.refcount = b.refcount +% rcIncrement
-        add(gch.decStack, b)
+        cell.refcount = cell.refcount +% rcIncrement
+        add(gch.decStack, cell)
 
 proc nimKeepAlive(p: PGenericSeq) {.compilerRtl, noinline.} =
   var c = usrToCell(p)
@@ -778,6 +777,7 @@ proc collectCT(gch: var TGcHeap) =
       gch.recGcLock == 0:
     gch.stat.maxStackSize = max(gch.stat.maxStackSize, stackSize())
     sysAssert(gch.decStack.len == 0, "collectCT")
+    prepareForInteriorPointerChecking(gch.region)
     markStackAndRegisters(gch)
     markThreadStacks(gch)
     gch.stat.maxStackCells = max(gch.stat.maxStackCells, gch.decStack.len)