C codegen: generate nimKeepAlive calls at strategic places to keep the C compiler from optimizing away all stack roots

compiler/ccgcalls.nim

@@ -0,0 +1,232 @@
+#
+#
+#           The Nimrod Compiler
+#        (c) Copyright 2011 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+type
+  TAfterCallActions = tuple[p: BProc, actions: PRope]
+
+proc fixupCall(p: BProc, t: PNode, d: var TLoc, pl: PRope) =
+  var pl = pl
+  var typ = t.sons[0].typ # getUniqueType() is too expensive here!
+  if typ.sons[0] != nil:
+    if isInvalidReturnType(typ.sons[0]):
+      if sonsLen(t) > 1: app(pl, ", ")
+      # beware of 'result = p(result)'. We always allocate a temporary:
+      if d.k in {locTemp, locNone}:
+        # We already got a temp. Great, special case it:
+        if d.k == locNone: getTemp(p, typ.sons[0], d)
+        app(pl, addrLoc(d))
+        app(pl, ")")
+        app(p.s[cpsStmts], pl)
+        appf(p.s[cpsStmts], ";$n")
+      else:
+        var tmp: TLoc
+        getTemp(p, typ.sons[0], tmp)
+        app(pl, addrLoc(tmp))
+        app(pl, ")")
+        app(p.s[cpsStmts], pl)
+        appf(p.s[cpsStmts], ";$n")
+        genAssignment(p, d, tmp, {}) # no need for deep copying
+    else:
+      app(pl, ")")
+      if d.k == locNone: getTemp(p, typ.sons[0], d)
+      assert(d.t != nil)        # generate an assignment to d:
+      var list: TLoc
+      initLoc(list, locCall, nil, OnUnknown)
+      list.r = pl
+      genAssignment(p, d, list, {}) # no need for deep copying
+  else:
+    app(pl, ")")
+    app(p.s[cpsStmts], pl)
+    appf(p.s[cpsStmts], ";$n")
+
+proc emitAfterCallActions(aca: TAfterCallActions) {.inline.} =
+  app(aca.p.s[cpsStmts], aca.actions)
+
+proc isInCurrentFrame(p: BProc, n: PNode): bool =
+  # checks if `n` is an expression that refers to the current frame;
+  # this does not work reliably because of forwarding + inlining can break it
+  case n.kind
+  of nkSym:
+    if n.sym.kind in {skVar, skResult, skTemp} and p.prc != nil:
+      result = p.prc.id == n.sym.owner.id
+  of nkDotExpr, nkBracketExpr:
+    if skipTypes(n.sons[0].typ, abstractInst).kind notin {tyVar,tyPtr,tyRef}:
+      result = isInCurrentFrame(p, n.sons[0])
+  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+    result = isInCurrentFrame(p, n.sons[1])
+  of nkHiddenDeref, nkDerefExpr:
+    # what about: var x = addr(y); callAsOpenArray(x[])?
+    # *shrug* ``addr`` is unsafe anyway.
+    result = false
+  of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
+    result = isInCurrentFrame(p, n.sons[0])
+  else: nil
+
+proc genKeepAlive(aca: var TAfterCallActions, n: PNode, a: TLoc) {.inline.} =
+  if a.s == onStack and optRefcGC in gGlobalOptions:
+    aca.p.module.appcg(aca.actions,
+                       "#nimKeepAlive((#TGenericSeq*)$1);$n", [a.rdLoc])
+
+proc openArrayLoc(aca: var TAfterCallActions, n: PNode): PRope =
+  var a: TLoc
+  initLocExpr(aca.p, n, a)
+  case skipTypes(a.t, abstractVar).kind
+  of tyOpenArray:
+    result = ropef("$1, $1Len0", [rdLoc(a)])
+  of tyString, tySequence:
+    result = ropef("$1->data, $1->$2", [a.rdLoc, lenField()])
+    genKeepAlive(aca, n, a)
+  of tyArray, tyArrayConstr:
+    result = ropef("$1, $2", [rdLoc(a), toRope(lengthOrd(a.t))])
+  else: InternalError("openArrayLoc: " & typeToString(a.t))
+
+proc genArgStringToCString(aca: var TAfterCallActions, 
+                           n: PNode): PRope {.inline.} =
+  var a: TLoc
+  initLocExpr(aca.p, n.sons[0], a)
+  result = ropef("$1->data", [a.rdLoc])
+  # we don't guarantee save string->cstring conversions anyway, so we use
+  # an additional check to improve performance:
+  if isInCurrentFrame(aca.p, n): genKeepAlive(aca, n, a)
+  
+proc genArg(aca: var TAfterCallActions, n: PNode, param: PSym): PRope =
+  var a: TLoc
+  if n.kind == nkStringToCString:
+    result = genArgStringToCString(aca, n)
+  elif skipTypes(param.typ, abstractVar).kind == tyOpenArray:
+    var n = if n.kind != nkHiddenAddr: n else: n.sons[0]
+    result = openArrayLoc(aca, n)
+  elif ccgIntroducedPtr(param):
+    initLocExpr(aca.p, n, a)
+    result = addrLoc(a)
+  else:
+    initLocExpr(aca.p, n, a)
+    result = rdLoc(a)
+
+proc genArgNoParam(aca: var TAfterCallActions, n: PNode): PRope =
+  var a: TLoc
+  if n.kind == nkStringToCString:
+    result = genArgStringToCString(aca, n)
+  else:
+    initLocExpr(aca.p, n, a)
+    result = rdLoc(a)
+
+proc genCall(p: BProc, t: PNode, d: var TLoc) =
+  var op: TLoc
+  var aca: TAfterCallActions
+  aca.p = p
+  # this is a hotspot in the compiler
+  initLocExpr(p, t.sons[0], op)
+  var pl = con(op.r, "(")
+  var typ = t.sons[0].typ # getUniqueType() is too expensive here!
+  assert(typ.kind == tyProc)
+  var length = sonsLen(t)
+  for i in countup(1, length - 1):
+    assert(sonsLen(typ) == sonsLen(typ.n))
+    if i < sonsLen(typ):
+      assert(typ.n.sons[i].kind == nkSym)
+      app(pl, genArg(aca, t.sons[i], typ.n.sons[i].sym))
+    else:
+      app(pl, genArgNoParam(aca, t.sons[i]))
+    if i < length - 1: app(pl, ", ")
+  fixupCall(p, t, d, pl)
+  emitAfterCallActions(aca)
+
+proc genInfixCall(p: BProc, t: PNode, d: var TLoc) =
+  var op, a: TLoc
+  var aca: TAfterCallActions
+  aca.p = p
+  initLocExpr(p, t.sons[0], op)
+  var pl: PRope = nil
+  var typ = t.sons[0].typ # getUniqueType() is too expensive here!
+  assert(typ.kind == tyProc)
+  var length = sonsLen(t)
+  assert(sonsLen(typ) == sonsLen(typ.n))
+  
+  var param = typ.n.sons[1].sym
+  app(pl, genArg(aca, t.sons[1], param))
+  
+  if skipTypes(param.typ, {tyGenericInst}).kind == tyPtr: app(pl, "->")
+  else: app(pl, ".")
+  app(pl, op.r)
+  app(pl, "(")
+  for i in countup(2, length - 1):
+    assert(sonsLen(typ) == sonsLen(typ.n))
+    if i < sonsLen(typ):
+      assert(typ.n.sons[i].kind == nkSym)
+      app(pl, genArg(aca, t.sons[i], typ.n.sons[i].sym))
+    else:
+      app(pl, genArgNoParam(aca, t.sons[i]))
+    if i < length - 1: app(pl, ", ")
+  fixupCall(p, t, d, pl)
+  emitAfterCallActions(aca)
+
+proc genNamedParamCall(p: BProc, t: PNode, d: var TLoc) =
+  # generates a crappy ObjC call
+  var op, a: TLoc
+  var aca: TAfterCallActions
+  aca.p = p
+  initLocExpr(p, t.sons[0], op)
+  var pl = toRope"["
+  var typ = t.sons[0].typ # getUniqueType() is too expensive here!
+  assert(typ.kind == tyProc)
+  var length = sonsLen(t)
+  assert(sonsLen(typ) == sonsLen(typ.n))
+  
+  if length > 1:
+    app(pl, genArg(aca, t.sons[1], typ.n.sons[1].sym))
+    app(pl, " ")
+  app(pl, op.r)
+  if length > 2:
+    app(pl, ": ")
+    app(pl, genArg(aca, t.sons[2], typ.n.sons[2].sym))
+  for i in countup(3, length-1):
+    assert(sonsLen(typ) == sonsLen(typ.n))
+    if i >= sonsLen(typ):
+      InternalError(t.info, "varargs for objective C method?")
+    assert(typ.n.sons[i].kind == nkSym)
+    var param = typ.n.sons[i].sym
+    app(pl, " ")
+    app(pl, param.name.s)
+    app(pl, ": ")
+    app(pl, genArg(aca, t.sons[i], param))
+  if typ.sons[0] != nil:
+    if isInvalidReturnType(typ.sons[0]):
+      if sonsLen(t) > 1: app(pl, " ")
+      # beware of 'result = p(result)'. We always allocate a temporary:
+      if d.k in {locTemp, locNone}:
+        # We already got a temp. Great, special case it:
+        if d.k == locNone: getTemp(p, typ.sons[0], d)
+        app(pl, "Result: ")
+        app(pl, addrLoc(d))
+        app(pl, "]")
+        app(p.s[cpsStmts], pl)
+        appf(p.s[cpsStmts], ";$n")
+      else:
+        var tmp: TLoc
+        getTemp(p, typ.sons[0], tmp)
+        app(pl, addrLoc(tmp))
+        app(pl, "]")
+        app(p.s[cpsStmts], pl)
+        appf(p.s[cpsStmts], ";$n")
+        genAssignment(p, d, tmp, {}) # no need for deep copying
+    else:
+      app(pl, "]")
+      if d.k == locNone: getTemp(p, typ.sons[0], d)
+      assert(d.t != nil)        # generate an assignment to d:
+      var list: TLoc
+      initLoc(list, locCall, nil, OnUnknown)
+      list.r = pl
+      genAssignment(p, d, list, {}) # no need for deep copying
+  else:
+    app(pl, "]")
+    app(p.s[cpsStmts], pl)
+    appf(p.s[cpsStmts], ";$n")
+  emitAfterCallActions(aca)
+

compiler/ccgexprs.nim

@@ -784,169 +784,7 @@ proc genEcho(p: BProc, n: PNode) =
   appcg(p, cpsStmts, "printf($1$2);$n", [
     makeCString(repeatStr(n.len-1, "%s") & tnl), args])
 
-proc fixupCall(p: BProc, t: PNode, d: var TLoc, pl: PRope) =
+include ccgcalls
-  var pl = pl
-  var typ = t.sons[0].typ # getUniqueType() is too expensive here!
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(typ.sons[0]):
-      if sonsLen(t) > 1: app(pl, ", ")
-      # beware of 'result = p(result)'. We always allocate a temporary:
-      if d.k in {locTemp, locNone}:
-        # We already got a temp. Great, special case it:
-        if d.k == locNone: getTemp(p, typ.sons[0], d)
-        app(pl, addrLoc(d))
-        app(pl, ")")
-        app(p.s[cpsStmts], pl)
-        appf(p.s[cpsStmts], ";$n")
-      else:
-        var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp)
-        app(pl, addrLoc(tmp))
-        app(pl, ")")
-        app(p.s[cpsStmts], pl)
-        appf(p.s[cpsStmts], ";$n")
-        genAssignment(p, d, tmp, {}) # no need for deep copying
-    else:
-      app(pl, ")")
-      if d.k == locNone: getTemp(p, typ.sons[0], d)
-      assert(d.t != nil)        # generate an assignment to d:
-      var list: TLoc
-      initLoc(list, locCall, nil, OnUnknown)
-      list.r = pl
-      genAssignment(p, d, list, {}) # no need for deep copying
-  else:
-    app(pl, ")")
-    app(p.s[cpsStmts], pl)
-    appf(p.s[cpsStmts], ";$n")
-
-proc openArrayLoc(a: TLoc): PRope =
-  case skipTypes(a.t, abstractVar).kind
-  of tyOpenArray:
-    result = ropef("$1, $1Len0", [rdLoc(a)])
-  of tyString, tySequence:
-    result = ropef("$1->data, $1->$2", [rdLoc(a), lenField()])
-  of tyArray, tyArrayConstr:
-    result = ropef("$1, $2", [rdLoc(a), toRope(lengthOrd(a.t))])
-  else: InternalError("openArrayLoc: " & typeToString(a.t))
-
-proc genArg(p: BProc, n: PNode, param: PSym): PRope =
-  var a: TLoc
-  if skipTypes(param.typ, abstractVar).kind == tyOpenArray:
-    var n = if n.kind != nkHiddenAddr: n else: n.sons[0]
-    initLocExpr(p, n, a)
-    result = openArrayLoc(a)
-  elif ccgIntroducedPtr(param): 
-    initLocExpr(p, n, a)
-    result = addrLoc(a)
-  else:
-    initLocExpr(p, n, a)
-    result = rdLoc(a)
-
-proc genCall(p: BProc, t: PNode, d: var TLoc) =
-  var op, a: TLoc
-  # this is a hotspot in the compiler
-  initLocExpr(p, t.sons[0], op)
-  var pl = con(op.r, "(")
-  var typ = t.sons[0].typ # getUniqueType() is too expensive here!
-  assert(typ.kind == tyProc)
-  var length = sonsLen(t)
-  for i in countup(1, length - 1):
-    assert(sonsLen(typ) == sonsLen(typ.n))
-    if i < sonsLen(typ):
-      assert(typ.n.sons[i].kind == nkSym)
-      app(pl, genArg(p, t.sons[i], typ.n.sons[i].sym))
-    else:
-      initLocExpr(p, t.sons[i], a) # generate expression for param
-      app(pl, rdLoc(a))
-    if i < length - 1: app(pl, ", ")
-  fixupCall(p, t, d, pl)
-
-proc genInfixCall(p: BProc, t: PNode, d: var TLoc) =
-  var op, a: TLoc
-  initLocExpr(p, t.sons[0], op)
-  var pl: PRope = nil
-  var typ = t.sons[0].typ # getUniqueType() is too expensive here!
-  assert(typ.kind == tyProc)
-  var length = sonsLen(t)
-  assert(sonsLen(typ) == sonsLen(typ.n))
-  
-  var param = typ.n.sons[1].sym
-  app(pl, genArg(p, t.sons[1], param))
-  
-  if skipTypes(param.typ, {tyGenericInst}).kind == tyPtr: app(pl, "->")
-  else: app(pl, ".")
-  app(pl, op.r)
-  app(pl, "(")
-  for i in countup(2, length - 1):
-    assert(sonsLen(typ) == sonsLen(typ.n))
-    if i < sonsLen(typ):
-      assert(typ.n.sons[i].kind == nkSym)
-      app(pl, genArg(p, t.sons[i], typ.n.sons[i].sym))
-    else:
-      initLocExpr(p, t.sons[i], a) # generate expression for param      
-      app(pl, rdLoc(a))
-    if i < length - 1: app(pl, ", ")
-  fixupCall(p, t, d, pl)
-
-proc genNamedParamCall(p: BProc, t: PNode, d: var TLoc) =
-  # generates a crappy ObjC call
-  var op, a: TLoc
-  initLocExpr(p, t.sons[0], op)
-  var pl = toRope"["
-  var typ = t.sons[0].typ # getUniqueType() is too expensive here!
-  assert(typ.kind == tyProc)
-  var length = sonsLen(t)
-  assert(sonsLen(typ) == sonsLen(typ.n))
-  
-  if length > 1:
-    app(pl, genArg(p, t.sons[1], typ.n.sons[1].sym))
-    app(pl, " ")
-  app(pl, op.r)
-  if length > 2:
-    app(pl, ": ")
-    app(pl, genArg(p, t.sons[2], typ.n.sons[2].sym))
-  for i in countup(3, length-1):
-    assert(sonsLen(typ) == sonsLen(typ.n))
-    if i >= sonsLen(typ):
-      InternalError(t.info, "varargs for objective C method?")
-    assert(typ.n.sons[i].kind == nkSym)
-    var param = typ.n.sons[i].sym
-    app(pl, " ")
-    app(pl, param.name.s)
-    app(pl, ": ")
-    app(pl, genArg(p, t.sons[i], param))
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(typ.sons[0]):
-      if sonsLen(t) > 1: app(pl, " ")
-      # beware of 'result = p(result)'. We always allocate a temporary:
-      if d.k in {locTemp, locNone}:
-        # We already got a temp. Great, special case it:
-        if d.k == locNone: getTemp(p, typ.sons[0], d)
-        app(pl, "Result: ")
-        app(pl, addrLoc(d))
-        app(pl, "]")
-        app(p.s[cpsStmts], pl)
-        appf(p.s[cpsStmts], ";$n")
-      else:
-        var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp)
-        app(pl, addrLoc(tmp))
-        app(pl, "]")
-        app(p.s[cpsStmts], pl)
-        appf(p.s[cpsStmts], ";$n")
-        genAssignment(p, d, tmp, {}) # no need for deep copying
-    else:
-      app(pl, "]")
-      if d.k == locNone: getTemp(p, typ.sons[0], d)
-      assert(d.t != nil)        # generate an assignment to d:
-      var list: TLoc
-      initLoc(list, locCall, nil, OnUnknown)
-      list.r = pl
-      genAssignment(p, d, list, {}) # no need for deep copying
-  else:
-    app(pl, "]")
-    app(p.s[cpsStmts], pl)
-    appf(p.s[cpsStmts], ";$n")
 
 proc genStrConcat(p: BProc, e: PNode, d: var TLoc) =
   #   <Nimrod code>

compiler/msgs.nim

@@ -564,18 +564,18 @@ var
   lastError = UnknownLineInfo()
 
 proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string, 
-               eh: TErrorHandling) = 
+               eh: TErrorHandling) =
   var frmt: string
   var ignoreMsg = false
   case msg
-  of errMin..errMax: 
+  of errMin..errMax:
     writeContext(info)
     frmt = posErrorFormat
     # we try to filter error messages so that not two error message
     # in the same file and line are produced:
     ignoreMsg = lastError == info and eh != doAbort
     lastError = info
-  of warnMin..warnMax: 
+  of warnMin..warnMax:
     ignoreMsg = optWarns notin gOptions or msg notin gNotes
     if not ignoreMsg: writeContext(info)
     frmt = posWarningFormat

lib/system/gc.nim

@@ -31,6 +31,8 @@ const
   rcWhite = 0b010  # member of a garbage cycle
   rcPurple = 0b011 # possible root of a cycle
   rcZct = 0b100    # in ZCT
+  rcMarked = 0b101 # dummy write to keep C code generator from
+                   # eliminating the root
   rcRed = 0b101    # Candidate cycle undergoing sigma-computation
   rcOrange = 0b110 # Candidate cycle awaiting epoch boundary
   rcShift = 3      # shift by rcShift to get the reference counter
@@ -517,12 +519,19 @@ proc gcMark(gch: var TGcHeap, p: pointer) {.inline.} =
       # mark the cell:
       cell.refcount = cell.refcount +% rcIncrement
       add(gch.decStack, cell)
-    # care for string->cstring and seq->openArray conversions:
+    when false:
-    var b = cast[PCell](c -% sizeof(TGenericSeq))
+      # Care for string->cstring and seq->openArray conversions.
-    if isAllocatedPtr(gch.region, b):
+      # Now the codegen deals with it, it generated ``nimKeepAlive`` calls.
-      # mark the cell:
+      var b = cast[PCell](c -% sizeof(TGenericSeq))
-      b.refcount = b.refcount +% rcIncrement
+      if isAllocatedPtr(gch.region, b):
-      add(gch.decStack, b)
+        # mark the cell:
+        b.refcount = b.refcount +% rcIncrement
+        add(gch.decStack, b)
+
+proc nimKeepAlive(p: PGenericSeq) {.compilerRtl, noinline.} =
+  var c = usrToCell(p)
+  if isAllocatedPtr(gch.region, c):
+    c.refcount = c.refcount or rcMarked
 
 proc markThreadStacks(gch: var TGcHeap) = 
   when hasThreadSupport and hasSharedHeap:

todo.txt

@@ -1,4 +1,4 @@
-Version 0.8.14
+version 0.8.14
 ==============
 
 - fix thread tests
@@ -20,7 +20,7 @@ version 0.9.0
 - change overloading resolution
 - implement closures; implement proper coroutines
 - implement ``partial`` pragma for partial evaluation
-- implicit invokation of `items` seems nice
+- implicit invokation of `items`/`pairs` seems nice
 - we need to support iteration of 2 different data structures in parallel
 - make exceptions compatible with C++ exceptions
 - ``=`` should be overloadable; requires specialization for ``=``
@@ -68,6 +68,7 @@ version 0.9.XX
   per default?
 - fix implicit generic routines
 - improve docgen to use the semantic pass
+- 'export' feature (requires improved docgen)
 - think about ``{:}.toTable[int, string]()``
 - mocking support with ``tyProxy`` that does:
   o.p(x) --> p(o, x) -->  myMacro(o, p, x)

web/news.txt

@@ -18,6 +18,8 @@ Bugfixes
 - Bugfix c2nim, c2pas: the ``--out`` option has never worked properly.
 - Bugfix: forwarding of generic procs never worked.
 - Some more bugfixes for macros and compile-time evaluation.
+- The compiler now generates code that tries to keep the C compiler from
+  optimizing stack allocated GC roots away.
 
 
 Changes affecting backwards compatibility