code gen can generate code to keep alive stack roots

compiler/ast.nim

@@ -273,6 +273,7 @@ type
 
 const
   tyPureObject* = tyTuple
+  GcTypeKinds* = {tyRef, tySequence, tyString}
 
 type
   TTypeKinds* = set[TTypeKind]

compiler/ccgcalls.nim

@@ -252,7 +252,8 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) =
     genNamedParamCall(p, e, d)
   else:
     genPrefixCall(p, nil, e, d)
-  if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d)
+  when false:
+    if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d)
 
 proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
   if ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags and
@@ -262,5 +263,6 @@ proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
     genNamedParamCall(p, ri, d)
   else:
     genPrefixCall(p, le, ri, d)
-  if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d)
+  when false:
+    if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d)
 

compiler/cgen.nim

@@ -297,23 +297,36 @@ proc getTemp(p: BProc, t: PType, result: var TLoc) =
   initTemp(p, result)
 
 proc keepAlive(p: BProc, toKeepAlive: TLoc) =
-  if optRefcGC notin gGlobalOptions: return
+  when false:
-  var result: TLoc
+    # deactivated because of the huge slowdown this causes; GC will take care
-  inc(p.labels)
+    # of interior pointers instead
-  result.r = con("LOC", toRope(p.labels))
+    if optRefcGC notin gGlobalOptions: return
-  appf(p.s[cpsLocals], "volatile $1 $2;$n",
+    var result: TLoc
-      [getTypeDesc(p.module, toKeepAlive.t), result.r])
+    var fid = toRope(p.gcFrameId)
-  result.k = locTemp
+    result.r = con("GCFRAME.F", fid)
-  result.a = -1
+    appf(p.gcFrameType, "  $1 F$2;$n",
-  result.t = toKeepAlive.t
+        [getTypeDesc(p.module, toKeepAlive.t), fid])
-  result.s = OnStack
+    inc(p.gcFrameId)
-  result.flags = {}
+    result.k = locTemp
-  if skipTypes(toKeepAlive.t, abstractVarRange).Kind notin complexValueType:
+    result.a = -1
-    appf(p.s[cpsStmts], "$1 = $2;$n", [rdLoc(result), rdLoc(toKeepAlive)])
+    result.t = toKeepAlive.t
-  else:
+    result.s = OnStack
-    appcg(p, cpsStmts,
+    result.flags = {}
-         "memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n",
+
-         [addrLoc(result), addrLoc(toKeepAlive), rdLoc(result)])
+    if skipTypes(toKeepAlive.t, abstractVarRange).Kind notin complexValueType:
+      appf(p.s[cpsStmts], "$1 = $2;$n", [rdLoc(result), rdLoc(toKeepAlive)])
+    else:
+      appcg(p, cpsStmts,
+           "memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n",
+           [addrLoc(result), addrLoc(toKeepAlive), rdLoc(result)])
+
+proc initGCFrame(p: BProc): PRope =
+  if p.gcFrameId > 0: result = ropef("struct {$1} GCFRAME;$n", p.gcFrameType)
+
+proc deinitGCFrame(p: BProc): PRope =
+  if p.gcFrameId > 0:
+    result = ropecg(p.module,
+                    "if (((NU)&GCFRAME) < 4096) #nimGCFrame(&GCFRAME);$n")
 
 proc cstringLit(p: BProc, r: var PRope, s: string): PRope = 
   if gCmd == cmdCompileToLLVM: 
@@ -370,6 +383,8 @@ proc assignLocalVar(p: BProc, s: PSym) =
     if s.kind == skLet: incl(s.loc.flags, lfNoDeepCopy)
   app(p.s[cpsLocals], getTypeDesc(p.module, s.loc.t))
   if sfRegister in s.flags: app(p.s[cpsLocals], " register")
+  elif skipTypes(s.typ, abstractInst).kind in GcTypeKinds:
+    app(p.s[cpsLocals], " GC_GUARD")
   if (sfVolatile in s.flags) or (p.nestedTryStmts.len > 0): 
     app(p.s[cpsLocals], " volatile")
   appf(p.s[cpsLocals], " $1;$n", [s.loc.r])
@@ -585,6 +600,7 @@ proc genProcAux(m: BModule, prc: PSym) =
         [header, p.s[cpsLocals], p.s[cpsInit], p.s[cpsStmts]])
   else:
     generatedProc = ropeff("$1 {$n", "define $1 {$n", [header])
+    app(generatedProc, initGCFrame(p))
     if optStackTrace in prc.options: 
       getFrameDecl(p)
       app(generatedProc, p.s[cpsLocals])
@@ -608,6 +624,7 @@ proc genProcAux(m: BModule, prc: PSym) =
     app(generatedProc, p.s[cpsInit])
     app(generatedProc, p.s[cpsStmts])
     if p.beforeRetNeeded: appf(generatedProc, "BeforeRet: $n;")
+    app(generatedProc, deinitGCFrame(p))
     if optStackTrace in prc.options: app(generatedProc, deinitFrame(p))
     if (optProfiler in prc.options) and (gCmd != cmdCompileToLLVM): 
       appf(generatedProc, 
@@ -816,6 +833,8 @@ proc genInitCode(m: BModule) =
     m.FrameDeclared = true
     getFrameDecl(m.initProc)
   
+  app(prc, initGCFrame(m.initProc))
+  
   app(prc, genSectionStart(cpsLocals))
   app(prc, m.initProc.s[cpsLocals])
   app(prc, genSectionEnd(cpsLocals))
@@ -842,6 +861,7 @@ proc genInitCode(m: BModule) =
   if optStackTrace in m.initProc.options and not m.PreventStackTrace:
     app(prc, deinitFrame(m.initProc))
   app(prc, genSectionEnd(cpsStmts))
+  app(prc, deinitGCFrame(m.initProc))
   appf(prc, "}$n$n")
   # we cannot simply add the init proc to ``m.s[cfsProcs]`` anymore because
   # that would lead to a *nesting* of merge sections which the merger does

compiler/cgendata.nim

@@ -72,6 +72,8 @@ type
     receiveClosure*: PType    # closure record type that we get
     module*: BModule          # used to prevent excessive parameter passing
     withinLoop*: int          # > 0 if we are within a loop
+    gcFrameId*: natural       # for the GC stack marking
+    gcFrameType*: PRope       # the struct {} we put the GC markers into
   
   TTypeSeq* = seq[PType]
   TCGen = object of TPassContext # represents a C source file

compiler/commands.nim

@@ -207,7 +207,7 @@ proc addPathRec(dir: string, info: TLineInfo) =
 
 proc track(arg: string, info: TLineInfo) = 
   var a = arg.split(',')
-  if a.len != 3: LocalError(info, errTokenExpected, "FILE,LINE,COLMUN")
+  if a.len != 3: LocalError(info, errTokenExpected, "FILE,LINE,COLUMN")
   var line, column: int
   if parseUtils.parseInt(a[1], line) <= 0:
     LocalError(info, errInvalidNumber, a[1])

compiler/extccomp.nim

@@ -21,7 +21,8 @@ type
     hasSwitchRange,           # CC allows ranges in switch statements (GNU C)
     hasComputedGoto,          # CC has computed goto (GNU C extension)
     hasCpp,                   # CC is/contains a C++ compiler
-    hasAssume                 # CC has __assume (Visual C extension)
+    hasAssume,                # CC has __assume (Visual C extension)
+    hasGcGuard                # CC supports GC_GUARD to keep stack roots
   TInfoCCProps* = set[TInfoCCProp]
   TInfoCC* = tuple[
     name: string,        # the short name of the compiler
@@ -71,7 +72,7 @@ compiler gcc:
     debug: "",
     pic: "-fPIC",
     asmStmtFrmt: "asm($1);$n",
-    props: {hasSwitchRange, hasComputedGoto, hasCpp})
+    props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard})
     
 compiler gpp:
   result = gcc()
@@ -80,10 +81,9 @@ compiler gpp:
   result.compilerExe = "g++"
   result.linkerExe = "g++"  
 
-  result.debug.add " -g " # XXX: Why is this default for g++, but not for gcc?
+  result.buildDll = " -mdll" 
-
+  # XXX: Hmm, I'm keeping this from the previos version, 
-  result.buildDll = " -mdll" # XXX: Hmm, I'm keeping this from the previos version, 
+  # but my gcc doesn't even have such an option (is this mingw?)
-                             # but my gcc doesn't even have such an option (is this mingw?)
 
 compiler llvmGcc:
   result = gcc()

compiler/types.nim

@@ -277,7 +277,7 @@ proc analyseObjectWithTypeField(t: PType): TTypeFieldResult =
   result = analyseObjectWithTypeFieldAux(t, marker)
 
 proc isGBCRef(t: PType): bool = 
-  result = t.kind in {tyRef, tySequence, tyString}
+  result = t.kind in GcTypeKinds
 
 proc containsGarbageCollectedRef(typ: PType): bool = 
   # returns true if typ contains a reference, sequence or string (all the

lib/nimbase.h

@@ -16,6 +16,7 @@ __GNUC__
 __DMC__
 __POCC__
 __TINYC__
+__clang__
 */
 
 
@@ -437,4 +438,11 @@ __declspec(naked) int __fastcall NimXadd(volatile int* pNum, int val) {
 #  define unlikely(x) (x)
 #endif
 
+#if defined(__GNUC__) || defined(__clang__)
+static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); }
+#  define GC_GUARD __attribute__ ((cleanup(GCGuard)))
+#else
+#  define GC_GUARD
+#endif
+
 #endif

lib/pure/strutils.nim

@@ -699,7 +699,7 @@ proc find*(s, sub: string, start: int = 0): int {.noSideEffect,
   rtl, extern: "nsuFindStr".} =
   ## Searches for `sub` in `s` starting at position `start`. Searching is
   ## case-sensitive. If `sub` is not in `s`, -1 is returned.
-  var a: TSkipTable
+  var a {.noinit.}: TSkipTable
   preprocessSub(sub, a)
   result = findAux(s, sub, start, a)
 
@@ -742,7 +742,7 @@ proc contains*(s: string, chars: set[char]): bool {.noSideEffect.} =
 proc replace*(s, sub: string, by = ""): string {.noSideEffect,
   rtl, extern: "nsuReplaceStr".} =
   ## Replaces `sub` in `s` by the string `by`.
-  var a: TSkipTable
+  var a {.noinit.}: TSkipTable
   result = ""
   preprocessSub(sub, a)
   var i = 0
@@ -989,7 +989,7 @@ proc formatBiggestFloat*(f: BiggestFloat, format: TFloatFormat = ffDefault,
   ## after the decimal point for Nimrod's ``biggestFloat`` type.
   const floatFormatToChar: array[TFloatFormat, char] = ['g', 'f', 'e']
   var
-    frmtstr: array[0..5, char]
+    frmtstr {.noinit.}: array[0..5, char]
     buf: array[0..80, char]
   frmtstr[0] = '%'
   frmtstr[1] = '#'
@@ -1018,16 +1018,41 @@ proc formatFloat*(f: float, format: TFloatFormat = ffDefault,
   ## after the decimal point for Nimrod's ``float`` type.
   result = formatBiggestFloat(f, format, precision)
 
+proc formatSize*(bytes: biggestInt, decimalSep = '.'): string =
+  ## Rounds and formats `bytes`. Examples:
+  ##
+  ## .. code-block:: nimrod
+  ##
+  ##    formatSize(1'i64 shl 31 + 300'i64) == "4GB"
+  ##    formatSize(4096) == "4KB"
+  ##
+  template frmt(a, b, c: expr): expr =
+    let bs = $b
+    insertSep($a) & decimalSep & bs.substr(0, 2) & c
+  let gigabytes = bytes shr 30
+  let megabytes = bytes shr 20
+  let kilobytes = bytes shr 10
+  if gigabytes != 0:
+    result = frmt(gigabytes, megabytes, "GB")
+  elif megabytes != 0:
+    result = frmt(megabytes, kilobytes, "MB")
+  elif kilobytes != 0:
+    result = frmt(kilobytes, bytes, "KB")
+  else:
+    result = insertSep($bytes) & "B"
+
 {.pop.}
 
 when isMainModule:
-  assert align("abc", 4) == " abc"
+  doAssert align("abc", 4) == " abc"
-  assert align("a", 0) == "a"
+  doAssert align("a", 0) == "a"
-  assert align("1232", 6) == "  1232"
+  doAssert align("1232", 6) == "  1232"
   echo wordWrap(""" this is a long text --  muchlongerthan10chars and here
                    it goes""", 10, false)
-  assert formatBiggestFloat(0.00000000001, ffDecimal, 11) == "0.00000000001"
+  doAssert formatBiggestFloat(0.00000000001, ffDecimal, 11) == "0.00000000001"
-  assert formatBiggestFloat(0.00000000001, ffScientific, 1) == "1.0e-11"
+  doAssert formatBiggestFloat(0.00000000001, ffScientific, 1) == "1.0e-11"
 
-  assert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c"
+  doAssert "$# $3 $# $#" % ["a", "b", "c"] == "a c b c"
+  echo formatSize(1'i64 shl 31 + 300'i64) # == "4,GB"
+  echo formatSize(1'i64 shl 31)
 

lib/system/alloc.nim

@@ -171,7 +171,7 @@ proc getMaxMem(a: var TMemRegion): int =
   # Since we update maxPagesCount only when freeing pages, 
   # maxPagesCount may not be up to date. Thus we use the
   # maximum of these both values here:
-  return max(a.currMem, a.maxMem)
+  result = max(a.currMem, a.maxMem)
   
 proc llAlloc(a: var TMemRegion, size: int): pointer =
   # *low-level* alloc for the memory managers data structures. Deallocation
@@ -550,6 +550,23 @@ proc isAllocatedPtr(a: TMemRegion, p: pointer): bool =
         var c = cast[PBigChunk](c)
         result = p == addr(c.data) and cast[ptr TFreeCell](p).zeroField >% 1
 
+proc interiorAllocatedPtr(a: TMemRegion, p: pointer): pointer =
+  if isAccessible(a, p):
+    var c = pageAddr(p)
+    if not chunkUnused(c):
+      if isSmallChunk(c):
+        var c = cast[PSmallChunk](c)
+        var offset = (cast[TAddress](p) and (PageSize-1)) -% 
+                     smallChunkOverhead()
+        if c.acc >% offset:
+          var d = cast[ptr TFreeCell](cast[TAddress](addr(c.data)) +% 
+                    offset -% (offset %% c.size))
+          if d.zeroField >% 1: result = d
+      else:
+        var c = cast[PBigChunk](c)
+        var d = addr(c.data)
+        if p >= d and cast[ptr TFreeCell](d).zeroField >% 1: result = d
+
 proc ptrSize(p: pointer): int =
   var x = cast[pointer](cast[TAddress](p) -% sizeof(TFreeCell))
   result = pageAddr(x).size - sizeof(TFreeCell)

lib/system/gc.nim

@@ -579,6 +579,13 @@ proc nimKeepAlive(p: PGenericSeq) {.compilerRtl, noinline.} =
   if isAllocatedPtr(gch.region, c):
     c.refcount = c.refcount or rcMarked
 
+proc nimGCFrame(p: pointer) {.compilerRtl, noinline.} =
+  # 'cast' is correct here! no offset to add:
+  var c = cast[PCell](p)
+  var x = cast[TAddress](c)
+  if x <% PageSize and (x and (MemAlign-1)) == 0:
+    c.refcount = c.refcount or rcMarked
+
 proc markThreadStacks(gch: var TGcHeap) = 
   when hasThreadSupport and hasSharedHeap:
     {.error: "not fully implemented".}

todo.txt

@@ -1,10 +1,9 @@
 version 0.8.14
 ==============
 
-- compiler/GC interaction need to generate code to prevent tail call
+- GC should care about interior pointers on the stack
-  optimization
+- BUG: type TX = TTable[string, int]
-
+- warning for implicit openArray -> varargs conversion
-- warning for implicit openArray -> varargs convention
 - implement explicit varargs; **but** ``len(varargs)`` problem remains! 
   --> solve by implicit conversion from varargs to openarray
 - implicit invokation of `items`/`pairs` seems nice
@@ -93,7 +92,7 @@ Library
 -------
 
 - wrappers for poppler; libharu
-- radix tree for strings; maybe suffix tree
+- suffix trees
 - locale support
 - bignums