Merge branch 'devel' into araq

lib/system/alloc.nim

@@ -9,7 +9,6 @@
 
 # Low level allocator for Nim. Has been designed to support the GC.
 # TODO:
-# - eliminate "used" field
 # - make searching for block O(1)
 {.push profiler:off.}
 

lib/system/excpt.nim

@@ -90,12 +90,11 @@ proc popSafePoint {.compilerRtl, inl.} =
   excHandler = excHandler.prev
 
 proc pushCurrentException(e: ref Exception) {.compilerRtl, inl.} =
-  #if e.parent.isNil:
-  #  e.parent = currException
+  e.up = currException
   currException = e
 
 proc popCurrentException {.compilerRtl, inl.} =
-  currException = nil # currException.parent
+  currException = currException.up
 
 # some platforms have native support for stack traces:
 const
@@ -247,6 +246,18 @@ when false:
     pushCurrentException(e)
     c_longjmp(excHandler.context, 1)
 
+var onUnhandledException*: (proc (errorMsg: string) {.
+  nimcall.}) ## set this error \
+  ## handler to override the existing behaviour on an unhandled exception.
+  ## The default is to write a stacktrace to ``stderr`` and then call ``quit(1)``.
+  ## Unstable API.
+
+template unhandled(buf, body) =
+  if onUnhandledException != nil:
+    onUnhandledException($buf)
+  else:
+    body
+
 proc raiseExceptionAux(e: ref Exception) =
   if localRaiseHook != nil:
     if not localRaiseHook(e): return
@@ -277,7 +288,9 @@ proc raiseExceptionAux(e: ref Exception) =
         add(buf, " [")
         add(buf, $e.name)
         add(buf, "]\n")
-        showErrorMessage(buf)
+        unhandled(buf):
+          showErrorMessage(buf)
+          quitOrDebug()
       else:
         # ugly, but avoids heap allocations :-)
         template xadd(buf, s, slen: expr) =
@@ -293,8 +306,9 @@ proc raiseExceptionAux(e: ref Exception) =
         add(buf, " [")
         xadd(buf, e.name, e.name.len)
         add(buf, "]\n")
-        showErrorMessage(buf)
-      quitOrDebug()
+        unhandled(buf):
+          showErrorMessage(buf)
+          quitOrDebug()
 
 proc raiseException(e: ref Exception, ename: cstring) {.compilerRtl.} =
   if e.name.isNil: e.name = ename

lib/system/gc_common.nim

@@ -25,6 +25,13 @@ when defined(nimTypeNames):
         c_fprintf(stdout, "[Heap] %s: #%ld; bytes: %ld\n", it.name, it.instances, it.sizes)
       it = it.nextType
 
+  when defined(nimGcRefLeak):
+    proc oomhandler() =
+      c_fprintf(stdout, "[Heap] ROOTS: #%ld\n", gch.additionalRoots.len)
+      writeLeaks()
+
+    outOfMemHook = oomhandler
+
 template decTypeSize(cell, t) =
   # XXX this needs to use atomics for multithreaded apps!
   when defined(nimTypeNames):

lib/system/gc_ms.nim

@@ -142,11 +142,54 @@ proc doOperation(p: pointer, op: WalkOp) {.benign.}
 proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) {.benign.}
 # we need the prototype here for debugging purposes
 
+when defined(nimGcRefLeak):
+  const
+    MaxTraceLen = 20 # tracking the last 20 calls is enough
+
+  type
+    GcStackTrace = object
+      lines: array[0..MaxTraceLen-1, cstring]
+      files: array[0..MaxTraceLen-1, cstring]
+
+  proc captureStackTrace(f: PFrame, st: var GcStackTrace) =
+    const
+      firstCalls = 5
+    var
+      it = f
+      i = 0
+      total = 0
+    while it != nil and i <= high(st.lines)-(firstCalls-1):
+      # the (-1) is for the "..." entry
+      st.lines[i] = it.procname
+      st.files[i] = it.filename
+      inc(i)
+      inc(total)
+      it = it.prev
+    var b = it
+    while it != nil:
+      inc(total)
+      it = it.prev
+    for j in 1..total-i-(firstCalls-1):
+      if b != nil: b = b.prev
+    if total != i:
+      st.lines[i] = "..."
+      st.files[i] = "..."
+      inc(i)
+    while b != nil and i <= high(st.lines):
+      st.lines[i] = b.procname
+      st.files[i] = b.filename
+      inc(i)
+      b = b.prev
+
+  var ax: array[10_000, GcStackTrace]
+
 proc nimGCref(p: pointer) {.compilerProc.} =
   # we keep it from being collected by pretending it's not even allocated:
   when false:
     when withBitvectors: excl(gch.allocated, usrToCell(p))
     else: usrToCell(p).refcount = rcBlack
+  when defined(nimGcRefLeak):
+    captureStackTrace(framePtr, ax[gch.additionalRoots.len])
   add(gch.additionalRoots, usrToCell(p))
 
 proc nimGCunref(p: pointer) {.compilerProc.} =
@@ -157,6 +200,8 @@ proc nimGCunref(p: pointer) {.compilerProc.} =
   while i >= 0:
     if d[i] == cell:
       d[i] = d[L]
+      when defined(nimGcRefLeak):
+        ax[i] = ax[L]
       dec gch.additionalRoots.len
       break
     dec(i)
@@ -164,6 +209,16 @@ proc nimGCunref(p: pointer) {.compilerProc.} =
     when withBitvectors: incl(gch.allocated, usrToCell(p))
     else: usrToCell(p).refcount = rcWhite
 
+when defined(nimGcRefLeak):
+  proc writeLeaks() =
+    for i in 0..gch.additionalRoots.len-1:
+      c_fprintf(stdout, "[Heap] NEW STACK TRACE\n")
+      for ii in 0..MaxTraceLen-1:
+        let line = ax[i].lines[ii]
+        let file = ax[i].files[ii]
+        if isNil(line): break
+        c_fprintf(stdout, "[Heap] %s(%s)\n", file, line)
+
 include gc_common
 
 proc prepareDealloc(cell: PCell) =

lib/system/gc_stack.nim

@@ -79,7 +79,7 @@ template withRegion*(r: MemRegion; body: untyped) =
   try:
     body
   finally:
-    r = tlRegion
+    #r = tlRegion
     tlRegion = oldRegion
 
 template inc(p: pointer, s: int) =

lib/system/jssys.nim

@@ -446,7 +446,7 @@ when defined(kwin):
       print(buf);
     """
 
-elif defined(nodejs):
+elif not defined(nimOldEcho):
   proc ewriteln(x: cstring) = log(x)
 
   proc rawEcho {.compilerproc, asmNoStackFrame.} =

lib/system/mmdisp.nim

@@ -255,12 +255,21 @@ elif defined(gogc):
         next_gc: uint64          # next GC (in heap_alloc time)
         last_gc: uint64          # last GC (in absolute time)
         pause_total_ns: uint64
-        pause_ns: array[256, uint64]
+        pause_ns: array[256, uint64] # circular buffer of recent gc pause lengths
+        pause_end: array[256, uint64] # circular buffer of recent gc end times (nanoseconds since 1970)
         numgc: uint32
+        numforcedgc: uint32      # number of user-forced GCs
+        gc_cpu_fraction: float64 # fraction of CPU time used by GC
         enablegc: cbool
         debuggc: cbool
         # Statistics about allocation size classes.
         by_size: array[goNumSizeClasses, goMStats_inner_struct]
+        # Statistics below here are not exported to MemStats directly.
+        tinyallocs: uint64       # number of tiny allocations that didn't cause actual allocation; not exported to go directly
+        gc_trigger: uint64
+        heap_live: uint64
+        heap_scan: uint64
+        heap_marked: uint64
 
   proc goRuntime_ReadMemStats(a2: ptr goMStats) {.cdecl,
     importc: "runtime_ReadMemStats",

lib/system/repr.nim

@@ -49,7 +49,7 @@ proc reprStrAux(result: var string, s: cstring; len: int) =
     of '"': add result, "\\\""
     of '\\': add result, "\\\\" # BUGFIX: forgotten
     of '\10': add result, "\\10\"\n\"" # " \n " # better readability
-    of '\128' .. '\255', '\0'..'\9', '\11'..'\31':
+    of '\127' .. '\255', '\0'..'\9', '\11'..'\31':
       add result, "\\" & reprInt(ord(c))
     else:
       result.add(c)
@@ -68,7 +68,7 @@ proc reprChar(x: char): string {.compilerRtl.} =
   case x
   of '"': add result, "\\\""
   of '\\': add result, "\\\\"
-  of '\128' .. '\255', '\0'..'\31': add result, "\\" & reprInt(ord(x))
+  of '\127' .. '\255', '\0'..'\31': add result, "\\" & reprInt(ord(x))
   else: add result, x
   add result, "\'"
 

lib/system/reprjs.nim

@@ -44,7 +44,7 @@ proc reprChar(x: char): string {.compilerRtl.} =
   case x
   of '"': add(result, "\\\"")
   of '\\': add(result, "\\\\")
-  of '\128'..'\255', '\0'..'\31': add( result, "\\" & reprInt(ord(x)) )
+  of '\127'..'\255', '\0'..'\31': add( result, "\\" & reprInt(ord(x)) )
   else: add(result, x)
   add(result, "\'")
 
@@ -56,7 +56,7 @@ proc reprStrAux(result: var string, s: cstring, len: int) =
     of '"': add(result, "\\\"")
     of '\\': add(result, "\\\\")
     of '\10': add(result, "\\10\"\n\"")
-    of '\128'..'\255', '\0'..'\9', '\11'..'\31':
+    of '\127'..'\255', '\0'..'\9', '\11'..'\31':
       add( result, "\\" & reprInt(ord(c)) )
     else:
       add( result, reprInt(ord(c)) ) # Not sure about this.

lib/system/syslocks.nim

@@ -117,6 +117,12 @@ else:
       when defined(linux) and defined(amd64):
         abi: array[48 div sizeof(clonglong), clonglong]
 
+    SysCondAttr {.importc: "pthread_condattr_t", pure, final
+               header: """#include <sys/types.h>
+                          #include <pthread.h>""".} = object
+      when defined(linux) and defined(amd64):
+        abi: array[4 div sizeof(cint), cint]  # actually a cint
+
     SysLockType = distinct cint
 
   proc initSysLockAux(L: var SysLockObj, attr: ptr SysLockAttr) {.
@@ -185,7 +191,7 @@ else:
       importc: "pthread_mutexattr_settype", header: "<pthread.h>", noSideEffect.}
 
   else:
-    proc initSysCondAux(cond: var SysCondObj, cond_attr: pointer) {.
+    proc initSysCondAux(cond: var SysCondObj, cond_attr: ptr SysCondAttr = nil) {.
       importc: "pthread_cond_init", header: "<pthread.h>", noSideEffect.}
     proc deinitSysCondAux(cond: var SysCondObj) {.noSideEffect,
       importc: "pthread_cond_destroy", header: "<pthread.h>".}
@@ -196,7 +202,7 @@ else:
       importc: "pthread_cond_signal", header: "<pthread.h>", noSideEffect.}
 
     when defined(ios):
-      proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) =
+      proc initSysCond(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) =
         cond = cast[SysCond](c_malloc(sizeof(SysCondObj)))
         initSysCondAux(cond[], cond_attr)
 
@@ -209,7 +215,7 @@ else:
       template signalSysCond(cond: var SysCond) =
         signalSysCondAux(cond[])
     else:
-      template initSysCond(cond: var SysCond, cond_attr: pointer = nil) =
+      template initSysCond(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) =
         initSysCondAux(cond, cond_attr)
       template deinitSysCond(cond: var SysCond) =
         deinitSysCondAux(cond)

lib/system/threads.nim

@@ -264,7 +264,17 @@ else:
 
     proc getThreadId*(): int =
       result = int(lwp_gettid())
-  elif defined(macosx) or defined(freebsd) or defined(openbsd) or defined(netbsd):
+  elif defined(openbsd):
+    proc getthrid(): int32 {.importc: "getthrid", header: "<unistd.h>".}
+
+    proc getThreadId*(): int =
+      result = int(getthrid())
+  elif defined(netbsd):
+    proc lwp_self(): int32 {.importc: "_lwp_self", header: "<lwp.h>".}
+
+    proc getThreadId*(): int =
+      result = int(lwp_self())
+  elif defined(macosx) or defined(freebsd):
     proc pthread_threadid_np(y: pointer; x: var uint64): cint {.importc, header: "pthread.h".}
 
     proc getThreadId*(): int =