first steps to explicit channels for thread communication; added mainThreadId

compiler/lexer.nim

@@ -267,7 +267,7 @@ proc GetNumber(L: var TLexer): TToken =
         inc(L.bufpos)
       matchUnderscoreChars(L, result, {'0'..'9'})
   endpos = L.bufpos
-  if L.buf[endpos] == '\'': 
+  if L.buf[endpos] == '\'':
     #matchUnderscoreChars(L, result, ['''', 'f', 'F', 'i', 'I', '0'..'9']);
     inc(endpos)
     L.bufpos = pos            # restore position
@@ -281,7 +281,7 @@ proc GetNumber(L: var TLexer): TToken =
         result.tokType = tkFloat32Lit
         inc(endpos, 2)
       else: 
-        lexMessage(L, errInvalidNumber, result.literal)
+        lexMessage(L, errInvalidNumber, result.literal & "'f" & L.buf[endpos])
     of 'i', 'I': 
       inc(endpos)
       if (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'): 
@@ -297,9 +297,9 @@ proc GetNumber(L: var TLexer): TToken =
         result.tokType = tkInt8Lit
         inc(endpos)
       else: 
-        lexMessage(L, errInvalidNumber, result.literal)
+        lexMessage(L, errInvalidNumber, result.literal & "'i" & L.buf[endpos])
-    else: lexMessage(L, errInvalidNumber, result.literal)
+    else: lexMessage(L, errInvalidNumber, result.literal & "'" & L.buf[endpos])
-  else: 
+  else:
     L.bufpos = pos            # restore position
   try: 
     if (L.buf[pos] == '0') and

compiler/nversion.nim

@@ -15,6 +15,6 @@ const
   defaultAsmMarkerSymbol* = '!'
   VersionMajor* = 0
   VersionMinor* = 8
-  VersionPatch* = 12
+  VersionPatch* = 13
   VersionAsString* = $VersionMajor & "." & $VersionMinor & "." & $VersionPatch
 

compiler/semcall.nim

@@ -83,6 +83,13 @@ proc explicitGenericInstError(n: PNode): PNode =
   LocalError(n.info, errCannotInstantiateX, renderTree(n))
   result = n
 
+proc explicitGenericSym(c: PContext, n: PNode, s: PSym): PNode =
+  var x: TCandidate
+  initCandidate(x, s, n)
+  var newInst = generateInstance(c, s, x.bindings, n.info)
+  markUsed(n, s)
+  result = newSymNode(newInst, n.info)
+
 proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode = 
   assert n.kind == nkBracketExpr
   for i in 1..sonsLen(n)-1:
@@ -94,27 +101,30 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
     # number of generic type parameters:
     if safeLen(s.ast.sons[genericParamsPos]) != n.len-1:
       return explicitGenericInstError(n)
+    result = explicitGenericSym(c, n, s)
   elif a.kind == nkSymChoice:
     # choose the generic proc with the proper number of type parameters.
     # XXX I think this could be improved by reusing sigmatch.ParamTypesMatch.
     # It's good enough for now.
-    var candidateCount = 0
+    result = newNodeI(nkSymChoice, n.info)
     for i in countup(0, len(a)-1): 
       var candidate = a.sons[i].sym
       if candidate.kind in {skProc, skMethod, skConverter, skIterator}: 
         # if suffices that the candidate has the proper number of generic 
         # type parameters:
         if safeLen(candidate.ast.sons[genericParamsPos]) == n.len-1:
-          s = candidate
+          result.add(explicitGenericSym(c, n, candidate))
-          inc(candidateCount)
+    # get rid of nkSymChoice if not ambigious:
-    if candidateCount != 1: return explicitGenericInstError(n)
+    if result.len == 1: result = result[0]
+    # candidateCount != 1: return explicitGenericInstError(n)
   else:
     assert false
   
-  var x: TCandidate
+  when false:
-  initCandidate(x, s, n)
+    var x: TCandidate
-  var newInst = generateInstance(c, s, x.bindings, n.info)
+    initCandidate(x, s, n)
-  
+    var newInst = generateInstance(c, s, x.bindings, n.info)
-  markUsed(n, s)
+    
-  result = newSymNode(newInst, n.info)
+    markUsed(n, s)
+    result = newSymNode(newInst, n.info)
 

compiler/seminst.nim

@@ -25,7 +25,7 @@ proc instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable) =
     if t == nil: 
       LocalError(a.info, errCannotInstantiateX, s.name.s)
       break
-    if (t.kind == tyGenericParam): 
+    if t.kind == tyGenericParam: 
       InternalError(a.info, "instantiateGenericParamList: " & q.name.s)
     s.typ = t
     addDecl(c, s)

compiler/semstmts.nim

@@ -152,42 +152,32 @@ proc semCase(c: PContext, n: PNode): PNode =
   closeScope(c.tab)
 
 proc SemReturn(c: PContext, n: PNode): PNode = 
-  var 
-    restype: PType
-    a: PNode                  # temporary assignment for code generator
   result = n
   checkSonsLen(n, 1)
-  if not (c.p.owner.kind in {skConverter, skMethod, skProc, skMacro}): 
+  if c.p.owner.kind notin {skConverter, skMethod, skProc, skMacro}:
     globalError(n.info, errXNotAllowedHere, "\'return\'")
-  if n.sons[0].kind != nkEmpty: 
+  if n.sons[0].kind != nkEmpty:
-    n.sons[0] = SemExprWithType(c, n.sons[0]) # check for type compatibility:
+    # transform ``return expr`` to ``result = expr; return``
-    restype = c.p.owner.typ.sons[0]
+    if c.p.resultSym == nil: InternalError(n.info, "semReturn")
-    if restype != nil: 
+    var a = newNodeI(nkAsgn, n.sons[0].info)
-      a = newNodeI(nkAsgn, n.sons[0].info)
+    addSon(a, newSymNode(c.p.resultSym))
-      n.sons[0] = fitNode(c, restype, n.sons[0])
+    addSon(a, n.sons[0])
-      # optimize away ``return result``, because it would be transformed
+    n.sons[0] = semAsgn(c, a)
-      # to ``result = result; return``:
+    # optimize away ``result = result``:
-      if (n.sons[0].kind == nkSym) and (sfResult in n.sons[0].sym.flags): 
+    if n[0][1].kind == nkSym and sfResult in n[0][1].sym.flags: 
-        n.sons[0] = ast.emptyNode
+      n.sons[0] = ast.emptyNode
-      else: 
-        if (c.p.resultSym == nil): InternalError(n.info, "semReturn")
-        addSon(a, semExprWithType(c, newSymNode(c.p.resultSym)))
-        addSon(a, n.sons[0])
-        n.sons[0] = a
-    else: 
-      localError(n.info, errCannotReturnExpr)
   
 proc SemYield(c: PContext, n: PNode): PNode = 
   result = n
   checkSonsLen(n, 1)
-  if (c.p.owner == nil) or (c.p.owner.kind != skIterator): 
+  if c.p.owner == nil or c.p.owner.kind != skIterator: 
     GlobalError(n.info, errYieldNotAllowedHere)
-  if n.sons[0].kind != nkEmpty: 
+  if n.sons[0].kind != nkEmpty:
     n.sons[0] = SemExprWithType(c, n.sons[0]) # check for type compatibility:
     var restype = c.p.owner.typ.sons[0]
     if restype != nil: 
       n.sons[0] = fitNode(c, restype, n.sons[0])
-      if (n.sons[0].typ == nil): InternalError(n.info, "semYield")
+      if n.sons[0].typ == nil: InternalError(n.info, "semYield")
     else: 
       localError(n.info, errCannotReturnExpr)
   

doc/endb.txt

@@ -6,10 +6,10 @@
 :Version: |nimrodversion|
 
 .. contents::
-
+
-
+
-**Note:** ENDB has not been maintained/tested since several versions. Help if
+**Note:** ENDB has not been maintained/tested since several versions. Help if
-you want this debugger to survive.
+you want this debugger to survive.
 
 Nimrod comes with a platform independant debugger -
 the `Embedded Nimrod Debugger`:idx: (`ENDB`:idx:). The debugger is
@@ -57,7 +57,7 @@ Executing Commands
     Continue execution until the next breakpoint.
 
 ``i``, ``ignore``
-    Continue execution, ignore all breakpoints. This is effectively quitting
+    Continue execution, ignore all breakpoints. This effectively quits
     the debugger and runs the program until it finishes.
 
 
@@ -143,7 +143,7 @@ Data Display Commands
     painfully slow, the debugger uses a *maximal display depth* concept for
     displaying.
 
-    You can alter the *maximal display depth* with the ``maxdisplay``
+    You can alter the maximal display depth with the ``maxdisplay``
     command.
 
 ``maxdisplay`` <natural>

doc/manual.txt

@@ -3153,7 +3153,7 @@ over *fine grained* concurrency.
 Threads and exceptions
 ----------------------
 
-The interaction between threads and exception is simple: A *handled* exception 
+The interaction between threads and exceptions is simple: A *handled* exception 
 in one thread cannot affect any other thread. However, an *unhandled* 
 exception in one thread terminates the whole *process*!
 

doc/tut2.txt

@@ -200,7 +200,7 @@ for any type:
   echo("abc".len) # is the same as echo(len("abc"))
   echo("abc".toUpper())
   echo({'a', 'b', 'c'}.card)
-  stdout.writeln("Hallo") # the same as write(stdout, "Hallo")
+  stdout.writeln("Hallo") # the same as writeln(stdout, "Hallo")
 
 (Another way to look at the method call syntax is that it provides the missing
 postfix notation.)

lib/pure/ropes.nim

@@ -145,17 +145,17 @@ proc rope*(i: BiggestInt): PRope {.rtl, extern: "nro$1BiggestInt".} =
 proc rope*(f: BiggestFloat): PRope {.rtl, extern: "nro$1BiggestFloat".} =
   ## Converts a float to a rope. 
   result = rope($f)
-
-proc disableCache*() {.rtl, extern: "nro$1".} =
-  ## the cache is discarded and disabled. The GC will reuse its used memory.
-  cache = nil
-  cacheEnabled = false
   
 proc enableCache*() {.rtl, extern: "nro$1".} =
   ## Enables the caching of leaves. This reduces the memory footprint at
   ## the cost of runtime efficiency.
   cacheEnabled = true
 
+proc disableCache*() {.rtl, extern: "nro$1".} =
+  ## the cache is discarded and disabled. The GC will reuse its used memory.
+  cache = nil
+  cacheEnabled = false
+
 proc `&`*(a, b: PRope): PRope {.rtl, extern: "nroConcRopeRope".} =
   ## the concatenation operator for ropes.
   if a == nil: 

lib/system.nim

@@ -783,7 +783,7 @@ const
   hasThreadSupport = compileOption("threads")
   hasSharedHeap = defined(boehmgc) # don't share heaps; every thread has its own
 
-when hasThreadSupport and not hasSharedHeap:
+when hasThreadSupport:
   {.pragma: rtlThreadVar, threadvar.}
 else:
   {.pragma: rtlThreadVar.}
@@ -835,7 +835,7 @@ proc insert*[T](x: var seq[T], item: T, i = 0) {.noSideEffect.} =
   setLen(x, xl+1)
   var j = xl-1
   while j >= i:
-    x[j+1] = x[j]
+    shallowCopy(x[j+1], x[j])
     dec(j)
   x[i] = item
 
@@ -1411,7 +1411,8 @@ var
     ## writes an error message and terminates the program. `outOfMemHook` can
     ## be used to raise an exception in case of OOM like so:
     ## 
-    ## code-block:: nimrod
+    ## .. code-block:: nimrod
+    ##
     ##   var gOutOfMem: ref EOutOfMemory
     ##   new(gOutOfMem) # need to be allocated *before* OOM really happened!
     ##   gOutOfMem.msg = "out of memory"

lib/system/inboxes.nim

@@ -205,9 +205,8 @@ proc send*[TMsg](receiver: TThreadId[TMsg], msg: TMsg) =
   var q = cast[PInbox](getInBoxMem(receiver[]))
   sendImpl(q)
 
-proc llRecv(res: pointer, typ: PNimType) =
+proc llRecv(q: PInbox, res: pointer, typ: PNimType) =
   # to save space, the generic is as small as possible
-  var q = cast[PInbox](getInBoxMem())
   acquireSys(q.lock)
   q.ready = true
   while q.count <= 0:
@@ -222,7 +221,8 @@ proc llRecv(res: pointer, typ: PNimType) =
 proc recv*[TMsg](): TMsg =
   ## receives a message from its internal message queue. This blocks until
   ## a message has arrived! You may use ``peek`` to avoid the blocking.
-  llRecv(addr(result), cast[PNimType](getTypeInfo(result)))
+  var q = cast[PInbox](getInBoxMem())
+  llRecv(q, addr(result), cast[PNimType](getTypeInfo(result)))
 
 proc peek*(): int =
   ## returns the current number of messages in the inbox.
@@ -242,3 +242,56 @@ proc ready*[TMsg](t: var TThread[TMsg]): bool =
   var q = cast[PInbox](getInBoxMem(t))
   result = q.ready
 
+# ---------------------- channel support -------------------------------------
+
+type
+  TChannel*[TMsg] = TInbox ## a channel for thread communication
+  TChannelId*[TMsg] = ptr TChannel[TMsg] ## the current implementation uses
+                                         ## a pointer as a channel ID.
+
+proc open*[TMsg](c: var TChannel[TMsg]) =
+  ## opens a channel `c` for inter thread communication.
+  initInbox(addr(c))
+
+proc close*[TMsg](c: var TChannel[TMsg]) =
+  ## closes a channel `c` and frees its associated resources.
+  freeInbox(addr(c))
+
+proc channelId*[TMsg](c: var TChannel[TMsg]): TChannelId[TMsg] {.inline.} =
+  ## returns the channel ID of `c`.
+  result = addr(c)
+  
+proc send*[TMsg](c: var TChannel[TMsg], msg: TMsg) =
+  ## sends a message to a channel. `msg` is deeply copied.
+  var q = cast[PInbox](addr(c))
+  sendImpl(q)
+
+proc send*[TMsg](c: TChannelId[TMsg], msg: TMsg) =
+  ## sends a message to a thread. `msg` is deeply copied.
+  var q = cast[PInbox](c)
+  sendImpl(q)
+
+proc peek*[TMsg](c: var TChannel[TMsg]): int =
+  ## returns the current number of messages in the channel `c`.
+  var q = cast[PInbox](addr(c))
+  lockInbox(q):
+    result = q.count
+
+proc peek*[TMsg](c: TChannelId[TMsg]): int =
+  ## returns the current number of messages in the channel `c`.
+  var q = cast[PInbox](c)
+  lockInbox(q):
+    result = q.count
+
+proc recv*[TMsg](c: TChannelId[TMsg]): TMsg =
+  ## receives a message from the channel `c`. This blocks until
+  ## a message has arrived! You may use ``peek`` to avoid the blocking.
+  var q = cast[PInbox](c)
+  llRecv(q, addr(result), cast[PNimType](getTypeInfo(result)))
+
+proc recv*[TMsg](c: var TChannel[TMsg]): TMsg =
+  ## receives a message from the channel `c`. This blocks until
+  ## a message has arrived! You may use ``peek`` to avoid the blocking.
+  var q = cast[PInbox](addr(c))
+  llRecv(q, addr(result), cast[PNimType](getTypeInfo(result)))
+

lib/system/mmdisp.nim

@@ -133,6 +133,26 @@ when defined(boehmgc):
   proc asgnRefNoCycle(dest: ppointer, src: pointer) {.compilerproc, inline.} =
     dest[] = src
 
+  type
+    TMemRegion = object {.final, pure.}
+  
+  var
+    dummy {.rtlThreadVar.}: int
+  
+  proc rawAlloc(r: var TMemRegion, size: int): pointer =
+    result = boehmAlloc(size)
+    if result == nil: raiseOutOfMem()
+  proc rawAlloc0(r: var TMemRegion, size: int): pointer =
+    result = alloc(size)
+    zeroMem(result, size)
+  proc realloc(r: var TMemRegion, p: Pointer, newsize: int): pointer =
+    result = boehmRealloc(p, newsize)
+    if result == nil: raiseOutOfMem()
+  proc rawDealloc(r: var TMemRegion, p: Pointer) = boehmDealloc(p)
+
+  proc deallocOsPages(r: var TMemRegion) {.inline.} = nil
+  proc deallocOsPages() {.inline.} = nil
+
   include "system/cellsets"
 elif defined(nogc):
   # Even though we don't want the GC, we cannot simply use C's memory manager

lib/system/threads.nim

@@ -266,7 +266,10 @@ proc initInbox(p: pointer)
 proc freeInbox(p: pointer)
 when not defined(boehmgc) and not hasSharedHeap:
   proc deallocOsPages()
-  
+
+when defined(mainThread):
+  initInbox(addr(mainThread.inbox))
+
 template ThreadProcWrapperBody(closure: expr) =
   ThreadVarSetValue(globalsSlot, closure)
   var t = cast[ptr TThread[TMsg]](closure)
@@ -379,6 +382,10 @@ proc myThreadId*[TMsg](): TThreadId[TMsg] =
   ## returns the thread ID of the thread that calls this proc.
   result = cast[TThreadId[TMsg]](ThreadVarGetValue(globalsSlot))
 
+proc mainThreadId*[TMsg](): TThreadId[TMsg] =
+  ## returns the thread ID of the main thread.
+  result = cast[TThreadId[TMsg]](addr(mainThread))
+
 when useStackMaskHack:
   proc runMain(tp: proc () {.thread.}) {.compilerproc.} =
     var mainThread: TThread[pointer]
@@ -388,7 +395,8 @@ when useStackMaskHack:
 # --------------------------- lock handling ----------------------------------
 
 type
-  TLock* = TSysLock ## Nimrod lock; not re-entrant!
+  TLock* = TSysLock ## Nimrod lock; whether this is re-entrant
+                    ## or not is unspecified!
   
 const
   noDeadlocks = false # compileOption("deadlockPrevention")

tests/threads/threadex.nim

@@ -9,7 +9,6 @@ type
 
 var
   consumer: TThread[TMsg]
-  producer: TThread[int]
   printedLines = 0
 
 proc consume() {.thread.} =
@@ -21,7 +20,7 @@ proc consume() {.thread.} =
     echo x.data
     discard atomicInc(printedLines)
 
-proc produce() {.thread.} =
+proc produce() =
   var m: TMsg
   var input = open("readme.txt")
   while not endOfFile(input):
@@ -30,15 +29,14 @@ proc produce() {.thread.} =
       consumer.send(m)
   close(input)
   m.k = mEof
-  m.backTo = myThreadId[int]()
+  m.backTo = mainThreadId[int]()
   consumer.send(m)
   var result = recv[int]()
   echo result
   
 createThread(consumer, consume)
-createThread(producer, produce)
+produce()
 joinThread(consumer)
-joinThread(producer)
 
 echo printedLines
 

todo.txt

@@ -31,7 +31,8 @@ version 0.9.XX
 - distinct types for array/seq indexes
 - GC: marker procs for native Nimrod GC and Boehm GC; precise stack marking
 - implicit ref/ptr->var conversion; the compiler may store an object
-  implicitly on the heap for write barrier efficiency
+  implicitly on the heap for write barrier efficiency; better: 
+  proc specialization in the code gen
 - resizing of strings/sequences could take into account the memory that
   is allocated
 - typeAllowed() for parameters...
@@ -50,13 +51,14 @@ version 0.9.XX
 Library
 -------
 
+- proper URL-parser
+- wrappers for poppler; libharu
 - radix tree for strings; maybe suffix tree
 - locale support
 - bignums
 - ftp (and other internet protocols)
 
 - pdcurses bindings
-- queues additional to streams: have two positions (read/write) instead of one
 
 - for system:
   proc `@` [T](a: openArray[T]): seq[T] = 
@@ -70,7 +72,7 @@ Low priority
 ------------
 
 - ``when T is int`` for generic code
-- ``when validCode( proc ()  )`` for generic code
+- ``when validCode(proc())`` for generic code
 - find a way for easy constructors and destructors; (destructors are much more
   important than constructors)
 - code generated for type information is wasteful

web/news.txt

@@ -22,11 +22,13 @@ Compiler Additions
 Library Additions
 -----------------
 
+- Added ``system.mainThreadId``.
+- Added explicit channels for thread communication.
+
 
 2011-07-10 Version 0.8.12 released
 ==================================
 
-
 Bugfixes
 --------