Merge branch 'devel' of https://github.com/Araq/Nimrod into devel

doc/lib.txt

@@ -223,6 +223,30 @@ Internet Protocols and Support
 * `uri <uri.html>`_
   This module provides functions for working with URIs.
 
+* `asyncdispatch <asyncdispatch.html>`_
+  This module implements an asynchronous dispatcher for IO operations.
+
+  **Note:** This module is still largely experimental.
+
+* `asyncnet <asyncnet.html>`_
+  This module implements asynchronous sockets based on the ``asyncdispatch``
+  module.
+
+  **Note:** This module is still largely experimental.
+
+* `asynchttpserver <asynchttpserver.html>`_
+  This module implements an asynchronous HTTP server using the ``asyncnet``
+  module.
+
+  **Note:** This module is still largely experimental.
+
+* `net <net.html>`_
+  This module implements a high-level sockets API. It will replace the
+  ``sockets`` module in the future.
+
+* `rawsockets <rawsockets.html>`_
+  This module implements a low-level sockets API.
+
 Parsers
 -------
 

lib/nimbase.h

@@ -376,7 +376,7 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); }
 #  define GC_GUARD
 #endif
 
-/* Test to see if nimrod and the C compiler agrees on the size of a pointer.
+/* Test to see if nimrod and the C compiler agree on the size of a pointer.
    On disagreement, your C compiler will say something like: 
    "error: 'assert_numbits' declared as an array with a negative size" */
 typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1];

lib/pure/asyncdispatch.nim

@@ -13,13 +13,19 @@ import rawsockets
 export TPort
 
 ## AsyncDispatch
-## --------
+## -------------
 ##
 ## This module implements a brand new dispatcher based on Futures.
-## On Windows IOCP is used and on other operating systems the selectors module
+## On Windows IOCP is used and on other operating systems the ``selectors``
-## is used instead.
+## module is used instead.
+##
+## **Note:** This module is still largely experimental.
+
 
 # TODO: Discarded void PFutures need to checked for exception.
+# TODO: Exceptions are currently uncatchable due to the limitation that 
+# you cannot have yield in a try stmt. Perhaps I can get the macro to put
+# a user's try except around ``future.read``.
 
 # -- Futures
 
@@ -394,7 +400,6 @@ when defined(windows) or defined(nimdoc):
           size
         else:
           bytesReceived
-      assert dataBuf.buf[0] != '\0'
       var data = newString(realSize)
       copyMem(addr data[0], addr dataBuf.buf[0], realSize)
       retFuture.complete($data)

lib/pure/asynchttpserver.nim

@@ -8,6 +8,8 @@
 #
 
 ## This module implements a high performance asynchronous HTTP server.
+##
+## **Note:** This module is still largely experimental.
 
 import strtabs, asyncnet, asyncdispatch, parseutils, parseurl, strutils
 type

lib/pure/asyncnet.nim

@@ -6,6 +6,44 @@
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 #
+
+## This module implements a high-level asynchronous sockets API based on the
+## asynchronous dispatcher defined in the ``asyncdispatch`` module.
+##
+## Example
+## =======
+## 
+## The following example demonstrates a simple chat server.
+##
+## .. code-block::nimrod
+##
+##   import asyncnet, asyncdispatch
+##
+##   var clients: seq[PAsyncSocket] = @[]
+##
+##   proc processClient(client: PAsyncSocket) {.async.} =
+##     while true:
+##       let line = await client.recvLine()
+##       for c in clients:
+##         await c.send(line & "\c\L")
+##
+##   proc serve() {.async.} =
+##     var server = newAsyncSocket()
+##     server.bindAddr(TPort(12345))
+##     server.listen()
+##
+##     while true:
+##       let client = await server.accept()
+##       clients.add client
+##
+##       processClient(client)
+##
+##   serve()
+##   runForever()
+##
+##
+## **Note:** This module is still largely experimental.
+
 import asyncdispatch
 import rawsockets
 import net

lib/pure/httpclient.nim

@@ -435,15 +435,25 @@ type
     connected: bool
     currentURL: TURL ## Where we are currently connected.
     headers: PStringTable
+    maxRedirects: int
     userAgent: string
 
-proc newAsyncHttpClient*(): PAsyncHttpClient =
+proc newAsyncHttpClient*(userAgent = defUserAgent,
+    maxRedirects = 5): PAsyncHttpClient =
+  ## Creates a new PAsyncHttpClient instance.
+  ##
+  ## ``userAgent`` specifies the user agent that will be used when making
+  ## requests.
+  ##
+  ## ``maxRedirects`` specifies the maximum amount of redirects to follow,
+  ## default is 5.
   new result
   result.headers = newStringTable(modeCaseInsensitive)
   result.userAgent = defUserAgent
+  result.maxRedirects = maxRedirects
 
 proc close*(client: PAsyncHttpClient) =
-  ## Closes any connections held by the HttpClient.
+  ## Closes any connections held by the HTTP client.
   if client.connected:
     client.socket.close()
     client.connected = false
@@ -588,6 +598,14 @@ proc newConnection(client: PAsyncHttpClient, url: TURL) {.async.} =
 
 proc request*(client: PAsyncHttpClient, url: string, httpMethod = httpGET,
               body = ""): PFuture[TResponse] {.async.} =
+  ## Connects to the hostname specified by the URL and performs a request
+  ## using the method specified.
+  ##
+  ## Connection will kept alive. Further requests on the same ``client`` to
+  ## the same hostname will not require a new connection to be made. The
+  ## connection can be closed by using the ``close`` procedure.
+  ##
+  ## The returned future will complete once the request is completed.
   let r = parseUrl(url)
   await newConnection(client, r)
 
@@ -602,6 +620,19 @@ proc request*(client: PAsyncHttpClient, url: string, httpMethod = httpGET,
   
   result = await parseResponse(client, httpMethod != httpHEAD)
 
+proc get*(client: PAsyncHttpClient, url: string): PFuture[TResponse] {.async.} =
+  ## Connects to the hostname specified by the URL and performs a GET request.
+  ##
+  ## This procedure will follow redirects up to a maximum number of redirects
+  ## specified in ``newAsyncHttpClient``.
+  result = await client.request(url, httpGET)
+  var lastURL = url
+  for i in 1..client.maxRedirects:
+    if result.status.redirection():
+      let redirectTo = getNewLocation(lastURL, result.headers)
+      result = await client.request(redirectTo, httpGET)
+      lastUrl = redirectTo
+
 when isMainModule:
   when true:
     # Async

lib/pure/math.nim

@@ -75,28 +75,30 @@ proc binom*(n, k: int): int {.noSideEffect.} =
     result = (result * (n + 1 - i)) div i
     
 proc fac*(n: int): int {.noSideEffect.} = 
-  ## computes the faculty function
+  ## computes the faculty/factorial function.
   result = 1
   for i in countup(2, n):
     result = result * i
 
 proc isPowerOfTwo*(x: int): bool {.noSideEffect.} =
-  ## returns true, if x is a power of two, false otherwise.
+  ## returns true, if `x` is a power of two, false otherwise.
-  ## Negative numbers are not a power of two.
+  ## Zero and negative numbers are not a power of two.
-  return (x and -x) == x
+  return (x != 0) and ((x and (x - 1)) == 0)
 
-proc nextPowerOfTwo*(x: int): int =
+proc nextPowerOfTwo*(x: int): int {.noSideEffect.} =
-  ## returns the nearest power of two, so that
+  ## returns `x` rounded up to the nearest power of two.
-  ## result**2 >= x > (result-1)**2.
+  ## Zero and negative numbers get rounded up to 1.
   result = x - 1 
   when defined(cpu64):
     result = result or (result shr 32)
+  when sizeof(int) > 16:
     result = result or (result shr 16)
+  when sizeof(int) > 8:
     result = result or (result shr 8)
   result = result or (result shr 4)
   result = result or (result shr 2)
   result = result or (result shr 1)
-  inc(result)
+  result += 1 + ord(x<=0)
 
 proc countBits32*(n: int32): int {.noSideEffect.} =
   ## counts the set bits in `n`.

lib/pure/net.nim

@@ -12,6 +12,9 @@
 {.deadCodeElim: on.}
 import rawsockets, os, strutils, unsigned, parseutils, times
 export TPort
+
+const useWinVersion = defined(Windows) or defined(nimdoc)
+
 type
   IpAddressFamily* {.pure.} = enum ## Describes the type of an IP address
     IPv6, ## IPv6 address
@@ -499,7 +502,7 @@ proc socketError*(socket: PSocket, err: int = -1, async = false) =
   if err == -1 and not (when defined(ssl): socket.isSSL else: false):
     let lastError = osLastError()
     if async:
-      when defined(windows):
+      when useWinVersion:
         if lastError.int32 == WSAEWOULDBLOCK:
           return
         else: osError(lastError)
@@ -525,7 +528,7 @@ proc bindAddr*(socket: PSocket, port = TPort(0), address = "") {.
 
   if address == "":
     var name: TSockaddr_in
-    when defined(windows):
+    when useWinVersion:
       name.sin_family = toInt(AF_INET).int16
     else:
       name.sin_family = toInt(AF_INET)
@@ -1009,7 +1012,7 @@ proc send*(socket: PSocket, data: pointer, size: int): int {.
     if socket.isSSL:
       return SSLWrite(socket.sslHandle, cast[cstring](data), size)
   
-  when defined(windows) or defined(macosx):
+  when useWinVersion or defined(macosx):
     result = send(socket.fd, data, size.cint, 0'i32)
   else:
     when defined(solaris): 
@@ -1088,7 +1091,7 @@ proc connectAsync(socket: PSocket, name: string, port = TPort(0),
       break
     else:
       lastError = osLastError()
-      when defined(windows):
+      when useWinVersion:
         # Windows EINTR doesn't behave same as POSIX.
         if lastError.int32 == WSAEWOULDBLOCK:
           success = true

lib/pure/parseurl.nim

@@ -1,13 +1,16 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2010 Dominik Picheta
+#        (c) Copyright 2014 Dominik Picheta
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 #
 
 ## Parses & constructs URLs.
+##
+## **Note**: This module will be deprecated in the future and merged into a
+## new ``url`` module.
 
 import strutils
 

lib/pure/selectors.nim

@@ -204,9 +204,9 @@ elif defined(windows):
     
     var retCode = 0
     if timeout != -1:
-      retCode = int(select(TSocketHandle(m+1), addr(rd), addr(wr), nil, addr(tv)))
+      retCode = int(select(cint(m+1), addr(rd), addr(wr), nil, addr(tv)))
     else:
-      retCode = int(select(TSocketHandle(m+1), addr(rd), addr(wr), nil, nil))
+      retCode = int(select(cint(m+1), addr(rd), addr(wr), nil, nil))
     
     if retCode < 0:
       OSError(OSLastError())
@@ -242,7 +242,7 @@ when isMainModule:
       sock: TSocket
   
   var sock = socket()
-  sock.setBlocking(false)
+  #sock.setBlocking(false)
   sock.connect("irc.freenode.net", TPort(6667))
   
   var selector = newSelector()

lib/pure/uri.nim

@@ -1,11 +1,15 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2012 Dominik Picheta
+#        (c) Copyright 2014 Dominik Picheta
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 #
+
+## **Note**: This module will be deprecated in the future and merged into a
+## new ``url`` module.
+
 import strutils
 type
   TUrl* = distinct string

web/news.txt

@@ -3,94 +3,149 @@ News
 ====
 
 
-..
-    2014-XX-XX Version 0.9.4 released
-    =================================
 
+2014-XX-XX Version 0.9.4 released
+=================================
 
-    Bugfixes
+The Nimrod development community is proud to announce the release of version
-    --------
+0.9.4 of the Nimrod compiler and tools.
 
+This release includes about 1300 changes in total including various bug
+fixes, new languages features and standard library additions and improvements.
+This release brings with it support for user-defined type classes, a brand
+new VM for executing Nimrod code at compile-time and new symbol binding
+rules for clean templates.
+It also introduces support for the brand new
+`Babel package manager <https://github.com/nimrod-code/babel>`_ which
+has itself seen its first release recently. Many of the wrappers that were
+present in the standard library have been moved to separate repositories
+and should now be installed using Babel.
 
-    Library Additions
+Apart from that a new **experimental** Asynchronous IO API has been added via
-    -----------------
+the ``asyncdispatch`` and ``asyncnet`` modules. The ``net`` and ``rawsockets``
+modules have also been added and they will likely replace the sockets
+module in the next release. The Asynchronous IO API has been designed to
+take advantage of Linux's epoll and Windows' IOCP APIs, support for BSD's
+kqueue has not been implemented yet but will be in the future.
+The Asynchronous IO API provides both
+a callback interface and an interface which allows you to write code as you
+would if you were writing synchronous code. The latter is done through
+the use of an ``await`` keyword which behaves similar to C#'s await. The
+following is a very simple chat server demonstrating Nimrod's new async
+capabilities.
 
-    - Added ``macros.genSym`` builtin for AST generation.
+.. code-block::nimrod
-    - Added ``macros.newLit`` procs for easier AST generation.
+  import asyncnet, asyncdispatch
 
+  var clients: seq[PAsyncSocket] = @[]
 
-    Changes affecting backwards compatibility
+  proc processClient(client: PAsyncSocket) {.async.} =
-    -----------------------------------------
+    while true:
+      let line = await client.recvLine()
+      for c in clients:
+        await c.send(line & "\c\L")
 
-    - The scoping rules for the ``if`` statement changed for better interaction 
+  proc serve() {.async.} =
+    var server = newAsyncSocket()
+    server.bindAddr(TPort(12345))
+    server.listen()
+
+    while true:
+      let client = await server.accept()
+      clients.add client
+
+      processClient(client)
+
+  serve()
+  runForever()
+
+Library Additions
+-----------------
+
+- Added ``macros.genSym`` builtin for AST generation.
+- Added ``macros.newLit`` procs for easier AST generation.
+- Added module ``logging``.
+- Added module ``asyncdispatch``.
+- Added module ``asyncnet``.
+- Added module ``net``.
+- Added module ``rawsockets``.
+- Added module ``selectors``.
+- Added module ``asynchttpserver``.
+- Added support for the new asynchronous IO in the ``httpclient`` module.
+
+Changes affecting backwards compatibility
+-----------------------------------------
+
+- The scoping rules for the ``if`` statement changed for better interaction 
   with the new syntactic construct ``(;)``.
-    - ``OSError`` family of procedures has been deprecated. Procedures with the same
+- ``OSError`` family of procedures has been deprecated. Procedures with the same
   name but which take different parameters have been introduced. These procs now
   require an error code to be passed to them. This error code can be retrieved
   using the new ``OSLastError`` proc.
-    - ``os.parentDir`` now returns "" if there is no parent dir.
+- ``os.parentDir`` now returns "" if there is no parent dir.
-    - In CGI scripts stacktraces are shown to the user only 
+- In CGI scripts stacktraces are shown to the user only 
   if ``cgi.setStackTraceStdout`` is used.
-    - The symbol binding rules for clean templates changed: ``bind`` for any
+- The symbol binding rules for clean templates changed: ``bind`` for any
   symbol that's not a parameter is now the default. ``mixin`` can be used
   to require instantiation scope for a symbol.
-    - ``quoteIfContainsWhite`` now escapes argument in such way that it can be safely
+- ``quoteIfContainsWhite`` now escapes argument in such way that it can be safely
   passed to shell, instead of just adding double quotes.
-    - ``macros.dumpTree`` and ``macros.dumpLisp`` have been made ``immediate``,
+- ``macros.dumpTree`` and ``macros.dumpLisp`` have been made ``immediate``,
   ``dumpTreeImm`` and ``dumpLispImm`` are now deprecated.
-    - The ``nil`` statement has been deprecated, use an empty ``discard`` instead.
+- The ``nil`` statement has been deprecated, use an empty ``discard`` instead.
-    - ``sockets.select`` now prunes sockets that are **not** ready from the list
+- ``sockets.select`` now prunes sockets that are **not** ready from the list
   of sockets given to it.
-    - The ``noStackFrame`` pragma has been renamed to ``asmNoStackFrame`` to
+- The ``noStackFrame`` pragma has been renamed to ``asmNoStackFrame`` to
   ensure you only use it when you know what you're doing.
 
 
-    Compiler Additions
+Compiler Additions
-    ------------------
+------------------
 
-    - The compiler can now warn about "uninitialized" variables. (There are no
+- The compiler can now warn about "uninitialized" variables. (There are no
   real uninitialized variables in Nimrod as they are initialized to binary
   zero). Activate via ``{.warning[Uninit]:on.}``.
-    - The compiler now enforces the ``not nil`` constraint.
+- The compiler now enforces the ``not nil`` constraint.
-    - The compiler now supports a ``codegenDecl`` pragma for even more control
+- The compiler now supports a ``codegenDecl`` pragma for even more control
   over the generated code.
-    - The compiler now supports a ``computedGoto`` pragma to support very fast
+- The compiler now supports a ``computedGoto`` pragma to support very fast
   dispatching for interpreters and the like.
-    - The old evaluation engine has been replaced by a proper register based
+- The old evaluation engine has been replaced by a proper register based
   virtual machine. This fixes numerous bugs for ``nimrod i`` and for macro
   evaluation.
-    - ``--gc:none`` produces warnings when code uses the GC.
+- ``--gc:none`` produces warnings when code uses the GC.
-    - A ``union`` pragma for better C interoperability is now supported.
+- A ``union`` pragma for better C interoperability is now supported.
-    - A ``packed`` pragma to control the memory packing/alignment of fields in 
+- A ``packed`` pragma to control the memory packing/alignment of fields in 
   an object.
-    - Arrays can be annotated to be ``unchecked`` for easier low level
+- Arrays can be annotated to be ``unchecked`` for easier low level
   manipulations of memory.
+- Support for the new Babel package manager.
 
 
-    Language Additions
+Language Additions
-    ------------------
+------------------
 
-    - Arrays can now be declared with a single integer literal ``N`` instead of a
+- Arrays can now be declared with a single integer literal ``N`` instead of a
   range; the range is then ``0..N-1``.
-    - Added ``requiresInit`` pragma to enforce explicit initialization.
+- Added ``requiresInit`` pragma to enforce explicit initialization.
-    - Exported templates are allowed to access hidden fields.
+- Exported templates are allowed to access hidden fields.
-    - The ``using statement`` enables you to more easily author domain-specific
+- The ``using statement`` enables you to more easily author domain-specific
   languages and libraries providing OOP-like syntactic sugar.
-    - Added the possibility to override various dot operators in order to handle
+- Added the possibility to override various dot operators in order to handle
   calls to missing procs and reads from undeclared fields at compile-time.
-    - The overload resolution now supports ``static[T]`` params that must be
+- The overload resolution now supports ``static[T]`` params that must be
   evaluable at compile-time.
-    - Support for user-defined type classes has been added.
+- Support for user-defined type classes has been added.
-    - The *command syntax* is supported in a lot more contexts.
+- The *command syntax* is supported in a lot more contexts.
-    - Anonymous iterators are now supported and iterators can capture variables
+- Anonymous iterators are now supported and iterators can capture variables
   of an outer proc.
-    - The experimental ``strongSpaces`` parsing mode has been implemented.
+- The experimental ``strongSpaces`` parsing mode has been implemented.
 
 
-    Tools improvements
+Tools improvements
-    ------------------
+------------------
 
-    - c2nim can deal with a subset of C++. Use the ``--cpp`` command line option
+- c2nim can deal with a subset of C++. Use the ``--cpp`` command line option
   to activate.
 
 

web/nimrod.ini

@@ -62,7 +62,8 @@ srcdoc2: "pure/ftpclient;pure/memfiles;pure/subexes;pure/collections/critbits"
 srcdoc2: "pure/asyncio;pure/actors;core/locks;pure/oids;pure/endians;pure/uri"
 srcdoc2: "pure/nimprof;pure/unittest;packages/docutils/highlite"
 srcdoc2: "packages/docutils/rst;packages/docutils/rstast"
-srcdoc2: "packages/docutils/rstgen;pure/logging"
+srcdoc2: "packages/docutils/rstgen;pure/logging;pure/asyncdispatch;pure/asyncnet"
+srcdoc2: "pure/rawsockets;pure/asynchttpserver;pure/net"
 
 webdoc: "wrappers/libcurl;pure/md5;wrappers/mysql;wrappers/iup"
 webdoc: "wrappers/sqlite3;wrappers/postgres;wrappers/tinyc"