documentation improvements; higher level Mongodb wrapper

config/nimrod.cfg

@@ -66,6 +66,8 @@ hint[LineTooLong]=off
   @else:
     # BSD got posix_spawn only recently, so we deactivate it for osproc:
     define:useFork
+    # at least NetBSD has problems with thread local storage:
+    tlsEmulation:on
   @end
 @end
 

doc/lib.txt

@@ -340,8 +340,11 @@ Database support
   A higher level SQLite database wrapper. The same interface is implemented
   for other databases too.
 
+* `db_mongo <db_mongo.html>`_
+  A higher level **mongodb** wrapper. 
+
 * `mongodb <mongo.html>`_
-  Wrapper for the **mongodb** client C library.
+  Lower level wrapper for the **mongodb** client C library.
   
 
 Other

examples/cgiex.nim

@@ -1,9 +1,6 @@
 # Test/show CGI module
 import strtabs, cgi
 
-
-#setTestData("name", "the andreas", "password", "rumpf\t\ttab")
-
 var myData = readData()
 validateData(myData, "name", "password")
 writeContentType()

lib/impure/db_mongo.nim

@@ -0,0 +1,213 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2012 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements a higher level wrapper for `mongodb`:idx:. Example:
+##
+## .. code-block:: nimrod
+##
+##    import mongo, db_mongo, oids, json
+##
+##    var conn = db_mongo.open()
+##
+##    # construct JSON data:
+##    var data = %{"a": %13, "b": %"my string value", 
+##                 "inner": %{"i": %71} }
+##
+##    var id = insertID(conn, "test.test", data)
+##
+##    for v in find(conn, "test.test", "this.a == 13"):
+##      print v
+##
+##    delete(conn, "test.test", id)
+##    close(conn)
+
+import mongo, oids, json
+
+type
+  EDb* = object of EIO ## exception that is raised if a database error occurs
+  TDbConn* = TMongo    ## a database connection; alias for ``TMongo``
+
+proc dbError*(db: TDbConn, msg: string) {.noreturn.} = 
+  ## raises an EDb exception with message `msg`.
+  var e: ref EDb
+  new(e)
+  if db.errstr[0] != '\0':
+    e.msg = $db.errstr
+  else:
+    e.msg = $db.err & " " & msg
+  raise e
+
+proc Close*(db: var TDbConn) = 
+  ## closes the database connection.
+  disconnect(db)
+  destroy(db)
+
+proc Open*(host: string = defaultHost, port: int = defaultPort): TDbConn =
+  ## opens a database connection. Raises `EDb` if the connection could not
+  ## be established.
+  init(result)
+  
+  let x = connect(result, host, port.cint)
+  if x != 0'i32:
+    dbError(result, "cannot open: " & host)
+
+proc jsonToBSon(b: var TBSon, key: string, j: PJsonNode) =
+  case j.kind
+  of JString:
+    add(b, key, j.str)
+  of JInt:
+    add(b, key, j.num)
+  of JFloat:
+    add(b, key, j.fnum)
+  of JBool:
+    addBool(b, key, ord(j.bval))
+  of JNull:
+    addNull(b, key)
+  of JObject:
+    addStartObject(b, key)
+    for k, v in items(j.fields):
+      jsonToBSon(b, k, v)
+    addFinishObject(b)
+  of JArray:
+    addStartArray(b, key)
+    for i, e in pairs(j.elems):
+      jsonToBSon(b, $i, e)
+    addFinishArray(b)
+
+proc jsonToBSon*(j: PJsonNode, oid: TOid): TBSon =
+  ## converts a JSON value into the BSON format. The result must be
+  ## ``destroyed`` explicitely!
+  init(result)
+  assert j.kind == JObject
+  add(result, "_id", oid)
+  for key, val in items(j.fields):
+    jsonToBSon(result, key, val)
+  finish(result)
+
+proc `[]`*(obj: var TBSon, fieldname: cstring): TBSon =
+  ## retrieves the value belonging to `fieldname`. Raises `EInvalidKey` if
+  ## the attribute does not exist.
+  var it = initIter(obj)
+  let res = find(it, result, fieldname)
+  if res == bkEOO:
+    raise newException(EInvalidIndex, "key not in object")
+
+proc getId*(obj: var TBSon): TOid =
+  ## retrieves the ``_id`` attribute of `obj`.
+  var it = initIter(obj)
+  var b: TBSon
+  let res = find(it, b, "_id")
+  if res == bkOID:
+    result = oidVal(it)[]
+  else:
+    raise newException(EInvalidIndex, "_id not in object")
+
+proc insertID*(db: var TDbConn, namespace: string, data: PJsonNode): TOid =
+  ## converts `data` to BSON format and inserts it in `namespace`. Returns
+  ## the generated OID for the ``_id`` field.
+  result = genOid()
+  var x = jsonToBSon(data, result)
+  insert(db, namespace, x)
+  destroy(x)
+
+proc insert*(db: var TDbConn, namespace: string, data: PJsonNode) =
+  ## converts `data` to BSON format and inserts it in `namespace`.  
+  discard InsertID(db, namespace, data)
+
+proc update*(db: var TDbConn, namespace: string, obj: var TBSon) =
+  ## updates `obj` in `namespace`.
+  var cond: TBson
+  init(cond)
+  cond.add("_id", getId(obj))
+  finish(cond)
+  update(db, namespace, cond, obj, ord(UPDATE_UPSERT))
+  destroy(cond)
+
+proc update*(db: var TDbConn, namespace: string, oid: TOid, obj: PJsonNode) =
+  ## updates the data with `oid` to have the new data `obj`.
+  var a = jsonToBSon(obj, oid)
+  Update(db, namespace, a)
+  destroy(a)
+
+proc delete*(db: var TDbConn, namespace: string, oid: TOid) =
+  ## Deletes the object belonging to `oid`.
+  var cond: TBson
+  init(cond)
+  cond.add("_id", oid)
+  finish(cond)
+  discard remove(db, namespace, cond)
+  destroy(cond)
+
+proc delete*(db: var TDbConn, namespace: string, obj: var TBSon) =
+  ## Deletes the object `obj`.
+  delete(db, namespace, getId(obj))
+
+iterator find*(db: var TDbConn, namespace: string): var TBSon =
+  ## iterates over any object in `namespace`.
+  var cursor: TCursor
+  init(cursor, db, namespace)
+  while next(cursor) == mongo.OK:
+    yield bson(cursor)[]
+  destroy(cursor)
+
+iterator find*(db: var TDbConn, namespace: string, 
+               query, fields: var TBSon): var TBSon =
+  ## yields the `fields` of any document that suffices `query`.
+  var cursor = find(db, namespace, query, fields, 0'i32, 0'i32, 0'i32)
+  if cursor != nil:
+    while next(cursor[]) == mongo.OK:
+      yield bson(cursor[])[]
+    destroy(cursor[])
+
+proc setupFieldnames(fields: openArray[string]): TBSon =
+  init(result)
+  for x in fields: add(result, x, 1'i32)
+  finish(result)
+
+iterator find*(db: var TDbConn, namespace: string, 
+               query: var TBSon, fields: openArray[string]): var TBSon =
+  ## yields the `fields` of any document that suffices `query`. If `fields` 
+  ## is ``[]`` the whole document is yielded.
+  var f = setupFieldnames(fields)
+  var cursor = find(db, namespace, query, f, 0'i32, 0'i32, 0'i32)
+  if cursor != nil:
+    while next(cursor[]) == mongo.OK:
+      yield bson(cursor[])[]
+    destroy(cursor[])
+  destroy(f)
+
+proc setupQuery(query: string): TBSon =
+  init(result)
+  add(result, "$where", query)
+  finish(result)
+
+iterator find*(db: var TDbConn, namespace: string, 
+               query: string, fields: openArray[string]): var TBSon =
+  ## yields the `fields` of any document that suffices `query`. If `fields` 
+  ## is ``[]`` the whole document is yielded.
+  var f = setupFieldnames(fields)
+  var q = setupQuery(query)
+  var cursor = find(db, namespace, q, f, 0'i32, 0'i32, 0'i32)
+  if cursor != nil:
+    while next(cursor[]) == mongo.OK:
+      yield bson(cursor[])[]
+    destroy(cursor[])
+  destroy(q)
+  destroy(f)
+
+when false:
+  # this doesn't work this way; would require low level hacking
+  iterator fieldPairs*(obj: var TBSon): tuple[key: cstring, value: TBSon] =
+    ## iterates over `obj` and yields all (key, value)-Pairs.
+    var it = initIter(obj)
+    var v: TBSon
+    while next(it) != bkEOO:
+      let key = key(it)
+      discard init(v, value(it))
+      yield (key, v)

lib/pure/json.nim

@@ -495,16 +495,16 @@ type
     JArray
     
   PJsonNode* = ref TJsonNode ## JSON node 
-  TJsonNode* {.final, pure.} = object
+  TJsonNode* {.final, pure, acyclic.} = object
     case kind*: TJsonNodeKind
     of JString:
-      str*: String
+      str*: string
     of JInt:
       num*: biggestInt
     of JFloat:
-      fnum: Float
+      fnum*: float
     of JBool:
-      bval*: Bool
+      bval*: bool
     of JNull:
       nil
     of JObject:
@@ -558,6 +558,45 @@ proc newJArray*(): PJsonNode =
   result.kind = JArray
   result.elems = @[]
 
+
+proc `%`*(s: string): PJsonNode =
+  ## Generic constructor for JSON data. Creates a new `JString PJsonNode`.
+  new(result)
+  result.kind = JString
+  result.str = s
+
+proc `%`*(n: biggestInt): PJsonNode =
+  ## Generic constructor for JSON data. Creates a new `JInt PJsonNode`.
+  new(result)
+  result.kind = JInt
+  result.num  = n
+
+proc `%`*(n: float): PJsonNode =
+  ## Generic constructor for JSON data. Creates a new `JFloat PJsonNode`.
+  new(result)
+  result.kind = JFloat
+  result.fnum  = n
+
+proc `%`*(b: bool): PJsonNode =
+  ## Generic constructor for JSON data. Creates a new `JBool PJsonNode`.
+  new(result)
+  result.kind = JBool
+  result.bval = b
+
+proc `%`*(keyVals: openArray[tuple[key: string, val: PJsonNode]]): PJsonNode =
+  ## Generic constructor for JSON data. Creates a new `JObject PJsonNode`
+  new(result)
+  result.kind = JObject
+  newSeq(result.fields, keyVals.len)
+  for i, p in pairs(keyVals): result.fields[i] = p
+
+proc `%`*(elements: openArray[PJSonNode]): PJsonNode =
+  ## Generic constructor for JSON data. Creates a new `JArray PJsonNode`
+  new(result)
+  result.kind = JArray
+  newSeq(result.elems, elements.len)
+  for i, p in pairs(elements): result.elems[i] = p
+
 proc len*(n: PJsonNode): int = 
   ## If `n` is a `JArray`, it returns the number of elements.
   ## If `n` is a `JObject`, it returns the number of pairs.

lib/pure/os.nim

@@ -1355,6 +1355,7 @@ when defined(macosx):
 
 proc getAppFilename*(): string {.rtl, extern: "nos$1".} =
   ## Returns the filename of the application's executable.
+  ## **Note**: This does not work reliably on BSD.
 
   # Linux: /proc/<pid>/exe
   # Solaris:
@@ -1410,6 +1411,7 @@ proc getApplicationDir*(): string {.rtl, extern: "nos$1", deprecated.} =
 
 proc getAppDir*(): string {.rtl, extern: "nos$1".} =
   ## Returns the directory of the application's executable.
+  ## **Note**: This does not work reliably on BSD.
   result = splitFile(getAppFilename()).dir
 
 proc sleep*(milsecs: int) {.rtl, extern: "nos$1".} =

lib/pure/pegs.nim

@@ -1486,10 +1486,10 @@ proc primary(p: var TPegParser): TPeg =
     return !primary(p)
   of tkAt:
     getTok(p)
-    return @primary(p)
+    return @(primary(p))
   of tkCurlyAt:
     getTok(p)
-    return @@primary(p).token(p)
+    return @@(primary(p).token(p))
   else: nil
   case p.tok.kind
   of tkIdentifier:

tests/compile/tmongo.nim

@@ -0,0 +1,16 @@
+
+import mongo, db_mongo, oids, json
+
+var conn = db_mongo.open()
+
+var data = %{"a": %13, "b": %"my string value", 
+             "inner": %{"i": %71} }
+
+var id = insertID(conn, "test.test", data)
+
+for v in find(conn, "test.test", "this.a == 13"):
+  print v
+
+delete(conn, "test.test", id)
+
+close(conn)

todo.txt

@@ -1,6 +1,7 @@
 version 0.9.0
 =============
 
+- sigil-like operator: @x.abc parsed as (@x).abc
 - document AVR/embedded systems better
 - implement ``--script:sh|bat`` command line option
 - make GC realtime capable: GC_step(ms: int)
@@ -14,7 +15,6 @@ version 0.9.0
   - test constant closures
 
 - dead code elim for JS backend; 'of' operator for JS backend
-- const ptr/ref
 - unsigned ints and bignums; requires abstract integer literal type: 
   use tyInt+node for that
 - implement the high level optimizer
@@ -25,10 +25,7 @@ version 0.9.0
 - we need to support iteration of 2 different data structures in parallel
 - make exceptions compatible with C++ exceptions
 - change how comments are part of the AST
-- optional indentation for 'case' statement; hm, keep in mind other syntax
-  changes that people want; may turn out to be a bad idea
 - activate more thread tests
-- optimize method dispatchers
 - rethink the syntax: distinction between expr and stmt is unfortunate; 
   indentation handling is quite complex too; problem with exception handling
   is that often the scope of ``try`` is wrong and apart from that ``try`` is
@@ -63,18 +60,11 @@ version 0.9.XX
 - implicit ref/ptr->var conversion; the compiler may store an object
   implicitly on the heap for write barrier efficiency; better: 
   proc specialization in the code gen
-- shared memory heap: ``shared ref`` etc. The only hard part in the GC is to
-  "stop the world". However, it may be worthwhile to generate explicit 
-  (or implicit) syncGC() calls in loops. Automatic loop injection seems
-  troublesome, but maybe we can come up with a simple heuristic. (All procs
-  that `new` shared memory are syncGC() candidates...)
 - EcmaScript needs a new and better code gen: simply adapt the C code gen to it
 - tlastmod returns wrong results on BSD (Linux, MacOS X: works)
 - nested tuple unpacking; tuple unpacking in non-var-context
 - 'nimrod def': does not always work?
-- implement/generalize the effect system
 - test branch coverage
-- checked exceptions
 - make pegs support a compile-time option and make c2nim use regexes instead
   per default?
 - 'const' objects including case objects
@@ -103,7 +93,6 @@ version 0.9.XX
 Library
 -------
 
-- wrappers for mongodb
 - suffix trees
 - locale support; i18n module
 - bignums
@@ -144,6 +133,7 @@ Low priority
 - implement explicit varargs; **but** ``len(varargs)`` problem remains! 
   --> solve by implicit conversion from varargs to openarray
 - implement closures that support nesting > 1
+- optimize method dispatchers
 
 
 Further optimization ideas
@@ -153,8 +143,21 @@ Further optimization ideas
   information inlining would provide, but don't inline for code size reasons.
 
 
-Version 2
+Version 2 and beyond
-=========
+====================
+
+
+- shared memory heap: ``shared ref`` etc. The only hard part in the GC is to
+  "stop the world". However, it may be worthwhile to generate explicit 
+  (or implicit) syncGC() calls in loops. Automatic loop injection seems
+  troublesome, but maybe we can come up with a simple heuristic. (All procs
+  that `new` shared memory are syncGC() candidates...)
+
+- implement/generalize the effect system; checked exceptions
+- const ptr/ref
+
+- optional indentation for 'case' statement; hm, keep in mind other syntax
+  changes that people want; may turn out to be a bad idea
 
 - language change: inheritance should only work with reference types, so that
   the ``type`` field is not needed for objects! --> zero overhead aggregation

web/download.txt

@@ -9,10 +9,10 @@ Please choose your platform:
 
 The source-based installation has been tested on these systems:
 * Linux: i386, AMD64, PowerPC
+* BSD: AMD64
 * Mac OS X: i386, AMD64
 
-Other UNIX-based operating systems may work. An older version was tested on 
+Other UNIX-based operating systems may work.
-FreeBSD (i386).
 
 .. include:: ../install.txt
 

web/index.txt

@@ -18,13 +18,11 @@ Welcome to Nimrod
   *Nimrod looks like this:*
   
   .. code-block:: nimrod
-    # Filter key=value pairs
+    import strutils
-    import re
+    
-
+    echo "List of ints (separate by whitespace): "
-    for x in lines("myfile.txt"):
+    echo stdin.readLine.split.each(parseInt).max, 
-      if x =~ re"(\w+)=(.*)":
+      " is the maximum"
-        echo "Key: ", matches[0],
-             " Value: ", matches[1]
 
 **Nimrod** is a new statically typed, imperative
 programming language, that supports procedural, object oriented, functional 

web/news.txt

@@ -26,6 +26,7 @@ Library Additions
 - Added ``macros.emit`` that can emit an arbitrary computed string as nimrod
   code during compilation.
 - Added ``strutils.parseEnum``.
+- Added ``json.%`` constructor operator.
 - The stdlib can now be avoided to a point where C code generation for 16bit 
   micro controllers is feasible.
 - Added module ``oids``.
@@ -55,6 +56,8 @@ Compiler Additions
 ------------------
 
 - Win64 is now an officially supported target.
+- The Nimrod compiler works on BSD again, but has some issues 
+  as ``os.getAppFilename`` and ``os.getAppDir`` cannot work reliably on BSD.
 - The compiler can detect and evaluate calls that can be evaluated at compile
   time for optimization purposes with the ``--implicitStatic`` command line
   option or pragma.

web/nimrod.ini

@@ -33,7 +33,7 @@ srcdoc: "pure/parseopt;pure/hashes;pure/strtabs;pure/lexbase"
 srcdoc: "pure/parsecfg;pure/parsexml;pure/parsecsv;pure/parsesql"
 srcdoc: "pure/streams;pure/terminal;pure/cgi;impure/web;pure/unicode"
 srcdoc: "impure/zipfiles;pure/htmlgen;pure/parseutils;pure/browsers"
-srcdoc: "impure/db_postgres;impure/db_mysql;impure/db_sqlite"
+srcdoc: "impure/db_postgres;impure/db_mysql;impure/db_sqlite;impure/db_mongo"
 srcdoc: "pure/httpserver;pure/httpclient;pure/smtp;impure/ssl"
 srcdoc: "pure/ropes;pure/unidecode/unidecode;pure/xmldom;pure/xmldomparser"
 srcdoc: "pure/xmlparser;pure/htmlparser;pure/xmltree;pure/colors"