Merge branch 'devel' of https://github.com/Araq/Nimrod into devel

compiler/ccgexprs.nim

@@ -1060,6 +1060,8 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) =
     t = t.sons[0].skipTypes(abstractInst)
     r = ropef("(*$1)", r)
     gcUsage(e)
+  else:
+    constructLoc(p, tmp)
   discard getTypeDesc(p.module, t)
   for i in 1 .. <e.len:
     let it = e.sons[i]
@@ -1082,6 +1084,7 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) =
     tmp2.s = if isRef: OnHeap else: OnStack
     tmp2.heapRoot = tmp.r
     expr(p, it.sons[1], tmp2)
+
   if d.k == locNone:
     d = tmp
   else:

compiler/msgs.nim

@@ -170,7 +170,7 @@ const
     errInvalidNumberOfYieldExpr: "invalid number of \'yield\' expressions", 
     errCannotReturnExpr: "current routine cannot return an expression", 
     errAttemptToRedefine: "redefinition of \'$1\'", 
-    errStmtInvalidAfterReturn: "statement not allowed after \'return\', \'break\' or \'raise\'", 
+    errStmtInvalidAfterReturn: "statement not allowed after \'return\', \'break\', \'raise\' or \'continue'", 
     errStmtExpected: "statement expected", 
     errInvalidLabel: "\'$1\' is no label", 
     errInvalidCmdLineOption: "invalid command line option: \'$1\'", 

compiler/options.nim

@@ -8,7 +8,7 @@
 #
 
 import
-  os, lists, strutils, strtabs
+  os, lists, strutils, strtabs, osproc, sets
   
 const
   hasTinyCBackend* = defined(tinyc)
@@ -16,6 +16,7 @@ const
   hasFFI* = defined(useFFI)
   newScopeForIf* = true
   useCaas* = not defined(noCaas)
+  noTimeMachine = defined(avoidTimeMachine) and defined(macosx)
 
 type                          # please make sure we have under 32 options
                               # (improves code efficiency a lot!)
@@ -263,6 +264,28 @@ proc toGeneratedFile*(path, ext: string): string =
   result = joinPath([getGeneratedPath(), changeFileExt(tail, ext)])
   #echo "toGeneratedFile(", path, ", ", ext, ") = ", result
 
+when noTimeMachine:
+  var alreadyExcludedDirs = initSet[string]()
+  proc excludeDirFromTimeMachine(dir: string) {.raises: [].} =
+    ## Calls a macosx command on the directory to exclude it from backups.
+    ##
+    ## The macosx tmutil command is invoked to mark the specified path as an
+    ## item to be excluded from time machine backups. If a path already exists
+    ## with files before excluding it, newer files won't be added to the
+    ## directory, but previous files won't be removed from the backup until the
+    ## user deletes that directory.
+    ##
+    ## The whole proc is optional and will ignore all kinds of errors. The only
+    ## way to be sure that it works is to call ``tmutil isexcluded path``.
+    if alreadyExcludedDirs.contains(dir): return
+    alreadyExcludedDirs.incl(dir)
+    try:
+      var p = startProcess("/usr/bin/tmutil", args = ["addexclusion", dir])
+      discard p.waitForExit
+      p.close
+    except E_Base, EOS:
+      discard
+
 proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string = 
   var (head, tail) = splitPath(f)
   #if len(head) > 0: head = removeTrailingDirSep(shortenDir(head & dirSep))
@@ -270,6 +293,8 @@ proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string =
   if createSubDir:
     try: 
       createDir(subdir)
+      when noTimeMachine:
+       excludeDirFromTimeMachine(subdir)
     except EOS: 
       writeln(stdout, "cannot create directory: " & subdir)
       quit(1)

compiler/service.nim

@@ -84,6 +84,7 @@ proc serve*(action: proc (){.nimcall.}) =
   of "tcp", "":
     when useCaas:
       var server = socket()
+      if server == invalidSocket: osError(osLastError())
       let p = getConfigVar("server.port")
       let port = if p.len > 0: parseInt(p).TPort else: 6000.TPort
       server.bindAddr(port, getConfigVar("server.address"))

doc/advopt.txt

@@ -4,6 +4,7 @@ Advanced commands:
   //compileToOC, objc       compile project to Objective C code
   //rst2html                convert a reStructuredText file to HTML
   //rst2tex                 convert a reStructuredText file to TeX
+  //jsondoc                 extract the documentation to a json file
   //buildIndex              build an index for the whole documentation
   //run                     run the project (with Tiny C backend; buggy!)
   //genDepend               generate a DOT file containing the

koch.nim

@@ -53,6 +53,7 @@ Boot options:
                            (not needed on Windows)
   -d:nativeStacktrace      use native stack traces (only for Mac OS X or Linux)
   -d:noCaas                build Nimrod without CAAS support
+  -d:avoidTimeMachine      only for Mac OS X, excludes nimcache dir from backups
 """
 
 proc exe(f: string): string = return addFileExt(f, ExeExt)

lib/pure/collections/tables.nim

@@ -196,7 +196,7 @@ proc `==`*[A, B](s, t: TTable[A, B]): bool =
     # to use the slow route here:
     for key, val in s:
       if not hasKey(t, key): return false
-      if mget(t, key) != val: return false
+      if t[key] != val: return false
     return true
   
 proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): TTable[C, B] =

lib/pure/ftpclient.nim

@@ -107,6 +107,7 @@ proc ftpClient*(address: string, port = TPort(21),
   result.isAsync = false
   result.dsockConnected = false
   result.csock = socket()
+  if result.csock == InvalidSocket: osError(osLastError())
 
 proc getDSock(ftp: PFTPClient): TSocket =
   if ftp.isAsync: return ftp.asyncDSock else: return ftp.dsock
@@ -213,6 +214,7 @@ proc pasv(ftp: PFTPClient) =
   ## Negotiate a data connection.
   if not ftp.isAsync:
     ftp.dsock = socket()
+    if ftp.dsock == InvalidSocket: osError(osLastError())
   else:
     ftp.asyncDSock = AsyncSocket()
     ftp.asyncDSock.handleRead =

lib/pure/httpclient.nim

@@ -75,7 +75,7 @@
 ## constructor should be used for this purpose. However,
 ## currently only basic authentication is supported.
 
-import sockets, strutils, parseurl, parseutils, strtabs, base64
+import sockets, strutils, parseurl, parseutils, strtabs, base64, os
 import asyncnet, asyncdispatch
 import rawsockets
 
@@ -288,6 +288,7 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "",
   add(headers, "\c\L")
   
   var s = socket()
+  if s == InvalidSocket: osError(osLastError())
   var port = sockets.TPort(80)
   if r.scheme == "https":
     when defined(ssl):

lib/pure/irc.nim

@@ -249,6 +249,7 @@ proc reconnect*(irc: PIRC, timeout = 5000) =
   if secSinceReconnect < timeout:
     sleep(timeout - secSinceReconnect)
   irc.sock = socket()
+  if irc.sock == InvalidSocket: osError(osLastError())
   irc.connect()
   irc.lastReconnect = epochTime()
 
@@ -274,6 +275,7 @@ proc irc*(address: string, port: TPort = 6667.TPort,
   result.messageBuffer = @[]
   result.status = SockIdle
   result.sock = socket()
+  if result.sock == InvalidSocket: osError(osLastError())
 
 proc processLine(irc: PIRC, line: string): TIRCEvent =
   if line.len == 0:

lib/pure/osproc.nim

@@ -137,6 +137,14 @@ proc startProcess*(command: string,
   ## to `startProcess`. See the documentation of ``TProcessOption`` for the
   ## meaning of these flags. You need to `close` the process when done.
   ##
+  ## Note that you can't pass any `args` if you use the option
+  ## ``poEvalCommand``, which invokes the system shell to run the specified
+  ## `command`. In this situation you have to concatenate manually the contents
+  ## of `args` to `command` carefully escaping/quoting any special characters,
+  ## since it will be passed *as is* to the system shell. Each system/shell may
+  ## feature different escaping rules, so try to avoid this kind of shell
+  ## invokation if possible as it leads to non portable software.
+  ##
   ## Return value: The newly created process object. Nil is never returned,
   ## but ``EOS`` is raised in case of an error.
 
@@ -633,7 +641,7 @@ elif not defined(useNimRtl):
     if poEvalCommand in options:
       sysCommand = "/bin/sh"
       sysArgsRaw = @[sysCommand, "-c", command]
-      assert args.len == 0
+      assert args.len == 0, "`args` has to be empty when using poEvalCommand."
     else:
       sysCommand = command
       sysArgsRaw = @[command]

lib/pure/scgi.nim

@@ -102,6 +102,7 @@ proc open*(s: var TScgiState, port = TPort(4000), address = "127.0.0.1",
   s.input = newString(s.buflen) # will be reused
   
   s.server = socket()
+  if s.server == InvalidSocket: osError(osLastError())
   new(s.client) # Initialise s.client for `next`
   if s.server == InvalidSocket: scgiError("could not open socket")
   #s.server.connect(connectionName, port)

lib/pure/selectors.nim

@@ -238,6 +238,7 @@ when isMainModule:
       sock: TSocket
   
   var sock = socket()
+  if sock == sockets.InvalidSocket: osError(osLastError())
   #sock.setBlocking(false)
   sock.connect("irc.freenode.net", TPort(6667))
   

lib/system/debugger.nim

@@ -178,7 +178,7 @@ proc hash(data: pointer, size: int): THash =
   while s > 0:
     h = h !& ord(p[i])
     inc(i)
-    cec(s)
+    dec(s)
   result = !$h
 
 proc hashGcHeader(data: pointer): THash =

lib/wrappers/sqlite3.nim

@@ -90,6 +90,7 @@ const
   SQLITE_IGNORE* = 2          # Original from sqlite3.h: 
                               ##define SQLITE_STATIC      ((void(*)(void *))0)
                               ##define SQLITE_TRANSIENT   ((void(*)(void *))-1)
+  SQLITE_DETERMINISTIC* = 0x800
 
 const 
   SQLITE_STATIC* = nil

tests/assign/tobjasgn.nim

@@ -1,16 +1,21 @@
 discard """
-  output: '''0
+  output: '''8 5 0 0
 pre test a:test b:1 c:2 haha:3
 assignment test a:test b:1 c:2 haha:3
 '''
 """
 
-type TSomeObj = object of TObject
+# bug #1005
-  Variable: int
+
+type
+  TSomeObj = object of TObject
+    a, b: int
+  PSomeObj = ref object
+    a, b: int
  
-var a = TSomeObj()
+var a = TSomeObj(a: 8)
- 
+var b = PSomeObj(a: 5)
-echo a.Variable.`$`
+echo a.a, " ", b.a, " ", a.b, " ", b.b
 
 # bug #575
 

tests/async/tasyncawait.nim

@@ -1,6 +1,5 @@
 discard """
   file: "tasyncawait.nim"
-  cmd: "nimrod cc --hints:on $# $#"
   output: "5000"
 """
 import asyncdispatch, rawsockets, net, strutils, os

tests/async/tasynciossl.nim

@@ -1,6 +1,6 @@
 discard """
   file: "tasynciossl.nim"
-  cmd: "nimrod cc --hints:on --define:ssl $# $#"
+  cmd: "nimrod $target --hints:on --define:ssl $options $file"
   output: "20000"
 """
 import sockets, asyncio, strutils, times
@@ -88,4 +88,4 @@ while true:
     break
 
 assert msgCount == (swarmSize * messagesToSend) * serverCount
-echo(msgCount)
+echo(msgCount)

tests/concurrency/tnodeadlocks.nim

@@ -1,6 +1,6 @@
 discard """
   outputsub: "101"
-  cmd: "nimrod cc --hints:on --threads:on $# $#"
+  cmd: "nimrod $target --hints:on --threads:on $options $file"
 """
 
 import os, locks

tests/dll/client.nim

@@ -1,6 +1,6 @@
 discard """
   output: "Done"
-  cmd: "nimrod cc --debuginfo --hints:on --define:useNimRtl $# $#"
+  cmd: "nimrod $target --debuginfo --hints:on --define:useNimRtl $options $file"
 """
 
 type

tests/dll/server.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nimrod cc --debuginfo --hints:on --define:useNimRtl --app:lib $# $#"
+  cmd: "nimrod $target --debuginfo --hints:on --define:useNimRtl --app:lib $options $file"
 """
 
 type

tests/generics/tmetafield.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nimrod check $# $#"
+  cmd: "nimrod check $options $file"
   errormsg: "'proc' is not a concrete type"
   errormsg: "'Foo' is not a concrete type."
   errormsg: "invalid type: 'TBaseMed'"

tests/generics/tthread_generic.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nimrod cc --hints:on --threads:on $# $#"
+  cmd: "nimrod $target --hints:on --threads:on $options $file"
 """
 
 type

tests/iter/titer2.nim

@@ -1,6 +1,6 @@
 discard """
   output: '''true'''
-  cmd: "nimrod cc --gc:none --hints:on --warnings:off $# $#"
+  cmd: "nimrod $target --gc:none --hints:on --warnings:off $options $file"
 """
 
 import hashes

tests/js.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nimrod js --hints:on $# $#"
+  cmd: "nimrod js --hints:on $options $file"
 """
 
 # This file tests the JavaScript generator

tests/js/test1.nim

@@ -1,5 +1,4 @@
 discard """
-  cmd: "nimrod js --hints:on $# $#"
   output: "1261129"
 """
 

tests/js/test2.nim

@@ -1,5 +1,4 @@
 discard """
-  cmd: "nimrod js --hints:on -r $# $#"
   output: '''foo
 js 3.14'''
 """

tests/js/testmagic.nim

@@ -1,5 +1,4 @@
 discard """
-  cmd: "nimrod js --hints:on -r $# $#"
   output: '''true'''
 """
 

tests/misc/tgtk.nim

@@ -1,4 +1,6 @@
-
+discard """
+  disabled: true
+"""
 import
   gtk2, glib2, atk, gdk2, gdk2pixbuf, libglade2, pango,
   pangoutils

tests/misc/tmandelbrot.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nimrod cc --hints:on -d:release $# $#"
+  cmd: "nimrod $target --hints:on -d:release $options $file"
 """
 
 # -*- nimrod -*-

tests/sets/tsets2.nim

@@ -1,6 +1,5 @@
 discard """
   output: '''true'''
-  cmd: "nimrod cc --hints:on $# $#"
 """
 
 import hashes, sets

tests/stdlib/tsockets.nim

@@ -1,6 +1,7 @@
-import sockets
+import sockets, os
 var s: TSocket
 s = socket()
+if s == InvalidSocket: osError(osLastError())
 
 s.connect("www.google.com", TPort(80))
 

tests/stdlib/tunidecode.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nimrod cc --hints:on -d:embedUnidecodeTable $# $#"
+  cmd: "nimrod $target --hints:on -d:embedUnidecodeTable $options $file"
   output: "Ausserst"
 """
 

tests/table/ttables.nim

@@ -1,6 +1,5 @@
 discard """
   output: '''true'''
-  cmd: "nimrod cc --hints:on $# $#"
 """
 
 import hashes, tables

tests/table/ttables2.nim

@@ -1,6 +1,5 @@
 discard """
   output: '''true'''
-  cmd: "nimrod cc --hints:on $# $#"
 """
 
 import tables

tests/testament/categories.nim

@@ -179,10 +179,12 @@ proc jsTests(r: var TResults, cat: Category, options: string) =
     
   for t in os.walkFiles("tests/js/t*.nim"):
     test(t)
-  for testfile in ["texceptions", "texcpt1", "texcsub", "tfinally",
+  for testfile in ["exception/texceptions", "exception/texcpt1",
-                   "tfinally2", "tfinally3", "tactiontable", "tmultim1",
+                   "exception/texcsub", "exception/tfinally",
-                   "tmultim3", "tmultim4"]:
+                   "exception/tfinally2", "exception/tfinally3",
-    test "tests/run/" & testfile & ".nim"
+                   "actiontable/tactiontable", "method/tmultim1",
+                   "method/tmultim3", "method/tmultim4"]:
+    test "tests/" & testfile & ".nim"
 
 # ------------------------- manyloc -------------------------------------------
 #proc runSpecialTests(r: var TResults, options: string) =

tests/testament/specs.nim

@@ -10,7 +10,7 @@
 import parseutils, strutils, os, osproc, streams, parsecfg
 
 const
-  cmdTemplate* = r"nimrod cc --hints:on $# $#"
+  cmdTemplate* = r"nimrod $target --hints:on $options $file"
 
 type
   TTestAction* = enum
@@ -51,6 +51,7 @@ type
 
 const
   targetToExt*: array[TTarget, string] = ["c", "cpp", "m", "js"]
+  targetToCmd*: array[TTarget, string] = ["c", "cpp", "objc", "js"]
 
 when not defined(parseCfgBool):
   # candidate for the stdlib:

tests/testament/tester.nim

@@ -53,8 +53,10 @@ let
   pegSuccess = peg"'Hint: operation successful'.*"
   pegOfInterest = pegLineError / pegOtherError
 
-proc callCompiler(cmdTemplate, filename, options: string): TSpec =
+proc callCompiler(cmdTemplate, filename, options: string,
-  let c = parseCmdLine(cmdTemplate % [options, filename])
+                  target: TTarget): TSpec =
+  let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target],
+                       "options", options, "file", filename])
   var p = startProcess(command=c[0], args=c[1.. -1],
                        options={poStdErrToStdOut, poUseShell})
   let outp = p.outputStream
@@ -111,10 +113,10 @@ proc addResult(r: var TResults, test: TTest,
   r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
   if success notin {reSuccess, reIgnored}:
     styledEcho styleBright, name, fgRed, " [", $success, "]"
-    styledEcho styleDim, "EXPECTED:"
+    echo"Expected:"
-    echo expected
+    styledEcho styleBright, expected
-    styledEcho styleDim, "GIVEN:"
+    echo"Given:"
-    echo given
+    styledEcho styleBright, given
 
 proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest) =
   if strip(expected.msg) notin strip(given.msg):
@@ -152,7 +154,7 @@ proc testSpec(r: var TResults, test: TTest) =
   # major entry point for a single test
   let tname = test.name.addFileExt(".nim")
   inc(r.total)
-  echo extractFilename(tname)
+  styledEcho "Processing ", fgCyan, extractFilename(tname)
   var expected = parseSpec(tname)
   if expected.err == reIgnored:
     r.addResult(test, "", "", reIgnored)
@@ -160,13 +162,15 @@ proc testSpec(r: var TResults, test: TTest) =
   else:
     case expected.action
     of actionCompile:
-      var given = callCompiler(expected.cmd, test.name, test.options)
+      var given = callCompiler(expected.cmd, test.name, test.options,
+                               test.target)
       if given.err == reSuccess:
         codegenCheck(test, expected.ccodeCheck, given)
       r.addResult(test, "", given.msg, given.err)
       if given.err == reSuccess: inc(r.passed)
     of actionRun:
-      var given = callCompiler(expected.cmd, test.name, test.options)
+      var given = callCompiler(expected.cmd, test.name, test.options,
+                               test.target)
       if given.err != reSuccess:
         r.addResult(test, "", given.msg, given.err)
       else:
@@ -176,10 +180,13 @@ proc testSpec(r: var TResults, test: TTest) =
           exeFile = dir / "nimcache" / file & ".js"
         else:
           exeFile = changeFileExt(tname, ExeExt)
-        
         if existsFile(exeFile):
+          if test.target == targetJS and findExe("nodejs") == "":
+            r.addResult(test, expected.outp, "nodejs binary not in PATH",
+                        reExeNotFound)
+            return
           var (buf, exitCode) = execCmdEx(
-            (if test.target==targetJS: "node " else: "") & exeFile)
+            (if test.target == targetJS: "nodejs " else: "") & exeFile)
           if exitCode != expected.ExitCode:
             r.addResult(test, "exitcode: " & $expected.exitCode,
                               "exitcode: " & $exitCode, reExitCodesDiffer)
@@ -194,7 +201,8 @@ proc testSpec(r: var TResults, test: TTest) =
         else:
           r.addResult(test, expected.outp, "executable not found", reExeNotFound)
     of actionReject:
-      var given = callCompiler(expected.cmd, test.name, test.options)
+      var given = callCompiler(expected.cmd, test.name, test.options,
+                               test.target)
       cmpMsgs(r, expected, given, test)
 
 proc testNoSpec(r: var TResults, test: TTest) =
@@ -202,7 +210,7 @@ proc testNoSpec(r: var TResults, test: TTest) =
   let tname = test.name.addFileExt(".nim")
   inc(r.total)
   echo extractFilename(tname)
-  let given = callCompiler(cmdTemplate, test.name, test.options)
+  let given = callCompiler(cmdTemplate, test.name, test.options, test.target)
   r.addResult(test, "", given.msg, given.err)
   if given.err == reSuccess: inc(r.passed)
 
@@ -223,7 +231,7 @@ proc main() =
   os.putenv "NIMTEST_NO_COLOR", "1"
   os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
 
-  backend.open()  
+  backend.open()
   var optPrintResults = false
   var p = initOptParser()
   p.next()
@@ -238,9 +246,12 @@ proc main() =
   var r = initResults()
   case action
   of "all":
-    for kind, dir in walkDir("tests"):
+    let testsDir = "tests" & dirSep
-      if kind == pcDir and dir notin ["testament", "testdata", "nimcache"]:
+    for kind, dir in walkDir(testsDir):
-        processCategory(r, Category(dir), p.cmdLineRest.string)
+      assert testsDir.startsWith(testsDir)
+      let cat = dir[testsDir.len .. -1]
+      if kind == pcDir and cat notin ["testament", "testdata", "nimcache"]:
+        processCategory(r, Category(cat), p.cmdLineRest.string)
     for a in AdditionalCategories:
       processCategory(r, Category(a), p.cmdLineRest.string)
   of "c", "cat", "category":
@@ -255,11 +266,11 @@ proc main() =
   else:
     quit usage
 
-  if optPrintResults: 
+  if optPrintResults:
     if action == "html": openDefaultBrowser(resultsFile)
     else: echo r, r.data
   backend.close()
-  
+
 if paramCount() == 0:
   quit usage
 main()

tests/threads/tthreadanalysis.nim

@@ -2,7 +2,7 @@ discard """
   outputsub: "101"
   msg: "Warning: write to foreign heap"
   line: 37
-  cmd: "nimrod cc --hints:on --threads:on $# $#"
+  cmd: "nimrod $target --hints:on --threads:on $options $file"
 """
 
 import os

tests/threads/tthreadanalysis2.nim

@@ -2,7 +2,7 @@ discard """
   file: "tthreadanalysis2.nim"
   line: 42
   errormsg: "write to foreign heap"
-  cmd: "nimrod cc --hints:on --threads:on $# $#"
+  cmd: "nimrod $target --hints:on --threads:on $options $file"
 """
 
 import os

tests/threads/tthreadanalysis3.nim

@@ -2,7 +2,7 @@ discard """
   file: "tthreadanalysis3.nim"
   line: 35
   errormsg: "write to foreign heap"
-  cmd: "nimrod cc --hints:on --threads:on $# $#"
+  cmd: "nimrod $target --hints:on --threads:on $options $file"
 """
 
 import os

tests/threads/tthreadheapviolation1.nim

@@ -1,7 +1,7 @@
 discard """
   line: 12
   errormsg: "write to foreign heap"
-  cmd: "nimrod cc --hints:on --threads:on $# $#"
+  cmd: "nimrod $target --hints:on --threads:on $options $file"
 """
 
 var 

tests/types/tillegaltyperecursion.nim

@@ -1,5 +1,5 @@
 discard """
-  cmd: "nimrod c --threads:on $# $#"
+  cmd: "nimrod $target --threads:on $options $file"
   errormsg: "illegal recursion in type 'TIRC'"
   line: 16
 """