tester checks exitcode; osproc additions; DLL fixes; taint mode fixes

koch.nim

@@ -177,7 +177,9 @@ proc clean(args: string) =
       RemoveDir(path)
 
 proc tests(args: string) =
-  exec("nimrod cc tests/tester")
+  # we compile the tester with taintMode:on to have a basic
+  # taint mode test :-)
+  exec("nimrod cc --taintMode:on tests/tester")
   exec("tests/tester reject")
   exec("tests/tester compile")
   exec("tests/tester examples")

lib/impure/re.nim

@@ -344,7 +344,7 @@ proc transformFile*(infile, outfile: string,
   ## reads in the file `infile`, performs a parallel replacement (calls
   ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an
   ## error occurs. This is supposed to be used for quick scripting.
-  var x = readFile(infile)
+  var x = readFile(infile).string
   writeFile(outfile, x.parallelReplace(subs))
   
 iterator split*(s: string, sep: TRegEx): string =

lib/pure/browsers.nim

@@ -34,7 +34,7 @@ proc openDefaultBrowser*(url: string) =
     var u = quoteIfContainsWhite(url)
     for a in items(attempts):
       if execShellCmd(a & u) == 0: return
-    for b in getEnv("BROWSER").split(PathSep):
+    for b in getEnv("BROWSER").string.split(PathSep):
       try:
         # we use ``startProcess`` here because we don't want to block!
         discard startProcess(command=b, args=[url], options={poUseShell})

lib/pure/os.nim

@@ -74,6 +74,10 @@ when defined(Nimdoc): # only for proper documentation:
       ## The file extension of a script file. For example: "" for POSIX,
       ## "bat" on Windows.
 
+    DynlibFormat* = "lib$1.so"
+      ## The format string to turn a filename into a `DLL`:idx: file (also
+      ## called `shared object`:idx: on some operating systems).
+
 elif defined(macos):
   const
     curdir* = ':'
@@ -84,6 +88,7 @@ elif defined(macos):
     FileSystemCaseSensitive* = false
     ExeExt* = ""
     ScriptExt* = ""
+    DynlibFormat* = "$1.dylib"
 
   #  MacOS paths
   #  ===========
@@ -113,6 +118,7 @@ elif doslike:
     FileSystemCaseSensitive* = false
     ExeExt* = "exe"
     ScriptExt* = "bat"
+    DynlibFormat* = "$1.dll"
 elif defined(PalmOS) or defined(MorphOS):
   const
     dirsep* = '/'
@@ -122,6 +128,7 @@ elif defined(PalmOS) or defined(MorphOS):
     FileSystemCaseSensitive* = false
     ExeExt* = ""
     ScriptExt* = ""
+    DynlibFormat* = "$1.prc"
 elif defined(RISCOS):
   const
     dirsep* = '.'
@@ -131,6 +138,7 @@ elif defined(RISCOS):
     FileSystemCaseSensitive* = true
     ExeExt* = ""
     ScriptExt* = ""
+    DynlibFormat* = "lib$1.so"
 else: # UNIX-like operating system
   const
     curdir* = '.'
@@ -141,6 +149,7 @@ else: # UNIX-like operating system
     FileSystemCaseSensitive* = true
     ExeExt* = ""
     ScriptExt* = ""
+    DynlibFormat* = "lib$1.so"
 
 const
   ExtSep* = '.'

lib/pure/osproc.nim

@@ -48,7 +48,8 @@ proc execProcess*(command: string,
 
 proc execCmd*(command: string): int {.rtl, extern: "nosp$1".}
   ## Executes ``command`` and returns its error code. Standard input, output,
-  ## error streams are inherited from the calling process.
+  ## error streams are inherited from the calling process. This operation
+  ## is also often called `system`:idx:.
 
 proc startProcess*(command: string,
                    workingDir: string = "",
@@ -70,6 +71,17 @@ proc startProcess*(command: string,
   ## Return value: The newly created process object. Nil is never returned,
   ## but ``EOS`` is raised in case of an error.
 
+proc startCmd*(command: string, options: set[TProcessOption] = {
+               poStdErrToStdOut, poUseShell}): PProcess =
+  ## a simpler version of `startProcess` that parses the command line into
+  ## program and arguments and then calls `startProcess` with the empty string
+  ## for `workingDir` and the nil string table for `env`.
+  var c = parseCmdLine(command)
+  var a: seq[string]
+  newSeq(a, c.len-1) # avoid slicing for now (still unstable)
+  for i in 1 .. c.len-1: a[i-1] = c[i]
+  result = startProcess(command=c[0], args=a, options=options)
+
 proc close*(p: PProcess) {.rtl, extern: "nosp$1".}
   ## When the process has finished executing, cleanup related handles
 
@@ -140,12 +152,6 @@ proc countProcessors*(): int {.rtl, extern: "nosp$1".} =
     result = sysconf(SC_NPROCESSORS_ONLN)
   if result <= 0: result = 1
 
-proc startProcessAux(cmd: string, options: set[TProcessOption]): PProcess =
-  var c = parseCmdLine(cmd)
-  var a: seq[string] = @[] # slicing is not yet implemented :-(
-  for i in 1 .. c.len-1: add(a, c[i])
-  result = startProcess(command=c[0], args=a, options=options)
-
 proc execProcesses*(cmds: openArray[string],
                     options = {poStdErrToStdOut, poParentStreams},
                     n = countProcessors()): int {.rtl, extern: "nosp$1".} =
@@ -158,7 +164,7 @@ proc execProcesses*(cmds: openArray[string],
     newSeq(q, n)
     var m = min(n, cmds.len)
     for i in 0..m-1:
-      q[i] = startProcessAux(cmds[i], options=options)
+      q[i] = startCmd(cmds[i], options=options)
     when defined(noBusyWaiting):
       var r = 0
       for i in m..high(cmds):
@@ -171,7 +177,7 @@ proc execProcesses*(cmds: openArray[string],
           echo(err)
         result = max(waitForExit(q[r]), result)
         if q[r] != nil: close(q[r])
-        q[r] = startProcessAux(cmds[i], options=options)
+        q[r] = startCmd(cmds[i], options=options)
         r = (r + 1) mod n
     else:
       var i = m
@@ -182,7 +188,7 @@ proc execProcesses*(cmds: openArray[string],
             #echo(outputStream(q[r]).readLine())
             result = max(waitForExit(q[r]), result)
             if q[r] != nil: close(q[r])
-            q[r] = startProcessAux(cmds[i], options=options)
+            q[r] = startCmd(cmds[i], options=options)
             inc(i)
             if i > high(cmds): break
     for i in 0..m-1:
@@ -190,7 +196,7 @@ proc execProcesses*(cmds: openArray[string],
       result = max(waitForExit(q[i]), result)
   else:
     for i in 0..high(cmds):
-      var p = startProcessAux(cmds[i], options=options)
+      var p = startCmd(cmds[i], options=options)
       result = max(waitForExit(p), result)
       close(p)
 
@@ -204,7 +210,7 @@ when not defined(useNimRtl):
   proc execProcess(command: string,
                    options: set[TProcessOption] = {poStdErrToStdOut,
                                                    poUseShell}): TaintedString =
-    var p = startProcessAux(command, options=options)
+    var p = startCmd(command, options=options)
     var outp = outputStream(p)
     result = TaintedString""
     while running(p) or not outp.atEnd(outp):
@@ -625,6 +631,27 @@ elif not defined(useNimRtl):
     
     pruneProcessSet(readfds, (rd))
 
+
+proc execCmdEx*(command: string, options: set[TProcessOption] = {
+                poStdErrToStdOut, poUseShell}): tuple[
+                output: TaintedString, 
+                exitCode: int] =
+  ## a convenience proc that runs the `command`, grabs all its output and
+  ## exit code and returns both.
+  var p = startCmd(command, options)
+  var outp = outputStream(p)
+  result = (TaintedString"", -1)
+  while not outp.atEnd(outp):
+    result[0].string.add(outp.readLine().string)
+    result[0].string.add("\n")
+    result[1] = peekExitCode(p)
+    if result[1] != -1: break
+  outp.close(outp)
+  if result[1] == -1:
+    result[1] = peekExitCode(p)
+  close(p)
+
+
 when isMainModule:
   var x = execProcess("gcc -v")
   echo "ECHO ", x

lib/pure/pegs.nim

@@ -969,7 +969,7 @@ proc transformFile*(infile, outfile: string,
   ## reads in the file `infile`, performs a parallel replacement (calls
   ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an
   ## error occurs. This is supposed to be used for quick scripting.
-  var x = readFile(infile)
+  var x = readFile(infile).string
   writeFile(outfile, x.parallelReplace(subs))
   
 iterator split*(s: string, sep: TPeg): string =

lib/system/mmdisp.nim

@@ -78,45 +78,49 @@ when defined(boehmgc):
   proc boehmRealloc(p: pointer, size: int): pointer {.
     importc: "GC_realloc", dynlib: boehmLib.}
   proc boehmDealloc(p: pointer) {.importc: "GC_free", dynlib: boehmLib.}
+  
+  when not defined(useNimRtl):
     
-  proc alloc(size: int): pointer =
+    proc alloc(size: int): pointer =
-    result = boehmAlloc(size)
+      result = boehmAlloc(size)
-    if result == nil: raiseOutOfMem()
+      if result == nil: raiseOutOfMem()
-  proc alloc0(size: int): pointer =
+    proc alloc0(size: int): pointer =
-    result = alloc(size)
+      result = alloc(size)
-    zeroMem(result, size)
+      zeroMem(result, size)
-  proc realloc(p: Pointer, newsize: int): pointer =
+    proc realloc(p: Pointer, newsize: int): pointer =
-    result = boehmRealloc(p, newsize)
+      result = boehmRealloc(p, newsize)
-    if result == nil: raiseOutOfMem()
+      if result == nil: raiseOutOfMem()
-  proc dealloc(p: Pointer) = boehmDealloc(p)
+    proc dealloc(p: Pointer) = boehmDealloc(p)
 
-  proc allocShared(size: int): pointer =
+    proc allocShared(size: int): pointer =
-    result = boehmAlloc(size)
+      result = boehmAlloc(size)
-    if result == nil: raiseOutOfMem()
+      if result == nil: raiseOutOfMem()
-  proc allocShared0(size: int): pointer =
+    proc allocShared0(size: int): pointer =
-    result = alloc(size)
+      result = alloc(size)
-    zeroMem(result, size)
+      zeroMem(result, size)
-  proc reallocShared(p: Pointer, newsize: int): pointer =
+    proc reallocShared(p: Pointer, newsize: int): pointer =
-    result = boehmRealloc(p, newsize)
+      result = boehmRealloc(p, newsize)
-    if result == nil: raiseOutOfMem()
+      if result == nil: raiseOutOfMem()
-  proc deallocShared(p: Pointer) = boehmDealloc(p)
+    proc deallocShared(p: Pointer) = boehmDealloc(p)
+
+    #boehmGCincremental()
+
+    proc GC_disable() = boehmGC_disable()
+    proc GC_enable() = boehmGC_enable()
+    proc GC_fullCollect() = boehmGCfullCollect()
+    proc GC_setStrategy(strategy: TGC_Strategy) = nil
+    proc GC_enableMarkAndSweep() = nil
+    proc GC_disableMarkAndSweep() = nil
+    proc GC_getStatistics(): string = return ""
+    
+    proc getOccupiedMem(): int = return -1
+    proc getFreeMem(): int = return -1
+    proc getTotalMem(): int = return -1
+
+    proc setStackBottom(theStackBottom: pointer) = nil
 
   proc initGC() = 
     when defined(macosx): boehmGCinit()
-  
-  #boehmGCincremental()
-
-  proc GC_disable() = boehmGC_disable()
-  proc GC_enable() = boehmGC_enable()
-  proc GC_fullCollect() = boehmGCfullCollect()
-  proc GC_setStrategy(strategy: TGC_Strategy) = nil
-  proc GC_enableMarkAndSweep() = nil
-  proc GC_disableMarkAndSweep() = nil
-  proc GC_getStatistics(): string = return ""
-  
-  proc getOccupiedMem(): int = return -1
-  proc getFreeMem(): int = return -1
-  proc getTotalMem(): int = return -1
 
   proc newObj(typ: PNimType, size: int): pointer {.compilerproc.} =
     result = alloc(size)
@@ -128,7 +132,6 @@ when defined(boehmgc):
   proc growObj(old: pointer, newsize: int): pointer =
     result = realloc(old, newsize)
 
-  proc setStackBottom(theStackBottom: pointer) = nil
   proc nimGCref(p: pointer) {.compilerproc, inline.} = nil
   proc nimGCunref(p: pointer) {.compilerproc, inline.} = nil
   

tests/dll/client.nim

@@ -0,0 +1,41 @@
+discard """
+  output: "Done"
+  cmd: "nimrod cc --debuginfo --hints:on --define:useNimRtl $# $#"
+"""
+
+type
+  TNodeKind = enum nkLit, nkSub, nkAdd, nkDiv, nkMul
+  TNode = object
+    case k: TNodeKind
+    of nkLit: x: int
+    else: a, b: ref TNode
+
+  PNode = ref TNode
+
+
+when defined(windows):
+  const dllname = "server.dll"
+elif defined(macosx):
+  const dllname = "libserver.dylib"
+else:
+  const dllname = "libserver.so"
+
+proc newLit(x: int): PNode {.importc: "newLit", dynlib: dllname.}
+proc newOp(k: TNodeKind, a, b: PNode): PNode {.
+  importc: "newOp", dynlib: dllname.}  
+proc buildTree(x: int): PNode {.importc: "buildTree", dynlib: dllname.} 
+
+proc eval(n: PNode): int =
+  case n.k
+  of nkLit: result = n.x
+  of nkSub: result = eval(n.a) - eval(n.b)
+  of nkAdd: result = eval(n.a) + eval(n.b)
+  of nkDiv: result = eval(n.a) div eval(n.b)
+  of nkMul: result = eval(n.a) * eval(n.b)
+
+# Test the GC:
+for i in 0..100_000:
+  discard eval(buildTree(2))
+  
+echo "Done"
+

tests/dll/dllsimple.nim

@@ -1,5 +0,0 @@
-discard """
-  file: tdllgen.nim
-"""
-proc test() {.exportc.} =
-  echo("Hello World!")

tests/dll/server.nim

@@ -0,0 +1,27 @@
+discard """
+  cmd: "nimrod cc --debuginfo --hints:on --define:useNimRtl --app:lib $# $#"
+"""
+
+type
+  TNodeKind = enum nkLit, nkSub, nkAdd, nkDiv, nkMul
+  TNode = object
+    case k: TNodeKind
+    of nkLit: x: int
+    else: a, b: ref TNode
+
+  PNode = ref TNode
+    
+proc newLit(x: int): PNode {.exportc: "newLit", dynlib.} =
+  new(result)
+  result.x = x
+  
+proc newOp(k: TNodeKind, a, b: PNode): PNode {.exportc: "newOp", dynlib.} =
+  assert a != nil
+  assert b != nil
+  new(result)
+  result.k = k
+  result.a = a
+  result.b = b
+  
+proc buildTree(x: int): PNode {.exportc: "buildTree", dynlib.} = 
+  result = newOp(nkMul, newOp(nkAdd, newLit(x), newLit(x)), newLit(x))

tests/tester.nim

@@ -50,8 +50,7 @@ when not defined(parseCfgBool):
 
 proc extractSpec(filename: string): string =
   const tripleQuote = "\"\"\""
-  var x = readFile(filename)
+  var x = readFile(filename).string
-  if isNil(x): quit "cannot open file: " & filename
   var a = x.find(tripleQuote)
   var b = x.find(tripleQuote, a+3)
   if a >= 0 and b > a:
@@ -101,9 +100,6 @@ proc parseSpec(filename: string): TSpec =
 
 # ----------------------------------------------------------------------------
 
-proc myExec(cmd: string): string =
-  result = osproc.execProcess(cmd)
-
 var
   pegLineError = peg"{[^(]*} '(' {\d+} ', ' \d+ ') Error:' \s* {.*}"
   pegOtherError = peg"'Error:' \s* {.*}"
@@ -119,7 +115,7 @@ proc callCompiler(cmdTemplate, filename, options: string): TSpec =
   var outp = p.outputStream
   var s = ""
   while running(p) or not outp.atEnd(outp):
-    var x = outp.readLine()
+    var x = outp.readLine().string
     if x =~ pegOfInterest:
       # `s` should contain the last error message
       s = x
@@ -145,7 +141,7 @@ proc initResults: TResults =
   result.data = ""
 
 proc readResults(filename: string): TResults =
-  result = marshal.to[TResults](readFile(filename))
+  result = marshal.to[TResults](readFile(filename).string)
 
 proc writeResults(filename: string, r: TResults) =
   writeFile(filename, $$r)
@@ -255,12 +251,16 @@ proc runSingleTest(r: var TResults, test, options: string) =
     else:
       var exeFile = changeFileExt(test, ExeExt)
       if existsFile(exeFile):
-        var buf = myExec(exeFile)
+        var (buf, exitCode) = execCmdEx(exeFile)
-        var success = strip(buf) == strip(expected.outp)
+        if exitCode != 0:
-        if expected.substr: success = expected.outp in buf
+          r.addResult(t, expected.outp, "exitCode: " & $exitCode, reFailure)
-        if success: inc(r.passed)
+        else:
-        r.addResult(t, expected.outp,
+          var success = strip(buf.string) == strip(expected.outp)
-            buf, if success: reSuccess else: reFailure)
+          if expected.substr and not success: 
+            success = expected.outp in buf.string
+          if success: inc(r.passed)
+          r.addResult(t, expected.outp,
+              buf.string, if success: reSuccess else: reFailure)
       else:
         r.addResult(t, expected.outp, "executable not found", reFailure)
 
@@ -320,16 +320,35 @@ proc compileRodFiles(r: var TResults, options: string) =
   test "gtkex2"
   delNimCache()
 
-# -----------------------------------------------------------------------------
+# --------------------- DLL generation tests ----------------------------------
 
-# DLL generation tests
+proc runBasicDLLTest(r: var TResults, options: string) =
-proc testDLLGen(r: var TResults, options: string) =
+  compileSingleTest r, "lib/nimrtl.nim", options & " --app:lib -d:createNimRtl"
-  compileSingleTest(r, "lib/nimrtl.nim", "--app:lib -d:createNimRtl")
+  compileSingleTest r, "tests/dll/server.nim", 
+    options & " --app:lib -d:useNimRtl"
   
-  template test(filename: expr): stmt =
+  when defined(Windows): 
-    compileSingleTest(r, "tests/dll/" / filename, options)
+    # windows looks in the dir of the exe (yay!):
+    var nimrtlDll = DynlibFormat % "nimrtl"
+    copyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll)
+  else:
+    # posix relies on crappy LD_LIBRARY_PATH (ugh!):
+    var libpath = getenv"LD_LIBRARY_PATH".string
+    if peg"\i '/nimrod' (!'/')* '/lib'" notin libpath:
+      echo "[Warning] insufficient LD_LIBRARY_PATH"
+    var serverDll = DynlibFormat % "server"
+    copyFile("tests/dll" / serverDll, "lib" / serverDll)
   
-  test "dllsimple.nim"
+  runSingleTest r, "tests/dll/client.nim", options & " -d:useNimRtl"
+
+proc runDLLTests(r: var TResults, options: string) =
+  runBasicDLLTest(r, options)
+  runBasicDLLTest(r, options & " -d:release")
+  runBasicDLLTest(r, options & " --gc:boehm")
+  runBasicDLLTest(r, options & " -d:release --gc:boehm")
+  
+  
+# -----------------------------------------------------------------------------
    
 proc compileExample(r: var TResults, pattern, options: string) =
   for test in os.walkFiles(pattern): compileSingleTest(r, test, options)
@@ -356,7 +375,7 @@ proc main(action: string) =
   var options = ""
   for i in 2.. paramCount():
     add(options, " ")
-    add(options, paramStr(i))
+    add(options, paramStr(i).string)
 
   case action
   of "reject":
@@ -368,7 +387,6 @@ proc main(action: string) =
     compile(compileRes, "tests/accept/compile/t*.nim", options)
     compile(compileRes, "tests/ecmas.nim", options)
     compileRodFiles(compileRes, options)
-    testDllGen(compileRes, options)
     writeResults(compileJson, compileRes)
   of "examples":
     var compileRes = readResults(compileJson)
@@ -380,6 +398,7 @@ proc main(action: string) =
     var runRes = initResults()
     run(runRes, "tests/accept/run", options)
     runRodFiles(runRes, options)
+    runDLLTests(runRes, options)
     writeResults(runJson, runRes)
   of "merge":
     var rejectRes = readResults(rejectJson)
@@ -387,10 +406,14 @@ proc main(action: string) =
     var runRes = readResults(runJson)
     listResults(rejectRes, compileRes, runRes)
     outputJSON(rejectRes, compileRes, runRes)
+  of "dll":
+    var runRes = initResults()
+    runDLLTests runRes, ""
+    writeResults(runJson, runRes)
   else:
     quit usage
 
 if paramCount() == 0:
   quit usage
-main(paramStr(1))
+main(paramStr(1).string)
 

todo.txt

@@ -2,7 +2,6 @@ Version 0.8.14
 ==============
 
 - bug: s[1..n] = @[] produces wrong C code
-- test suite should contain DLL tests
 - optimize unused constants away (affected by HLO)
 - fix actors.nim; test with different thread var implementations
 - dead code elim for JS backend
@@ -25,6 +24,8 @@ version 0.9.0
 - tests: run the GC tests
 - change overloading resolution
 - implement closures; implement proper coroutines
+- implicit invokation of `items` seems nice
+- we need to support iteration of 2 different data structures in parallel
 - make exceptions compatible with C++ exceptions
 - ``=`` should be overloadable; requires specialization for ``=``
 - 'const' objects including case objects

web/news.txt

@@ -101,7 +101,7 @@ Library Additions
 - Added ``system.slurp`` for easy resource embedding.
 - Added ``system.running`` for threads.
 - Added ``xmltree.innerText``.
-- Added ``os.isAbsolute``.
+- Added ``os.isAbsolute``, ``os.dynLibFormat``.
 - Added ``parseutils.interpolatedFragments``.
 - Added ``macros.treeRepr``, ``macros.lispRepr``, ``macros.dumpTree``, 
   ``macros.dumpLisp``, ``macros.parseExpr``, ``macros.parseStmt``, ``macros.getAst``.
@@ -109,6 +109,7 @@ Library Additions
 - Added ``irc`` module.
 - Added ``ftpclient`` module.
 - Added ``memfiles`` module.
+- Added ``osproc.startCmd``, ``osproc.execCmdEx``.
 
 
 2011-07-10 Version 0.8.12 released