Add support for private tests and captured output

src/private/captureoutput.nim

@@ -0,0 +1,58 @@
+import os, oids, tables
+
+when defined(windows):
+  proc dup(fd: cint): cint {.importc: "_dup", header: "<io.h>".}
+  proc dup2(fd1, fd2: cint): cint {.importc: "_dup2", header: "<io.h>".}
+else:
+  import posix
+
+proc getTempFile(): string =
+  return getTempDir() / $genOid() & ".tmp"
+
+template captureOutput*(code: untyped): (string, ref Exception) =
+  var backupFds = newTable[File, cint]()
+
+  let
+    streams = [stdout, stderr]
+    fname = getTempFile()
+    tempFile = open(fname, fmReadWrite)
+    tempHandle = tempFile.getFileHandle()
+
+  for stream in streams:
+    let handle = stream.getFileHandle()
+
+    let save = dup(handle)
+    backupFds[stream] = save
+
+    discard dup2(tempHandle, handle)
+
+  var exc: ref Exception
+  var output = ""
+
+  try:
+    code
+
+    tempFile.setFilePos(0)
+    output = tempFile.readAll()
+    tempFile.close()
+
+    (output, exc)
+  except Exception as e:
+    exc = e
+
+    tempFile.setFilePos(0)
+    output = tempFile.readAll()
+    tempFile.close()
+
+    (output, exc)
+  finally:
+    for stream in streams:
+      let handle = stream.getFileHandle()
+      let save = backupFds[stream]
+      discard dup2(save, handle)
+
+    try:
+      os.removeFile(fname)
+    except:
+      discard
+

src/tani.nim

@@ -1,9 +1,9 @@
 import macros
-import tables, strutils, os
+import tables, strutils, os, typetraits
 
-export tables, strutils, os
+export tables, strutils, os, typetraits
 
-when defined(ECMAScript):
+when defined(js):
   const noColors = true
 else:
   const noColors = defined(noColors)
@@ -11,6 +11,8 @@ else:
   export terminal
 
 import private/utils
+when not defined(js):
+  import private/captureoutput
 
 type
   Test = ref object
@@ -39,12 +41,12 @@ type
     checkFuncName: string
     valTable: Table[string, string]
 
-  PrivateTestError = object of TestAssertError
-    ## Raised when a test tries to access something
-    ## that is private to a module
-
 var testsModuleMap {.compileTime.} = newTable[string, TestsModule]()
 
+var
+  totalTestsPassed = 0
+  totalTests = 0
+
 proc `==`*[T](ar: openarray[T], ar2: openarray[T]): bool =
   ## helper proc to compare arrays
   if len(ar) != len(ar2):
@@ -54,7 +56,7 @@ proc `==`*[T](ar: openarray[T], ar2: openarray[T]): bool =
       return false
   return true
 
-template returnException(name, testName, snip, vals, pos, posRel) =
+template returnException(name, tname, snip, vals, pos, posRel) =
   ## private template for raising an exception
   var
     filename = posRel.filename
@@ -76,7 +78,7 @@ template returnException(name, testName, snip, vals, pos, posRel) =
   exc.lineNumber = line
   exc.column = col
   exc.codeSnip = snip
-  exc.testName = testName
+  exc.testName = tname
   exc.valTable = vals
   exc.checkFuncName = name
   raise exc
@@ -127,6 +129,9 @@ proc `$`*(s: ref object): string =
 proc objToStr*[T: object](obj: var T): string =
   tupleObjToStr(obj)
 
+proc objToStr*[T: object](obj: T): string =
+  tupleObjToStr(obj)
+
 proc objToStr*[T: tuple](obj: T): string =
   result = "tuple " & tupleObjToStr(obj)
 
@@ -141,7 +146,7 @@ macro toString*(obj: typed): untyped =
   case kind:
     of ntyTuple, ntyObject:
       template toStrAst(obj): string =
-        einheit.objToStr(obj)
+        tani.objToStr(obj)
       result = getAst(toStrAst(obj))
     of ntyString:
       template toStrAst(obj): string =
@@ -220,24 +225,14 @@ macro getSyms(code:untyped): untyped =
       initTable[string, string]()
     result = getAst(emptyTable())
 
-template check*(code: untyped) =
-  ## Assertions for tests
-  if not code:
-    let
-      pos = instantiationInfo(fullpaths=true)
-      posRel = instantiationInfo()
-      vals = getSyms(code)
-      # get ast string with extra spaces ignored
-      snip = astToStr(code).strip().split({'\t', '\v', '\c', '\n', '\f'}).join("; ")
-
-    returnException("check", testName, snip, vals, pos, posRel)
 
 template wrapCode(code): untyped =
   # This is needed to prevent an "unreachable code" error
   # if the code block raises an exception
   (proc () = code)()
 
-template checkRaises*(error: untyped, code: untyped): untyped =
+template genCheckRaises(tname) =
+  template checkRaises(error: typed, code: typed): untyped {.used.}=
     ## Raises a TestAssertError when the exception "error" is
     ## not thrown in the code
     let
@@ -250,8 +245,7 @@ template checkRaises*(error: untyped, code: untyped): untyped =
         codeStr = astToStr(code).strip().split().join(" ")
         snip = "$1, $2".format(astToStr(error), codeStr)
         vals = {codeStr: "No Exception Raised"}.toTable()
-      testName = testName
+      returnException("checkRaises", tname, snip, vals, pos, posRel)
-    returnException("checkRaises", testName, snip, vals, pos, posRel)
 
     except error:
       discard
@@ -263,19 +257,20 @@ template checkRaises*(error: untyped, code: untyped): untyped =
         codeStr = astToStr(code).strip().split().join(" ")
         snip = "$1, $2".format(astToStr(error), codeStr)
         vals = {codeStr: $e.name}.toTable()
-      testName = testName
 
-    returnException("checkRaises", testName, snip, vals, pos, posRel)
+      returnException("checkRaises", tname, snip, vals, pos, posRel)
 
-
+proc getTicks(): string {.used.}=
-proc printRunning*(testsInfo: TestsInfo) =
   let termSize = getTermSize()
+
   var
     numTicks = termSize.width
-    ticks = ""
 
   for i in 0..<numTicks:
-    ticks &= "-"
+    result &= "-"
+
+proc printRunning*(testsInfo: TestsInfo) =
+  let ticks = getTicks()
 
   when not defined(quiet):
     when not noColors:
@@ -283,7 +278,7 @@ proc printRunning*(testsInfo: TestsInfo) =
         styleBright,
         fgYellow, "\l" & ticks,
         fgYellow, "\l\l[Running]",
-        fgWhite, " tests in $1 ".format(testsInfo.fileName)
+        fgWhite, " tests in ", fgBlue, testsInfo.fileName
       )
     else:
       echo "\l$1\l".format(ticks)
@@ -301,22 +296,18 @@ proc printPassedTests*(info: TestsInfo) =
     if info.testsPassed != info.numTests:
       color = fgRed
 
-  var passedStr = "[" & $info.testsPassed & "/" & $info.numTests & "]"
+  var passedStr {.used.} = "[" & $info.testsPassed & "/" & $info.numTests & "]"
 
   when not defined(quiet):
     when not noColors:
       styledEcho(
         styleBright, color, "\l", passedStr,
-        fgWhite, " tests passed for ", info.fileName, "."
+        fgWhite, " tests passed for ", fgBlue, info.fileName
       )
     else:
       echo "\l$1 tests passed for $2.".format(passedStr, info.fileName)
 
-
+template printTestProgress(info, testName) =
-template runTest(procCall, info, testName) =
-
-  try:
-    procCall
   when defined(quiet):
     when noColors:
       stdout.write(".")
@@ -335,49 +326,7 @@ template runTest(procCall, info, testName) =
     else:
       echo "$1     $2".format(okStr, testName)
 
-    info.testsPassed += 1
+template printTestFailed(info, testName, e) =
-    info.lastTestFailed = false
-  except PrivateTestError as e:
-    info.numTests -= 1
-    when defined(quiet):
-      when noColors:
-        stdout.write("N")
-      else:
-        setForegroundColor(fgBlue)
-        writeStyled("N", {styleBright})
-        setForegroundColor(fgWhite)
-    else:
-      when not noColors:
-        styledEcho(styleBright,
-                   fgBlue, "\l[Not run]",
-                   fgWhite, " ", testName)
-      else:
-        echo "\l[Not run] $1".format(testName)
-
-      let
-        name = e.checkFuncName
-        snip = e.codeSnip
-        line = e.lineNumber
-        col = e.column
-        filename = e.fileName
-        vals = e.valTable
-
-      when not noColors:
-        styledEcho(styleDim, fgWhite,
-          "  Test code contains private or non accessible symbols:")
-
-        styledEcho(styleDim, fgGreen, "    " & snip)
-
-        styledEcho(
-          styleDim, fgWhite,
-          "  Location: $1; line $2; col $3".format(filename, line, col))
-      else:
-        echo "  Test code contains private or non accessible symbols:\l    $1".format(snip)
-        echo "  Location: $1; line $2; col: $3".format(filename, line, col)
-
-    info.lastTestFailed = true
-
-  except TestAssertError as e:
   when defined(quiet):
     when noColors:
       stdout.write("F")
@@ -422,10 +371,54 @@ template runTest(procCall, info, testName) =
 
       echo "  Location: $1; line $2; col: $3".format(filename, line, col)
 
+
+template printTestOutput(info, testName, output) =
+  when not defined(quiet):
+    if output.len > 0:
+      let ticks = getTicks()
+      when noColors:
+        echo "\l" & ticks
+        echo "\l[Captured output for test \"$1\"]\l".format(testName)
+        echo "  " & output.strip().replace("\l", "\l  ")
+        echo "\l$1\l".format(ticks)
+      else:
+        styledEcho(styleDim,
+                   fgBlue, "\l$1\l".format(ticks),
+                   fgBlue, "\l[Captured output for test \"$1\"]".format(testName))
+
+        styledEcho(styleDim,
+                   fgWhite, "\l  " & output.strip().replace("\l", "\l  "))
+
+        styledEcho(styleDim,
+                   fgBlue, "\l$1\l".format(ticks))
+
+
+template runTest(procCall, info, testName) =
+
+  var output = ""
+  try:
+    when not defined(js):
+      let (testOut, exc) = captureOutput(procCall)
+      output = testOut
+
+      if not exc.isNil:
+        raise exc
+    else:
+      procCall
+
+    printTestProgress(info, testName)
+    printTestOutput(info, testName, output)
+
+    info.testsPassed += 1
+    info.lastTestFailed = false
+
+  except TestAssertError as e:
+    printTestFailed(info, testName, e)
+    printTestOutput(info, testName, output)
     info.lastTestFailed = true
 
 
-proc printSummary(totalTestsPassed: int, totalTests: int) =
+proc printSummary*() =
   when not noColors:
     var summaryColor = fgGreen
 
@@ -443,14 +436,7 @@ proc printSummary(totalTestsPassed: int, totalTests: int) =
       echo "\l\l$1 tests passed.".format(passedStr)
   else:
 
-    let termSize = getTermSize()
+    let ticks = getTicks()
-
-    var
-      ticks = ""
-      numTicks = termSize.width
-
-    for i in 0..<numTicks:
-      ticks &= "-"
 
     when not noColors:
       styledEcho(styleBright,
@@ -477,63 +463,43 @@ template createRunTests(tests, testsInfo, totalTests, totalTestsPassed) =
       writeStyled(testsInfo.fileName & " ", {styleBright})
 
   tests
+
   testsInfo.printPassedTests()
 
   totalTests += testsInfo.numTests
   totalTestsPassed += testsInfo.testsPassed
 
 template makeProc(body, tnameSym, tName, lineInfo, currentDir) =
-  when not compiles((proc (tnameSym: string) = body)(tName)):
-    (proc (tnameSym: string) =
-      let
-        #vals = getSyms(body)
-        # get ast string with extra spaces ignored
-        astBody = astToStr(body).strip().split({'\t', '\v', '\c', '\n', '\f'})
-        snip = astBody[1..<astBody.len].join("\l    ")
-
-      var
-        filename = lineInfo.fileName.relativePath(currentDir)
-        line = lineInfo.line
-        col = lineInfo.column
-
-      var message = "\lCode contains private or non accessible symbols."
-      message &= "  Location: $1; line $2; col: $3".format(filename, line, col)
-
-      var exc = newException(PrivateTestError, message)
-      exc.fileName = filename
-      exc.lineNumber = line
-      exc.column = col
-      exc.codeSnip = snip
-      exc.testName = tName
-      #exc.valTable = vals
-      exc.checkFuncName = tnameSym
-      raise exc
-    )(tName)
-  else:
   (proc (tnameSym: string) = body)(tName)
 
 template createInfo(infoSym, name, ntests, currentDir) =
-  let infoSym = TestsInfo(fileName: name.relativePath(currentDir), numTests: ntests)
+  let infoSym = TestsInfo(
+    fileName: name.relativePath(currentDir),
+    numTests: ntests
+  )
 
-template createTotalVars(totalTestsPassed, totalTests) =
+template genCheck(tname) =
-  var
+  template check(code: untyped) {.used.}=
-    totalTests = 0
+    ## Assertions for tests
-    totalTestsPassed = 0
+    if not code:
+      let
+        pos = instantiationInfo(fullpaths=true)
+        posRel = instantiationInfo()
+        vals = getSyms(code)
+        # get ast string with extra spaces ignored
+        snip = astToStr(code).strip().split({'\t', '\v', '\c', '\n', '\f'}).join("; ")
 
-template printFinalSummary(totalTestsPassed, totalTests) =
+      returnException("check", tname, snip, vals, pos, posRel)
-  printSummary(totalTestsPassed, totalTests)
 
-proc expandTests(returnErrorCode: bool, currentDir: string): NimNode =
+proc expandTests(currentDir, currentFile: string): NimNode {.used.} =
 
   result = newNimNode(nnkStmtList)
 
   let
-    totalTestsPassed = genSym(nskVar, "totalPassed")
+    totalTestsPassed = bindSym("totalTestsPassed")
-    totalTests = genSym(nskVar, "totalTests")
+    totalTests = bindSym("totalTests")
 
-  result.add(getAst(createTotalVars(totalTestsPassed, totalTests)))
+  let name = currentFile
-
-  for name in testsModuleMap.keys():
   let
     testsModule = testsModuleMap[name]
     infoSym = genSym(nskLet, "testsInfo")
@@ -548,13 +514,10 @@ proc expandTests(returnErrorCode: bool, currentDir: string): NimNode =
       tname = genSym(nskParam, "tname")
 
     body.insert(0,
-        nnkLetSection.newTree(
+      getAst(genCheckRaises(tname))
-          nnkIdentDefs.newTree(
-            ident("testName"),
-            newEmptyNode(),
-            tname
-          )
     )
+    body.insert(0,
+      getAst(genCheck(tname))
     )
 
     let
@@ -566,9 +529,14 @@ proc expandTests(returnErrorCode: bool, currentDir: string): NimNode =
 
   result.add(getAst(createRunTests(allTests, infoSym, totalTests, totalTestsPassed)))
 
-  result.add(getAst(printFinalSummary(totalTestsPassed, totalTests)))
+  result = quote do:
-  if returnErrorCode:
+    (proc () = `result`)()
-    result.add(quote do: `totalTests` - `totalTestsPassed`)
+
+proc getResult*(): int =
+  return totalTests - totalTestsPassed
+
+macro fixture*(procDef: untyped): untyped =
+  return procDef
 
 macro test*(name: string, body: untyped): untyped =
   ## The test macro. No runtime overhead is introduced using it and
@@ -578,11 +546,34 @@ macro test*(name: string, body: untyped): untyped =
     testsInfo = getTestsModule(fileName)
   testsInfo.addTest(body, name)
 
-macro runTests*(returnErrorCode=false): untyped =
+macro runTests*(site: varargs[untyped]): untyped =
-  ## This macro must be run in the main testing module that imports
+  ## This macro needs to be put in all modules that have tests.
-  ## all of the other tests.
   ##
-  ## If returnErrorCode is true, the number of failed tests will
+  ## Tests will be run when the symbol ``test`` is defined to the compiler (-d: test)
-  ## be returned.
+  when defined(test):
-  let currentDir = getProjectPath()
+    let
-  return expandTests(returnErrorCode.boolVal, currentDir)
+      currentDir = getProjectPath()
+      currentFile = site.lineInfoObj.fileName
+    return expandTests(currentDir, currentFile)
+
+macro runTestsMain*(site: varargs[untyped]): untyped =
+  ## This macro must be run in the main testing module that imports
+  ## all of the other test modules.
+  ##
+  ## After this is run, quitting with an error code of ``totalTests - totalPassedTests``
+  ## is the default behavior. To change this, define ``noErrorCode`` in the compiler
+  ## options and manage the error code yourself with ``getResult()``. This only executes
+  ## when the ``test`` symbol is defined.
+  when defined(test):
+    let
+      currentDir = getProjectPath()
+      currentFile = site.lineInfoObj.fileName
+      expTests = expandTests(currentDir, currentFile)
+
+    template runAll(expTests) =
+      expTests
+      printSummary()
+      when not defined(noErrorCode):
+        quit(getResult())
+
+    return getAst(runAll(expTests))

tests/test.nim

@@ -1,6 +1,23 @@
-import tani, tables, strutils
+import tani, test2
+import terminal
+import os
 
+proc foo() =
   test "test 2":
     let foo = 100
+    foo()
     check(foo < 4)
-runTests()
+
+proc y(): int =
+  test "y test1":
+    check y() > 0
+
+  test "foo bar":
+    echo os.joinPath("test", "output")
+    check y() > 0
+    checkRaises(Exception):
+      raise newException(Exception, "Foo Exception")
+
+  return 10
+
+runTestsMain()

tests/test2.nim

@@ -0,0 +1,29 @@
+import tani
+
+type
+  MyObj* = object
+    stuff*: int
+
+proc stuff2*(): int =
+  test "adfsadf":
+    check stuff2() == 200
+  return 200
+
+
+proc testing(): MyObj =
+  test "my cool test":
+    check testing().stuff == 200
+
+  return MyObj(stuff: 200)
+
+proc stuff() =
+  test "my cool test":
+    var x = 20
+    if x == 20:
+      echo "hoo"
+    stuff()
+    check x == 30
+    check testing().stuff == 200
+  echo "hello"
+
+runTests()