[Testament] Extend and document message testing aids (#19996)

* [Testament] Extend and document message testing aids

* Enable inline msgs when not reject action.
Eliminates the pain of changing the line and column numbers in `nimout`
or `output` while making changes to the test.
* Enable using inline msgs and nimout together.
Allows ease of inline msgs for the test as well as testing msgs from
other modules.
* Add path separator and test filename variable interpolation in
msgs.
Eases handling path separators in the msgs.
* Add some documentation.

* Fixed lots of broken tests

* Fixed more broken tests

* Support multiple inline messages per a line

* Fix a broken test

* Revert variable substitution in `output`

* Remove uneeded params

* Update doc/testament.md

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

* Update testament/specs.nim

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

* Update testament/specs.nim

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

* Fix indentation

Co-authored-by: quantimnot <quantimnot@users.noreply.github.com>
Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

doc/testament.md

@@ -113,7 +113,7 @@ Example "template" **to edit** and write a Testament unittest:
     # Provide an `output` string to assert that the test prints to standard out
     # exactly the expected string. Provide an `outputsub` string to assert that
     # the string given here is a substring of the standard out output of the
-    # test.
+    # test (the output includes both the compiler and test execution output).
     output: ""
     outputsub: ""
 
@@ -153,8 +153,7 @@ Example "template" **to edit** and write a Testament unittest:
     # Command the test should use to run. If left out or an empty string is
     # provided, the command is taken to be:
     # "nim $target --hints:on -d:testing --nimblePath:build/deps/pkgs $options $file"
-    # You can use the $target, $options, and $file placeholders in your own
+    # Subject to variable interpolation.
-    # command, too.
     cmd: "nim c -r $file"
 
     # Maximum generated temporary intermediate code file size for the test.
@@ -189,7 +188,76 @@ Example "template" **to edit** and write a Testament unittest:
 * `This is not the full spec of Testament, check the Testament Spec on GitHub, see parseSpec(). <https://github.com/nim-lang/Nim/blob/devel/testament/specs.nim#L238>`_
 * `Nim itself uses Testament, so there are plenty of test examples. <https://github.com/nim-lang/Nim/tree/devel/tests>`_
 * Has some built-in CI compatibility, like Azure Pipelines, etc.
-* `Testament supports inlined error messages on Unittests, basically comments with the expected error directly on the code. <https://github.com/nim-lang/Nim/blob/9a110047cbe2826b1d4afe63e3a1f5a08422b73f/tests/effects/teffects1.nim>`_
+
+
+Inline hints, warnings and errors (notes)
+-----------------------------------------
+
+Testing the line, column, kind and message of hints, warnings and errors can
+be written inline like so:
+
+.. code-block:: nim
+
+  {.warning: "warning!!"} #[tt.Warning
+           ^ warning!! [User] ]#
+
+The opening `#[tt.` marks the message line.
+The `^` marks the message column.
+
+Inline messages can be combined with `nimout` when `nimoutFull` is false (default).
+This allows testing for expected messages from other modules:
+
+.. code-block:: nim
+
+  discard """
+    nimout: "config.nims(1, 1) Hint: some hint message [User]"
+  """
+  {.warning: "warning!!"} #[tt.Warning
+           ^ warning!! [User] ]#
+
+Multiple messages for a line can be checked by delimiting messages with ';':
+
+.. code-block:: nim
+
+  discard """
+    matrix: "--errorMax:0 --styleCheck:error"
+  """
+
+  proc generic_proc*[T](a_a: int) = #[tt.Error
+       ^ 'generic_proc' should be: 'genericProc'; tt.Error
+                        ^ 'a_a' should be: 'aA' ]#
+    discard
+
+Use `--errorMax:0` in `matrix`, or `cmd: "nim check $file"` when testing
+for multiple 'Error' messages.
+
+Output message variable interpolation
+-------------------------------------
+
+`errormsg`, `nimout`, and inline messages are subject to these variable interpolations:
+
+* `${/}` - platform's directory separator
+* `$file` - the filename (without directory) of the test
+
+All other `$` characters need escaped as `$$`.
+
+Cmd variable interpolation
+--------------------------
+
+The `cmd` option is subject to these variable interpolations:
+
+* `$target` - the compilation target, e.g. `c`.
+* `$options` - the options for the compiler.
+* `$file` - the file path of the test.
+* `$filedir` - the directory of the test file.
+
+.. code-block:: nim
+
+  discard """
+    cmd: "nim $target --nimblePath:./nimbleDir/simplePkgs $options $file"
+  """
+
+All other `$` characters need escaped as `$$`.
 
 
 Unit test Examples

testament/specs.nim

@@ -75,6 +75,7 @@ type
     # xxx make sure `isJoinableSpec` takes into account each field here.
     action*: TTestAction
     file*, cmd*: string
+    filename*: string ## Test filename (without path).
     input*: string
     outputCheck*: TOutputCheck
     sortoutput*: bool
@@ -127,19 +128,56 @@ when not declared(parseCfgBool):
     of "n", "no", "false", "0", "off": result = false
     else: raise newException(ValueError, "cannot interpret as a bool: " & s)
 
+proc addLine*(self: var string; pieces: varargs[string]) =
+  for piece in pieces:
+    self.add piece
+  self.add "\n"
+
+
 const
-  inlineErrorMarker = "#[tt."
+  inlineErrorKindMarker = "tt."
+  inlineErrorMarker = "#[" & inlineErrorKindMarker
 
 proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int =
+  ## Extract inline error messages.
+  ##
+  ## Can parse a single message for a line:
+  ##
+  ## .. code-block:: nim
+  ##
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name] ]#
+  ##
+  ## Can parse multiple messages for a line when they are separated by ';':
+  ##
+  ## .. code-block:: nim
+  ##
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name]; tt.Error
+  ##                           ^ 'no_destroy' should be: 'nodestroy' [Name]; tt.Error
+  ##                                       ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]#
+  ##
+  ## .. code-block:: nim
+  ##
+  ##   proc generic_proc*[T] {.no_destroy, userPragma.} = #[tt.Error
+  ##        ^ 'generic_proc' should be: 'genericProc' [Name];
+  ##     tt.Error              ^ 'no_destroy' should be: 'nodestroy' [Name];
+  ##     tt.Error                          ^ 'userPragma' should be: 'user_pragma' [template declared in mstyleCheck.nim(10, 9)] [Name] ]#
+  ##
   result = i + len(inlineErrorMarker)
   inc col, len(inlineErrorMarker)
+  let msgLine = line
+  var msgCol = -1
+  var msg = ""
   var kind = ""
+
+  template parseKind =
     while result < s.len and s[result] in IdentChars:
       kind.add s[result]
       inc result
       inc col
-
+    if kind notin ["Hint", "Warning", "Error"]:
-  var caret = (line, -1)
+      spec.parseErrors.addLine "expected inline message kind: Hint, Warning, Error"
 
   template skipWhitespace =
     while result < s.len and s[result] in Whitespace:
@@ -150,34 +188,70 @@ proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var T
         inc col
       inc result
 
-  skipWhitespace()
+  template parseCaret =
     if result < s.len and s[result] == '^':
-    caret = (line-1, col)
+      msgCol = col
       inc result
       inc col
       skipWhitespace()
+    else:
+      spec.parseErrors.addLine "expected column marker ('^') for inline message"
 
-  var msg = ""
+  template isMsgDelimiter: bool =
-  while result < s.len-1:
+    s[result] == ';' and
-    if s[result] == '\n':
+    (block:
-      inc result
+      let nextTokenIdx = result + 1 + parseutils.skipWhitespace(s, result + 1)
-      inc line
+      if s.len > nextTokenIdx + len(inlineErrorKindMarker) and
-      col = 1
+         s[nextTokenIdx..(nextTokenIdx + len(inlineErrorKindMarker) - 1)] == inlineErrorKindMarker:
-    elif s[result] == ']' and s[result+1] == '#':
+        true
+      else:
+        false)
+
+  template trimTrailingMsgWhitespace =
     while msg.len > 0 and msg[^1] in Whitespace:
       setLen msg, msg.len - 1
 
-      inc result
+  template addInlineError =
-      inc col, 2
+    doAssert msg[^1] notin Whitespace
     if kind == "Error": spec.action = actionReject
-      spec.unjoinable = true
+    spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: msgLine, col: msgCol)
-      spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: caret[0], col: caret[1])
+
+  parseKind()
+  skipWhitespace()
+  parseCaret()
+
+  while result < s.len-1:
+    if s[result] == '\n':
+      msg.add '\n'
+      inc result
+      inc line
+      col = 1
+    elif isMsgDelimiter():
+      trimTrailingMsgWhitespace()
+      inc result
+      skipWhitespace()
+      addInlineError()
+      inc result, len(inlineErrorKindMarker)
+      inc col, 1 + len(inlineErrorKindMarker)
+      kind.setLen 0
+      msg.setLen 0
+      parseKind()
+      skipWhitespace()
+      parseCaret()
+    elif s[result] == ']' and s[result+1] == '#':
+      trimTrailingMsgWhitespace()
+      inc result, 2
+      inc col, 2
+      addInlineError()
       break
     else:
       msg.add s[result]
       inc result
       inc col
 
+  if spec.inlineErrors.len > 0:
+    spec.unjoinable = true
+
 proc extractSpec(filename: string; spec: var TSpec): string =
   const
     tripleQuote = "\"\"\""
@@ -229,15 +303,6 @@ proc parseTargets*(value: string): set[TTarget] =
     of "js": result.incl(targetJS)
     else: raise newException(ValueError, "invalid target: '$#'" % v)
 
-proc addLine*(self: var string; a: string) =
-  self.add a
-  self.add "\n"
-
-proc addLine*(self: var string; a, b: string) =
-  self.add a
-  self.add b
-  self.add "\n"
-
 proc initSpec*(filename: string): TSpec =
   result.file = filename
 
@@ -249,6 +314,7 @@ proc isCurrentBatch*(testamentData: TestamentData; filename: string): bool =
 
 proc parseSpec*(filename: string): TSpec =
   result.file = filename
+  result.filename = extractFilename(filename)
   let specStr = extractSpec(filename, result)
   var ss = newStringStream(specStr)
   var p: CfgParser
@@ -439,3 +505,15 @@ proc parseSpec*(filename: string): TSpec =
   result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable
   if not result.inCurrentBatch:
     result.err = reDisabled
+
+  # Interpolate variables in msgs:
+  template varSub(msg: string): string =
+    try:
+      msg % ["/", $DirSep, "file", result.filename]
+    except ValueError:
+      result.parseErrors.addLine "invalid variable interpolation (see 'https://nim-lang.github.io/Nim/testament.html#writing-unitests-output-message-variable-interpolation')"
+      msg
+  result.nimout = result.nimout.varSub
+  result.msg = result.msg.varSub
+  for inlineError in result.inlineErrors.mitems:
+    inlineError.msg = inlineError.msg.varSub

testament/testament.nim

@@ -338,46 +338,24 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
       discard waitForExit(p)
       close(p)
 
-proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest,
+proc toString(inlineError: InlineError, filename: string): string =
-                          target: TTarget, extraOptions: string) =
+  result.add "$file($line, $col) $kind: $msg" % [
-  let pegLine = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' {[^:]*} ':' \s* {.*}"
+    "file", filename,
-  var covered = initIntSet()
+    "line", $inlineError.line,
-  for line in splitLines(given.nimout):
+    "col", $inlineError.col,
+    "kind", $inlineError.kind,
+    "msg", $inlineError.msg
+  ]
 
-    if line =~ pegLine:
+proc inlineErrorsMsgs(expected: TSpec): string =
-      let file = extractFilename(matches[0])
+  for inlineError in expected.inlineErrors.items:
-      let line = try: parseInt(matches[1]) except: -1
+    result.addLine inlineError.toString(expected.filename)
-      let col = try: parseInt(matches[2]) except: -1
-      let kind = matches[3]
-      let msg = matches[4]
 
-      if file == extractFilename test.name:
+proc checkForInlineErrors(expected, given: TSpec): bool =
-        var i = 0
+  for inlineError in expected.inlineErrors:
-        for x in expected.inlineErrors:
+    if inlineError.toString(expected.filename) notin given.nimout:
-          if x.line == line and (x.col == col or x.col < 0) and
+      return false
-              x.kind == kind and x.msg in msg:
+  true
-            covered.incl i
-          inc i
-
-  block coverCheck:
-    for j in 0..high(expected.inlineErrors):
-      if j notin covered:
-        var e = test.name
-        e.add '('
-        e.addInt expected.inlineErrors[j].line
-        if expected.inlineErrors[j].col > 0:
-          e.add ", "
-          e.addInt expected.inlineErrors[j].col
-        e.add ") "
-        e.add expected.inlineErrors[j].kind
-        e.add ": "
-        e.add expected.inlineErrors[j].msg
-
-        r.addResult(test, target, extraOptions, e, given.nimout, reMsgsDiffer)
-        break coverCheck
-
-    r.addResult(test, target, extraOptions, "", given.msg, reSuccess)
-    inc(r.passed)
 
 proc nimoutCheck(expected, given: TSpec): bool =
   result = true
@@ -389,15 +367,16 @@ proc nimoutCheck(expected, given: TSpec): bool =
 
 proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest,
              target: TTarget, extraOptions: string) =
-  if expected.inlineErrors.len > 0:
+  if not checkForInlineErrors(expected, given) or
-    checkForInlineErrors(r, expected, given, test, target, extraOptions)
+    (not expected.nimoutFull and not nimoutCheck(expected, given)):
+      r.addResult(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer)
   elif strip(expected.msg) notin strip(given.msg):
     r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer)
   elif not nimoutCheck(expected, given):
     r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer)
   elif extractFilename(expected.file) != extractFilename(given.file) and
       "internal error:" notin expected.msg:
-    r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer)
+    r.addResult(test, target, extraOptions, expected.filename, given.file, reFilesDiffer)
   elif expected.line != given.line and expected.line != 0 or
        expected.column != given.column and expected.column != 0:
     r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column,
@@ -449,9 +428,10 @@ proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string,
     if expected.needsCodegenCheck:
       codegenCheck(test, target, expected, expectedmsg, given)
       givenmsg = given.msg
-    if not nimoutCheck(expected, given):
+    if not nimoutCheck(expected, given) or
+       not checkForInlineErrors(expected, given):
       given.err = reMsgsDiffer
-      expectedmsg = expected.nimout
+      expectedmsg = expected.nimout & inlineErrorsMsgs(expected)
       givenmsg = given.nimout.strip
   else:
     givenmsg = "$ " & given.cmd & '\n' & given.nimout
@@ -480,17 +460,11 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
     r.addResult(test, target, extraOptions, "", "", test.spec.err)
     inc(r.skipped)
     return
-
+  var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
-  template callNimCompilerImpl(): untyped =
-    # xxx this used to also pass: `--stdout --hint:Path:off`, but was done inconsistently
-    # with other branches
-    callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
   case expected.action
   of actionCompile:
-    var given = callNimCompilerImpl()
     compilerOutputTests(test, target, extraOptions, given, expected, r)
   of actionRun:
-    var given = callNimCompilerImpl()
     if given.err != reSuccess:
       r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr)
     else:
@@ -540,10 +514,8 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
               (expected.outputCheck == ocSubstr and expected.output notin bufB):
             given.err = reOutputsDiffer
             r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer)
-          else:
           compilerOutputTests(test, target, extraOptions, given, expected, r)
   of actionReject:
-    let given = callNimCompilerImpl()
     cmpMsgs(r, expected, given, test, target, extraOptions)
 
 proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) =

tests/arc/topt_cursor.nim

@@ -12,7 +12,7 @@ try:
     x_cursor = ("different", 54) else:
     x_cursor = ("string here", 80)
   echo [
-    :tmpD = `$`(x_cursor)
+    :tmpD = `$$`(x_cursor)
     :tmpD]
 finally:
   `=destroy`(:tmpD)

tests/arc/topt_no_cursor.nim

@@ -78,7 +78,7 @@ try:
     `=copy`(:tmpD_1, it_cursor.val)
     :tmpD_1)
   echo [
-    :tmpD_2 = `$`(a)
+    :tmpD_2 = `$$`(a)
     :tmpD_2]
 finally:
   `=destroy`(:tmpD_2)

tests/casestmt/tincompletecaseobject2.nim

@@ -1,12 +1,5 @@
 discard """
 cmd: "nim check $file"
-errormsg: "not all cases are covered; missing: {A, B}"
-nimout: '''
-tincompletecaseobject2.nim(18, 1) Error: not all cases are covered; missing: {' ', '!', '\"', '#', '$', '%', '&', '\'', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~'}
-tincompletecaseobject2.nim(22, 1) Error: not all cases are covered; missing: {B, C, D}
-tincompletecaseobject2.nim(25, 1) Error: not all cases are covered; missing: {A, C}
-tincompletecaseobject2.nim(28, 1) Error: not all cases are covered; missing: {A, B}
-'''
 """
 type
   ABCD = enum A, B, C, D
@@ -15,15 +8,19 @@ type
   AliasRangeABC = RangeABC
   PrintableChars = range[' ' .. '~']
 
-case PrintableChars 'x':
+case PrintableChars 'x': #[tt.Error
+^ not all cases are covered; missing: {' ', '!', '\"', '#', '$$', '%', '&', '\'', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~'}]#
 of '0'..'9', 'A'..'Z', 'a'..'z': discard
 of '(', ')': discard
 
-case AliasABCD A:
+case AliasABCD A: #[tt.Error
+^ not all cases are covered; missing: {B, C, D}]#
 of A: discard
 
-case RangeABC A:
+case RangeABC A: #[tt.Error
+^ not all cases are covered; missing: {A, C}]#
 of B: discard
 
-case AliasRangeABC A:
+case AliasRangeABC A: #[tt.Error
+^ not all cases are covered; missing: {A, B}]#
 of C: discard

tests/effects/teffects1.nim

@@ -1,10 +1,8 @@
 discard """
   cmd: "nim check $file"
-  nimout: '''teffects1.nim(22, 28) template/generic instantiation from here
+  nimout: '''
-teffects1.nim(23, 13) Error: can raise an unlisted exception: ref IOError
+teffects1.nim(17, 28) template/generic instantiation from here
-teffects1.nim(22, 29) Hint: 'lier' cannot raise 'IO2Error' [XCannotRaiseY]
+'''
-teffects1.nim(38, 21) Error: type mismatch: got <proc (x: int): string{.noSideEffect, gcsafe, locks: 0.}> but expected 'MyProcType = proc (x: int): string{.closure.}'
-.raise effects differ'''
 """
 {.push warningAsError[Effect]: on.}
 type
@@ -16,11 +14,10 @@ type
 
 proc forw: int {. .}
 
-proc lier(): int {.raises: [IO2Error].} =
+proc lier(): int {.raises: [IO2Error].} = #[tt.Hint
-  #[tt.Hint                 ^ 'lier' cannot raise 'IO2Error' [XCannotRaiseY] ]#
+                            ^ 'lier' cannot raise 'IO2Error' [XCannotRaiseY] ]#
   writeLine stdout, "arg" #[tt.Error
-            ^  can raise an unlisted exception: ref IOError
+            ^ writeLine stdout, ["arg"] can raise an unlisted exception: ref IOError ]#
-  ]#
 
 proc forw: int =
   raise newException(IOError, "arg")
@@ -38,7 +35,7 @@ proc foo(x: int): string {.raises: [ValueError].} =
 var p: MyProcType = foo #[tt.Error
                     ^
 type mismatch: got <proc (x: int): string{.noSideEffect, gcsafe, locks: 0.}> but expected 'MyProcType = proc (x: int): string{.closure.}'
-
+.raise effects differ
 ]#
 {.pop.}
 {.pop.}

tests/errmsgs/tinteger_literals.nim

@@ -1,15 +1,14 @@
 discard """
-cmd: "nim check $file"
+  cmd: "nim check $file"
-errormsg: "number out of range: '300'u8'"
-nimout: '''
-tinteger_literals.nim(12, 9) Error: number out of range: '18446744073709551616'u64'
-tinteger_literals.nim(13, 9) Error: number out of range: '9223372036854775808'i64'
-tinteger_literals.nim(14, 9) Error: number out of range: '9223372036854775808'
-tinteger_literals.nim(15, 9) Error: number out of range: '300'u8'
-'''
 """
-
+# high(uint64) + 1
-discard 18446744073709551616'u64 # high(uint64) + 1
+discard 18446744073709551616'u64 #[tt.Error
-discard 9223372036854775808'i64  # high(int64) + 1
+        ^ number out of range: '18446744073709551616'u64' ]#
-discard 9223372036854775808      # high(int64) + 1
+# high(int64) + 1
-discard 300'u8
+discard 9223372036854775808'i64 #[tt.Error
+        ^ number out of range: '9223372036854775808'i64' ]#
+# high(int64) + 1
+discard 9223372036854775808 #[tt.Error
+        ^ number out of range: '9223372036854775808' ]#
+discard 300'u8 #[tt.Error
+        ^ number out of range: '300'u8' ]#

tests/readme.md

@@ -27,25 +27,6 @@ Options:
 There are certain spec keys that imply ``run``, including ``output`` and
 ``outputsub``.
 
-## cmd
-
-Specifies the Nim command to use for compiling the test.
-
-There are a number of variables that are replaced in this spec option:
-
-* ``$target`` - the compilation target, e.g. ``c``.
-* ``$options`` - the options for the compiler.
-* ``$file`` - the filename of the test.
-* ``$filedir`` - the directory of the test file.
-
-Example:
-
-```nim
-discard """
-  cmd: "nim $target --nimblePath:./nimbleDir/simplePkgs $options $file"
-"""
-```
-
 # Categories
 
 Each folder under this directory represents a test category, which can be

tests/stdlib/t8925.nim

@@ -1,5 +1,5 @@
 discard """
-  errormsg: "type mismatch between pattern '$i' (position: 1) and HourRange var 'hour'"
+  errormsg: "type mismatch between pattern '$$i' (position: 1) and HourRange var 'hour'"
   file: "strscans.nim"
 """
 

tests/strictnotnil/tnilcheck.nim

@@ -1,6 +1,5 @@
 discard """
-cmd: "nim check $file"
+action: compile
-action: "reject"
 """
 
 import tables

tests/testament/tinlinemsg.nim

@@ -0,0 +1,8 @@
+discard """
+  matrix: "--errorMax:0 --styleCheck:error"
+"""
+
+proc generic_proc*[T](a_a: int) = #[tt.Error
+     ^ 'generic_proc' should be: 'genericProc'; tt.Error
+                      ^ 'a_a' should be: 'aA' ]#
+  discard

tests/testament/tshould_not_work.nim

@@ -43,8 +43,7 @@ import stdtest/testutils
 
 proc main =
   const nim = getCurrentCompilerExe()
-  # TODO: bin/testament instead? like other tools (eg bin/nim, bin/nimsuggest etc)
+  let testamentExe = "bin/testament"
-  let testamentExe = "testament/testament"
   let cmd = fmt"{testamentExe} --directory:testament --colors:off --backendLogging:off --nim:{nim} category shouldfail"
   let (outp, status) = execCmdEx(cmd)
   doAssert status == 1, $status

tests/views/tdont_mutate.nim

@@ -10,7 +10,7 @@ const
   Whitespace = {' ', '\t', '\n', '\r'}
 
 proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): Table[int, openArray[char]] #[tt.Error
-      'result' borrows from the immutable location 's' and attempts to mutate it
+^ 'result' borrows from the immutable location 's' and attempts to mutate it
     ]# =
   var last = 0
   var splits = maxsplit
@@ -35,7 +35,7 @@ proc `$`(x: openArray[char]): string =
 
 proc otherTest(x: int) =
   var y: var int = x #[tt.Error
-    'y' borrows from the immutable location 'x' and attempts to mutate it
+      ^ 'y' borrows from the immutable location 'x' and attempts to mutate it
   ]#
   y = 3