[Testament] Extend and document message testing aids (#19996)

* [Testament] Extend and document message testing aids

* Enable inline msgs when not reject action.
Eliminates the pain of changing the line and column numbers in `nimout`
or `output` while making changes to the test.
* Enable using inline msgs and nimout together.
Allows ease of inline msgs for the test as well as testing msgs from
other modules.
* Add path separator and test filename variable interpolation in
msgs.
Eases handling path separators in the msgs.
* Add some documentation.

* Fixed lots of broken tests

* Fixed more broken tests

* Support multiple inline messages per a line

* Fix a broken test

* Revert variable substitution in `output`

* Remove uneeded params

* Update doc/testament.md

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

* Update testament/specs.nim

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

* Update testament/specs.nim

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

* Fix indentation

Co-authored-by: quantimnot <quantimnot@users.noreply.github.com>
Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

tests/arc/topt_cursor.nim

@@ -12,7 +12,7 @@ try:
     x_cursor = ("different", 54) else:
     x_cursor = ("string here", 80)
   echo [
-    :tmpD = `$`(x_cursor)
+    :tmpD = `$$`(x_cursor)
     :tmpD]
 finally:
   `=destroy`(:tmpD)

tests/arc/topt_no_cursor.nim

@@ -78,7 +78,7 @@ try:
     `=copy`(:tmpD_1, it_cursor.val)
     :tmpD_1)
   echo [
-    :tmpD_2 = `$`(a)
+    :tmpD_2 = `$$`(a)
     :tmpD_2]
 finally:
   `=destroy`(:tmpD_2)

tests/casestmt/tincompletecaseobject2.nim

@@ -1,12 +1,5 @@
 discard """
 cmd: "nim check $file"
-errormsg: "not all cases are covered; missing: {A, B}"
-nimout: '''
-tincompletecaseobject2.nim(18, 1) Error: not all cases are covered; missing: {' ', '!', '\"', '#', '$', '%', '&', '\'', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~'}
-tincompletecaseobject2.nim(22, 1) Error: not all cases are covered; missing: {B, C, D}
-tincompletecaseobject2.nim(25, 1) Error: not all cases are covered; missing: {A, C}
-tincompletecaseobject2.nim(28, 1) Error: not all cases are covered; missing: {A, B}
-'''
 """
 type
   ABCD = enum A, B, C, D
@@ -15,15 +8,19 @@ type
   AliasRangeABC = RangeABC
   PrintableChars = range[' ' .. '~']
 
-case PrintableChars 'x':
+case PrintableChars 'x': #[tt.Error
+^ not all cases are covered; missing: {' ', '!', '\"', '#', '$$', '%', '&', '\'', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~'}]#
 of '0'..'9', 'A'..'Z', 'a'..'z': discard
 of '(', ')': discard
 
-case AliasABCD A:
+case AliasABCD A: #[tt.Error
+^ not all cases are covered; missing: {B, C, D}]#
 of A: discard
 
-case RangeABC A:
+case RangeABC A: #[tt.Error
+^ not all cases are covered; missing: {A, C}]#
 of B: discard
 
-case AliasRangeABC A:
+case AliasRangeABC A: #[tt.Error
+^ not all cases are covered; missing: {A, B}]#
 of C: discard

tests/effects/teffects1.nim

@@ -1,10 +1,8 @@
 discard """
   cmd: "nim check $file"
-  nimout: '''teffects1.nim(22, 28) template/generic instantiation from here
-teffects1.nim(23, 13) Error: can raise an unlisted exception: ref IOError
-teffects1.nim(22, 29) Hint: 'lier' cannot raise 'IO2Error' [XCannotRaiseY]
-teffects1.nim(38, 21) Error: type mismatch: got <proc (x: int): string{.noSideEffect, gcsafe, locks: 0.}> but expected 'MyProcType = proc (x: int): string{.closure.}'
-.raise effects differ'''
+  nimout: '''
+teffects1.nim(17, 28) template/generic instantiation from here
+'''
 """
 {.push warningAsError[Effect]: on.}
 type
@@ -16,11 +14,10 @@ type
 
 proc forw: int {. .}
 
-proc lier(): int {.raises: [IO2Error].} =
-  #[tt.Hint                 ^ 'lier' cannot raise 'IO2Error' [XCannotRaiseY] ]#
+proc lier(): int {.raises: [IO2Error].} = #[tt.Hint
+                            ^ 'lier' cannot raise 'IO2Error' [XCannotRaiseY] ]#
   writeLine stdout, "arg" #[tt.Error
-            ^  can raise an unlisted exception: ref IOError
-  ]#
+            ^ writeLine stdout, ["arg"] can raise an unlisted exception: ref IOError ]#
 
 proc forw: int =
   raise newException(IOError, "arg")
@@ -38,7 +35,7 @@ proc foo(x: int): string {.raises: [ValueError].} =
 var p: MyProcType = foo #[tt.Error
                     ^
 type mismatch: got <proc (x: int): string{.noSideEffect, gcsafe, locks: 0.}> but expected 'MyProcType = proc (x: int): string{.closure.}'
-
+.raise effects differ
 ]#
 {.pop.}
 {.pop.}

tests/errmsgs/tinteger_literals.nim

@@ -1,15 +1,14 @@
 discard """
-cmd: "nim check $file"
-errormsg: "number out of range: '300'u8'"
-nimout: '''
-tinteger_literals.nim(12, 9) Error: number out of range: '18446744073709551616'u64'
-tinteger_literals.nim(13, 9) Error: number out of range: '9223372036854775808'i64'
-tinteger_literals.nim(14, 9) Error: number out of range: '9223372036854775808'
-tinteger_literals.nim(15, 9) Error: number out of range: '300'u8'
-'''
+  cmd: "nim check $file"
 """
-
-discard 18446744073709551616'u64 # high(uint64) + 1
-discard 9223372036854775808'i64  # high(int64) + 1
-discard 9223372036854775808      # high(int64) + 1
-discard 300'u8
+# high(uint64) + 1
+discard 18446744073709551616'u64 #[tt.Error
+        ^ number out of range: '18446744073709551616'u64' ]#
+# high(int64) + 1
+discard 9223372036854775808'i64 #[tt.Error
+        ^ number out of range: '9223372036854775808'i64' ]#
+# high(int64) + 1
+discard 9223372036854775808 #[tt.Error
+        ^ number out of range: '9223372036854775808' ]#
+discard 300'u8 #[tt.Error
+        ^ number out of range: '300'u8' ]#

tests/pragmas/tvar_macro.nim

@@ -85,7 +85,7 @@ block: # with other pragmas
   let importedFooBar {.importc: "exportedFooBar", nodecl.}: set[char]
 
   doAssert importedFooBar == fooBar #[tt.Warning
-                            ^ fooBar is deprecated
+                             ^ fooBar is deprecated
   ]#
   
 

tests/readme.md

@@ -27,25 +27,6 @@ Options:
 There are certain spec keys that imply ``run``, including ``output`` and
 ``outputsub``.
 
-## cmd
-
-Specifies the Nim command to use for compiling the test.
-
-There are a number of variables that are replaced in this spec option:
-
-* ``$target`` - the compilation target, e.g. ``c``.
-* ``$options`` - the options for the compiler.
-* ``$file`` - the filename of the test.
-* ``$filedir`` - the directory of the test file.
-
-Example:
-
-```nim
-discard """
-  cmd: "nim $target --nimblePath:./nimbleDir/simplePkgs $options $file"
-"""
-```
-
 # Categories
 
 Each folder under this directory represents a test category, which can be

tests/stdlib/t8925.nim

@@ -1,5 +1,5 @@
 discard """
-  errormsg: "type mismatch between pattern '$i' (position: 1) and HourRange var 'hour'"
+  errormsg: "type mismatch between pattern '$$i' (position: 1) and HourRange var 'hour'"
   file: "strscans.nim"
 """
 

tests/strictnotnil/tnilcheck.nim

@@ -1,6 +1,5 @@
 discard """
-cmd: "nim check $file"
-action: "reject"
+action: compile
 """
 
 import tables

tests/testament/tinlinemsg.nim

@@ -0,0 +1,8 @@
+discard """
+  matrix: "--errorMax:0 --styleCheck:error"
+"""
+
+proc generic_proc*[T](a_a: int) = #[tt.Error
+     ^ 'generic_proc' should be: 'genericProc'; tt.Error
+                      ^ 'a_a' should be: 'aA' ]#
+  discard

tests/testament/tshould_not_work.nim

@@ -43,8 +43,7 @@ import stdtest/testutils
 
 proc main =
   const nim = getCurrentCompilerExe()
-  # TODO: bin/testament instead? like other tools (eg bin/nim, bin/nimsuggest etc)
-  let testamentExe = "testament/testament"
+  let testamentExe = "bin/testament"
   let cmd = fmt"{testamentExe} --directory:testament --colors:off --backendLogging:off --nim:{nim} category shouldfail"
   let (outp, status) = execCmdEx(cmd)
   doAssert status == 1, $status

tests/views/tdont_mutate.nim

@@ -10,7 +10,7 @@ const
   Whitespace = {' ', '\t', '\n', '\r'}
 
 proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): Table[int, openArray[char]] #[tt.Error
-      'result' borrows from the immutable location 's' and attempts to mutate it
+^ 'result' borrows from the immutable location 's' and attempts to mutate it
     ]# =
   var last = 0
   var splits = maxsplit
@@ -35,7 +35,7 @@ proc `$`(x: openArray[char]): string =
 
 proc otherTest(x: int) =
   var y: var int = x #[tt.Error
-    'y' borrows from the immutable location 'x' and attempts to mutate it
+      ^ 'y' borrows from the immutable location 'x' and attempts to mutate it
   ]#
   y = 3