[Testament] Extend and document message testing aids (#19996)

* [Testament] Extend and document message testing aids

* Enable inline msgs when not reject action.
Eliminates the pain of changing the line and column numbers in `nimout`
or `output` while making changes to the test.
* Enable using inline msgs and nimout together.
Allows ease of inline msgs for the test as well as testing msgs from
other modules.
* Add path separator and test filename variable interpolation in
msgs.
Eases handling path separators in the msgs.
* Add some documentation.

* Fixed lots of broken tests

* Fixed more broken tests

* Support multiple inline messages per a line

* Fix a broken test

* Revert variable substitution in `output`

* Remove uneeded params

* Update doc/testament.md

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

* Update testament/specs.nim

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

* Update testament/specs.nim

Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

* Fix indentation

Co-authored-by: quantimnot <quantimnot@users.noreply.github.com>
Co-authored-by: Clay Sweetser <Varriount@users.noreply.github.com>

testament/testament.nim

@@ -338,46 +338,24 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
       discard waitForExit(p)
       close(p)
 
-proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest,
-                          target: TTarget, extraOptions: string) =
-  let pegLine = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' {[^:]*} ':' \s* {.*}"
-  var covered = initIntSet()
-  for line in splitLines(given.nimout):
+proc toString(inlineError: InlineError, filename: string): string =
+  result.add "$file($line, $col) $kind: $msg" % [
+    "file", filename,
+    "line", $inlineError.line,
+    "col", $inlineError.col,
+    "kind", $inlineError.kind,
+    "msg", $inlineError.msg
+  ]
 
-    if line =~ pegLine:
-      let file = extractFilename(matches[0])
-      let line = try: parseInt(matches[1]) except: -1
-      let col = try: parseInt(matches[2]) except: -1
-      let kind = matches[3]
-      let msg = matches[4]
+proc inlineErrorsMsgs(expected: TSpec): string =
+  for inlineError in expected.inlineErrors.items:
+    result.addLine inlineError.toString(expected.filename)
 
-      if file == extractFilename test.name:
-        var i = 0
-        for x in expected.inlineErrors:
-          if x.line == line and (x.col == col or x.col < 0) and
-              x.kind == kind and x.msg in msg:
-            covered.incl i
-          inc i
-
-  block coverCheck:
-    for j in 0..high(expected.inlineErrors):
-      if j notin covered:
-        var e = test.name
-        e.add '('
-        e.addInt expected.inlineErrors[j].line
-        if expected.inlineErrors[j].col > 0:
-          e.add ", "
-          e.addInt expected.inlineErrors[j].col
-        e.add ") "
-        e.add expected.inlineErrors[j].kind
-        e.add ": "
-        e.add expected.inlineErrors[j].msg
-
-        r.addResult(test, target, extraOptions, e, given.nimout, reMsgsDiffer)
-        break coverCheck
-
-    r.addResult(test, target, extraOptions, "", given.msg, reSuccess)
-    inc(r.passed)
+proc checkForInlineErrors(expected, given: TSpec): bool =
+  for inlineError in expected.inlineErrors:
+    if inlineError.toString(expected.filename) notin given.nimout:
+      return false
+  true
 
 proc nimoutCheck(expected, given: TSpec): bool =
   result = true
@@ -389,15 +367,16 @@ proc nimoutCheck(expected, given: TSpec): bool =
 
 proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest,
              target: TTarget, extraOptions: string) =
-  if expected.inlineErrors.len > 0:
-    checkForInlineErrors(r, expected, given, test, target, extraOptions)
+  if not checkForInlineErrors(expected, given) or
+    (not expected.nimoutFull and not nimoutCheck(expected, given)):
+      r.addResult(test, target, extraOptions, expected.nimout & inlineErrorsMsgs(expected), given.nimout, reMsgsDiffer)
   elif strip(expected.msg) notin strip(given.msg):
     r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer)
   elif not nimoutCheck(expected, given):
     r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer)
   elif extractFilename(expected.file) != extractFilename(given.file) and
       "internal error:" notin expected.msg:
-    r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer)
+    r.addResult(test, target, extraOptions, expected.filename, given.file, reFilesDiffer)
   elif expected.line != given.line and expected.line != 0 or
        expected.column != given.column and expected.column != 0:
     r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column,
@@ -449,9 +428,10 @@ proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string,
     if expected.needsCodegenCheck:
       codegenCheck(test, target, expected, expectedmsg, given)
       givenmsg = given.msg
-    if not nimoutCheck(expected, given):
+    if not nimoutCheck(expected, given) or
+       not checkForInlineErrors(expected, given):
       given.err = reMsgsDiffer
-      expectedmsg = expected.nimout
+      expectedmsg = expected.nimout & inlineErrorsMsgs(expected)
       givenmsg = given.nimout.strip
   else:
     givenmsg = "$ " & given.cmd & '\n' & given.nimout
@@ -480,17 +460,11 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
     r.addResult(test, target, extraOptions, "", "", test.spec.err)
     inc(r.skipped)
     return
-
-  template callNimCompilerImpl(): untyped =
-    # xxx this used to also pass: `--stdout --hint:Path:off`, but was done inconsistently
-    # with other branches
-    callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
+  var given = callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions)
   case expected.action
   of actionCompile:
-    var given = callNimCompilerImpl()
     compilerOutputTests(test, target, extraOptions, given, expected, r)
   of actionRun:
-    var given = callNimCompilerImpl()
     if given.err != reSuccess:
       r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr)
     else:
@@ -540,10 +514,8 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec,
               (expected.outputCheck == ocSubstr and expected.output notin bufB):
             given.err = reOutputsDiffer
             r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer)
-          else:
-            compilerOutputTests(test, target, extraOptions, given, expected, r)
+          compilerOutputTests(test, target, extraOptions, given, expected, r)
   of actionReject:
-    let given = callNimCompilerImpl()
     cmpMsgs(r, expected, given, test, target, extraOptions)
 
 proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) =