testament improvement: allow inline error messages inside test cases (#15294)

* testament support for inline error messages
* adapt teffects1.nim test to show the potential

testament/specs.nim

@@ -7,8 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
-import sequtils, parseutils, strutils, os, streams, parsecfg
+import sequtils, parseutils, strutils, os, streams, parsecfg,
-from hashes import hash
+  tables, hashes
 
 type TestamentData* = ref object
   # better to group globals under 1 object; could group the other ones here too
@@ -62,6 +62,11 @@ type
     targetObjC = "ObjC"
     targetJS = "JS"
 
+  InlineError* = object
+    kind*: string
+    msg*: string
+    line*, col*: int
+
   TSpec* = object
     action*: TTestAction
     file*, cmd*: string
@@ -90,6 +95,7 @@ type
     useValgrind*: bool
     timeout*: float # in seconds, fractions possible,
                     # but don't rely on much precision
+    inlineErrors*: seq[InlineError] # line information to error message
 
 proc getCmd*(s: TSpec): string =
   if s.cmd.len == 0:
@@ -116,14 +122,85 @@ when not declared(parseCfgBool):
     of "n", "no", "false", "0", "off": result = false
     else: raise newException(ValueError, "cannot interpret as a bool: " & s)
 
-proc extractSpec(filename: string): string =
+const
-  const tripleQuote = "\"\"\""
+  inlineErrorMarker = "#[tt."
-  var x = readFile(filename).string
+
-  var a = x.find(tripleQuote)
+proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int =
-  var b = x.find(tripleQuote, a+3)
+  result = i + len(inlineErrorMarker)
+  inc col, len(inlineErrorMarker)
+  var kind = ""
+  while result < s.len and s[result] in IdentChars:
+    kind.add s[result]
+    inc result
+    inc col
+
+  var caret = (line, -1)
+
+  template skipWhitespace =
+    while result < s.len and s[result] in Whitespace:
+      if s[result] == '\n':
+        col = 1
+        inc line
+      else:
+        inc col
+      inc result
+
+  skipWhitespace()
+  if result < s.len and s[result] == '^':
+    caret = (line-1, col)
+    inc result
+    inc col
+    skipWhitespace()
+
+  var msg = ""
+  while result < s.len-1:
+    if s[result] == '\n':
+      inc result
+      inc line
+      col = 1
+    elif s[result] == ']' and s[result+1] == '#':
+      while msg.len > 0 and msg[^1] in Whitespace:
+        setLen msg, msg.len - 1
+
+      inc result
+      inc col, 2
+      if kind == "Error": spec.action = actionReject
+      spec.unjoinable = true
+      spec.inlineErrors.add InlineError(kind: kind, msg: msg, line: caret[0], col: caret[1])
+      break
+    else:
+      msg.add s[result]
+      inc result
+      inc col
+
+proc extractSpec(filename: string; spec: var TSpec): string =
+  const
+    tripleQuote = "\"\"\""
+  var s = readFile(filename).string
+
+  var i = 0
+  var a = -1
+  var b = -1
+  var line = 1
+  var col = 1
+  while i < s.len:
+    if s.continuesWith(tripleQuote, i):
+      if a < 0: a = i
+      elif b < 0: b = i
+      inc i, 2
+      inc col
+    elif s[i] == '\n':
+      inc line
+      col = 1
+    elif s.continuesWith(inlineErrorMarker, i):
+      i = extractErrorMsg(s, i, line, col, spec)
+    else:
+      inc col
+    inc i
+
   # look for """ only in the first section
   if a >= 0 and b > a and a < 40:
-    result = x.substr(a+3, b-1).replace("'''", tripleQuote)
+    result = s.substr(a+3, b-1).replace("'''", tripleQuote)
   else:
     #echo "warning: file does not contain spec: " & filename
     result = ""
@@ -160,7 +237,7 @@ proc isCurrentBatch(testamentData: TestamentData, filename: string): bool =
 
 proc parseSpec*(filename: string): TSpec =
   result.file = filename
-  let specStr = extractSpec(filename)
+  let specStr = extractSpec(filename, result)
   var ss = newStringStream(specStr)
   var p: CfgParser
   open(p, ss, filename, 1)

testament/testament.nim

@@ -12,7 +12,7 @@
 import
   strutils, pegs, os, osproc, streams, json,
   backend, parseopt, specs, htmlgen, browsers, terminal,
-  algorithm, times, md5, sequtils, azure
+  algorithm, times, md5, sequtils, azure, intsets
 from std/sugar import dup
 import compiler/nodejs
 
@@ -71,6 +71,7 @@ type
 let
   pegLineError =
     peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}"
+
   pegLineTemplate =
     peg"""
       {[^(]*} '(' {\d+} ', ' {\d+} ') '
@@ -317,8 +318,50 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
       discard waitForExit(p)
       close(p)
 
+proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) =
+  let pegLine = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' {[^:]*} ':' \s* {.*}"
+  var covered = initIntSet()
+  for line in splitLines(given.nimout):
+
+    if line =~ pegLine:
+      let file = extractFilename(matches[0])
+      let line = try: parseInt(matches[1]) except: -1
+      let col = try: parseInt(matches[2]) except: -1
+      let kind = matches[3]
+      let msg = matches[4]
+
+      if file == extractFilename test.name:
+        var i = 0
+        for x in expected.inlineErrors:
+          if x.line == line and (x.col == col or x.col < 0) and
+              x.kind == kind and x.msg in msg:
+            covered.incl i
+          inc i
+
+  block coverCheck:
+    for j in 0..high(expected.inlineErrors):
+      if j notin covered:
+        var e = test.name
+        e.add "("
+        e.addInt expected.inlineErrors[j].line
+        if expected.inlineErrors[j].col > 0:
+          e.add ", "
+          e.addInt expected.inlineErrors[j].col
+        e.add ") "
+        e.add expected.inlineErrors[j].kind
+        e.add ": "
+        e.add expected.inlineErrors[j].msg
+
+        r.addResult(test, target, e, given.nimout, reMsgsDiffer)
+        break coverCheck
+
+    r.addResult(test, target, "", given.msg, reSuccess)
+    inc(r.passed)
+
 proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) =
-  if strip(expected.msg) notin strip(given.msg):
+  if expected.inlineErrors.len > 0:
+    checkForInlineErrors(r, expected, given, test, target)
+  elif strip(expected.msg) notin strip(given.msg):
     r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer)
   elif expected.nimout.len > 0 and expected.nimout.normalizeMsg notin given.nimout.normalizeMsg:
     r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer)
@@ -389,6 +432,8 @@ proc nimoutCheck(test: TTest; expectedNimout: string; given: var TSpec) =
       given.err = reMsgsDiffer
       break
 
+
+
 proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec,
                          expected: TSpec; r: var TResults) =
   var expectedmsg: string = ""

tests/effects/teffects1.nim

@@ -1,13 +1,5 @@
 discard """
-  errormsg: "type mismatch: got <proc (x: int): string{.noSideEffect, gcsafe, locks: 0.}> but expected 'MyProcType = proc (x: int): string{.closure.}'"
-  file: "teffects1.nim"
-  line: 38
   cmd: "nim check $file"
-  nimout: '''teffects1.nim(22, 28) template/generic instantiation from here
-teffects1.nim(23, 13) Error: can raise an unlisted exception: ref IOError
-teffects1.nim(22, 29) Hint: 'IO2Error' is declared but not used [XDeclaredButNotUsed]
-teffects1.nim(38, 21) Error: type mismatch: got <proc (x: int): string{.noSideEffect, gcsafe, locks: 0.}> but expected 'MyProcType = proc (x: int): string{.closure.}'
-.raise effects differ'''
 """
 
 type
@@ -20,7 +12,10 @@ type
 proc forw: int {. .}
 
 proc lier(): int {.raises: [IO2Error].} =
-  writeLine stdout, "arg"
+  #[tt.Hint                 ^ 'IO2Error' is declared but not used [XDeclaredButNotUsed] ]#
+  writeLine stdout, "arg" #[tt.Error
+            ^  can raise an unlisted exception: ref IOError
+  ]#
 
 proc forw: int =
   raise newException(IOError, "arg")
@@ -35,5 +30,9 @@ proc foo(x: int): string {.raises: [ValueError].} =
     raise newException(ValueError, "Use single digit")
   $x
 
-var p: MyProcType = foo
+var p: MyProcType = foo #[tt.Error
+                    ^
+type mismatch: got <proc (x: int): string{.noSideEffect, gcsafe, locks: 0.}> but expected 'MyProcType = proc (x: int): string{.closure.}'
+
+]#
 {.pop.}