nimsuggest: structured error reporting; EPC mode still fails

compiler/compilerlog.nim

@@ -1,9 +0,0 @@
-
-from os import getHomeDir, `/`
-
-proc logStr*(line: string) =
-  var f: File
-  if open(f, getHomeDir() / "nimsuggest.log", fmAppend):
-    f.writeLine(line)
-    f.close()
-

compiler/msgs.nim

@@ -631,12 +631,17 @@ proc unknownLineInfo*(): TLineInfo =
   result.col = int16(-1)
   result.fileIndex = -1
 
+type
+  Severity* {.pure.} = enum ## VS Code only supports these three
+    Hint, Warning, Error
+
 var
   msgContext: seq[TLineInfo] = @[]
   lastError = unknownLineInfo()
 
   errorOutputs* = {eStdOut, eStdErr}
   writelnHook*: proc (output: string) {.closure.}
+  structuredErrorHook*: proc (info: TLineInfo; msg: string; severity: Severity) {.closure.}
 
 proc suggestWriteln*(s: string) =
   if eStdOut in errorOutputs:
@@ -745,6 +750,8 @@ proc msgWriteln*(s: string, flags: MsgFlags = {}) =
   ## is present, then it is used to output message rather than stderr/stdout.
   ## This behavior can be altered by given optional flags.
 
+  ## This is used for 'nim dump' etc. where we don't have nimsuggest
+  ## support.
   #if gCmd == cmdIdeTools and optCDebug notin gGlobalOptions: return
 
   if not isNil(writelnHook) and msgSkipHook notin flags:
@@ -833,7 +840,7 @@ proc quit(msg: TMsgKind) =
 
 proc log*(s: string) {.procvar.} =
   var f: File
-  if open(f, "nimsuggest.log", fmAppend):
+  if open(f, getHomeDir() / "nimsuggest.log", fmAppend):
     f.writeLine(s)
     close(f)
 
@@ -858,12 +865,16 @@ proc writeContext(lastinfo: TLineInfo) =
   var info = lastinfo
   for i in countup(0, len(msgContext) - 1):
     if msgContext[i] != lastinfo and msgContext[i] != info:
-      styledMsgWriteln(styleBright,
+      if structuredErrorHook != nil:
-                       PosFormat % [toMsgFilename(msgContext[i]),
+        structuredErrorHook(msgContext[i], getMessageStr(errInstantiationFrom, ""),
-                                    coordToStr(msgContext[i].line),
+                            Severity.Error)
-                                    coordToStr(msgContext[i].col+1)],
+      else:
-                       resetStyle,
+        styledMsgWriteln(styleBright,
-                       getMessageStr(errInstantiationFrom, ""))
+                         PosFormat % [toMsgFilename(msgContext[i]),
+                                      coordToStr(msgContext[i].line),
+                                      coordToStr(msgContext[i].col+1)],
+                         resetStyle,
+                         getMessageStr(errInstantiationFrom, ""))
     info = msgContext[i]
 
 proc ignoreMsgBecauseOfIdeTools(msg: TMsgKind): bool =
@@ -873,13 +884,16 @@ proc rawMessage*(msg: TMsgKind, args: openArray[string]) =
   var
     title: string
     color: ForegroundColor
-    kind:  string
+    kind: string
+    sev: Severity
   case msg
   of errMin..errMax:
+    sev = Severity.Error
     writeContext(unknownLineInfo())
     title = ErrorTitle
     color = ErrorColor
   of warnMin..warnMax:
+    sev = Severity.Warning
     if optWarns notin gOptions: return
     if msg notin gNotes: return
     writeContext(unknownLineInfo())
@@ -888,13 +902,18 @@ proc rawMessage*(msg: TMsgKind, args: openArray[string]) =
     kind = WarningsToStr[ord(msg) - ord(warnMin)]
     inc(gWarnCounter)
   of hintMin..hintMax:
+    sev = Severity.Hint
     if optHints notin gOptions: return
     if msg notin gNotes: return
     title = HintTitle
     color = HintColor
     kind = HintsToStr[ord(msg) - ord(hintMin)]
     inc(gHintCounter)
-  let s = `%`(msgKindToString(msg), args)
+  let s = msgKindToString(msg) % args
+
+  if structuredErrorHook != nil:
+    structuredErrorHook(unknownLineInfo(), s & (if kind != nil: KindFormat % kind else: ""), sev)
+
   if not ignoreMsgBecauseOfIdeTools(msg):
     if kind != nil:
       styledMsgWriteln(color, title, resetStyle, s,
@@ -932,8 +951,10 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
     color: ForegroundColor
     kind:  string
     ignoreMsg = false
+    sev: Severity
   case msg
   of errMin..errMax:
+    sev = Severity.Error
     writeContext(info)
     title = ErrorTitle
     color = ErrorColor
@@ -942,6 +963,7 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
     #ignoreMsg = lastError == info and eh != doAbort
     lastError = info
   of warnMin..warnMax:
+    sev = Severity.Warning
     ignoreMsg = optWarns notin gOptions or msg notin gNotes
     if not ignoreMsg: writeContext(info)
     title = WarningTitle
@@ -949,6 +971,7 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
     kind = WarningsToStr[ord(msg) - ord(warnMin)]
     inc(gWarnCounter)
   of hintMin..hintMax:
+    sev = Severity.Hint
     ignoreMsg = optHints notin gOptions or msg notin gNotes
     title = HintTitle
     color = HintColor
@@ -960,14 +983,18 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string,
   let x = PosFormat % [toMsgFilename(info), coordToStr(info.line),
                        coordToStr(info.col+1)]
   let s = getMessageStr(msg, arg)
-  if not ignoreMsg and not ignoreMsgBecauseOfIdeTools(msg):
+
-    if kind != nil:
+  if not ignoreMsg:
-      styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s,
+    if structuredErrorHook != nil:
-                       KindColor, `%`(KindFormat, kind))
+      structuredErrorHook(info, s & (if kind != nil: KindFormat % kind else: ""), sev)
-    else:
+    if not ignoreMsgBecauseOfIdeTools(msg):
-      styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s)
+      if kind != nil:
-    if msg in errMin..errMax and hintSource in gNotes:
+        styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s,
-      info.writeSurroundingSrc
+                         KindColor, `%`(KindFormat, kind))
+      else:
+        styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s)
+      if msg in errMin..errMax and hintSource in gNotes:
+        info.writeSurroundingSrc
   handleError(msg, eh, s)
 
 proc fatal*(info: TLineInfo, msg: TMsgKind, arg = "") =
@@ -992,12 +1019,12 @@ proc message*(info: TLineInfo, msg: TMsgKind, arg = "") =
   liMessage(info, msg, arg, doNothing)
 
 proc internalError*(info: TLineInfo, errMsg: string) =
-  if gCmd == cmdIdeTools: return
+  if gCmd == cmdIdeTools and structuredErrorHook.isNil: return
   writeContext(info)
   liMessage(info, errInternal, errMsg, doAbort)
 
 proc internalError*(errMsg: string) =
-  if gCmd == cmdIdeTools: return
+  if gCmd == cmdIdeTools and structuredErrorHook.isNil: return
   writeContext(unknownLineInfo())
   rawMessage(errInternal, errMsg)
 

compiler/options.nim

@@ -100,7 +100,7 @@ type
 
   IdeCmd* = enum
     ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideMod,
-    ideHighlight, ideOutline, ideKnown
+    ideHighlight, ideOutline, ideKnown, ideMsg
 
   ConfigRef* = ref object ## eventually all global configuration should be moved here
     cppDefines*: HashSet[string]
@@ -349,7 +349,7 @@ proc rawFindFile2(f: string): string =
       # bring to front
       for j in countDown(i,1):
         swap(lazyPaths[j], lazyPaths[j-1])
-      
+
       return result.canonicalizePath
   result = ""
 
@@ -437,6 +437,7 @@ proc parseIdeCmd*(s: string): IdeCmd =
   of "highlight": ideHighlight
   of "outline": ideOutline
   of "known": ideKnown
+  of "msg": ideMsg
   else: ideNone
 
 proc `$`*(c: IdeCmd): string =
@@ -452,3 +453,4 @@ proc `$`*(c: IdeCmd): string =
   of ideHighlight: "highlight"
   of ideOutline: "outline"
   of ideKnown: "known"
+  of ideMsg: "msg"

compiler/semstmts.nim

@@ -1193,8 +1193,6 @@ type
     stepRegisterSymbol,
     stepDetermineType,
 
-import compilerlog
-
 proc hasObjParam(s: PSym): bool =
   var t = s.typ
   for col in countup(1, sonsLen(t)-1):

compiler/suggest.nim

@@ -9,7 +9,7 @@
 
 ## This file implements features required for IDE support.
 ##
-## Due to Nim's natures and the fact that ``system.nim`` is always imported,
+## Due to Nim's nature and the fact that ``system.nim`` is always imported,
 ## there are lots of potential symbols. Furthermore thanks to templates and
 ## macros even context based analysis does not help much: In a context like
 ## ``let x: |`` where a type has to follow, that type might be constructed from
@@ -126,7 +126,8 @@ proc `$`*(suggest: Suggest): string =
   else:
     result.add($suggest.symkind)
     result.add(sep)
-    result.add(suggest.qualifiedPath.join("."))
+    if suggest.qualifiedPath != nil:
+      result.add(suggest.qualifiedPath.join("."))
     result.add(sep)
     result.add(suggest.forth)
     result.add(sep)
@@ -161,8 +162,6 @@ proc filterSym(s: PSym; prefix: PNode): bool {.inline.} =
       if n.len > 0:
         result = prefixMatch(s, n[0])
     else: discard
-    if result:
-      echo "indeed a prefix match ", n
   if s.kind != skModule:
     result = prefix.isNil or prefixMatch(s, prefix)
 

tools/nimsuggest/nimsuggest.nim

@@ -17,7 +17,7 @@ import compiler / [options, commands, modules, sem,
   passes, passaux, msgs, nimconf,
   extccomp, condsyms,
   sigmatch, ast, scriptconfig,
-  idents, modulegraphs, compilerlog, vm]
+  idents, modulegraphs, vm]
 
 when defined(windows):
   import winlean
@@ -60,6 +60,20 @@ var
   gLogging = false
   gRefresh: bool
 
+  requests: Channel[string]
+  results: Channel[Suggest]
+
+proc writelnToChannel(line: string) =
+  results.send(Suggest(section: ideMsg, doc: line))
+
+proc sugResultHook(s: Suggest) =
+  results.send(s)
+
+proc errorHook(info: TLineInfo; msg: string; sev: Severity) =
+  results.send(Suggest(section: ideChk, filePath: toFullPath(info),
+    line: toLinenumber(info), column: toColumn(info), doc: msg,
+    forth: $sev))
+
 const
   seps = {':', ';', ' ', '\t'}
   Help = "usage: sug|con|def|use|dus|chk|mod|highlight|outline|known file.nim[;dirtyfile.nim]:line:col\n" &
@@ -133,9 +147,12 @@ proc symFromInfo(graph: ModuleGraph; gTrackPos: TLineInfo): PSym =
 proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
              graph: ModuleGraph; cache: IdentCache) =
   if gLogging:
-    logStr("cmd: " & $cmd & ", file: " & file & ", dirtyFile: " & dirtyfile &
+    log("cmd: " & $cmd & ", file: " & file & ", dirtyFile: " & dirtyfile &
           "[" & $line & ":" & $col & "]")
   gIdeCmd = cmd
+  if cmd == ideChk:
+    msgs.structuredErrorHook = errorHook
+    msgs.writelnHook = proc (s: string) = discard
   if cmd == ideUse and suggestVersion != 2:
     graph.resetAllModules()
   var isKnownFile = true
@@ -194,7 +211,7 @@ template sendEpc(results: typed, tdef, hook: untyped) =
   executeEpc(gIdeCmd, args, graph, cache)
   let res = sexp(results)
   if gLogging:
-    logStr($res)
+    log($res)
   returnEpc(client, uid, res)
 
 template checkSanity(client, sizeHex, size, messageBuffer: typed) =
@@ -205,16 +222,12 @@ template checkSanity(client, sizeHex, size, messageBuffer: typed) =
   if client.recv(messageBuffer, size) != size:
     raise newException(ValueError, "didn't get all the bytes")
 
-var
-  requests: Channel[string]
-  results: Channel[Suggest]
-
 proc toStdout() {.gcsafe.} =
   while true:
     let res = results.recv()
     case res.section
     of ideNone: break
-    of ideChk: echo res.doc
+    of ideMsg: echo res.doc
     of ideKnown: echo res.quality == 1
     else: echo res
 
@@ -223,7 +236,7 @@ proc toSocket(stdoutSocket: Socket) {.gcsafe.} =
     let res = results.recv()
     case res.section
     of ideNone: break
-    of ideChk: stdoutSocket.send(res.doc & "\c\L")
+    of ideMsg: stdoutSocket.send(res.doc & "\c\L")
     of ideKnown: stdoutSocket.send($(res.quality == 1) & "\c\L")
     else: stdoutSocket.send($res & "\c\L")
 
@@ -233,7 +246,7 @@ proc toEpc(client: Socket; uid: BiggestInt) {.gcsafe.} =
     let res = results.recv()
     case res.section
     of ideNone: break
-    of ideChk:
+    of ideMsg:
       list.add sexp(res.doc)
     of ideKnown:
       list.add sexp(res.quality == 1)
@@ -241,12 +254,6 @@ proc toEpc(client: Socket; uid: BiggestInt) {.gcsafe.} =
       list.add sexp(res)
   returnEpc(client, uid, list)
 
-proc writelnToChannel(line: string) =
-  results.send(Suggest(section: ideChk, doc: line))
-
-proc sugResultHook(s: Suggest) =
-  results.send(s)
-
 template setVerbosity(level: typed) =
   gVerbosity = level
   gNotes = NotesVerbosity[gVerbosity]
@@ -353,7 +360,7 @@ proc replEpc(x: ThreadParams) {.thread.} =
       else: discard
       let cmd = $gIdeCmd & " " & args.argsToStr
       if gLogging:
-        logStr "MSG CMD: " & cmd
+        log "MSG CMD: " & cmd
       requests.send(cmd)
       toEpc(client, uid)
     of "methods":
@@ -437,11 +444,11 @@ proc recompileFullProject(graph: ModuleGraph; cache: IdentCache) =
 proc mainThread(graph: ModuleGraph; cache: IdentCache) =
   if gLogging:
     for it in searchPaths:
-      logStr(it)
+      log(it)
 
   proc wrHook(line: string) {.closure.} =
     if gMode == mepc:
-      if gLogging: logStr(line)
+      if gLogging: log(line)
     else:
       writelnToChannel(line)
 
@@ -454,7 +461,6 @@ proc mainThread(graph: ModuleGraph; cache: IdentCache) =
     if hasData:
       msgs.writelnHook = wrHook
       suggestionResultHook = sugResultHook
-
       execCmd(req, graph, cache)
       idle = 0
     else:
@@ -464,6 +470,7 @@ proc mainThread(graph: ModuleGraph; cache: IdentCache) =
       # we use some nimsuggest activity to enable a lazy recompile:
       gIdeCmd = ideChk
       msgs.writelnHook = proc (s: string) = discard
+      msgs.structuredErrorHook = nil
       suggestionResultHook = proc (s: Suggest) = discard
       recompileFullProject(graph, cache)
 
@@ -482,6 +489,10 @@ proc mainCommand(graph: ModuleGraph; cache: IdentCache) =
 
   # do not stop after the first error:
   msgs.gErrorMax = high(int)
+  # do not print errors, but log them
+  msgs.writelnHook = proc (s: string) = log(s)
+  msgs.structuredErrorHook = nil
+
   # compile the project before showing any input so that we already
   # can answer questions right away:
   compileProject(graph, cache)
@@ -559,9 +570,9 @@ proc handleCmdLine(cache: IdentCache; config: ConfigRef) =
       raise newException(IOError,
           "Cannot find Nim standard library: Nim compiler not in PATH")
     gPrefixDir = binaryPath.splitPath().head.parentDir()
-    #msgs.writelnHook = proc (line: string) = logStr(line)
+    #msgs.writelnHook = proc (line: string) = log(line)
     if gLogging:
-      logStr("START " & gProjectFull)
+      log("START " & gProjectFull)
 
     loadConfigs(DefaultConfig, cache, config) # load all config files
     # now process command line arguments again, because some options in the

tools/nimsuggest/tests/tchk1.nim

@@ -0,0 +1,27 @@
+# test we get some suggestion at the end of the file
+
+
+
+
+
+
+
+type
+
+
+template foo() =
+
+proc main =
+
+#[!]#
+discard """
+$nimsuggest --tester $file
+>chk $1
+chk;;skUnknown;;;;Hint;;???;;-1;;-1;;"tchk1 [Processing]";;0
+chk;;skUnknown;;;;Error;;$file;;12;;0;;"identifier expected, but found \'keyword template\'";;0
+chk;;skUnknown;;;;Error;;$file;;14;;0;;"complex statement requires indentation";;0
+chk;;skUnknown;;;;Error;;$file;;12;;0;;"implementation of \'foo\' expected";;0
+chk;;skUnknown;;;;Error;;$file;;17;;0;;"invalid indentation";;0
+chk;;skUnknown;;;;Hint;;$file;;12;;9;;"\'foo\' is declared but not used [XDeclaredButNotUsed]";;0
+chk;;skUnknown;;;;Hint;;$file;;14;;5;;"\'tchk1.main()\' is declared but not used [XDeclaredButNotUsed]";;0
+"""

tools/nimsuggest/tests/tcursor_at_end.nim

@@ -0,0 +1,12 @@
+# test we get some suggestion at the end of the file
+
+discard """
+$nimsuggest --tester $file
+>sug $1
+sug;;skProc;;tcursor_at_end.main;;proc ();;$file;;10;;5;;"";;*
+"""
+
+
+proc main = discard
+
+#[!]#

tools/nimsuggest/tests/twithin_macro_prefix.nim

@@ -0,0 +1,209 @@
+
+import macros
+
+macro class*(head, body: untyped): untyped =
+  # The macro is immediate, since all its parameters are untyped.
+  # This means, it doesn't resolve identifiers passed to it.
+
+  var typeName, baseName: NimNode
+
+  # flag if object should be exported
+  var exported: bool
+
+  if head.kind == nnkInfix and head[0].ident == !"of":
+    # `head` is expression `typeName of baseClass`
+    # echo head.treeRepr
+    # --------------------
+    # Infix
+    #   Ident !"of"
+    #   Ident !"Animal"
+    #   Ident !"RootObj"
+    typeName = head[1]
+    baseName = head[2]
+
+  elif head.kind == nnkInfix and head[0].ident == !"*" and
+       head[2].kind == nnkPrefix and head[2][0].ident == !"of":
+    # `head` is expression `typeName* of baseClass`
+    # echo head.treeRepr
+    # --------------------
+    # Infix
+    #   Ident !"*"
+    #   Ident !"Animal"
+    #   Prefix
+    #     Ident !"of"
+    #     Ident !"RootObj"
+    typeName = head[1]
+    baseName = head[2][1]
+    exported = true
+
+  else:
+    quit "Invalid node: " & head.lispRepr
+
+  # The following prints out the AST structure:
+  #
+  # import macros
+  # dumptree:
+  #   type X = ref object of Y
+  #     z: int
+  # --------------------
+  # StmtList
+  #   TypeSection
+  #     TypeDef
+  #       Ident !"X"
+  #       Empty
+  #       RefTy
+  #         ObjectTy
+  #           Empty
+  #           OfInherit
+  #             Ident !"Y"
+  #           RecList
+  #             IdentDefs
+  #               Ident !"z"
+  #               Ident !"int"
+  #               Empty
+
+  # create a type section in the result
+  result =
+    if exported:
+      # mark `typeName` with an asterisk
+      quote do:
+        type `typeName`* = ref object of `baseName`
+    else:
+      quote do:
+        type `typeName` = ref object of `baseName`
+
+  # echo treeRepr(body)
+  # --------------------
+  # StmtList
+  #   VarSection
+  #     IdentDefs
+  #       Ident !"name"
+  #       Ident !"string"
+  #       Empty
+  #     IdentDefs
+  #       Ident !"age"
+  #       Ident !"int"
+  #       Empty
+  #   MethodDef
+  #     Ident !"vocalize"
+  #     Empty
+  #     Empty
+  #     FormalParams
+  #       Ident !"string"
+  #     Empty
+  #     Empty
+  #     StmtList
+  #       StrLit ...
+  #   MethodDef
+  #     Ident !"age_human_yrs"
+  #     Empty
+  #     Empty
+  #     FormalParams
+  #       Ident !"int"
+  #     Empty
+  #     Empty
+  #     StmtList
+  #       DotExpr
+  #         Ident !"this"
+  #         Ident !"age"
+
+  # var declarations will be turned into object fields
+  var recList = newNimNode(nnkRecList)
+
+  # expected name of constructor
+  let ctorName = newIdentNode("new" & $typeName)
+
+  # Iterate over the statements, adding `this: T`
+  # to the parameters of functions, unless the
+  # function is a constructor
+  for node in body.children:
+    case node.kind:
+
+      of nnkMethodDef, nnkProcDef:
+        # check if it is the ctor proc
+        if node.name.kind != nnkAccQuoted and node.name.basename == ctorName:
+          # specify the return type of the ctor proc
+          node.params[0] = typeName
+        else:
+          # inject `self: T` into the arguments
+          node.params.insert(1, newIdentDefs(ident("self"), typeName))
+        result.add(node)
+
+      of nnkVarSection:
+        # variables get turned into fields of the type.
+        for n in node.children:
+          recList.add(n)
+
+      else:
+        result.add(node)
+
+  # Inspect the tree structure:
+  #
+  # echo result.treeRepr
+  # --------------------
+  # StmtList
+  #   TypeSection
+  #     TypeDef
+  #       Ident !"Animal"
+  #       Empty
+  #       RefTy
+  #         ObjectTy
+  #           Empty
+  #           OfInherit
+  #             Ident !"RootObj"
+  #           Empty   <= We want to replace this
+  # MethodDef
+  # ...
+
+  result[0][0][2][0][2] = recList
+
+  # Lets inspect the human-readable version of the output
+  #echo repr(result)
+
+# ---
+
+class Animal of RootObj:
+  var name: string
+  var age: int
+  method vocalize: string {.base.} = "..." # use `base` pragma to annonate base methods
+  method age_human_yrs: int {.base.} = self.age # `this` is injected
+  proc `$`: string = "animal:" & self.name & ":" & $self.age
+
+class Dog of Animal:
+  method vocalize: string = "woof"
+  method age_human_yrs: int = self.age * 7
+  proc `$`: string = "dog:" & self.name & ":" & $self.age
+
+class Cat of Animal:
+  method vocalize: string = "meow"
+  proc `$`: string = "cat:" & self.name & ":" & $self.age
+
+class Rabbit of Animal:
+  proc newRabbit(name: string, age: int) = # the constructor doesn't need a return type
+    result = Rabbit(name: name, age: age)
+  method vocalize: string = "meep"
+  proc `$`: string =
+    self.ag#[!]#
+    result = "rabbit:" & self.name & ":" & $self.age
+
+# ---
+
+var animals: seq[Animal] = @[]
+animals.add(Dog(name: "Sparky", age: 10))
+animals.add(Cat(name: "Mitten", age: 10))
+
+for a in animals:
+  echo a.vocalize()
+  echo a.age_human_yrs()
+
+let r = newRabbit("Fluffy", 3)
+echo r.vocalize()
+echo r.age_human_yrs()
+echo r
+
+discard """
+$nimsuggest --tester $file
+>sug $1
+sug;;skField;;age;;int;;$file;;167;;6;;"";;100
+sug;;skMethod;;twithin_macro_prefix.age_human_yrs;;proc (self: Animal): int;;$file;;169;;9;;"";;100
+"""