compiler/[msgs, suggest]: improve highlighter accuracy (#10496)

Previously the compiler would generate suggestions based on the symbol
identifier length, but that might not reflect the actual representation
of it within the actual source code.

This commit implements a simple source scanner for the suggest module to
address the problem outlined above.

Fixes nim-lang/nimsuggest#24

compiler/msgs.nim

@@ -171,6 +171,7 @@ proc toFullPath*(conf: ConfigRef; fileIdx: FileIndex): string =
 proc setDirtyFile*(conf: ConfigRef; fileIdx: FileIndex; filename: AbsoluteFile) =
   assert fileIdx.int32 >= 0
   conf.m.fileInfos[fileIdx.int32].dirtyFile = filename
+  setLen conf.m.fileInfos[fileIdx.int32].lines, 0
 
 proc setHash*(conf: ConfigRef; fileIdx: FileIndex; hash: string) =
   assert fileIdx.int32 >= 0
@@ -194,6 +195,9 @@ template toFilename*(conf: ConfigRef; info: TLineInfo): string =
 template toFullPath*(conf: ConfigRef; info: TLineInfo): string =
   toFullPath(conf, info.fileIndex)
 
+template toFullPathConsiderDirty*(conf: ConfigRef; info: TLineInfo): string =
+  string toFullPathConsiderDirty(conf, info.fileIndex)
+
 proc toMsgFilename*(conf: ConfigRef; info: TLineInfo): string =
   if info.fileIndex.int32 < 0:
     result = "???"
@@ -434,7 +438,7 @@ proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
 
   if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
     try:
-      for line in lines(toFullPath(conf, i)):
+      for line in lines(toFullPathConsiderDirty(conf, i)):
         addSourceLine conf, i.fileIndex, line.string
     except IOError:
       discard

compiler/suggest.nim

@@ -32,7 +32,7 @@
 
 # included from sigmatch.nim
 
-import algorithm, prefixmatches, lineinfos, pathutils
+import algorithm, prefixmatches, lineinfos, pathutils, parseutils
 from wordrecg import wDeprecated, wError, wAddr, wYield, specialWords
 
 when defined(nimsuggest):
@@ -81,6 +81,38 @@ proc cmpSuggestions(a, b: Suggest): int =
   # independent of hashing order:
   result = cmp(a.name[], b.name[])
 
+proc getTokenLenFromSource(conf: ConfigRef; ident: string; info: TLineInfo): int =
+  let
+    line = sourceLine(conf, info)
+    column = toColumn(info)
+
+  proc isOpeningBacktick(col: int): bool =
+    if col >= 0 and col < line.len:
+      if line[col] == '`':
+        not isOpeningBacktick(col - 1)
+      else:
+        isOpeningBacktick(col - 1)
+    else:
+      false
+
+  if column > line.len:
+    result = 0
+  elif column > 0 and line[column - 1] == '`' and isOpeningBacktick(column - 1):
+    result = skipUntil(line, '`', column)
+    if cmpIgnoreStyle(line[column..column + result - 1], ident) != 0:
+      result = 0
+  elif ident[0] in linter.Letters and ident[^1] != '=':
+    result = identLen(line, column)
+    if cmpIgnoreStyle(line[column..column + result - 1], ident) != 0:
+      result = 0
+  else:
+    result = skipWhile(line, OpChars + {'[', '(', '{', ']', ')', '}'}, column)
+    if ident[^1] == '=' and ident[0] in linter.Letters:
+      if line[column..column + result - 1] != "=":
+        result = 0
+    elif line[column..column + result - 1] != ident:
+      result = 0
+
 proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo;
                   quality: range[0..100]; prefix: PrefixMatch;
                   inTypeContext: bool; scope: int): Suggest =
@@ -88,7 +120,6 @@ proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info
   result.section = section
   result.quality = quality
   result.isGlobal = sfGlobal in s.flags
-  result.tokenLen = s.name.s.len
   result.prefix = prefix
   result.contextFits = inTypeContext == (s.kind in {skType, skGenericParam})
   result.scope = scope
@@ -126,6 +157,10 @@ proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info
   result.line = toLinenumber(infox)
   result.column = toColumn(infox)
   result.version = conf.suggestVersion
+  result.tokenLen = if section != ideHighlight:
+                      s.name.s.len
+                    else:
+                      getTokenLenFromSource(conf, s.name.s, infox)
 
 proc `$`*(suggest: Suggest): string =
   result = $suggest.section

nimsuggest/tests/tsetter_highlight.nim

@@ -0,0 +1,10 @@
+proc `a=`(a, b: int) = discard
+10.a = 1000#[!]#
+
+discard """
+$nimsuggest --tester $file
+>highlight $1
+highlight;;skProc;;1;;6;;2
+highlight;;skType;;1;;16;;3
+highlight;;skProc;;2;;5;;1
+"""

nimsuggest/tests/tsi_highlight.nim

@@ -0,0 +1,11 @@
+proc a: int = 0
+e_c_h_o#[!]#
+
+discard """
+$nimsuggest --tester $file
+>highlight $1
+highlight;;skProc;;1;;5;;1
+highlight;;skType;;1;;8;;3
+highlight;;skResult;;1;;0;;0
+highlight;;skProc;;2;;0;;7
+"""