fixes #21232; highlite.getNextToken() returns gtOperator for '/' in C (#21234)

lib/packages/docutils/highlite.nim

@@ -498,6 +498,9 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string],
           of '\0':
             break
           else: inc(pos)
+      else:
+        g.kind = gtOperator
+        while g.buf[pos] in OpChars: inc(pos)
     of '#':
       inc(pos)
       if hasPreprocessor in flags:

tests/stdlib/thighlite.nim

@@ -1,6 +1,7 @@
 
 import unittest, strutils
 import ../../lib/packages/docutils/highlite
+import std/objectdollar
 
 block: # Nim tokenizing
   test "string literals and escape seq":
@@ -30,3 +31,12 @@ block: # Cmd (shell) tokenizing
         ("file.nim", gtIdentifier), ("\n", gtWhitespace),
         ("out: file [SuccessX]", gtProgramOutput)
       ])
+
+block: # bug #21232
+  let code = "/"
+  var toknizr: GeneralTokenizer
+
+  initGeneralTokenizer(toknizr, code)
+
+  getNextToken(toknizr, langC)
+  check $toknizr == """(kind: gtOperator, start: 0, length: 1, buf: "/", pos: 1, state: gtEof, lang: langC)"""