jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixes #21232; highlite.getNextToken() returns gtOperator for '/' in C (#21234)

Browse source
This commit is contained in:
ringabout 2023-01-08 22:40:27 +08:00 • committed by GitHub
commit a55604a3d8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 13 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

3
lib/packages/docutils/highlite.nim
View file

@ -498,6 +498,9 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string],
of '\0': of '\0':
break break
else: inc(pos) else: inc(pos)
else:
g.kind = gtOperator
while g.buf[pos] in OpChars: inc(pos)
of '#': of '#':
inc(pos) inc(pos)
if hasPreprocessor in flags: if hasPreprocessor in flags:

10
tests/stdlib/thighlite.nim
View file

@ -1,6 +1,7 @@
import unittest, strutils import unittest, strutils
import ../../lib/packages/docutils/highlite import ../../lib/packages/docutils/highlite
import std/objectdollar
block: # Nim tokenizing block: # Nim tokenizing
test "string literals and escape seq": test "string literals and escape seq":
@ -30,3 +31,12 @@ block: # Cmd (shell) tokenizing
("file.nim", gtIdentifier), ("\n", gtWhitespace), ("file.nim", gtIdentifier), ("\n", gtWhitespace),
("out: file [SuccessX]", gtProgramOutput) ("out: file [SuccessX]", gtProgramOutput)
]) ])
block: # bug #21232
let code = "/"
var toknizr: GeneralTokenizer
initGeneralTokenizer(toknizr, code)
getNextToken(toknizr, langC)
check $toknizr == """(kind: gtOperator, start: 0, length: 1, buf: "/", pos: 1, state: gtEof, lang: langC)"""