Merge pull request #7907 from GULPF/lexer-unicode-fix

Lexer fix for multi byte characters
2018-06-06 11:42:59 +02:00 · 2018-06-06 11:42:59 +02:00 · e957d369b1
commit e957d369b1
parent 8804ecd175 c69b88688f
5 changed files with 29 additions and 22 deletions
--- a/lib/packages/docutils/highlite.nim
+++ b/lib/packages/docutils/highlite.nim
@ -130,7 +130,7 @@ proc nimNumber(g: var GeneralTokenizer, position: int): int =

 const
  OpChars  = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.',
-              '|', '=', '%', '&', '$', '@', '~', ':', '\x80'..'\xFF'}
+              '|', '=', '%', '&', '$', '@', '~', ':'}

 proc nimNextToken(g: var GeneralTokenizer) =
  const