jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixes edge cases in the lexer

Browse source
This commit is contained in:
Andreas Rumpf 2016-07-12 01:05:52 +02:00
commit 56f99f56ac
1 changed files with 5 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
compiler/lexer.nim
View file

@ -735,7 +735,8 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
if c == '\226' and if c == '\226' and
buf[pos+1] == '\128' and buf[pos+1] == '\128' and
buf[pos+2] == '\147': # It's a 'magic separator' en-dash Unicode buf[pos+2] == '\147': # It's a 'magic separator' en-dash Unicode
if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars: if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars or
isMagicIdentSeparatorRune(buf, pos+magicIdentSeparatorRuneByteWidth) or pos == L.bufpos:
lexMessage(L, errInvalidToken, "–") lexMessage(L, errInvalidToken, "–")
break break
inc(pos, magicIdentSeparatorRuneByteWidth) inc(pos, magicIdentSeparatorRuneByteWidth)
@ -747,7 +748,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
h = h !& ord(c) h = h !& ord(c)
inc(pos) inc(pos)
of '_': of '_':
if buf[pos+1] notin SymChars: if buf[pos+1] notin SymChars or isMagicIdentSeparatorRune(buf, pos+1):
lexMessage(L, errInvalidToken, "_") lexMessage(L, errInvalidToken, "_")
break break
inc(pos) inc(pos)
@ -1056,7 +1057,8 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
inc(L.bufpos) inc(L.bufpos)
of '_': of '_':
inc(L.bufpos) inc(L.bufpos)
if L.buf[L.bufpos] notin SymChars: if L.buf[L.bufpos] notin SymChars+{'_'} and not
isMagicIdentSeparatorRune(L.buf, L.bufpos):
tok.tokType = tkSymbol tok.tokType = tkSymbol
tok.ident = getIdent("_") tok.ident = getIdent("_")
else: else: