beginnings of the new nimpretty tool; still unusable
This commit is contained in:
Andreas Rumpf
parent
0b7b116f04
commit
eddf9abd13
7 changed files with 288 additions and 187 deletions
|
|
@ -126,6 +126,9 @@ type
|
|||
literal*: string # the parsed (string) literal; and
|
||||
# documentation comments are here too
|
||||
line*, col*: int
|
||||
when defined(nimpretty):
|
||||
offsetA*, offsetB*: int # used for pretty printing so that literals
|
||||
# like 0b01 or r"\L" are unaffected
|
||||
|
||||
TErrorHandler* = proc (info: TLineInfo; msg: TMsgKind; arg: string)
|
||||
TLexer* = object of TBaseLexer
|
||||
|
|
@ -144,7 +147,10 @@ type
|
|||
var gLinesCompiled*: int # all lines that have been compiled
|
||||
|
||||
proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
|
||||
newLineInfo(L.fileIdx, tok.line, tok.col)
|
||||
result = newLineInfo(L.fileIdx, tok.line, tok.col)
|
||||
when defined(nimpretty):
|
||||
result.offsetA = tok.offsetA
|
||||
result.offsetB = tok.offsetB
|
||||
|
||||
proc isKeyword*(kind: TTokType): bool =
|
||||
result = (kind >= tokKeywordLow) and (kind <= tokKeywordHigh)
|
||||
|
|
@ -245,11 +251,13 @@ proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
|
|||
proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
|
||||
result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)
|
||||
|
||||
template tokenBegin(pos) {.dirty.} =
|
||||
template tokenBegin(tok, pos) {.dirty.} =
|
||||
when defined(nimsuggest):
|
||||
var colA = getColNumber(L, pos)
|
||||
when defined(nimpretty):
|
||||
tok.offsetA = L.offsetBase + pos
|
||||
|
||||
template tokenEnd(pos) {.dirty.} =
|
||||
template tokenEnd(tok, pos) {.dirty.} =
|
||||
when defined(nimsuggest):
|
||||
let colB = getColNumber(L, pos)+1
|
||||
if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and
|
||||
|
|
@ -257,8 +265,10 @@ template tokenEnd(pos) {.dirty.} =
|
|||
L.cursor = CursorPosition.InToken
|
||||
gTrackPos.col = colA.int16
|
||||
colA = 0
|
||||
when defined(nimpretty):
|
||||
tok.offsetB = L.offsetBase + pos
|
||||
|
||||
template tokenEndIgnore(pos) =
|
||||
template tokenEndIgnore(tok, pos) =
|
||||
when defined(nimsuggest):
|
||||
let colB = getColNumber(L, pos)
|
||||
if L.fileIdx == gTrackPos.fileIndex and gTrackPos.col in colA..colB and
|
||||
|
|
@ -266,8 +276,10 @@ template tokenEndIgnore(pos) =
|
|||
gTrackPos.fileIndex = trackPosInvalidFileIdx
|
||||
gTrackPos.line = -1
|
||||
colA = 0
|
||||
when defined(nimpretty):
|
||||
tok.offsetB = L.offsetBase + pos
|
||||
|
||||
template tokenEndPrevious(pos) =
|
||||
template tokenEndPrevious(tok, pos) =
|
||||
when defined(nimsuggest):
|
||||
# when we detect the cursor in whitespace, we attach the track position
|
||||
# to the token that came before that, but only if we haven't detected
|
||||
|
|
@ -279,6 +291,8 @@ template tokenEndPrevious(pos) =
|
|||
gTrackPos = L.previousToken
|
||||
gTrackPosAttached = true
|
||||
colA = 0
|
||||
when defined(nimpretty):
|
||||
tok.offsetB = L.offsetBase + pos
|
||||
|
||||
{.push overflowChecks: off.}
|
||||
# We need to parse the largest uint literal without overflow checks
|
||||
|
|
@ -363,7 +377,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
|
|||
result.literal = ""
|
||||
result.base = base10
|
||||
startpos = L.bufpos
|
||||
tokenBegin(startPos)
|
||||
tokenBegin(result, startPos)
|
||||
|
||||
# First stage: find out base, make verifications, build token literal string
|
||||
if L.buf[L.bufpos] == '0' and L.buf[L.bufpos + 1] in baseCodeChars + {'O'}:
|
||||
|
|
@ -573,7 +587,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
|
|||
lexMessageLitNum(L, errInvalidNumber, startpos)
|
||||
except OverflowError, RangeError:
|
||||
lexMessageLitNum(L, errNumberOutOfRange, startpos)
|
||||
tokenEnd(postPos-1)
|
||||
tokenEnd(result, postPos-1)
|
||||
L.bufpos = postPos
|
||||
|
||||
proc handleHexChar(L: var TLexer, xi: var int) =
|
||||
|
|
@ -691,10 +705,11 @@ proc handleCRLF(L: var TLexer, pos: int): int =
|
|||
else: result = pos
|
||||
|
||||
proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
|
||||
var pos = L.bufpos + 1 # skip "
|
||||
var pos = L.bufpos
|
||||
var buf = L.buf # put `buf` in a register
|
||||
var line = L.lineNumber # save linenumber for better error message
|
||||
tokenBegin(pos)
|
||||
tokenBegin(tok, pos)
|
||||
inc pos # skip "
|
||||
if buf[pos] == '\"' and buf[pos+1] == '\"':
|
||||
tok.tokType = tkTripleStrLit # long string literal:
|
||||
inc(pos, 2) # skip ""
|
||||
|
|
@ -710,18 +725,18 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
|
|||
of '\"':
|
||||
if buf[pos+1] == '\"' and buf[pos+2] == '\"' and
|
||||
buf[pos+3] != '\"':
|
||||
tokenEndIgnore(pos+2)
|
||||
tokenEndIgnore(tok, pos+2)
|
||||
L.bufpos = pos + 3 # skip the three """
|
||||
break
|
||||
add(tok.literal, '\"')
|
||||
inc(pos)
|
||||
of CR, LF:
|
||||
tokenEndIgnore(pos)
|
||||
tokenEndIgnore(tok, pos)
|
||||
pos = handleCRLF(L, pos)
|
||||
buf = L.buf
|
||||
add(tok.literal, tnl)
|
||||
of nimlexbase.EndOfFile:
|
||||
tokenEndIgnore(pos)
|
||||
tokenEndIgnore(tok, pos)
|
||||
var line2 = L.lineNumber
|
||||
L.lineNumber = line
|
||||
lexMessagePos(L, errClosingTripleQuoteExpected, L.lineStart)
|
||||
|
|
@ -742,11 +757,11 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
|
|||
inc(pos, 2)
|
||||
add(tok.literal, '"')
|
||||
else:
|
||||
tokenEndIgnore(pos)
|
||||
tokenEndIgnore(tok, pos)
|
||||
inc(pos) # skip '"'
|
||||
break
|
||||
elif c in {CR, LF, nimlexbase.EndOfFile}:
|
||||
tokenEndIgnore(pos)
|
||||
tokenEndIgnore(tok, pos)
|
||||
lexMessage(L, errClosingQuoteExpected)
|
||||
break
|
||||
elif (c == '\\') and not rawMode:
|
||||
|
|
@ -759,7 +774,7 @@ proc getString(L: var TLexer, tok: var TToken, rawMode: bool) =
|
|||
L.bufpos = pos
|
||||
|
||||
proc getCharacter(L: var TLexer, tok: var TToken) =
|
||||
tokenBegin(L.bufpos)
|
||||
tokenBegin(tok, L.bufpos)
|
||||
inc(L.bufpos) # skip '
|
||||
var c = L.buf[L.bufpos]
|
||||
case c
|
||||
|
|
@ -769,14 +784,14 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
|
|||
tok.literal = $c
|
||||
inc(L.bufpos)
|
||||
if L.buf[L.bufpos] != '\'': lexMessage(L, errMissingFinalQuote)
|
||||
tokenEndIgnore(L.bufpos)
|
||||
tokenEndIgnore(tok, L.bufpos)
|
||||
inc(L.bufpos) # skip '
|
||||
|
||||
proc getSymbol(L: var TLexer, tok: var TToken) =
|
||||
var h: Hash = 0
|
||||
var pos = L.bufpos
|
||||
var buf = L.buf
|
||||
tokenBegin(pos)
|
||||
tokenBegin(tok, pos)
|
||||
while true:
|
||||
var c = buf[pos]
|
||||
case c
|
||||
|
|
@ -793,7 +808,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
|
|||
break
|
||||
inc(pos)
|
||||
else: break
|
||||
tokenEnd(pos-1)
|
||||
tokenEnd(tok, pos-1)
|
||||
h = !$h
|
||||
tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
|
||||
L.bufpos = pos
|
||||
|
|
@ -814,7 +829,7 @@ proc endOperator(L: var TLexer, tok: var TToken, pos: int,
|
|||
proc getOperator(L: var TLexer, tok: var TToken) =
|
||||
var pos = L.bufpos
|
||||
var buf = L.buf
|
||||
tokenBegin(pos)
|
||||
tokenBegin(tok, pos)
|
||||
var h: Hash = 0
|
||||
while true:
|
||||
var c = buf[pos]
|
||||
|
|
@ -822,7 +837,7 @@ proc getOperator(L: var TLexer, tok: var TToken) =
|
|||
h = h !& ord(c)
|
||||
inc(pos)
|
||||
endOperator(L, tok, pos, h)
|
||||
tokenEnd(pos-1)
|
||||
tokenEnd(tok, pos-1)
|
||||
# advance pos but don't store it in L.bufpos so the next token (which might
|
||||
# be an operator too) gets the preceding spaces:
|
||||
tok.strongSpaceB = 0
|
||||
|
|
@ -837,7 +852,7 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
|
|||
var pos = start
|
||||
var buf = L.buf
|
||||
var toStrip = 0
|
||||
tokenBegin(pos)
|
||||
tokenBegin(tok, pos)
|
||||
# detect the amount of indentation:
|
||||
if isDoc:
|
||||
toStrip = getColNumber(L, pos)
|
||||
|
|
@ -864,36 +879,37 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
|
|||
if isDoc:
|
||||
if buf[pos+1] == '#' and buf[pos+2] == '#':
|
||||
if nesting == 0:
|
||||
tokenEndIgnore(pos+2)
|
||||
tokenEndIgnore(tok, pos+2)
|
||||
inc(pos, 3)
|
||||
break
|
||||
dec nesting
|
||||
tok.literal.add ']'
|
||||
elif buf[pos+1] == '#':
|
||||
if nesting == 0:
|
||||
tokenEndIgnore(pos+1)
|
||||
tokenEndIgnore(tok, pos+1)
|
||||
inc(pos, 2)
|
||||
break
|
||||
dec nesting
|
||||
inc pos
|
||||
of CR, LF:
|
||||
tokenEndIgnore(pos)
|
||||
tokenEndIgnore(tok, pos)
|
||||
pos = handleCRLF(L, pos)
|
||||
buf = L.buf
|
||||
# strip leading whitespace:
|
||||
when defined(nimpretty): tok.literal.add "\L"
|
||||
if isDoc:
|
||||
tok.literal.add "\n"
|
||||
when not defined(nimpretty): tok.literal.add "\n"
|
||||
inc tok.iNumber
|
||||
var c = toStrip
|
||||
while buf[pos] == ' ' and c > 0:
|
||||
inc pos
|
||||
dec c
|
||||
of nimlexbase.EndOfFile:
|
||||
tokenEndIgnore(pos)
|
||||
tokenEndIgnore(tok, pos)
|
||||
lexMessagePos(L, errGenerated, pos, "end of multiline comment expected")
|
||||
break
|
||||
else:
|
||||
if isDoc: tok.literal.add buf[pos]
|
||||
if isDoc or defined(nimpretty): tok.literal.add buf[pos]
|
||||
inc(pos)
|
||||
L.bufpos = pos
|
||||
|
||||
|
|
@ -907,7 +923,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
|
|||
if buf[pos+2] == '[':
|
||||
skipMultiLineComment(L, tok, pos+3, true)
|
||||
return
|
||||
tokenBegin(pos)
|
||||
tokenBegin(tok, pos)
|
||||
inc(pos, 2)
|
||||
|
||||
var toStrip = 0
|
||||
|
|
@ -921,7 +937,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
|
|||
if buf[pos] == '\\': lastBackslash = pos+1
|
||||
add(tok.literal, buf[pos])
|
||||
inc(pos)
|
||||
tokenEndIgnore(pos)
|
||||
tokenEndIgnore(tok, pos)
|
||||
pos = handleCRLF(L, pos)
|
||||
buf = L.buf
|
||||
var indent = 0
|
||||
|
|
@ -940,14 +956,14 @@ proc scanComment(L: var TLexer, tok: var TToken) =
|
|||
else:
|
||||
if buf[pos] > ' ':
|
||||
L.indentAhead = indent
|
||||
tokenEndIgnore(pos)
|
||||
tokenEndIgnore(tok, pos)
|
||||
break
|
||||
L.bufpos = pos
|
||||
|
||||
proc skip(L: var TLexer, tok: var TToken) =
|
||||
var pos = L.bufpos
|
||||
var buf = L.buf
|
||||
tokenBegin(pos)
|
||||
tokenBegin(tok, pos)
|
||||
tok.strongSpaceA = 0
|
||||
while true:
|
||||
case buf[pos]
|
||||
|
|
@ -958,7 +974,7 @@ proc skip(L: var TLexer, tok: var TToken) =
|
|||
if not L.allowTabs: lexMessagePos(L, errTabulatorsAreNotAllowed, pos)
|
||||
inc(pos)
|
||||
of CR, LF:
|
||||
tokenEndPrevious(pos)
|
||||
tokenEndPrevious(tok, pos)
|
||||
pos = handleCRLF(L, pos)
|
||||
buf = L.buf
|
||||
var indent = 0
|
||||
|
|
@ -985,12 +1001,12 @@ proc skip(L: var TLexer, tok: var TToken) =
|
|||
pos = L.bufpos
|
||||
buf = L.buf
|
||||
else:
|
||||
tokenBegin(pos)
|
||||
tokenBegin(tok, pos)
|
||||
while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos)
|
||||
tokenEndIgnore(pos+1)
|
||||
tokenEndIgnore(tok, pos+1)
|
||||
else:
|
||||
break # EndOfFile also leaves the loop
|
||||
tokenEndPrevious(pos-1)
|
||||
tokenEndPrevious(tok, pos-1)
|
||||
L.bufpos = pos
|
||||
|
||||
proc rawGetTok*(L: var TLexer, tok: var TToken) =
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue