fixes #2429
This commit is contained in:
Araq
parent
13259c669d
commit
0d7e0e1b4f
2 changed files with 178 additions and 156 deletions
|
|
@ -34,37 +34,15 @@ type
|
|||
lineNumber*: int ## the current line number
|
||||
sentinel: int
|
||||
lineStart: int # index of last line start in buffer
|
||||
fileOpened: bool
|
||||
refillChars: set[char]
|
||||
|
||||
{.deprecated: [TBaseLexer: BaseLexer].}
|
||||
|
||||
proc open*(L: var BaseLexer, input: Stream, bufLen: int = 8192)
|
||||
## inits the BaseLexer with a stream to read from
|
||||
|
||||
proc close*(L: var BaseLexer)
|
||||
## closes the base lexer. This closes `L`'s associated stream too.
|
||||
|
||||
proc getCurrentLine*(L: BaseLexer, marker: bool = true): string
|
||||
## retrieves the current line.
|
||||
|
||||
proc getColNumber*(L: BaseLexer, pos: int): int
|
||||
## retrieves the current column.
|
||||
|
||||
proc handleCR*(L: var BaseLexer, pos: int): int
|
||||
## Call this if you scanned over '\c' in the buffer; it returns the the
|
||||
## position to continue the scanning from. `pos` must be the position
|
||||
## of the '\c'.
|
||||
proc handleLF*(L: var BaseLexer, pos: int): int
|
||||
## Call this if you scanned over '\L' in the buffer; it returns the the
|
||||
## position to continue the scanning from. `pos` must be the position
|
||||
## of the '\L'.
|
||||
|
||||
# implementation
|
||||
|
||||
const
|
||||
chrSize = sizeof(char)
|
||||
|
||||
proc close(L: var BaseLexer) =
|
||||
proc close*(L: var BaseLexer) =
|
||||
## closes the base lexer. This closes `L`'s associated stream too.
|
||||
dealloc(L.buf)
|
||||
close(L.input)
|
||||
|
||||
|
|
@ -93,7 +71,7 @@ proc fillBuffer(L: var BaseLexer) =
|
|||
dec(s) # BUGFIX (valgrind)
|
||||
while true:
|
||||
assert(s < L.bufLen)
|
||||
while (s >= 0) and not (L.buf[s] in NewLines): dec(s)
|
||||
while s >= 0 and L.buf[s] notin L.refillChars: dec(s)
|
||||
if s >= 0:
|
||||
# we found an appropriate character for a sentinel:
|
||||
L.sentinel = s
|
||||
|
|
@ -121,31 +99,46 @@ proc fillBaseLexer(L: var BaseLexer, pos: int): int =
|
|||
fillBuffer(L)
|
||||
L.bufpos = 0 # XXX: is this really correct?
|
||||
result = 0
|
||||
L.lineStart = result
|
||||
|
||||
proc handleCR(L: var BaseLexer, pos: int): int =
|
||||
proc handleCR*(L: var BaseLexer, pos: int): int =
|
||||
## Call this if you scanned over '\c' in the buffer; it returns the the
|
||||
## position to continue the scanning from. `pos` must be the position
|
||||
## of the '\c'.
|
||||
assert(L.buf[pos] == '\c')
|
||||
inc(L.lineNumber)
|
||||
result = fillBaseLexer(L, pos)
|
||||
if L.buf[result] == '\L':
|
||||
result = fillBaseLexer(L, result)
|
||||
L.lineStart = result
|
||||
|
||||
proc handleLF(L: var BaseLexer, pos: int): int =
|
||||
proc handleLF*(L: var BaseLexer, pos: int): int =
|
||||
## Call this if you scanned over '\L' in the buffer; it returns the the
|
||||
## position to continue the scanning from. `pos` must be the position
|
||||
## of the '\L'.
|
||||
assert(L.buf[pos] == '\L')
|
||||
inc(L.lineNumber)
|
||||
result = fillBaseLexer(L, pos) #L.lastNL := result-1; // BUGFIX: was: result;
|
||||
L.lineStart = result
|
||||
|
||||
proc handleRefillChar*(L: var BaseLexer, pos: int): int =
|
||||
## To be documented.
|
||||
assert(L.buf[pos] in L.refillChars)
|
||||
result = fillBaseLexer(L, pos) #L.lastNL := result-1; // BUGFIX: was: result;
|
||||
|
||||
proc skipUtf8Bom(L: var BaseLexer) =
|
||||
if (L.buf[0] == '\xEF') and (L.buf[1] == '\xBB') and (L.buf[2] == '\xBF'):
|
||||
inc(L.bufpos, 3)
|
||||
inc(L.lineStart, 3)
|
||||
|
||||
proc open(L: var BaseLexer, input: Stream, bufLen: int = 8192) =
|
||||
proc open*(L: var BaseLexer, input: Stream, bufLen: int = 8192;
|
||||
refillChars: set[char] = NewLines) =
|
||||
## inits the BaseLexer with a stream to read from.
|
||||
assert(bufLen > 0)
|
||||
assert(input != nil)
|
||||
L.input = input
|
||||
L.bufpos = 0
|
||||
L.bufLen = bufLen
|
||||
L.refillChars = refillChars
|
||||
L.buf = cast[cstring](alloc(bufLen * chrSize))
|
||||
L.sentinel = bufLen - 1
|
||||
L.lineStart = 0
|
||||
|
|
@ -153,10 +146,12 @@ proc open(L: var BaseLexer, input: Stream, bufLen: int = 8192) =
|
|||
fillBuffer(L)
|
||||
skipUtf8Bom(L)
|
||||
|
||||
proc getColNumber(L: BaseLexer, pos: int): int =
|
||||
proc getColNumber*(L: BaseLexer, pos: int): int =
|
||||
## retrieves the current column.
|
||||
result = abs(pos - L.lineStart)
|
||||
|
||||
proc getCurrentLine(L: BaseLexer, marker: bool = true): string =
|
||||
proc getCurrentLine*(L: BaseLexer, marker: bool = true): string =
|
||||
## retrieves the current line.
|
||||
var i: int
|
||||
result = ""
|
||||
i = L.lineStart
|
||||
|
|
@ -166,4 +161,3 @@ proc getCurrentLine(L: BaseLexer, marker: bool = true): string =
|
|||
add(result, "\n")
|
||||
if marker:
|
||||
add(result, spaces(getColNumber(L, L.bufpos)) & "^\n")
|
||||
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ proc open*(my: var XmlParser, input: Stream, filename: string,
|
|||
## a whitespace token is reported as an ``xmlWhitespace`` event.
|
||||
## If `options` contains ``reportComments`` a comment token is reported as an
|
||||
## ``xmlComment`` event.
|
||||
lexbase.open(my, input)
|
||||
lexbase.open(my, input, 8192, {'\c', '\L', '/'})
|
||||
my.filename = filename
|
||||
my.state = stateStart
|
||||
my.kind = xmlError
|
||||
|
|
@ -243,6 +243,10 @@ proc parseCDATA(my: var XmlParser) =
|
|||
pos = lexbase.handleLF(my, pos)
|
||||
buf = my.buf
|
||||
add(my.a, '\L')
|
||||
of '/':
|
||||
pos = lexbase.handleRefillChar(my, pos)
|
||||
buf = my.buf
|
||||
add(my.a, '/')
|
||||
else:
|
||||
add(my.a, buf[pos])
|
||||
inc(pos)
|
||||
|
|
@ -271,6 +275,10 @@ proc parseComment(my: var XmlParser) =
|
|||
pos = lexbase.handleLF(my, pos)
|
||||
buf = my.buf
|
||||
if my.options.contains(reportComments): add(my.a, '\L')
|
||||
of '/':
|
||||
pos = lexbase.handleRefillChar(my, pos)
|
||||
buf = my.buf
|
||||
if my.options.contains(reportComments): add(my.a, '/')
|
||||
else:
|
||||
if my.options.contains(reportComments): add(my.a, buf[pos])
|
||||
inc(pos)
|
||||
|
|
@ -393,6 +401,10 @@ proc parsePI(my: var XmlParser) =
|
|||
pos = lexbase.handleLF(my, pos)
|
||||
buf = my.buf
|
||||
add(my.b, '\L')
|
||||
of '/':
|
||||
pos = lexbase.handleRefillChar(my, pos)
|
||||
buf = my.buf
|
||||
add(my.b, '/')
|
||||
else:
|
||||
add(my.b, buf[pos])
|
||||
inc(pos)
|
||||
|
|
@ -428,6 +440,10 @@ proc parseSpecial(my: var XmlParser) =
|
|||
pos = lexbase.handleLF(my, pos)
|
||||
buf = my.buf
|
||||
add(my.a, '\L')
|
||||
of '/':
|
||||
pos = lexbase.handleRefillChar(my, pos)
|
||||
buf = my.buf
|
||||
add(my.b, '/')
|
||||
else:
|
||||
add(my.a, buf[pos])
|
||||
inc(pos)
|
||||
|
|
@ -450,8 +466,11 @@ proc parseTag(my: var XmlParser) =
|
|||
my.c = my.a # save for later
|
||||
else:
|
||||
my.kind = xmlElementStart
|
||||
if my.buf[my.bufpos] == '/' and my.buf[my.bufpos+1] == '>':
|
||||
inc(my.bufpos, 2)
|
||||
let slash = my.buf[my.bufpos] == '/'
|
||||
if slash:
|
||||
my.bufpos = lexbase.handleRefillChar(my, my.bufpos)
|
||||
if slash and my.buf[my.bufpos] == '>':
|
||||
inc(my.bufpos)
|
||||
my.state = stateEmptyElementTag
|
||||
my.c = nil
|
||||
elif my.buf[my.bufpos] == '>':
|
||||
|
|
@ -460,7 +479,8 @@ proc parseTag(my: var XmlParser) =
|
|||
markError(my, errGtExpected)
|
||||
|
||||
proc parseEndTag(my: var XmlParser) =
|
||||
inc(my.bufpos, 2)
|
||||
my.bufpos = lexbase.handleRefillChar(my, my.bufpos+1)
|
||||
#inc(my.bufpos, 2)
|
||||
parseName(my, my.a)
|
||||
parseWhitespace(my, skip=true)
|
||||
if my.buf[my.bufpos] == '>':
|
||||
|
|
@ -545,6 +565,10 @@ proc parseCharData(my: var XmlParser) =
|
|||
pos = lexbase.handleLF(my, pos)
|
||||
buf = my.buf
|
||||
add(my.a, '\L')
|
||||
of '/':
|
||||
pos = lexbase.handleRefillChar(my, pos)
|
||||
buf = my.buf
|
||||
add(my.a, '/')
|
||||
else:
|
||||
add(my.a, buf[pos])
|
||||
inc(pos)
|
||||
|
|
@ -612,10 +636,14 @@ proc next*(my: var XmlParser) =
|
|||
my.kind = xmlElementClose
|
||||
inc(my.bufpos)
|
||||
my.state = stateNormal
|
||||
elif my.buf[my.bufpos] == '/' and my.buf[my.bufpos+1] == '>':
|
||||
elif my.buf[my.bufpos] == '/':
|
||||
my.bufpos = lexbase.handleRefillChar(my, my.bufpos)
|
||||
if my.buf[my.bufpos] == '>':
|
||||
my.kind = xmlElementClose
|
||||
inc(my.bufpos, 2)
|
||||
inc(my.bufpos)
|
||||
my.state = stateEmptyElementTag
|
||||
else:
|
||||
markError(my, errGtExpected)
|
||||
else:
|
||||
parseAttribute(my)
|
||||
# state remains the same
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue