jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

StringStream and parseJson, parseCfg, parseSql et al for the vm (#10746)

Browse source
This commit is contained in:
Arne Döring 2019-02-28 22:57:57 +01:00 • committed by Andreas Rumpf
commit 1102b8ac6e
15 changed files with 354 additions and 403 deletions
Download patch file Download diff file Expand all files Collapse all files

16
compiler/ccgmerge.nim
View file

@ -145,38 +145,34 @@ proc atEndMark(buf: cstring, pos: int): bool =
proc readVerbatimSection(L: var TBaseLexer): Rope = proc readVerbatimSection(L: var TBaseLexer): Rope =
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf
var r = newStringOfCap(30_000) var r = newStringOfCap(30_000)
while true: while true:
case buf[pos] case L.buf[pos]
of CR: of CR:
pos = nimlexbase.handleCR(L, pos) pos = nimlexbase.handleCR(L, pos)
buf = L.buf
r.add('\L') r.add('\L')
of LF: of LF:
pos = nimlexbase.handleLF(L, pos) pos = nimlexbase.handleLF(L, pos)
buf = L.buf
r.add('\L') r.add('\L')
of '\0': of '\0':
doAssert(false, "ccgmerge: expected: " & NimMergeEndMark) doAssert(false, "ccgmerge: expected: " & NimMergeEndMark)
break break
else: else:
if atEndMark(buf, pos): if atEndMark(L.buf, pos):
inc pos, NimMergeEndMark.len inc pos, NimMergeEndMark.len
break break
r.add(buf[pos]) r.add(L.buf[pos])
inc pos inc pos
L.bufpos = pos L.bufpos = pos
result = r.rope result = r.rope
proc readKey(L: var TBaseLexer, result: var string) = proc readKey(L: var TBaseLexer, result: var string) =
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf
setLen(result, 0) setLen(result, 0)
while buf[pos] in IdentChars: while L.buf[pos] in IdentChars:
result.add(buf[pos]) result.add(L.buf[pos])
inc pos inc pos
if buf[pos] != ':': doAssert(false, "ccgmerge: ':' expected") if L.buf[pos] != ':': doAssert(false, "ccgmerge: ':' expected")
L.bufpos = pos + 1 # skip ':' L.bufpos = pos + 1 # skip ':'
proc newFakeType(id: int): PType = proc newFakeType(id: int): PType =

123
compiler/lexer.nim
View file

@ -318,17 +318,16 @@ template eatChar(L: var TLexer, t: var TToken) =
proc getNumber(L: var TLexer, result: var TToken) = proc getNumber(L: var TLexer, result: var TToken) =
proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]): Natural = proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]): Natural =
var pos = L.bufpos # use registers for pos, buf var pos = L.bufpos # use registers for pos, buf
var buf = L.buf
result = 0 result = 0
while true: while true:
if buf[pos] in chars: if L.buf[pos] in chars:
add(tok.literal, buf[pos]) add(tok.literal, L.buf[pos])
inc(pos) inc(pos)
inc(result) inc(result)
else: else:
break break
if buf[pos] == '_': if L.buf[pos] == '_':
if buf[pos+1] notin chars: if L.buf[pos+1] notin chars:
lexMessage(L, errGenerated, lexMessage(L, errGenerated,
"only single underscores may occur in a token and token may not " & "only single underscores may occur in a token and token may not " &
"end with an underscore: e.g. '1__1' and '1_' are invalid") "end with an underscore: e.g. '1__1' and '1_' are invalid")
@ -339,9 +338,8 @@ proc getNumber(L: var TLexer, result: var TToken) =
proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) = proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) =
var pos = L.bufpos # use registers for pos, buf var pos = L.bufpos # use registers for pos, buf
var buf = L.buf while L.buf[pos] in chars:
while buf[pos] in chars: add(tok.literal, L.buf[pos])
add(tok.literal, buf[pos])
inc(pos) inc(pos)
L.bufpos = pos L.bufpos = pos
@ -800,25 +798,23 @@ type
proc getString(L: var TLexer, tok: var TToken, mode: StringMode) = proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf # put `buf` in a register
var line = L.lineNumber # save linenumber for better error message var line = L.lineNumber # save linenumber for better error message
tokenBegin(tok, pos - ord(mode == raw)) tokenBegin(tok, pos - ord(mode == raw))
inc pos # skip " inc pos # skip "
if buf[pos] == '\"' and buf[pos+1] == '\"': if L.buf[pos] == '\"' and L.buf[pos+1] == '\"':
tok.tokType = tkTripleStrLit # long string literal: tok.tokType = tkTripleStrLit # long string literal:
inc(pos, 2) # skip "" inc(pos, 2) # skip ""
# skip leading newline: # skip leading newline:
if buf[pos] in {' ', '\t'}: if L.buf[pos] in {' ', '\t'}:
var newpos = pos+1 var newpos = pos+1
while buf[newpos] in {' ', '\t'}: inc newpos while L.buf[newpos] in {' ', '\t'}: inc newpos
if buf[newpos] in {CR, LF}: pos = newpos if L.buf[newpos] in {CR, LF}: pos = newpos
pos = handleCRLF(L, pos) pos = handleCRLF(L, pos)
buf = L.buf
while true: while true:
case buf[pos] case L.buf[pos]
of '\"': of '\"':
if buf[pos+1] == '\"' and buf[pos+2] == '\"' and if L.buf[pos+1] == '\"' and L.buf[pos+2] == '\"' and
buf[pos+3] != '\"': L.buf[pos+3] != '\"':
tokenEndIgnore(tok, pos+2) tokenEndIgnore(tok, pos+2)
L.bufpos = pos + 3 # skip the three """ L.bufpos = pos + 3 # skip the three """
break break
@ -827,7 +823,6 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
of CR, LF: of CR, LF:
tokenEndIgnore(tok, pos) tokenEndIgnore(tok, pos)
pos = handleCRLF(L, pos) pos = handleCRLF(L, pos)
buf = L.buf
add(tok.literal, "\n") add(tok.literal, "\n")
of nimlexbase.EndOfFile: of nimlexbase.EndOfFile:
tokenEndIgnore(tok, pos) tokenEndIgnore(tok, pos)
@ -838,16 +833,16 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
L.bufpos = pos L.bufpos = pos
break break
else: else:
add(tok.literal, buf[pos]) add(tok.literal, L.buf[pos])
inc(pos) inc(pos)
else: else:
# ordinary string literal # ordinary string literal
if mode != normal: tok.tokType = tkRStrLit if mode != normal: tok.tokType = tkRStrLit
else: tok.tokType = tkStrLit else: tok.tokType = tkStrLit
while true: while true:
var c = buf[pos] var c = L.buf[pos]
if c == '\"': if c == '\"':
if mode != normal and buf[pos+1] == '\"': if mode != normal and L.buf[pos+1] == '\"':
inc(pos, 2) inc(pos, 2)
add(tok.literal, '"') add(tok.literal, '"')
else: else:
@ -885,10 +880,9 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
proc getSymbol(L: var TLexer, tok: var TToken) = proc getSymbol(L: var TLexer, tok: var TToken) =
var h: Hash = 0 var h: Hash = 0
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf
tokenBegin(tok, pos) tokenBegin(tok, pos)
while true: while true:
var c = buf[pos] var c = L.buf[pos]
case c case c
of 'a'..'z', '0'..'9', '\x80'..'\xFF': of 'a'..'z', '0'..'9', '\x80'..'\xFF':
h = h !& ord(c) h = h !& ord(c)
@ -898,7 +892,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
h = h !& ord(c) h = h !& ord(c)
inc(pos) inc(pos)
of '_': of '_':
if buf[pos+1] notin SymChars: if L.buf[pos+1] notin SymChars:
lexMessage(L, errGenerated, "invalid token: trailing underscore") lexMessage(L, errGenerated, "invalid token: trailing underscore")
break break
inc(pos) inc(pos)
@ -923,11 +917,10 @@ proc endOperator(L: var TLexer, tok: var TToken, pos: int,
proc getOperator(L: var TLexer, tok: var TToken) = proc getOperator(L: var TLexer, tok: var TToken) =
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf
tokenBegin(tok, pos) tokenBegin(tok, pos)
var h: Hash = 0 var h: Hash = 0
while true: while true:
var c = buf[pos] var c = L.buf[pos]
if c notin OpChars: break if c notin OpChars: break
h = h !& ord(c) h = h !& ord(c)
inc(pos) inc(pos)
@ -936,10 +929,10 @@ proc getOperator(L: var TLexer, tok: var TToken) =
# advance pos but don't store it in L.bufpos so the next token (which might # advance pos but don't store it in L.bufpos so the next token (which might
# be an operator too) gets the preceding spaces: # be an operator too) gets the preceding spaces:
tok.strongSpaceB = 0 tok.strongSpaceB = 0
while buf[pos] == ' ': while L.buf[pos] == ' ':
inc pos inc pos
inc tok.strongSpaceB inc tok.strongSpaceB
if buf[pos] in {CR, LF, nimlexbase.EndOfFile}: if L.buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
tok.strongSpaceB = -1 tok.strongSpaceB = -1
proc getPrecedence*(tok: TToken, strongSpaces: bool): int = proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
@ -980,9 +973,8 @@ proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
proc newlineFollows*(L: TLexer): bool = proc newlineFollows*(L: TLexer): bool =
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf
while true: while true:
case buf[pos] case L.buf[pos]
of ' ', '\t': of ' ', '\t':
inc(pos) inc(pos)
of CR, LF: of CR, LF:
@ -990,49 +982,47 @@ proc newlineFollows*(L: TLexer): bool =
break break
of '#': of '#':
inc(pos) inc(pos)
if buf[pos] == '#': inc(pos) if L.buf[pos] == '#': inc(pos)
if buf[pos] != '[': return true if L.buf[pos] != '[': return true
else: else:
break break
proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
isDoc: bool) = isDoc: bool) =
var pos = start var pos = start
var buf = L.buf
var toStrip = 0 var toStrip = 0
tokenBegin(tok, pos) tokenBegin(tok, pos)
# detect the amount of indentation: # detect the amount of indentation:
if isDoc: if isDoc:
toStrip = getColNumber(L, pos) toStrip = getColNumber(L, pos)
while buf[pos] == ' ': inc pos while L.buf[pos] == ' ': inc pos
if buf[pos] in {CR, LF}: if L.buf[pos] in {CR, LF}:
pos = handleCRLF(L, pos) pos = handleCRLF(L, pos)
buf = L.buf
toStrip = 0 toStrip = 0
while buf[pos] == ' ': while L.buf[pos] == ' ':
inc pos inc pos
inc toStrip inc toStrip
var nesting = 0 var nesting = 0
while true: while true:
case buf[pos] case L.buf[pos]
of '#': of '#':
if isDoc: if isDoc:
if buf[pos+1] == '#' and buf[pos+2] == '[': if L.buf[pos+1] == '#' and L.buf[pos+2] == '[':
inc nesting inc nesting
tok.literal.add '#' tok.literal.add '#'
elif buf[pos+1] == '[': elif L.buf[pos+1] == '[':
inc nesting inc nesting
inc pos inc pos
of ']': of ']':
if isDoc: if isDoc:
if buf[pos+1] == '#' and buf[pos+2] == '#': if L.buf[pos+1] == '#' and L.buf[pos+2] == '#':
if nesting == 0: if nesting == 0:
tokenEndIgnore(tok, pos+2) tokenEndIgnore(tok, pos+2)
inc(pos, 3) inc(pos, 3)
break break
dec nesting dec nesting
tok.literal.add ']' tok.literal.add ']'
elif buf[pos+1] == '#': elif L.buf[pos+1] == '#':
if nesting == 0: if nesting == 0:
tokenEndIgnore(tok, pos+1) tokenEndIgnore(tok, pos+1)
inc(pos, 2) inc(pos, 2)
@ -1042,14 +1032,13 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
of CR, LF: of CR, LF:
tokenEndIgnore(tok, pos) tokenEndIgnore(tok, pos)
pos = handleCRLF(L, pos) pos = handleCRLF(L, pos)
buf = L.buf
# strip leading whitespace: # strip leading whitespace:
when defined(nimpretty): tok.literal.add "\L" when defined(nimpretty): tok.literal.add "\L"
if isDoc: if isDoc:
when not defined(nimpretty): tok.literal.add "\n" when not defined(nimpretty): tok.literal.add "\n"
inc tok.iNumber inc tok.iNumber
var c = toStrip var c = toStrip
while buf[pos] == ' ' and c > 0: while L.buf[pos] == ' ' and c > 0:
inc pos inc pos
dec c dec c
of nimlexbase.EndOfFile: of nimlexbase.EndOfFile:
@ -1057,7 +1046,7 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
lexMessagePos(L, errGenerated, pos, "end of multiline comment expected") lexMessagePos(L, errGenerated, pos, "end of multiline comment expected")
break break
else: else:
if isDoc or defined(nimpretty): tok.literal.add buf[pos] if isDoc or defined(nimpretty): tok.literal.add L.buf[pos]
inc(pos) inc(pos)
L.bufpos = pos L.bufpos = pos
when defined(nimpretty): when defined(nimpretty):
@ -1065,49 +1054,47 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
proc scanComment(L: var TLexer, tok: var TToken) = proc scanComment(L: var TLexer, tok: var TToken) =
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf
tok.tokType = tkComment tok.tokType = tkComment
# iNumber contains the number of '\n' in the token # iNumber contains the number of '\n' in the token
tok.iNumber = 0 tok.iNumber = 0
assert buf[pos+1] == '#' assert L.buf[pos+1] == '#'
when defined(nimpretty): when defined(nimpretty):
tok.commentOffsetA = L.offsetBase + pos - 1 tok.commentOffsetA = L.offsetBase + pos - 1
if buf[pos+2] == '[': if L.buf[pos+2] == '[':
skipMultiLineComment(L, tok, pos+3, true) skipMultiLineComment(L, tok, pos+3, true)
return return
tokenBegin(tok, pos) tokenBegin(tok, pos)
inc(pos, 2) inc(pos, 2)
var toStrip = 0 var toStrip = 0
while buf[pos] == ' ': while L.buf[pos] == ' ':
inc pos inc pos
inc toStrip inc toStrip
while true: while true:
var lastBackslash = -1 var lastBackslash = -1
while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
if buf[pos] == '\\': lastBackslash = pos+1 if L.buf[pos] == '\\': lastBackslash = pos+1
add(tok.literal, buf[pos]) add(tok.literal, L.buf[pos])
inc(pos) inc(pos)
tokenEndIgnore(tok, pos) tokenEndIgnore(tok, pos)
pos = handleCRLF(L, pos) pos = handleCRLF(L, pos)
buf = L.buf
var indent = 0 var indent = 0
while buf[pos] == ' ': while L.buf[pos] == ' ':
inc(pos) inc(pos)
inc(indent) inc(indent)
if buf[pos] == '#' and buf[pos+1] == '#': if L.buf[pos] == '#' and L.buf[pos+1] == '#':
tok.literal.add "\n" tok.literal.add "\n"
inc(pos, 2) inc(pos, 2)
var c = toStrip var c = toStrip
while buf[pos] == ' ' and c > 0: while L.buf[pos] == ' ' and c > 0:
inc pos inc pos
dec c dec c
inc tok.iNumber inc tok.iNumber
else: else:
if buf[pos] > ' ': if L.buf[pos] > ' ':
L.indentAhead = indent L.indentAhead = indent
tokenEndIgnore(tok, pos) tokenEndIgnore(tok, pos)
break break
@ -1117,7 +1104,6 @@ proc scanComment(L: var TLexer, tok: var TToken) =
proc skip(L: var TLexer, tok: var TToken) = proc skip(L: var TLexer, tok: var TToken) =
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf
tokenBegin(tok, pos) tokenBegin(tok, pos)
tok.strongSpaceA = 0 tok.strongSpaceA = 0
when defined(nimpretty): when defined(nimpretty):
@ -1127,7 +1113,7 @@ proc skip(L: var TLexer, tok: var TToken) =
tok.commentOffsetB = tok.commentOffsetA tok.commentOffsetB = tok.commentOffsetA
tok.line = -1 tok.line = -1
while true: while true:
case buf[pos] case L.buf[pos]
of ' ': of ' ':
inc(pos) inc(pos)
inc(tok.strongSpaceA) inc(tok.strongSpaceA)
@ -1137,13 +1123,12 @@ proc skip(L: var TLexer, tok: var TToken) =
of CR, LF: of CR, LF:
tokenEndPrevious(tok, pos) tokenEndPrevious(tok, pos)
pos = handleCRLF(L, pos) pos = handleCRLF(L, pos)
buf = L.buf
var indent = 0 var indent = 0
while true: while true:
if buf[pos] == ' ': if L.buf[pos] == ' ':
inc(pos) inc(pos)
inc(indent) inc(indent)
elif buf[pos] == '#' and buf[pos+1] == '[': elif L.buf[pos] == '#' and L.buf[pos+1] == '[':
when defined(nimpretty): when defined(nimpretty):
hasComment = true hasComment = true
if tok.line < 0: if tok.line < 0:
@ -1151,32 +1136,30 @@ proc skip(L: var TLexer, tok: var TToken) =
commentIndent = indent commentIndent = indent
skipMultiLineComment(L, tok, pos+2, false) skipMultiLineComment(L, tok, pos+2, false)
pos = L.bufpos pos = L.bufpos
buf = L.buf
else: else:
break break
tok.strongSpaceA = 0 tok.strongSpaceA = 0
when defined(nimpretty): when defined(nimpretty):
if buf[pos] == '#' and tok.line < 0: commentIndent = indent if L.buf[pos] == '#' and tok.line < 0: commentIndent = indent
if buf[pos] > ' ' and (buf[pos] != '#' or buf[pos+1] == '#'): if L.buf[pos] > ' ' and (L.buf[pos] != '#' or L.buf[pos+1] == '#'):
tok.indent = indent tok.indent = indent
L.currLineIndent = indent L.currLineIndent = indent
break break
of '#': of '#':
# do not skip documentation comment: # do not skip documentation comment:
if buf[pos+1] == '#': break if L.buf[pos+1] == '#': break
when defined(nimpretty): when defined(nimpretty):
hasComment = true hasComment = true
if tok.line < 0: if tok.line < 0:
tok.line = L.lineNumber tok.line = L.lineNumber
if buf[pos+1] == '[': if L.buf[pos+1] == '[':
skipMultiLineComment(L, tok, pos+2, false) skipMultiLineComment(L, tok, pos+2, false)
pos = L.bufpos pos = L.bufpos
buf = L.buf
else: else:
tokenBegin(tok, pos) tokenBegin(tok, pos)
while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
when defined(nimpretty): tok.literal.add buf[pos] when defined(nimpretty): tok.literal.add L.buf[pos]
inc(pos) inc(pos)
tokenEndIgnore(tok, pos+1) tokenEndIgnore(tok, pos+1)
when defined(nimpretty): when defined(nimpretty):

32
compiler/nimlexbase.nim
View file

@ -39,8 +39,7 @@ const
type type
TBaseLexer* = object of RootObj TBaseLexer* = object of RootObj
bufpos*: int bufpos*: int
buf*: cstring buf*: string
bufLen*: int # length of buffer in characters
stream*: PLLStream # we read from this stream stream*: PLLStream # we read from this stream
lineNumber*: int # the current line number lineNumber*: int # the current line number
# private data: # private data:
@ -65,11 +64,7 @@ proc handleLF*(L: var TBaseLexer, pos: int): int
# of the LF. # of the LF.
# implementation # implementation
const
chrSize = sizeof(char)
proc closeBaseLexer(L: var TBaseLexer) = proc closeBaseLexer(L: var TBaseLexer) =
dealloc(L.buf)
llStreamClose(L.stream) llStreamClose(L.stream)
proc fillBuffer(L: var TBaseLexer) = proc fillBuffer(L: var TBaseLexer) =
@ -80,14 +75,13 @@ proc fillBuffer(L: var TBaseLexer) =
oldBufLen: int oldBufLen: int
# we know here that pos == L.sentinel, but not if this proc # we know here that pos == L.sentinel, but not if this proc
# is called the first time by initBaseLexer() # is called the first time by initBaseLexer()
assert(L.sentinel < L.bufLen) assert(L.sentinel < L.buf.len)
toCopy = L.bufLen - L.sentinel - 1 toCopy = L.buf.len - L.sentinel - 1
assert(toCopy >= 0) assert(toCopy >= 0)
if toCopy > 0: if toCopy > 0:
moveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize) moveMem(addr L.buf[0], addr L.buf[L.sentinel + 1], toCopy)
# "moveMem" handles overlapping regions # "moveMem" handles overlapping regions
charsRead = llStreamRead(L.stream, addr(L.buf[toCopy]), charsRead = llStreamRead(L.stream, addr L.buf[toCopy], L.sentinel + 1)
(L.sentinel + 1) * chrSize) div chrSize
s = toCopy + charsRead s = toCopy + charsRead
if charsRead < L.sentinel + 1: if charsRead < L.sentinel + 1:
L.buf[s] = EndOfFile # set end marker L.buf[s] = EndOfFile # set end marker
@ -96,7 +90,7 @@ proc fillBuffer(L: var TBaseLexer) =
# compute sentinel: # compute sentinel:
dec(s) # BUGFIX (valgrind) dec(s) # BUGFIX (valgrind)
while true: while true:
assert(s < L.bufLen) assert(s < L.buf.len)
while (s >= 0) and not (L.buf[s] in NewLines): dec(s) while (s >= 0) and not (L.buf[s] in NewLines): dec(s)
if s >= 0: if s >= 0:
# we found an appropriate character for a sentinel: # we found an appropriate character for a sentinel:
@ -105,17 +99,16 @@ proc fillBuffer(L: var TBaseLexer) =
else: else:
# rather than to give up here because the line is too long, # rather than to give up here because the line is too long,
# double the buffer's size and try again: # double the buffer's size and try again:
oldBufLen = L.bufLen oldBufLen = L.buf.len
L.bufLen = L.bufLen * 2 L.buf.setLen(L.buf.len * 2)
L.buf = cast[cstring](realloc(L.buf, L.bufLen * chrSize)) assert(L.buf.len - oldBufLen == oldBufLen)
assert(L.bufLen - oldBufLen == oldBufLen)
charsRead = llStreamRead(L.stream, addr(L.buf[oldBufLen]), charsRead = llStreamRead(L.stream, addr(L.buf[oldBufLen]),
oldBufLen * chrSize) div chrSize oldBufLen)
if charsRead < oldBufLen: if charsRead < oldBufLen:
L.buf[oldBufLen + charsRead] = EndOfFile L.buf[oldBufLen + charsRead] = EndOfFile
L.sentinel = oldBufLen + charsRead L.sentinel = oldBufLen + charsRead
break break
s = L.bufLen - 1 s = L.buf.len - 1
proc fillBaseLexer(L: var TBaseLexer, pos: int): int = proc fillBaseLexer(L: var TBaseLexer, pos: int): int =
assert(pos <= L.sentinel) assert(pos <= L.sentinel)
@ -149,8 +142,7 @@ proc openBaseLexer(L: var TBaseLexer, inputstream: PLLStream, bufLen = 8192) =
assert(bufLen > 0) assert(bufLen > 0)
L.bufpos = 0 L.bufpos = 0
L.offsetBase = 0 L.offsetBase = 0
L.bufLen = bufLen L.buf = newString(bufLen)
L.buf = cast[cstring](alloc(bufLen * chrSize))
L.sentinel = bufLen - 1 L.sentinel = bufLen - 1
L.lineStart = 0 L.lineStart = 0
L.lineNumber = 1 # lines start at 1 L.lineNumber = 1 # lines start at 1

2
compiler/vmgen.nim
View file

@ -2039,6 +2039,8 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
genConv(c, n, n.sons[1], dest) genConv(c, n, n.sons[1], dest)
of nkObjDownConv: of nkObjDownConv:
genConv(c, n, n.sons[0], dest) genConv(c, n, n.sons[0], dest)
of nkObjUpConv:
genConv(c, n, n.sons[0], dest)
of nkVarSection, nkLetSection: of nkVarSection, nkLetSection:
unused(c, n, dest) unused(c, n, dest)
genVarSection(c, n) genVarSection(c, n)

17
lib/packages/docutils/rst.nim
View file

@ -155,18 +155,17 @@ proc getAdornment(L: var Lexer, tok: var Token) =
proc getIndentAux(L: var Lexer, start: int): int = proc getIndentAux(L: var Lexer, start: int): int =
var pos = start var pos = start
var buf = L.buf
# skip the newline (but include it in the token!) # skip the newline (but include it in the token!)
if buf[pos] == '\x0D': if L.buf[pos] == '\x0D':
if buf[pos + 1] == '\x0A': inc(pos, 2) if L.buf[pos + 1] == '\x0A': inc(pos, 2)
else: inc(pos) else: inc(pos)
elif buf[pos] == '\x0A': elif L.buf[pos] == '\x0A':
inc(pos) inc(pos)
if L.skipPounds: if L.skipPounds:
if buf[pos] == '#': inc(pos) if L.buf[pos] == '#': inc(pos)
if buf[pos] == '#': inc(pos) if L.buf[pos] == '#': inc(pos)
while true: while true:
case buf[pos] case L.buf[pos]
of ' ', '\x0B', '\x0C': of ' ', '\x0B', '\x0C':
inc(pos) inc(pos)
inc(result) inc(result)
@ -175,9 +174,9 @@ proc getIndentAux(L: var Lexer, start: int): int =
result = result - (result mod 8) + 8 result = result - (result mod 8) + 8
else: else:
break # EndOfFile also leaves the loop break # EndOfFile also leaves the loop
if buf[pos] == '\0': if L.buf[pos] == '\0':
result = 0 result = 0
elif (buf[pos] == '\x0A') or (buf[pos] == '\x0D'): elif (L.buf[pos] == '\x0A') or (L.buf[pos] == '\x0D'):
# look at the next line for proper indentation: # look at the next line for proper indentation:
result = getIndentAux(L, pos) result = getIndentAux(L, pos)
L.bufpos = pos # no need to set back buf L.bufpos = pos # no need to set back buf

47
lib/pure/lexbase.nim
View file

@ -28,11 +28,7 @@ type
BaseLexer* = object of RootObj ## the base lexer. Inherit your lexer from BaseLexer* = object of RootObj ## the base lexer. Inherit your lexer from
## this object. ## this object.
bufpos*: int ## the current position within the buffer bufpos*: int ## the current position within the buffer
when defined(js): ## the buffer itself buf*: string ## the buffer itself
buf*: string
else:
buf*: cstring
bufLen*: int ## length of buffer in characters
input: Stream ## the input stream input: Stream ## the input stream
lineNumber*: int ## the current line number lineNumber*: int ## the current line number
sentinel: int sentinel: int
@ -40,13 +36,8 @@ type
offsetBase*: int # use ``offsetBase + bufpos`` to get the offset offsetBase*: int # use ``offsetBase + bufpos`` to get the offset
refillChars: set[char] refillChars: set[char]
const
chrSize = sizeof(char)
proc close*(L: var BaseLexer) = proc close*(L: var BaseLexer) =
## closes the base lexer. This closes `L`'s associated stream too. ## closes the base lexer. This closes `L`'s associated stream too.
when not defined(js):
dealloc(L.buf)
close(L.input) close(L.input)
proc fillBuffer(L: var BaseLexer) = proc fillBuffer(L: var BaseLexer) =
@ -57,17 +48,21 @@ proc fillBuffer(L: var BaseLexer) =
oldBufLen: int oldBufLen: int
# we know here that pos == L.sentinel, but not if this proc # we know here that pos == L.sentinel, but not if this proc
# is called the first time by initBaseLexer() # is called the first time by initBaseLexer()
assert(L.sentinel < L.bufLen) assert(L.sentinel + 1 <= L.buf.len)
toCopy = L.bufLen - L.sentinel - 1 toCopy = L.buf.len - (L.sentinel + 1)
assert(toCopy >= 0) assert(toCopy >= 0)
if toCopy > 0: if toCopy > 0:
when defined(js): when defined(js):
for i in 0 ..< toCopy: L.buf[i] = L.buf[L.sentinel + 1 + i] for i in 0 ..< toCopy:
L.buf[i] = L.buf[L.sentinel + 1 + i]
else:
when nimvm:
for i in 0 ..< toCopy:
L.buf[i] = L.buf[L.sentinel + 1 + i]
else: else:
# "moveMem" handles overlapping regions # "moveMem" handles overlapping regions
moveMem(L.buf, addr L.buf[L.sentinel + 1], toCopy * chrSize) moveMem(addr L.buf[0], addr L.buf[L.sentinel + 1], toCopy)
charsRead = readData(L.input, addr(L.buf[toCopy]), charsRead = L.input.readDataStr(L.buf, toCopy ..< toCopy + L.sentinel + 1)
(L.sentinel + 1) * chrSize) div chrSize
s = toCopy + charsRead s = toCopy + charsRead
if charsRead < L.sentinel + 1: if charsRead < L.sentinel + 1:
L.buf[s] = EndOfFile # set end marker L.buf[s] = EndOfFile # set end marker
@ -76,7 +71,7 @@ proc fillBuffer(L: var BaseLexer) =
# compute sentinel: # compute sentinel:
dec(s) # BUGFIX (valgrind) dec(s) # BUGFIX (valgrind)
while true: while true:
assert(s < L.bufLen) assert(s < L.buf.len)
while s >= 0 and L.buf[s] notin L.refillChars: dec(s) while s >= 0 and L.buf[s] notin L.refillChars: dec(s)
if s >= 0: if s >= 0:
# we found an appropriate character for a sentinel: # we found an appropriate character for a sentinel:
@ -85,20 +80,14 @@ proc fillBuffer(L: var BaseLexer) =
else: else:
# rather than to give up here because the line is too long, # rather than to give up here because the line is too long,
# double the buffer's size and try again: # double the buffer's size and try again:
oldBufLen = L.bufLen oldBufLen = L.buf.len
L.bufLen = L.bufLen * 2 L.buf.setLen(L.buf.len * 2)
when defined(js): charsRead = readDataStr(L.input, L.buf, oldBufLen ..< L.buf.len)
L.buf.setLen(L.bufLen)
else:
L.buf = cast[cstring](realloc(L.buf, L.bufLen * chrSize))
assert(L.bufLen - oldBufLen == oldBufLen)
charsRead = readData(L.input, addr(L.buf[oldBufLen]),
oldBufLen * chrSize) div chrSize
if charsRead < oldBufLen: if charsRead < oldBufLen:
L.buf[oldBufLen + charsRead] = EndOfFile L.buf[oldBufLen + charsRead] = EndOfFile
L.sentinel = oldBufLen + charsRead L.sentinel = oldBufLen + charsRead
break break
s = L.bufLen - 1 s = L.buf.len - 1
proc fillBaseLexer(L: var BaseLexer, pos: int): int = proc fillBaseLexer(L: var BaseLexer, pos: int): int =
assert(pos <= L.sentinel) assert(pos <= L.sentinel)
@ -148,12 +137,8 @@ proc open*(L: var BaseLexer, input: Stream, bufLen: int = 8192;
L.input = input L.input = input
L.bufpos = 0 L.bufpos = 0
L.offsetBase = 0 L.offsetBase = 0
L.bufLen = bufLen
L.refillChars = refillChars L.refillChars = refillChars
when defined(js):
L.buf = newString(bufLen) L.buf = newString(bufLen)
else:
L.buf = cast[cstring](alloc(bufLen * chrSize))
L.sentinel = bufLen - 1 L.sentinel = bufLen - 1
L.lineStart = 0 L.lineStart = 0
L.lineNumber = 1 # lines start at 1 L.lineNumber = 1 # lines start at 1

24
lib/pure/parsecfg.nim
View file

@ -261,35 +261,32 @@ proc handleCRLF(c: var CfgParser, pos: int): int =
proc getString(c: var CfgParser, tok: var Token, rawMode: bool) = proc getString(c: var CfgParser, tok: var Token, rawMode: bool) =
var pos = c.bufpos + 1 # skip " var pos = c.bufpos + 1 # skip "
var buf = c.buf # put `buf` in a register
tok.kind = tkSymbol tok.kind = tkSymbol
if (buf[pos] == '"') and (buf[pos + 1] == '"'): if (c.buf[pos] == '"') and (c.buf[pos + 1] == '"'):
# long string literal: # long string literal:
inc(pos, 2) # skip "" inc(pos, 2) # skip ""
# skip leading newline: # skip leading newline:
pos = handleCRLF(c, pos) pos = handleCRLF(c, pos)
buf = c.buf
while true: while true:
case buf[pos] case c.buf[pos]
of '"': of '"':
if (buf[pos + 1] == '"') and (buf[pos + 2] == '"'): break if (c.buf[pos + 1] == '"') and (c.buf[pos + 2] == '"'): break
add(tok.literal, '"') add(tok.literal, '"')
inc(pos) inc(pos)
of '\c', '\L': of '\c', '\L':
pos = handleCRLF(c, pos) pos = handleCRLF(c, pos)
buf = c.buf
add(tok.literal, "\n") add(tok.literal, "\n")
of lexbase.EndOfFile: of lexbase.EndOfFile:
tok.kind = tkInvalid tok.kind = tkInvalid
break break
else: else:
add(tok.literal, buf[pos]) add(tok.literal, c.buf[pos])
inc(pos) inc(pos)
c.bufpos = pos + 3 # skip the three """ c.bufpos = pos + 3 # skip the three """
else: else:
# ordinary string literal # ordinary string literal
while true: while true:
var ch = buf[pos] var ch = c.buf[pos]
if ch == '"': if ch == '"':
inc(pos) # skip '"' inc(pos) # skip '"'
break break
@ -307,26 +304,23 @@ proc getString(c: var CfgParser, tok: var Token, rawMode: bool) =
proc getSymbol(c: var CfgParser, tok: var Token) = proc getSymbol(c: var CfgParser, tok: var Token) =
var pos = c.bufpos var pos = c.bufpos
var buf = c.buf
while true: while true:
add(tok.literal, buf[pos]) add(tok.literal, c.buf[pos])
inc(pos) inc(pos)
if not (buf[pos] in SymChars): break if not (c.buf[pos] in SymChars): break
c.bufpos = pos c.bufpos = pos
tok.kind = tkSymbol tok.kind = tkSymbol
proc skip(c: var CfgParser) = proc skip(c: var CfgParser) =
var pos = c.bufpos var pos = c.bufpos
var buf = c.buf
while true: while true:
case buf[pos] case c.buf[pos]
of ' ', '\t': of ' ', '\t':
inc(pos) inc(pos)
of '#', ';': of '#', ';':
while not (buf[pos] in {'\c', '\L', lexbase.EndOfFile}): inc(pos) while not (c.buf[pos] in {'\c', '\L', lexbase.EndOfFile}): inc(pos)
of '\c', '\L': of '\c', '\L':
pos = handleCRLF(c, pos) pos = handleCRLF(c, pos)
buf = c.buf
else: else:
break # EndOfFile also leaves the loop break # EndOfFile also leaves the loop
c.bufpos = pos c.bufpos = pos

15
lib/pure/parsecsv.nim
View file

@ -156,44 +156,41 @@ proc open*(my: var CsvParser, filename: string,
proc parseField(my: var CsvParser, a: var string) = proc parseField(my: var CsvParser, a: var string) =
var pos = my.bufpos var pos = my.bufpos
var buf = my.buf
if my.skipWhite: if my.skipWhite:
while buf[pos] in {' ', '\t'}: inc(pos) while my.buf[pos] in {' ', '\t'}: inc(pos)
setLen(a, 0) # reuse memory setLen(a, 0) # reuse memory
if buf[pos] == my.quote and my.quote != '\0': if my.buf[pos] == my.quote and my.quote != '\0':
inc(pos) inc(pos)
while true: while true:
let c = buf[pos] let c = my.buf[pos]
if c == '\0': if c == '\0':
my.bufpos = pos # can continue after exception? my.bufpos = pos # can continue after exception?
error(my, pos, my.quote & " expected") error(my, pos, my.quote & " expected")
break break
elif c == my.quote: elif c == my.quote:
if my.esc == '\0' and buf[pos+1] == my.quote: if my.esc == '\0' and my.buf[pos+1] == my.quote:
add(a, my.quote) add(a, my.quote)
inc(pos, 2) inc(pos, 2)
else: else:
inc(pos) inc(pos)
break break
elif c == my.esc: elif c == my.esc:
add(a, buf[pos+1]) add(a, my.buf[pos+1])
inc(pos, 2) inc(pos, 2)
else: else:
case c case c
of '\c': of '\c':
pos = handleCR(my, pos) pos = handleCR(my, pos)
buf = my.buf
add(a, "\n") add(a, "\n")
of '\l': of '\l':
pos = handleLF(my, pos) pos = handleLF(my, pos)
buf = my.buf
add(a, "\n") add(a, "\n")
else: else:
add(a, c) add(a, c)
inc(pos) inc(pos)
else: else:
while true: while true:
let c = buf[pos] let c = my.buf[pos]
if c == my.sep: break if c == my.sep: break
if c in {'\c', '\l', '\0'}: break if c in {'\c', '\l', '\0'}: break
add(a, c) add(a, c)

76
lib/pure/parsejson.nim
View file

@ -182,11 +182,10 @@ proc parseEscapedUTF16*(buf: cstring, pos: var int): int =
proc parseString(my: var JsonParser): TokKind = proc parseString(my: var JsonParser): TokKind =
result = tkString result = tkString
var pos = my.bufpos + 1 var pos = my.bufpos + 1
var buf = my.buf
if my.rawStringLiterals: if my.rawStringLiterals:
add(my.a, '"') add(my.a, '"')
while true: while true:
case buf[pos] case my.buf[pos]
of '\0': of '\0':
my.err = errQuoteExpected my.err = errQuoteExpected
result = tkError result = tkError
@ -199,9 +198,9 @@ proc parseString(my: var JsonParser): TokKind =
of '\\': of '\\':
if my.rawStringLiterals: if my.rawStringLiterals:
add(my.a, '\\') add(my.a, '\\')
case buf[pos+1] case my.buf[pos+1]
of '\\', '"', '\'', '/': of '\\', '"', '\'', '/':
add(my.a, buf[pos+1]) add(my.a, my.buf[pos+1])
inc(pos, 2) inc(pos, 2)
of 'b': of 'b':
add(my.a, '\b') add(my.a, '\b')
@ -223,17 +222,17 @@ proc parseString(my: var JsonParser): TokKind =
add(my.a, 'u') add(my.a, 'u')
inc(pos, 2) inc(pos, 2)
var pos2 = pos var pos2 = pos
var r = parseEscapedUTF16(buf, pos) var r = parseEscapedUTF16(my.buf, pos)
if r < 0: if r < 0:
my.err = errInvalidToken my.err = errInvalidToken
break break
# Deal with surrogates # Deal with surrogates
if (r and 0xfc00) == 0xd800: if (r and 0xfc00) == 0xd800:
if buf[pos] != '\\' or buf[pos+1] != 'u': if my.buf[pos] != '\\' or my.buf[pos+1] != 'u':
my.err = errInvalidToken my.err = errInvalidToken
break break
inc(pos, 2) inc(pos, 2)
var s = parseEscapedUTF16(buf, pos) var s = parseEscapedUTF16(my.buf, pos)
if (s and 0xfc00) == 0xdc00 and s > 0: if (s and 0xfc00) == 0xdc00 and s > 0:
r = 0x10000 + (((r - 0xd800) shl 10) or (s - 0xdc00)) r = 0x10000 + (((r - 0xd800) shl 10) or (s - 0xdc00))
else: else:
@ -242,8 +241,8 @@ proc parseString(my: var JsonParser): TokKind =
if my.rawStringLiterals: if my.rawStringLiterals:
let length = pos - pos2 let length = pos - pos2
for i in 1 .. length: for i in 1 .. length:
if buf[pos2] in {'0'..'9', 'A'..'F', 'a'..'f'}: if my.buf[pos2] in {'0'..'9', 'A'..'F', 'a'..'f'}:
add(my.a, buf[pos2]) add(my.a, my.buf[pos2])
inc pos2 inc pos2
else: else:
break break
@ -251,61 +250,54 @@ proc parseString(my: var JsonParser): TokKind =
add(my.a, toUTF8(Rune(r))) add(my.a, toUTF8(Rune(r)))
else: else:
# don't bother with the error # don't bother with the error
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
of '\c': of '\c':
pos = lexbase.handleCR(my, pos) pos = lexbase.handleCR(my, pos)
buf = my.buf
add(my.a, '\c') add(my.a, '\c')
of '\L': of '\L':
pos = lexbase.handleLF(my, pos) pos = lexbase.handleLF(my, pos)
buf = my.buf
add(my.a, '\L') add(my.a, '\L')
else: else:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
my.bufpos = pos # store back my.bufpos = pos # store back
proc skip(my: var JsonParser) = proc skip(my: var JsonParser) =
var pos = my.bufpos var pos = my.bufpos
var buf = my.buf
while true: while true:
case buf[pos] case my.buf[pos]
of '/': of '/':
if buf[pos+1] == '/': if my.buf[pos+1] == '/':
# skip line comment: # skip line comment:
inc(pos, 2) inc(pos, 2)
while true: while true:
case buf[pos] case my.buf[pos]
of '\0': of '\0':
break break
of '\c': of '\c':
pos = lexbase.handleCR(my, pos) pos = lexbase.handleCR(my, pos)
buf = my.buf
break break
of '\L': of '\L':
pos = lexbase.handleLF(my, pos) pos = lexbase.handleLF(my, pos)
buf = my.buf
break break
else: else:
inc(pos) inc(pos)
elif buf[pos+1] == '*': elif my.buf[pos+1] == '*':
# skip long comment: # skip long comment:
inc(pos, 2) inc(pos, 2)
while true: while true:
case buf[pos] case my.buf[pos]
of '\0': of '\0':
my.err = errEOC_Expected my.err = errEOC_Expected
break break
of '\c': of '\c':
pos = lexbase.handleCR(my, pos) pos = lexbase.handleCR(my, pos)
buf = my.buf
of '\L': of '\L':
pos = lexbase.handleLF(my, pos) pos = lexbase.handleLF(my, pos)
buf = my.buf
of '*': of '*':
inc(pos) inc(pos)
if buf[pos] == '/': if my.buf[pos] == '/':
inc(pos) inc(pos)
break break
else: else:
@ -316,51 +308,47 @@ proc skip(my: var JsonParser) =
inc(pos) inc(pos)
of '\c': of '\c':
pos = lexbase.handleCR(my, pos) pos = lexbase.handleCR(my, pos)
buf = my.buf
of '\L': of '\L':
pos = lexbase.handleLF(my, pos) pos = lexbase.handleLF(my, pos)
buf = my.buf
else: else:
break break
my.bufpos = pos my.bufpos = pos
proc parseNumber(my: var JsonParser) = proc parseNumber(my: var JsonParser) =
var pos = my.bufpos var pos = my.bufpos
var buf = my.buf if my.buf[pos] == '-':
if buf[pos] == '-':
add(my.a, '-') add(my.a, '-')
inc(pos) inc(pos)
if buf[pos] == '.': if my.buf[pos] == '.':
add(my.a, "0.") add(my.a, "0.")
inc(pos) inc(pos)
else: else:
while buf[pos] in Digits: while my.buf[pos] in Digits:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
if buf[pos] == '.': if my.buf[pos] == '.':
add(my.a, '.') add(my.a, '.')
inc(pos) inc(pos)
# digits after the dot: # digits after the dot:
while buf[pos] in Digits: while my.buf[pos] in Digits:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
if buf[pos] in {'E', 'e'}: if my.buf[pos] in {'E', 'e'}:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
if buf[pos] in {'+', '-'}: if my.buf[pos] in {'+', '-'}:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
while buf[pos] in Digits: while my.buf[pos] in Digits:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
my.bufpos = pos my.bufpos = pos
proc parseName(my: var JsonParser) = proc parseName(my: var JsonParser) =
var pos = my.bufpos var pos = my.bufpos
var buf = my.buf if my.buf[pos] in IdentStartChars:
if buf[pos] in IdentStartChars: while my.buf[pos] in IdentChars:
while buf[pos] in IdentChars: add(my.a, my.buf[pos])
add(my.a, buf[pos])
inc(pos) inc(pos)
my.bufpos = pos my.bufpos = pos

101
lib/pure/parsesql.nim
View file

@ -148,35 +148,33 @@ proc handleCRLF(c: var SqlLexer, pos: int): int =
proc skip(c: var SqlLexer) = proc skip(c: var SqlLexer) =
var pos = c.bufpos var pos = c.bufpos
var buf = c.buf
var nested = 0 var nested = 0
while true: while true:
case buf[pos] case c.buf[pos]
of ' ', '\t': of ' ', '\t':
inc(pos) inc(pos)
of '-': of '-':
if buf[pos+1] == '-': if c.buf[pos+1] == '-':
while not (buf[pos] in {'\c', '\L', lexbase.EndOfFile}): inc(pos) while not (c.buf[pos] in {'\c', '\L', lexbase.EndOfFile}): inc(pos)
else: else:
break break
of '/': of '/':
if buf[pos+1] == '*': if c.buf[pos+1] == '*':
inc(pos,2) inc(pos,2)
while true: while true:
case buf[pos] case c.buf[pos]
of '\0': break of '\0': break
of '\c', '\L': of '\c', '\L':
pos = handleCRLF(c, pos) pos = handleCRLF(c, pos)
buf = c.buf
of '*': of '*':
if buf[pos+1] == '/': if c.buf[pos+1] == '/':
inc(pos, 2) inc(pos, 2)
if nested <= 0: break if nested <= 0: break
dec(nested) dec(nested)
else: else:
inc(pos) inc(pos)
of '/': of '/':
if buf[pos+1] == '*': if c.buf[pos+1] == '*':
inc(pos, 2) inc(pos, 2)
inc(nested) inc(nested)
else: else:
@ -185,21 +183,19 @@ proc skip(c: var SqlLexer) =
else: break else: break
of '\c', '\L': of '\c', '\L':
pos = handleCRLF(c, pos) pos = handleCRLF(c, pos)
buf = c.buf
else: else:
break # EndOfFile also leaves the loop break # EndOfFile also leaves the loop
c.bufpos = pos c.bufpos = pos
proc getString(c: var SqlLexer, tok: var Token, kind: TokKind) = proc getString(c: var SqlLexer, tok: var Token, kind: TokKind) =
var pos = c.bufpos + 1 var pos = c.bufpos + 1
var buf = c.buf
tok.kind = kind tok.kind = kind
block parseLoop: block parseLoop:
while true: while true:
while true: while true:
var ch = buf[pos] var ch = c.buf[pos]
if ch == '\'': if ch == '\'':
if buf[pos+1] == '\'': if c.buf[pos+1] == '\'':
inc(pos, 2) inc(pos, 2)
add(tok.literal, '\'') add(tok.literal, '\'')
else: else:
@ -221,30 +217,27 @@ proc getString(c: var SqlLexer, tok: var Token, kind: TokKind) =
if c.lineNumber > line: if c.lineNumber > line:
# a new line whitespace has been parsed, so we check if the string # a new line whitespace has been parsed, so we check if the string
# continues after the whitespace: # continues after the whitespace:
buf = c.buf # may have been reallocated
pos = c.bufpos pos = c.bufpos
if buf[pos] == '\'': inc(pos) if c.buf[pos] == '\'': inc(pos)
else: break parseLoop else: break parseLoop
else: break parseLoop else: break parseLoop
c.bufpos = pos c.bufpos = pos
proc getDollarString(c: var SqlLexer, tok: var Token) = proc getDollarString(c: var SqlLexer, tok: var Token) =
var pos = c.bufpos + 1 var pos = c.bufpos + 1
var buf = c.buf
tok.kind = tkDollarQuotedConstant tok.kind = tkDollarQuotedConstant
var tag = "$" var tag = "$"
while buf[pos] in IdentChars: while c.buf[pos] in IdentChars:
add(tag, buf[pos]) add(tag, c.buf[pos])
inc(pos) inc(pos)
if buf[pos] == '$': inc(pos) if c.buf[pos] == '$': inc(pos)
else: else:
tok.kind = tkInvalid tok.kind = tkInvalid
return return
while true: while true:
case buf[pos] case c.buf[pos]
of '\c', '\L': of '\c', '\L':
pos = handleCRLF(c, pos) pos = handleCRLF(c, pos)
buf = c.buf
add(tok.literal, "\L") add(tok.literal, "\L")
of '\0': of '\0':
tok.kind = tkInvalid tok.kind = tkInvalid
@ -252,37 +245,35 @@ proc getDollarString(c: var SqlLexer, tok: var Token) =
of '$': of '$':
inc(pos) inc(pos)
var tag2 = "$" var tag2 = "$"
while buf[pos] in IdentChars: while c.buf[pos] in IdentChars:
add(tag2, buf[pos]) add(tag2, c.buf[pos])
inc(pos) inc(pos)
if buf[pos] == '$': inc(pos) if c.buf[pos] == '$': inc(pos)
if tag2 == tag: break if tag2 == tag: break
add(tok.literal, tag2) add(tok.literal, tag2)
add(tok.literal, '$') add(tok.literal, '$')
else: else:
add(tok.literal, buf[pos]) add(tok.literal, c.buf[pos])
inc(pos) inc(pos)
c.bufpos = pos c.bufpos = pos
proc getSymbol(c: var SqlLexer, tok: var Token) = proc getSymbol(c: var SqlLexer, tok: var Token) =
var pos = c.bufpos var pos = c.bufpos
var buf = c.buf
while true: while true:
add(tok.literal, buf[pos]) add(tok.literal, c.buf[pos])
inc(pos) inc(pos)
if buf[pos] notin {'a'..'z','A'..'Z','0'..'9','_','$', '\128'..'\255'}: if c.buf[pos] notin {'a'..'z','A'..'Z','0'..'9','_','$', '\128'..'\255'}:
break break
c.bufpos = pos c.bufpos = pos
tok.kind = tkIdentifier tok.kind = tkIdentifier
proc getQuotedIdentifier(c: var SqlLexer, tok: var Token, quote='\"') = proc getQuotedIdentifier(c: var SqlLexer, tok: var Token, quote='\"') =
var pos = c.bufpos + 1 var pos = c.bufpos + 1
var buf = c.buf
tok.kind = tkQuotedIdentifier tok.kind = tkQuotedIdentifier
while true: while true:
var ch = buf[pos] var ch = c.buf[pos]
if ch == quote: if ch == quote:
if buf[pos+1] == quote: if c.buf[pos+1] == quote:
inc(pos, 2) inc(pos, 2)
add(tok.literal, quote) add(tok.literal, quote)
else: else:
@ -298,11 +289,10 @@ proc getQuotedIdentifier(c: var SqlLexer, tok: var Token, quote='\"') =
proc getBitHexString(c: var SqlLexer, tok: var Token, validChars: set[char]) = proc getBitHexString(c: var SqlLexer, tok: var Token, validChars: set[char]) =
var pos = c.bufpos + 1 var pos = c.bufpos + 1
var buf = c.buf
block parseLoop: block parseLoop:
while true: while true:
while true: while true:
var ch = buf[pos] var ch = c.buf[pos]
if ch in validChars: if ch in validChars:
add(tok.literal, ch) add(tok.literal, ch)
inc(pos) inc(pos)
@ -318,9 +308,8 @@ proc getBitHexString(c: var SqlLexer, tok: var Token, validChars: set[char]) =
if c.lineNumber > line: if c.lineNumber > line:
# a new line whitespace has been parsed, so we check if the string # a new line whitespace has been parsed, so we check if the string
# continues after the whitespace: # continues after the whitespace:
buf = c.buf # may have been reallocated
pos = c.bufpos pos = c.bufpos
if buf[pos] == '\'': inc(pos) if c.buf[pos] == '\'': inc(pos)
else: break parseLoop else: break parseLoop
else: break parseLoop else: break parseLoop
c.bufpos = pos c.bufpos = pos
@ -328,29 +317,28 @@ proc getBitHexString(c: var SqlLexer, tok: var Token, validChars: set[char]) =
proc getNumeric(c: var SqlLexer, tok: var Token) = proc getNumeric(c: var SqlLexer, tok: var Token) =
tok.kind = tkInteger tok.kind = tkInteger
var pos = c.bufpos var pos = c.bufpos
var buf = c.buf while c.buf[pos] in Digits:
while buf[pos] in Digits: add(tok.literal, c.buf[pos])
add(tok.literal, buf[pos])
inc(pos) inc(pos)
if buf[pos] == '.': if c.buf[pos] == '.':
tok.kind = tkNumeric tok.kind = tkNumeric
add(tok.literal, buf[pos]) add(tok.literal, c.buf[pos])
inc(pos) inc(pos)
while buf[pos] in Digits: while c.buf[pos] in Digits:
add(tok.literal, buf[pos]) add(tok.literal, c.buf[pos])
inc(pos) inc(pos)
if buf[pos] in {'E', 'e'}: if c.buf[pos] in {'E', 'e'}:
tok.kind = tkNumeric tok.kind = tkNumeric
add(tok.literal, buf[pos]) add(tok.literal, c.buf[pos])
inc(pos) inc(pos)
if buf[pos] == '+': if c.buf[pos] == '+':
inc(pos) inc(pos)
elif buf[pos] == '-': elif c.buf[pos] == '-':
add(tok.literal, buf[pos]) add(tok.literal, c.buf[pos])
inc(pos) inc(pos)
if buf[pos] in Digits: if c.buf[pos] in Digits:
while buf[pos] in Digits: while c.buf[pos] in Digits:
add(tok.literal, buf[pos]) add(tok.literal, c.buf[pos])
inc(pos) inc(pos)
else: else:
tok.kind = tkInvalid tok.kind = tkInvalid
@ -361,24 +349,23 @@ proc getOperator(c: var SqlLexer, tok: var Token) =
'^', '&', '|', '`', '?'} '^', '&', '|', '`', '?'}
tok.kind = tkOperator tok.kind = tkOperator
var pos = c.bufpos var pos = c.bufpos
var buf = c.buf
var trailingPlusMinus = false var trailingPlusMinus = false
while true: while true:
case buf[pos] case c.buf[pos]
of '-': of '-':
if buf[pos] == '-': break if c.buf[pos] == '-': break
if not trailingPlusMinus and buf[pos+1] notin operators and if not trailingPlusMinus and c.buf[pos+1] notin operators and
tok.literal.len > 0: break tok.literal.len > 0: break
of '/': of '/':
if buf[pos] == '*': break if c.buf[pos] == '*': break
of '~', '!', '@', '#', '%', '^', '&', '|', '`', '?': of '~', '!', '@', '#', '%', '^', '&', '|', '`', '?':
trailingPlusMinus = true trailingPlusMinus = true
of '+': of '+':
if not trailingPlusMinus and buf[pos+1] notin operators and if not trailingPlusMinus and c.buf[pos+1] notin operators and
tok.literal.len > 0: break tok.literal.len > 0: break
of '*', '<', '>', '=': discard of '*', '<', '>', '=': discard
else: break else: break
add(tok.literal, buf[pos]) add(tok.literal, c.buf[pos])
inc(pos) inc(pos)
c.bufpos = pos c.bufpos = pos

129
lib/pure/parsexml.nim
View file

@ -345,11 +345,10 @@ proc markError(my: var XmlParser, kind: XmlErrorKind) {.inline.} =
proc parseCDATA(my: var XmlParser) = proc parseCDATA(my: var XmlParser) =
var pos = my.bufpos + len("<![CDATA[") var pos = my.bufpos + len("<![CDATA[")
var buf = my.buf
while true: while true:
case buf[pos] case my.buf[pos]
of ']': of ']':
if buf[pos+1] == ']' and buf[pos+2] == '>': if my.buf[pos+1] == ']' and my.buf[pos+2] == '>':
inc(pos, 3) inc(pos, 3)
break break
add(my.a, ']') add(my.a, ']')
@ -359,29 +358,25 @@ proc parseCDATA(my: var XmlParser) =
break break
of '\c': of '\c':
pos = lexbase.handleCR(my, pos) pos = lexbase.handleCR(my, pos)
buf = my.buf
add(my.a, '\L') add(my.a, '\L')
of '\L': of '\L':
pos = lexbase.handleLF(my, pos) pos = lexbase.handleLF(my, pos)
buf = my.buf
add(my.a, '\L') add(my.a, '\L')
of '/': of '/':
pos = lexbase.handleRefillChar(my, pos) pos = lexbase.handleRefillChar(my, pos)
buf = my.buf
add(my.a, '/') add(my.a, '/')
else: else:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
my.bufpos = pos # store back my.bufpos = pos # store back
my.kind = xmlCData my.kind = xmlCData
proc parseComment(my: var XmlParser) = proc parseComment(my: var XmlParser) =
var pos = my.bufpos + len("<!--") var pos = my.bufpos + len("<!--")
var buf = my.buf
while true: while true:
case buf[pos] case my.buf[pos]
of '-': of '-':
if buf[pos+1] == '-' and buf[pos+2] == '>': if my.buf[pos+1] == '-' and my.buf[pos+2] == '>':
inc(pos, 3) inc(pos, 3)
break break
if my.options.contains(reportComments): add(my.a, '-') if my.options.contains(reportComments): add(my.a, '-')
@ -391,38 +386,32 @@ proc parseComment(my: var XmlParser) =
break break
of '\c': of '\c':
pos = lexbase.handleCR(my, pos) pos = lexbase.handleCR(my, pos)
buf = my.buf
if my.options.contains(reportComments): add(my.a, '\L') if my.options.contains(reportComments): add(my.a, '\L')
of '\L': of '\L':
pos = lexbase.handleLF(my, pos) pos = lexbase.handleLF(my, pos)
buf = my.buf
if my.options.contains(reportComments): add(my.a, '\L') if my.options.contains(reportComments): add(my.a, '\L')
of '/': of '/':
pos = lexbase.handleRefillChar(my, pos) pos = lexbase.handleRefillChar(my, pos)
buf = my.buf
if my.options.contains(reportComments): add(my.a, '/') if my.options.contains(reportComments): add(my.a, '/')
else: else:
if my.options.contains(reportComments): add(my.a, buf[pos]) if my.options.contains(reportComments): add(my.a, my.buf[pos])
inc(pos) inc(pos)
my.bufpos = pos my.bufpos = pos
my.kind = xmlComment my.kind = xmlComment
proc parseWhitespace(my: var XmlParser, skip=false) = proc parseWhitespace(my: var XmlParser, skip=false) =
var pos = my.bufpos var pos = my.bufpos
var buf = my.buf
while true: while true:
case buf[pos] case my.buf[pos]
of ' ', '\t': of ' ', '\t':
if not skip: add(my.a, buf[pos]) if not skip: add(my.a, my.buf[pos])
inc(pos) inc(pos)
of '\c': of '\c':
# the specification says that CR-LF, CR are to be transformed to LF # the specification says that CR-LF, CR are to be transformed to LF
pos = lexbase.handleCR(my, pos) pos = lexbase.handleCR(my, pos)
buf = my.buf
if not skip: add(my.a, '\L') if not skip: add(my.a, '\L')
of '\L': of '\L':
pos = lexbase.handleLF(my, pos) pos = lexbase.handleLF(my, pos)
buf = my.buf
if not skip: add(my.a, '\L') if not skip: add(my.a, '\L')
else: else:
break break
@ -434,53 +423,51 @@ const
proc parseName(my: var XmlParser, dest: var string) = proc parseName(my: var XmlParser, dest: var string) =
var pos = my.bufpos var pos = my.bufpos
var buf = my.buf if my.buf[pos] in NameStartChar:
if buf[pos] in NameStartChar:
while true: while true:
add(dest, buf[pos]) add(dest, my.buf[pos])
inc(pos) inc(pos)
if buf[pos] notin NameChar: break if my.buf[pos] notin NameChar: break
my.bufpos = pos my.bufpos = pos
else: else:
markError(my, errNameExpected) markError(my, errNameExpected)
proc parseEntity(my: var XmlParser, dest: var string) = proc parseEntity(my: var XmlParser, dest: var string) =
var pos = my.bufpos+1 var pos = my.bufpos+1
var buf = my.buf
my.kind = xmlCharData my.kind = xmlCharData
if buf[pos] == '#': if my.buf[pos] == '#':
var r: int var r: int
inc(pos) inc(pos)
if buf[pos] == 'x': if my.buf[pos] == 'x':
inc(pos) inc(pos)
while true: while true:
case buf[pos] case my.buf[pos]
of '0'..'9': r = (r shl 4) or (ord(buf[pos]) - ord('0')) of '0'..'9': r = (r shl 4) or (ord(my.buf[pos]) - ord('0'))
of 'a'..'f': r = (r shl 4) or (ord(buf[pos]) - ord('a') + 10) of 'a'..'f': r = (r shl 4) or (ord(my.buf[pos]) - ord('a') + 10)
of 'A'..'F': r = (r shl 4) or (ord(buf[pos]) - ord('A') + 10) of 'A'..'F': r = (r shl 4) or (ord(my.buf[pos]) - ord('A') + 10)
else: break else: break
inc(pos) inc(pos)
else: else:
while buf[pos] in {'0'..'9'}: while my.buf[pos] in {'0'..'9'}:
r = r * 10 + (ord(buf[pos]) - ord('0')) r = r * 10 + (ord(my.buf[pos]) - ord('0'))
inc(pos) inc(pos)
add(dest, toUTF8(Rune(r))) add(dest, toUTF8(Rune(r)))
elif buf[pos] == 'l' and buf[pos+1] == 't' and buf[pos+2] == ';': elif my.buf[pos] == 'l' and my.buf[pos+1] == 't' and my.buf[pos+2] == ';':
add(dest, '<') add(dest, '<')
inc(pos, 2) inc(pos, 2)
elif buf[pos] == 'g' and buf[pos+1] == 't' and buf[pos+2] == ';': elif my.buf[pos] == 'g' and my.buf[pos+1] == 't' and my.buf[pos+2] == ';':
add(dest, '>') add(dest, '>')
inc(pos, 2) inc(pos, 2)
elif buf[pos] == 'a' and buf[pos+1] == 'm' and buf[pos+2] == 'p' and elif my.buf[pos] == 'a' and my.buf[pos+1] == 'm' and my.buf[pos+2] == 'p' and
buf[pos+3] == ';': my.buf[pos+3] == ';':
add(dest, '&') add(dest, '&')
inc(pos, 3) inc(pos, 3)
elif buf[pos] == 'a' and buf[pos+1] == 'p' and buf[pos+2] == 'o' and elif my.buf[pos] == 'a' and my.buf[pos+1] == 'p' and my.buf[pos+2] == 'o' and
buf[pos+3] == 's' and buf[pos+4] == ';': my.buf[pos+3] == 's' and my.buf[pos+4] == ';':
add(dest, '\'') add(dest, '\'')
inc(pos, 4) inc(pos, 4)
elif buf[pos] == 'q' and buf[pos+1] == 'u' and buf[pos+2] == 'o' and elif my.buf[pos] == 'q' and my.buf[pos+1] == 'u' and my.buf[pos+2] == 'o' and
buf[pos+3] == 't' and buf[pos+4] == ';': my.buf[pos+3] == 't' and my.buf[pos+4] == ';':
add(dest, '"') add(dest, '"')
inc(pos, 4) inc(pos, 4)
else: else:
@ -491,7 +478,7 @@ proc parseEntity(my: var XmlParser, dest: var string) =
my.kind = xmlEntity my.kind = xmlEntity
else: else:
add(dest, '&') add(dest, '&')
if buf[pos] == ';': if my.buf[pos] == ';':
inc(pos) inc(pos)
else: else:
markError(my, errSemicolonExpected) markError(my, errSemicolonExpected)
@ -501,15 +488,14 @@ proc parsePI(my: var XmlParser) =
inc(my.bufpos, "<?".len) inc(my.bufpos, "<?".len)
parseName(my, my.a) parseName(my, my.a)
var pos = my.bufpos var pos = my.bufpos
var buf = my.buf
setLen(my.b, 0) setLen(my.b, 0)
while true: while true:
case buf[pos] case my.buf[pos]
of '\0': of '\0':
markError(my, errQmGtExpected) markError(my, errQmGtExpected)
break break
of '?': of '?':
if buf[pos+1] == '>': if my.buf[pos+1] == '>':
inc(pos, 2) inc(pos, 2)
break break
add(my.b, '?') add(my.b, '?')
@ -517,18 +503,15 @@ proc parsePI(my: var XmlParser) =
of '\c': of '\c':
# the specification says that CR-LF, CR are to be transformed to LF # the specification says that CR-LF, CR are to be transformed to LF
pos = lexbase.handleCR(my, pos) pos = lexbase.handleCR(my, pos)
buf = my.buf
add(my.b, '\L') add(my.b, '\L')
of '\L': of '\L':
pos = lexbase.handleLF(my, pos) pos = lexbase.handleLF(my, pos)
buf = my.buf
add(my.b, '\L') add(my.b, '\L')
of '/': of '/':
pos = lexbase.handleRefillChar(my, pos) pos = lexbase.handleRefillChar(my, pos)
buf = my.buf
add(my.b, '/') add(my.b, '/')
else: else:
add(my.b, buf[pos]) add(my.b, my.buf[pos])
inc(pos) inc(pos)
my.bufpos = pos my.bufpos = pos
my.kind = xmlPI my.kind = xmlPI
@ -536,10 +519,9 @@ proc parsePI(my: var XmlParser) =
proc parseSpecial(my: var XmlParser) = proc parseSpecial(my: var XmlParser) =
# things that start with <! # things that start with <!
var pos = my.bufpos + 2 var pos = my.bufpos + 2
var buf = my.buf
var opentags = 0 var opentags = 0
while true: while true:
case buf[pos] case my.buf[pos]
of '\0': of '\0':
markError(my, errGtExpected) markError(my, errGtExpected)
break break
@ -556,18 +538,15 @@ proc parseSpecial(my: var XmlParser) =
add(my.a, '>') add(my.a, '>')
of '\c': of '\c':
pos = lexbase.handleCR(my, pos) pos = lexbase.handleCR(my, pos)
buf = my.buf
add(my.a, '\L') add(my.a, '\L')
of '\L': of '\L':
pos = lexbase.handleLF(my, pos) pos = lexbase.handleLF(my, pos)
buf = my.buf
add(my.a, '\L') add(my.a, '\L')
of '/': of '/':
pos = lexbase.handleRefillChar(my, pos) pos = lexbase.handleRefillChar(my, pos)
buf = my.buf
add(my.b, '/') add(my.b, '/')
else: else:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
my.bufpos = pos my.bufpos = pos
my.kind = xmlSpecial my.kind = xmlSpecial
@ -635,13 +614,12 @@ proc parseAttribute(my: var XmlParser) =
parseWhitespace(my, skip=true) parseWhitespace(my, skip=true)
var pos = my.bufpos var pos = my.bufpos
var buf = my.buf if my.buf[pos] in {'\'', '"'}:
if buf[pos] in {'\'', '"'}: var quote = my.buf[pos]
var quote = buf[pos]
var pendingSpace = false var pendingSpace = false
inc(pos) inc(pos)
while true: while true:
case buf[pos] case my.buf[pos]
of '\0': of '\0':
markError(my, errQuoteExpected) markError(my, errQuoteExpected)
break break
@ -658,31 +636,28 @@ proc parseAttribute(my: var XmlParser) =
inc(pos) inc(pos)
of '\c': of '\c':
pos = lexbase.handleCR(my, pos) pos = lexbase.handleCR(my, pos)
buf = my.buf
pendingSpace = true pendingSpace = true
of '\L': of '\L':
pos = lexbase.handleLF(my, pos) pos = lexbase.handleLF(my, pos)
buf = my.buf
pendingSpace = true pendingSpace = true
of '/': of '/':
pos = lexbase.handleRefillChar(my, pos) pos = lexbase.handleRefillChar(my, pos)
buf = my.buf
add(my.b, '/') add(my.b, '/')
else: else:
if buf[pos] == quote: if my.buf[pos] == quote:
inc(pos) inc(pos)
break break
else: else:
if pendingSpace: if pendingSpace:
add(my.b, ' ') add(my.b, ' ')
pendingSpace = false pendingSpace = false
add(my.b, buf[pos]) add(my.b, my.buf[pos])
inc(pos) inc(pos)
elif allowUnquotedAttribs in my.options: elif allowUnquotedAttribs in my.options:
const disallowedChars = {'"', '\'', '`', '=', '<', '>', ' ', const disallowedChars = {'"', '\'', '`', '=', '<', '>', ' ',
'\0', '\t', '\L', '\F', '\f'} '\0', '\t', '\L', '\F', '\f'}
let startPos = pos let startPos = pos
while (let c = buf[pos]; c notin disallowedChars): while (let c = my.buf[pos]; c notin disallowedChars):
if c == '&': if c == '&':
my.bufpos = pos my.bufpos = pos
parseEntity(my, my.b) parseEntity(my, my.b)
@ -696,33 +671,29 @@ proc parseAttribute(my: var XmlParser) =
else: else:
markError(my, errQuoteExpected) markError(my, errQuoteExpected)
# error corrections: guess what was meant # error corrections: guess what was meant
while buf[pos] != '>' and buf[pos] > ' ': while my.buf[pos] != '>' and my.buf[pos] > ' ':
add(my.b, buf[pos]) add(my.b, my.buf[pos])
inc pos inc pos
my.bufpos = pos my.bufpos = pos
parseWhitespace(my, skip=true) parseWhitespace(my, skip=true)
proc parseCharData(my: var XmlParser) = proc parseCharData(my: var XmlParser) =
var pos = my.bufpos var pos = my.bufpos
var buf = my.buf
while true: while true:
case buf[pos] case my.buf[pos]
of '\0', '<', '&': break of '\0', '<', '&': break
of '\c': of '\c':
# the specification says that CR-LF, CR are to be transformed to LF # the specification says that CR-LF, CR are to be transformed to LF
pos = lexbase.handleCR(my, pos) pos = lexbase.handleCR(my, pos)
buf = my.buf
add(my.a, '\L') add(my.a, '\L')
of '\L': of '\L':
pos = lexbase.handleLF(my, pos) pos = lexbase.handleLF(my, pos)
buf = my.buf
add(my.a, '\L') add(my.a, '\L')
of '/': of '/':
pos = lexbase.handleRefillChar(my, pos) pos = lexbase.handleRefillChar(my, pos)
buf = my.buf
add(my.a, '/') add(my.a, '/')
else: else:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
my.bufpos = pos my.bufpos = pos
my.kind = xmlCharData my.kind = xmlCharData
@ -731,18 +702,17 @@ proc rawGetTok(my: var XmlParser) =
my.kind = xmlError my.kind = xmlError
setLen(my.a, 0) setLen(my.a, 0)
var pos = my.bufpos var pos = my.bufpos
var buf = my.buf case my.buf[pos]
case buf[pos]
of '<': of '<':
case buf[pos+1] case my.buf[pos+1]
of '/': of '/':
parseEndTag(my) parseEndTag(my)
of '!': of '!':
if buf[pos+2] == '[' and buf[pos+3] == 'C' and buf[pos+4] == 'D' and if my.buf[pos+2] == '[' and my.buf[pos+3] == 'C' and my.buf[pos+4] == 'D' and
buf[pos+5] == 'A' and buf[pos+6] == 'T' and buf[pos+7] == 'A' and my.buf[pos+5] == 'A' and my.buf[pos+6] == 'T' and my.buf[pos+7] == 'A' and
buf[pos+8] == '[': my.buf[pos+8] == '[':
parseCDATA(my) parseCDATA(my)
elif buf[pos+2] == '-' and buf[pos+3] == '-': elif my.buf[pos+2] == '-' and my.buf[pos+3] == '-':
parseComment(my) parseComment(my)
else: else:
parseSpecial(my) parseSpecial(my)
@ -841,4 +811,3 @@ when not defined(testing) and isMainModule:
of xmlSpecial: of xmlSpecial:
echo("SPECIAL: " & x.charData) echo("SPECIAL: " & x.charData)
close(x) close(x)

43
lib/pure/pegs.nim
View file

@ -1545,20 +1545,17 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) =
proc skip(c: var PegLexer) = proc skip(c: var PegLexer) =
var pos = c.bufpos var pos = c.bufpos
var buf = c.buf
while pos < c.buf.len: while pos < c.buf.len:
case buf[pos] case c.buf[pos]
of ' ', '\t': of ' ', '\t':
inc(pos) inc(pos)
of '#': of '#':
while (pos < c.buf.len) and while (pos < c.buf.len) and
not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) not (c.buf[pos] in {'\c', '\L', '\0'}): inc(pos)
of '\c': of '\c':
pos = handleCR(c, pos) pos = handleCR(c, pos)
buf = c.buf
of '\L': of '\L':
pos = handleLF(c, pos) pos = handleLF(c, pos)
buf = c.buf
else: else:
break # EndOfFile also leaves the loop break # EndOfFile also leaves the loop
c.bufpos = pos c.bufpos = pos
@ -1566,10 +1563,9 @@ proc skip(c: var PegLexer) =
proc getString(c: var PegLexer, tok: var Token) = proc getString(c: var PegLexer, tok: var Token) =
tok.kind = tkStringLit tok.kind = tkStringLit
var pos = c.bufpos + 1 var pos = c.bufpos + 1
var buf = c.buf var quote = c.buf[pos-1]
var quote = buf[pos-1]
while pos < c.buf.len: while pos < c.buf.len:
case buf[pos] case c.buf[pos]
of '\\': of '\\':
c.bufpos = pos c.bufpos = pos
getEscapedChar(c, tok) getEscapedChar(c, tok)
@ -1577,22 +1573,21 @@ proc getString(c: var PegLexer, tok: var Token) =
of '\c', '\L', '\0': of '\c', '\L', '\0':
tok.kind = tkInvalid tok.kind = tkInvalid
break break
elif buf[pos] == quote: elif c.buf[pos] == quote:
inc(pos) inc(pos)
break break
else: else:
add(tok.literal, buf[pos]) add(tok.literal, c.buf[pos])
inc(pos) inc(pos)
c.bufpos = pos c.bufpos = pos
proc getDollar(c: var PegLexer, tok: var Token) = proc getDollar(c: var PegLexer, tok: var Token) =
var pos = c.bufpos + 1 var pos = c.bufpos + 1
var buf = c.buf if c.buf[pos] in {'0'..'9'}:
if buf[pos] in {'0'..'9'}:
tok.kind = tkBackref tok.kind = tkBackref
tok.index = 0 tok.index = 0
while pos < c.buf.len and buf[pos] in {'0'..'9'}: while pos < c.buf.len and c.buf[pos] in {'0'..'9'}:
tok.index = tok.index * 10 + ord(buf[pos]) - ord('0') tok.index = tok.index * 10 + ord(c.buf[pos]) - ord('0')
inc(pos) inc(pos)
else: else:
tok.kind = tkDollar tok.kind = tkDollar
@ -1602,14 +1597,13 @@ proc getCharSet(c: var PegLexer, tok: var Token) =
tok.kind = tkCharSet tok.kind = tkCharSet
tok.charset = {} tok.charset = {}
var pos = c.bufpos + 1 var pos = c.bufpos + 1
var buf = c.buf
var caret = false var caret = false
if buf[pos] == '^': if c.buf[pos] == '^':
inc(pos) inc(pos)
caret = true caret = true
while pos < c.buf.len: while pos < c.buf.len:
var ch: char var ch: char
case buf[pos] case c.buf[pos]
of ']': of ']':
if pos < c.buf.len: inc(pos) if pos < c.buf.len: inc(pos)
break break
@ -1622,11 +1616,11 @@ proc getCharSet(c: var PegLexer, tok: var Token) =
tok.kind = tkInvalid tok.kind = tkInvalid
break break
else: else:
ch = buf[pos] ch = c.buf[pos]
inc(pos) inc(pos)
incl(tok.charset, ch) incl(tok.charset, ch)
if buf[pos] == '-': if c.buf[pos] == '-':
if pos+1 < c.buf.len and buf[pos+1] == ']': if pos+1 < c.buf.len and c.buf[pos+1] == ']':
incl(tok.charset, '-') incl(tok.charset, '-')
inc(pos) inc(pos)
else: else:
@ -1635,7 +1629,7 @@ proc getCharSet(c: var PegLexer, tok: var Token) =
else: else:
break break
var ch2: char var ch2: char
case buf[pos] case c.buf[pos]
of '\\': of '\\':
c.bufpos = pos c.bufpos = pos
getEscapedChar(c, tok) getEscapedChar(c, tok)
@ -1646,7 +1640,7 @@ proc getCharSet(c: var PegLexer, tok: var Token) =
break break
else: else:
if pos+1 < c.buf.len: if pos+1 < c.buf.len:
ch2 = buf[pos] ch2 = c.buf[pos]
inc(pos) inc(pos)
else: else:
break break
@ -1657,11 +1651,10 @@ proc getCharSet(c: var PegLexer, tok: var Token) =
proc getSymbol(c: var PegLexer, tok: var Token) = proc getSymbol(c: var PegLexer, tok: var Token) =
var pos = c.bufpos var pos = c.bufpos
var buf = c.buf
while pos < c.buf.len: while pos < c.buf.len:
add(tok.literal, buf[pos]) add(tok.literal, c.buf[pos])
inc(pos) inc(pos)
if pos < buf.len and buf[pos] notin strutils.IdentChars: break if pos < c.buf.len and c.buf[pos] notin strutils.IdentChars: break
c.bufpos = pos c.bufpos = pos
tok.kind = tkIdentifier tok.kind = tkIdentifier

31
lib/pure/streams.nim
View file

@ -53,12 +53,17 @@ type
{.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.}
getPositionImpl*: proc (s: Stream): int getPositionImpl*: proc (s: Stream): int
{.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.}
readDataStrImpl*: proc (s: Stream, buffer: var string, slice: Slice[int]): int
{.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.}
readDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int): int readDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int): int
{.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.} {.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.}
peekDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int): int peekDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int): int
{.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.} {.nimcall, raises: [Defect, IOError, OSError], tags: [ReadIOEffect], gcsafe.}
writeDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int) writeDataImpl*: proc (s: Stream, buffer: pointer, bufLen: int)
{.nimcall, raises: [Defect, IOError, OSError], tags: [WriteIOEffect], gcsafe.} {.nimcall, raises: [Defect, IOError, OSError], tags: [WriteIOEffect], gcsafe.}
flushImpl*: proc (s: Stream) flushImpl*: proc (s: Stream)
{.nimcall, raises: [Defect, IOError, OSError], tags: [WriteIOEffect], gcsafe.} {.nimcall, raises: [Defect, IOError, OSError], tags: [WriteIOEffect], gcsafe.}
@ -87,6 +92,14 @@ proc readData*(s: Stream, buffer: pointer, bufLen: int): int =
## low level proc that reads data into an untyped `buffer` of `bufLen` size. ## low level proc that reads data into an untyped `buffer` of `bufLen` size.
result = s.readDataImpl(s, buffer, bufLen) result = s.readDataImpl(s, buffer, bufLen)
proc readDataStr*(s: Stream, buffer: var string, slice: Slice[int]): int =
## low level proc that reads data into a string ``buffer`` at ``slice``.
if s.readDataStrImpl != nil:
result = s.readDataStrImpl(s, buffer, slice)
else:
# fallback
result = s.readData(addr buffer[0], buffer.len)
when not defined(js): when not defined(js):
proc readAll*(s: Stream): string = proc readAll*(s: Stream): string =
## Reads all available data. ## Reads all available data.
@ -344,6 +357,19 @@ when not defined(js):
var s = StringStream(s) var s = StringStream(s)
return s.pos return s.pos
proc ssReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int =
var s = StringStream(s)
result = min(slice.b + 1 - slice.a, s.data.len - s.pos)
if result > 0:
when nimvm:
for i in 0 ..< result: # sorry, but no fast string splicing on the vm.
buffer[slice.a + i] = s.data[s.pos + i]
else:
copyMem(unsafeAddr buffer[slice.a], addr s.data[s.pos], result)
inc(s.pos, result)
else:
result = 0
proc ssReadData(s: Stream, buffer: pointer, bufLen: int): int = proc ssReadData(s: Stream, buffer: pointer, bufLen: int): int =
var s = StringStream(s) var s = StringStream(s)
result = min(bufLen, s.data.len - s.pos) result = min(bufLen, s.data.len - s.pos)
@ -389,6 +415,7 @@ when not defined(js):
result.readDataImpl = ssReadData result.readDataImpl = ssReadData
result.peekDataImpl = ssPeekData result.peekDataImpl = ssPeekData
result.writeDataImpl = ssWriteData result.writeDataImpl = ssWriteData
result.readDataStrImpl = ssReadDataStr
type type
FileStream* = ref FileStreamObj ## a stream that encapsulates a `File` FileStream* = ref FileStreamObj ## a stream that encapsulates a `File`
@ -407,6 +434,9 @@ when not defined(js):
proc fsReadData(s: Stream, buffer: pointer, bufLen: int): int = proc fsReadData(s: Stream, buffer: pointer, bufLen: int): int =
result = readBuffer(FileStream(s).f, buffer, bufLen) result = readBuffer(FileStream(s).f, buffer, bufLen)
proc fsReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int =
result = readBuffer(FileStream(s).f, addr buffer[slice.a], slice.b + 1 - slice.a)
proc fsPeekData(s: Stream, buffer: pointer, bufLen: int): int = proc fsPeekData(s: Stream, buffer: pointer, bufLen: int): int =
let pos = fsGetPosition(s) let pos = fsGetPosition(s)
defer: fsSetPosition(s, pos) defer: fsSetPosition(s, pos)
@ -424,6 +454,7 @@ when not defined(js):
result.atEndImpl = fsAtEnd result.atEndImpl = fsAtEnd
result.setPositionImpl = fsSetPosition result.setPositionImpl = fsSetPosition
result.getPositionImpl = fsGetPosition result.getPositionImpl = fsGetPosition
result.readDataStrImpl = fsReadDataStr
result.readDataImpl = fsReadData result.readDataImpl = fsReadData
result.peekDataImpl = fsPeekData result.peekDataImpl = fsPeekData
result.writeDataImpl = fsWriteData result.writeDataImpl = fsWriteData

55
nimsuggest/sexp.nim
View file

@ -124,9 +124,8 @@ proc handleHexChar(c: char, x: var int): bool =
proc parseString(my: var SexpParser): TTokKind = proc parseString(my: var SexpParser): TTokKind =
result = tkString result = tkString
var pos = my.bufpos + 1 var pos = my.bufpos + 1
var buf = my.buf
while true: while true:
case buf[pos] case my.buf[pos]
of '\0': of '\0':
my.err = errQuoteExpected my.err = errQuoteExpected
result = tkError result = tkError
@ -135,9 +134,9 @@ proc parseString(my: var SexpParser): TTokKind =
inc(pos) inc(pos)
break break
of '\\': of '\\':
case buf[pos+1] case my.buf[pos+1]
of '\\', '"', '\'', '/': of '\\', '"', '\'', '/':
add(my.a, buf[pos+1]) add(my.a, my.buf[pos+1])
inc(pos, 2) inc(pos, 2)
of 'b': of 'b':
add(my.a, '\b') add(my.a, '\b')
@ -157,65 +156,61 @@ proc parseString(my: var SexpParser): TTokKind =
of 'u': of 'u':
inc(pos, 2) inc(pos, 2)
var r: int var r: int
if handleHexChar(buf[pos], r): inc(pos) if handleHexChar(my.buf[pos], r): inc(pos)
if handleHexChar(buf[pos], r): inc(pos) if handleHexChar(my.buf[pos], r): inc(pos)
if handleHexChar(buf[pos], r): inc(pos) if handleHexChar(my.buf[pos], r): inc(pos)
if handleHexChar(buf[pos], r): inc(pos) if handleHexChar(my.buf[pos], r): inc(pos)
add(my.a, toUTF8(Rune(r))) add(my.a, toUTF8(Rune(r)))
else: else:
# don't bother with the error # don't bother with the error
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
of '\c': of '\c':
pos = lexbase.handleCR(my, pos) pos = lexbase.handleCR(my, pos)
buf = my.buf
add(my.a, '\c') add(my.a, '\c')
of '\L': of '\L':
pos = lexbase.handleLF(my, pos) pos = lexbase.handleLF(my, pos)
buf = my.buf
add(my.a, '\L') add(my.a, '\L')
else: else:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
my.bufpos = pos # store back my.bufpos = pos # store back
proc parseNumber(my: var SexpParser) = proc parseNumber(my: var SexpParser) =
var pos = my.bufpos var pos = my.bufpos
var buf = my.buf if my.buf[pos] == '-':
if buf[pos] == '-':
add(my.a, '-') add(my.a, '-')
inc(pos) inc(pos)
if buf[pos] == '.': if my.buf[pos] == '.':
add(my.a, "0.") add(my.a, "0.")
inc(pos) inc(pos)
else: else:
while buf[pos] in Digits: while my.buf[pos] in Digits:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
if buf[pos] == '.': if my.buf[pos] == '.':
add(my.a, '.') add(my.a, '.')
inc(pos) inc(pos)
# digits after the dot: # digits after the dot:
while buf[pos] in Digits: while my.buf[pos] in Digits:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
if buf[pos] in {'E', 'e'}: if my.buf[pos] in {'E', 'e'}:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
if buf[pos] in {'+', '-'}: if my.buf[pos] in {'+', '-'}:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
while buf[pos] in Digits: while my.buf[pos] in Digits:
add(my.a, buf[pos]) add(my.a, my.buf[pos])
inc(pos) inc(pos)
my.bufpos = pos my.bufpos = pos
proc parseSymbol(my: var SexpParser) = proc parseSymbol(my: var SexpParser) =
var pos = my.bufpos var pos = my.bufpos
var buf = my.buf if my.buf[pos] in IdentStartChars:
if buf[pos] in IdentStartChars: while my.buf[pos] in IdentChars:
while buf[pos] in IdentChars: add(my.a, my.buf[pos])
add(my.a, buf[pos])
inc(pos) inc(pos)
my.bufpos = pos my.bufpos = pos

40
tests/stdlib/tjsonmacro.nim
View file

@ -520,3 +520,43 @@ when true:
block test_tuple: block test_tuple:
doAssert $(%* (a1: 10, a2: "foo")) == """{"a1":10,"a2":"foo"}""" doAssert $(%* (a1: 10, a2: "foo")) == """{"a1":10,"a2":"foo"}"""
doAssert $(%* (10, "foo")) == """[10,"foo"]""" doAssert $(%* (10, "foo")) == """[10,"foo"]"""
# TODO: when the issue with the limeted vm registers is solved, the
# exact same test as above should be evaluated at compile time as
# well, to ensure that the vm functionality won't diverge from the
# runtime functionality. Until then, the following test should do it.
static:
var t = parseJson("""
{
"name":"Bongo",
"email":"bongo@bingo.com",
"list": [11,7,15],
"year": 1975,
"dict": {"a": 1, "b": 2},
"arr": [1.0, 2.0, 7.0],
"person": {"name": "boney"},
"dog": {"name": "honey"},
"fruit": {"color": 10},
"distfruit": {"color": 11},
"emails": ["abc", "123"]
}
""")
doAssert t["name"].getStr == "Bongo"
doAssert t["email"].getStr == "bongo@bingo.com"
doAssert t["list"][0].getInt == 11
doAssert t["list"][1].getInt == 7
doAssert t["list"][2].getInt == 15
doAssert t["year"].getInt == 1975
doAssert t["dict"]["a"].getInt == 1
doAssert t["dict"]["b"].getInt == 2
doAssert t["arr"][0].getFloat == 1.0
doAssert t["arr"][1].getFloat == 2.0
doAssert t["arr"][2].getFloat == 7.0
doAssert t["person"]["name"].getStr == "boney"
doAssert t["distfruit"]["color"].getInt == 11
doAssert t["dog"]["name"].getStr == "honey"
doAssert t["fruit"]["color"].getInt == 10
doAssert t["emails"][0].getStr == "abc"
doAssert t["emails"][1].getStr == "123"