jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

StringStream and parseJson, parseCfg, parseSql et al for the vm (#10746)

Browse source
This commit is contained in:
Arne Döring 2019-02-28 22:57:57 +01:00 • committed by Andreas Rumpf
commit 1102b8ac6e
15 changed files with 354 additions and 403 deletions
Download patch file Download diff file Expand all files Collapse all files

32
compiler/nimlexbase.nim
View file

@ -39,8 +39,7 @@ const
type
TBaseLexer* = object of RootObj
bufpos*: int
buf*: cstring
bufLen*: int # length of buffer in characters
buf*: string
stream*: PLLStream # we read from this stream
lineNumber*: int # the current line number
# private data:
@ -65,11 +64,7 @@ proc handleLF*(L: var TBaseLexer, pos: int): int
# of the LF.
# implementation
const
chrSize = sizeof(char)
proc closeBaseLexer(L: var TBaseLexer) =
dealloc(L.buf)
llStreamClose(L.stream)
proc fillBuffer(L: var TBaseLexer) =
@ -80,14 +75,13 @@ proc fillBuffer(L: var TBaseLexer) =
oldBufLen: int
# we know here that pos == L.sentinel, but not if this proc
# is called the first time by initBaseLexer()
assert(L.sentinel < L.bufLen)
toCopy = L.bufLen - L.sentinel - 1
assert(L.sentinel < L.buf.len)
toCopy = L.buf.len - L.sentinel - 1
assert(toCopy >= 0)
if toCopy > 0:
moveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize)
moveMem(addr L.buf[0], addr L.buf[L.sentinel + 1], toCopy)
# "moveMem" handles overlapping regions
charsRead = llStreamRead(L.stream, addr(L.buf[toCopy]),
(L.sentinel + 1) * chrSize) div chrSize
charsRead = llStreamRead(L.stream, addr L.buf[toCopy], L.sentinel + 1)
s = toCopy + charsRead
if charsRead < L.sentinel + 1:
L.buf[s] = EndOfFile # set end marker
@ -96,7 +90,7 @@ proc fillBuffer(L: var TBaseLexer) =
# compute sentinel:
dec(s) # BUGFIX (valgrind)
while true:
assert(s < L.bufLen)
assert(s < L.buf.len)
while (s >= 0) and not (L.buf[s] in NewLines): dec(s)
if s >= 0:
# we found an appropriate character for a sentinel:
@ -105,17 +99,16 @@ proc fillBuffer(L: var TBaseLexer) =
else:
# rather than to give up here because the line is too long,
# double the buffer's size and try again:
oldBufLen = L.bufLen
L.bufLen = L.bufLen * 2
L.buf = cast[cstring](realloc(L.buf, L.bufLen * chrSize))
assert(L.bufLen - oldBufLen == oldBufLen)
oldBufLen = L.buf.len
L.buf.setLen(L.buf.len * 2)
assert(L.buf.len - oldBufLen == oldBufLen)
charsRead = llStreamRead(L.stream, addr(L.buf[oldBufLen]),
oldBufLen * chrSize) div chrSize
oldBufLen)
if charsRead < oldBufLen:
L.buf[oldBufLen + charsRead] = EndOfFile
L.sentinel = oldBufLen + charsRead
break
s = L.bufLen - 1
s = L.buf.len - 1
proc fillBaseLexer(L: var TBaseLexer, pos: int): int =
assert(pos <= L.sentinel)
@ -149,8 +142,7 @@ proc openBaseLexer(L: var TBaseLexer, inputstream: PLLStream, bufLen = 8192) =
assert(bufLen > 0)
L.bufpos = 0
L.offsetBase = 0
L.bufLen = bufLen
L.buf = cast[cstring](alloc(bufLen * chrSize))
L.buf = newString(bufLen)
L.sentinel = bufLen - 1
L.lineStart = 0
L.lineNumber = 1 # lines start at 1