fixes #2429

lib/pure/lexbase.nim

@@ -34,37 +34,15 @@ type
     lineNumber*: int          ## the current line number
     sentinel: int
     lineStart: int            # index of last line start in buffer
-    fileOpened: bool
+    refillChars: set[char]
 
 {.deprecated: [TBaseLexer: BaseLexer].}
 
-proc open*(L: var BaseLexer, input: Stream, bufLen: int = 8192)
-  ## inits the BaseLexer with a stream to read from
-
-proc close*(L: var BaseLexer)
-  ## closes the base lexer. This closes `L`'s associated stream too.
-
-proc getCurrentLine*(L: BaseLexer, marker: bool = true): string
-  ## retrieves the current line.
-
-proc getColNumber*(L: BaseLexer, pos: int): int
-  ## retrieves the current column.
-
-proc handleCR*(L: var BaseLexer, pos: int): int
-  ## Call this if you scanned over '\c' in the buffer; it returns the the
-  ## position to continue the scanning from. `pos` must be the position
-  ## of the '\c'.
-proc handleLF*(L: var BaseLexer, pos: int): int
-  ## Call this if you scanned over '\L' in the buffer; it returns the the
-  ## position to continue the scanning from. `pos` must be the position
-  ## of the '\L'.
-
-# implementation
-
 const
   chrSize = sizeof(char)
 
-proc close(L: var BaseLexer) =
+proc close*(L: var BaseLexer) =
+  ## closes the base lexer. This closes `L`'s associated stream too.
   dealloc(L.buf)
   close(L.input)
 
@@ -80,7 +58,7 @@ proc fillBuffer(L: var BaseLexer) =
   toCopy = L.bufLen - L.sentinel - 1
   assert(toCopy >= 0)
   if toCopy > 0:
-    moveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize) 
+    moveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize)
     # "moveMem" handles overlapping regions
   charsRead = readData(L.input, addr(L.buf[toCopy]),
                        (L.sentinel + 1) * chrSize) div chrSize
@@ -93,7 +71,7 @@ proc fillBuffer(L: var BaseLexer) =
     dec(s)                    # BUGFIX (valgrind)
     while true:
       assert(s < L.bufLen)
-      while (s >= 0) and not (L.buf[s] in NewLines): dec(s)
+      while s >= 0 and L.buf[s] notin L.refillChars: dec(s)
       if s >= 0:
         # we found an appropriate character for a sentinel:
         L.sentinel = s
@@ -121,31 +99,46 @@ proc fillBaseLexer(L: var BaseLexer, pos: int): int =
     fillBuffer(L)
     L.bufpos = 0              # XXX: is this really correct?
     result = 0
-  L.lineStart = result
 
-proc handleCR(L: var BaseLexer, pos: int): int =
+proc handleCR*(L: var BaseLexer, pos: int): int =
+  ## Call this if you scanned over '\c' in the buffer; it returns the the
+  ## position to continue the scanning from. `pos` must be the position
+  ## of the '\c'.
   assert(L.buf[pos] == '\c')
   inc(L.lineNumber)
   result = fillBaseLexer(L, pos)
   if L.buf[result] == '\L':
     result = fillBaseLexer(L, result)
+  L.lineStart = result
 
-proc handleLF(L: var BaseLexer, pos: int): int =
+proc handleLF*(L: var BaseLexer, pos: int): int =
+  ## Call this if you scanned over '\L' in the buffer; it returns the the
+  ## position to continue the scanning from. `pos` must be the position
+  ## of the '\L'.
   assert(L.buf[pos] == '\L')
   inc(L.lineNumber)
   result = fillBaseLexer(L, pos) #L.lastNL := result-1; // BUGFIX: was: result;
+  L.lineStart = result
+
+proc handleRefillChar*(L: var BaseLexer, pos: int): int =
+  ## To be documented.
+  assert(L.buf[pos] in L.refillChars)
+  result = fillBaseLexer(L, pos) #L.lastNL := result-1; // BUGFIX: was: result;
 
 proc skipUtf8Bom(L: var BaseLexer) =
   if (L.buf[0] == '\xEF') and (L.buf[1] == '\xBB') and (L.buf[2] == '\xBF'):
     inc(L.bufpos, 3)
     inc(L.lineStart, 3)
 
-proc open(L: var BaseLexer, input: Stream, bufLen: int = 8192) =
+proc open*(L: var BaseLexer, input: Stream, bufLen: int = 8192;
+           refillChars: set[char] = NewLines) =
+  ## inits the BaseLexer with a stream to read from.
   assert(bufLen > 0)
   assert(input != nil)
   L.input = input
   L.bufpos = 0
   L.bufLen = bufLen
+  L.refillChars = refillChars
   L.buf = cast[cstring](alloc(bufLen * chrSize))
   L.sentinel = bufLen - 1
   L.lineStart = 0
@@ -153,10 +146,12 @@ proc open(L: var BaseLexer, input: Stream, bufLen: int = 8192) =
   fillBuffer(L)
   skipUtf8Bom(L)
 
-proc getColNumber(L: BaseLexer, pos: int): int =
+proc getColNumber*(L: BaseLexer, pos: int): int =
+  ## retrieves the current column.
   result = abs(pos - L.lineStart)
 
-proc getCurrentLine(L: BaseLexer, marker: bool = true): string =
+proc getCurrentLine*(L: BaseLexer, marker: bool = true): string =
+  ## retrieves the current line.
   var i: int
   result = ""
   i = L.lineStart
@@ -166,4 +161,3 @@ proc getCurrentLine(L: BaseLexer, marker: bool = true): string =
   add(result, "\n")
   if marker:
     add(result, spaces(getColNumber(L, L.bufpos)) & "^\n")
-