Merge pull request #3146 from c-blake/devel

Add MemFile text interface factored so users can optionally skip Nim …

lib/pure/memfiles.nim

@@ -11,6 +11,9 @@
 ##
 ## This module provides support for `memory mapped files`:idx:
 ## (Posix's `mmap`:idx:) on the different operating systems.
+##
+## It also provides some fast iterators over lines in text files (or
+## other "line-like", variable length, delimited records).
 
 when defined(windows):
   import winlean
@@ -245,3 +248,96 @@ proc close*(f: var MemFile) =
   
   if error: raiseOSError(lastErr)
 
+type MemSlice* = object  ## represent slice of a MemFile for iteration over delimited lines/records
+  data*: pointer
+  size*: int
+
+proc c_memcpy(a, b: pointer, n: int) {.importc: "memcpy", header: "<string.h>".}
+
+proc `$`*(ms: MemSlice): string {.inline.} =
+  ## Return a Nim string built from a MemSlice.
+  var buf = newString(ms.size)
+  c_memcpy(addr(buf[0]), ms.data, ms.size)
+  buf[ms.size] = '\0'
+  result = buf
+
+iterator memSlices*(mfile: MemFile, delim='\l', eat='\r'): MemSlice {.inline.} =
+  ## Iterates over [optional `eat`] `delim`-delimited slices in MemFile `mfile`.
+  ##
+  ## Default parameters parse lines ending in either Unix(\\l) or Windows(\\r\\l)
+  ## style on on a line-by-line basis.  I.e., not every line needs the same ending.
+  ## Unlike readLine(File) & lines(File), archaic MacOS9 \\r-delimited lines
+  ## are not supported as a third option for each line.  Such archaic MacOS9
+  ## files can be handled by passing delim='\\r', eat='\\0', though.
+  ##
+  ## Delimiters are not part of the returned slice.  A final, unterminated line
+  ## or record is returned just like any other.
+  ##
+  ## Non-default delimiters can be passed to allow iteration over other sorts
+  ## of "line-like" variable length records.  Pass eat='\\0' to be strictly
+  ## `delim`-delimited. (Eating an optional prefix equal to '\\0' is not
+  ## supported.)
+  ##
+  ## This zero copy, memchr-limited interface is probably the fastest way to
+  ## iterate over line-like records in a file.  However, returned (data,size)
+  ## objects are not Nim strings, bounds checked Nim arrays, or even terminated
+  ## C strings.  So, care is required to access the data (e.g., think C mem*
+  ## functions, not str* functions).  Example:
+  ##
+  ## .. code-block:: nim
+  ##   var count = 0
+  ##   for slice in memSlices(memfiles.open("foo")):
+  ##     if slice.size > 0 and cast[cstring](slice.data)[0] != '#':
+  ##       inc(count)
+  ##   echo count
+
+  proc c_memchr(cstr: pointer, c: char, n: csize): pointer {.
+       importc: "memchr", header: "<string.h>" .}
+  proc `-!`(p, q: pointer): int {.inline.} = return cast[int](p) -% cast[int](q)
+  var ms: MemSlice
+  var ending: pointer
+  ms.data = mfile.mem
+  var remaining = mfile.size
+  while remaining > 0:
+    ending = c_memchr(ms.data, delim, remaining)
+    if ending == nil:                               # unterminated final slice
+      ms.size = remaining                           # Weird case..check eat?
+      yield ms
+      break
+    ms.size = ending -! ms.data                     # delim is NOT included
+    if eat != '\0' and ms.size > 0 and cast[cstring](ms.data)[ms.size - 1] == eat:
+      dec(ms.size)                                  # trim pre-delim char
+    yield ms
+    ms.data = cast[pointer](cast[int](ending) +% 1)     # skip delim
+    remaining = mfile.size - (ms.data -! mfile.mem)
+
+iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): TaintedString {.inline.} =
+  ## Replace contents of passed buffer with each new line, like
+  ## `readLine(File) <system.html#readLine,File,TaintedString>`_.
+  ## `delim`, `eat`, and delimiting logic is exactly as for
+  ## `memSlices <#memSlices>`_, but Nim strings are returned.  Example:
+  ##
+  ## .. code-block:: nim
+  ##   var buffer: TaintedString = ""
+  ##   for line in lines(memfiles.open("foo"), buffer):
+  ##     echo line
+
+  for ms in memSlices(mfile, delim, eat):
+    buf.setLen(ms.size)
+    c_memcpy(addr(buf[0]), ms.data, ms.size)
+    buf[ms.size] = '\0'
+    yield buf
+
+iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
+  ## Return each line in a file as a Nim string, like
+  ## `lines(File) <system.html#lines.i,File>`_.
+  ## `delim`, `eat`, and delimiting logic is exactly as for
+  ## `memSlices <#memSlices>`_, but Nim strings are returned.  Example:
+  ##
+  ## .. code-block:: nim
+  ##   for line in lines(memfiles.open("foo")):
+  ##     echo line
+
+  var buf = TaintedString(newStringOfCap(80))
+  for line in lines(mfile, buf, delim, eat):
+    yield buf

tests/stdlib/tmemlines.nim

@@ -0,0 +1,5 @@
+import memfiles
+var inp = memfiles.open("readme.txt")
+for line in lines(inp):
+  echo("#" & line & "#")
+close(inp)

tests/stdlib/tmemlinesBuf.nim

@@ -0,0 +1,6 @@
+import memfiles
+var inp = memfiles.open("readme.txt")
+var buffer: TaintedString = ""
+for line in lines(inp, buffer):
+  echo("#" & line & "#")
+close(inp)

tests/stdlib/tmemslices.nim

@@ -0,0 +1,6 @@
+import memfiles
+var inp = memfiles.open("readme.txt")
+for mem in memSlices(inp):
+  if mem.size > 3:
+    echo("#" & $mem & "#")
+close(inp)