Merge pull request #3143 from def-/readline-faster

Improve performance of readLine by using fgets

lib/system.nim

@@ -2635,17 +2635,17 @@ when not defined(JS): #and not defined(nimscript):
 
     proc readLine*(f: File): TaintedString  {.tags: [ReadIOEffect], benign.}
       ## reads a line of text from the file `f`. May throw an IO exception.
-      ## A line of text may be delimited by ``CR``, ``LF`` or
+      ## A line of text may be delimited by ``LF`` or ``CRLF``. The newline
-      ## ``CRLF``. The newline character(s) are not part of the returned string.
+      ## character(s) are not part of the returned string.
 
     proc readLine*(f: File, line: var TaintedString): bool {.tags: [ReadIOEffect],
                   benign.}
       ## reads a line of text from the file `f` into `line`. `line` must not be
       ## ``nil``! May throw an IO exception.
-      ## A line of text may be delimited by ``CR``, ``LF`` or
+      ## A line of text may be delimited by ``LF`` or ``CRLF``. The newline
-      ## ``CRLF``. The newline character(s) are not part of the returned string.
+      ## character(s) are not part of the returned string. Returns ``false``
-      ## Returns ``false`` if the end of the file has been reached, ``true``
+      ## if the end of the file has been reached, ``true`` otherwise. If
-      ## otherwise. If ``false`` is returned `line` contains no new data.
+      ## ``false`` is returned `line` contains no new data.
 
     proc writeLn*[Ty](f: File, x: varargs[Ty, `$`]) {.inline,
                              tags: [WriteIOEffect], benign, deprecated.}

lib/system/sysio.nim

@@ -31,26 +31,6 @@ proc fprintf(f: File, frmt: cstring) {.importc: "fprintf",
 proc strlen(c: cstring): int {.
   importc: "strlen", header: "<string.h>", tags: [].}
 
-when defined(posix):
-  proc getc_unlocked(stream: File): cint {.importc: "getc_unlocked",
-    header: "<stdio.h>", tags: [ReadIOEffect].}
-
-  proc flockfile(stream: File) {.importc: "flockfile", header: "<stdio.h>",
-    tags: [ReadIOEffect].}
-
-  proc funlockfile(stream: File) {.importc: "funlockfile", header: "<stdio.h>",
-    tags: [ReadIOEffect].}
-elif false:
-  # doesn't work on Windows yet:
-  proc getc_unlocked(stream: File): cint {.importc: "_fgetc_nolock",
-    header: "<stdio.h>", tags: [ReadIOEffect].}
-
-  proc flockfile(stream: File) {.importc: "_lock_file", header: "<stdio.h>",
-    tags: [ReadIOEffect].}
-
-  proc funlockfile(stream: File) {.importc: "_unlock_file", header: "<stdio.h>",
-    tags: [ReadIOEffect].}
-
 # C routine that is used here:
 proc fread(buf: pointer, size, n: int, f: File): int {.
   importc: "fread", header: "<stdio.h>", tags: [ReadIOEffect].}
@@ -59,6 +39,10 @@ proc fseek(f: File, offset: clong, whence: int): int {.
 proc ftell(f: File): int {.importc: "ftell", header: "<stdio.h>", tags: [].}
 proc setvbuf(stream: File, buf: pointer, typ, size: cint): cint {.
   importc, header: "<stdio.h>", tags: [].}
+proc memchr(s: pointer, c: cint, n: csize): pointer {.
+  importc: "memchr", header: "<string.h>", tags: [].}
+proc memset(s: pointer, c: cint, n: csize) {.
+  header: "<string.h>", importc: "memset", tags: [].}
 
 {.push stackTrace:off, profiler:off.}
 proc write(f: File, c: cstring) = fputs(c, f)
@@ -86,40 +70,32 @@ const
 proc raiseEIO(msg: string) {.noinline, noreturn.} =
   sysFatal(IOError, msg)
 
-when declared(getc_unlocked):
+proc readLine(f: File, line: var TaintedString): bool =
-  proc readLine(f: File, line: var TaintedString): bool =
+  var pos = 0
-    setLen(line.string, 0) # reuse the buffer!
+  # Use the currently reserved space for a first try
-    flockfile(f)
+  var space = cast[PGenericSeq](line.string).space
+  line.string.setLen(space)
+
   while true:
-      var c = getc_unlocked(f)
+    # memset to \l so that we can tell how far fgets wrote, even on EOF, where
-      if c < 0'i32:
+    # fgets doesn't append an \l
-        if line.len > 0: break
+    memset(addr line.string[pos], '\l'.ord, space)
-        else: return false
+    if fgets(addr line.string[pos], space, f) == nil:
-      if c == 10'i32: break # LF
+      line.string.setLen(0)
-      if c == 13'i32:  # CR
+      return false
-        c = getc_unlocked(f) # is the next char LF?
+    let m = memchr(addr line.string[pos], '\l'.ord, space)
-        if c != 10'i32: ungetc(c, f) # no, put the character back
+    if m != nil:
-        break
+      # \l found: Could be our own or the one by fgets, in any case, we're done
-      add line.string, chr(int(c))
+      let last = cast[ByteAddress](m) - cast[ByteAddress](addr line.string[0])
-    result = true
+      if last > 0 and line.string[last-1] == '\c':
-    funlockfile(f)
+        line.string.setLen(last-1)
-else:
+        return true
-  proc readLine(f: File, line: var TaintedString): bool =
+      line.string.setLen(last)
-    # of course this could be optimized a bit; but IO is slow anyway...
+      return true
-    # and it was difficult to get this CORRECT with Ansi C's methods
+    # No \l found: Increase buffer and read more
-    setLen(line.string, 0) # reuse the buffer!
+    inc pos, space
-    while true:
+    space = 128 # read in 128 bytes at a time
-      var c = fgetc(f)
+    line.string.setLen(pos+space)
-      if c < 0'i32:
-        if line.len > 0: break
-        else: return false
-      if c == 10'i32: break # LF
-      if c == 13'i32:  # CR
-        c = fgetc(f) # is the next char LF?
-        if c != 10'i32: ungetc(c, f) # no, put the character back
-        break
-      add line.string, chr(int(c))
-    result = true
 
 proc readLine(f: File): TaintedString =
   result = TaintedString(newStringOfCap(80))