From 3943fba34b317abf2c2421fdfd0f042dd34bbbc6 Mon Sep 17 00:00:00 2001 From: def Date: Fri, 24 Jul 2015 00:29:09 +0200 Subject: [PATCH 1/3] Improve performance of readLine by using fgets This drops compatibility with pure CR line endings of old Mac systems --- lib/system.nim | 12 +++---- lib/system/sysio.nim | 77 +++++++++++++------------------------------- 2 files changed, 29 insertions(+), 60 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index c5b0e0cc7..2ebfc6d73 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2603,17 +2603,17 @@ when not defined(JS): #and not defined(NimrodVM): proc readLine*(f: File): TaintedString {.tags: [ReadIOEffect], benign.} ## reads a line of text from the file `f`. May throw an IO exception. - ## A line of text may be delimited by ``CR``, ``LF`` or - ## ``CRLF``. The newline character(s) are not part of the returned string. + ## A line of text may be delimited by ``LF`` or ``CRLF``. The newline + ## character(s) are not part of the returned string. proc readLine*(f: File, line: var TaintedString): bool {.tags: [ReadIOEffect], benign.} ## reads a line of text from the file `f` into `line`. `line` must not be ## ``nil``! May throw an IO exception. - ## A line of text may be delimited by ``CR``, ``LF`` or - ## ``CRLF``. The newline character(s) are not part of the returned string. - ## Returns ``false`` if the end of the file has been reached, ``true`` - ## otherwise. If ``false`` is returned `line` contains no new data. + ## A line of text may be delimited by ``LF`` or ``CRLF``. The newline + ## character(s) are not part of the returned string. Returns ``false`` + ## if the end of the file has been reached, ``true`` otherwise. If + ## ``false`` is returned `line` contains no new data. proc writeLn*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, tags: [WriteIOEffect], benign, deprecated.} diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index 5464ee126..dccb13303 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -31,26 +31,6 @@ proc fprintf(f: File, frmt: cstring) {.importc: "fprintf", proc strlen(c: cstring): int {. importc: "strlen", header: "", tags: [].} -when defined(posix): - proc getc_unlocked(stream: File): cint {.importc: "getc_unlocked", - header: "", tags: [ReadIOEffect].} - - proc flockfile(stream: File) {.importc: "flockfile", header: "", - tags: [ReadIOEffect].} - - proc funlockfile(stream: File) {.importc: "funlockfile", header: "", - tags: [ReadIOEffect].} -elif false: - # doesn't work on Windows yet: - proc getc_unlocked(stream: File): cint {.importc: "_fgetc_nolock", - header: "", tags: [ReadIOEffect].} - - proc flockfile(stream: File) {.importc: "_lock_file", header: "", - tags: [ReadIOEffect].} - - proc funlockfile(stream: File) {.importc: "_unlock_file", header: "", - tags: [ReadIOEffect].} - # C routine that is used here: proc fread(buf: pointer, size, n: int, f: File): int {. importc: "fread", header: "", tags: [ReadIOEffect].} @@ -86,40 +66,29 @@ const proc raiseEIO(msg: string) {.noinline, noreturn.} = sysFatal(IOError, msg) -when declared(getc_unlocked): - proc readLine(f: File, line: var TaintedString): bool = - setLen(line.string, 0) # reuse the buffer! - flockfile(f) - while true: - var c = getc_unlocked(f) - if c < 0'i32: - if line.len > 0: break - else: return false - if c == 10'i32: break # LF - if c == 13'i32: # CR - c = getc_unlocked(f) # is the next char LF? - if c != 10'i32: ungetc(c, f) # no, put the character back - break - add line.string, chr(int(c)) - result = true - funlockfile(f) -else: - proc readLine(f: File, line: var TaintedString): bool = - # of course this could be optimized a bit; but IO is slow anyway... - # and it was difficult to get this CORRECT with Ansi C's methods - setLen(line.string, 0) # reuse the buffer! - while true: - var c = fgetc(f) - if c < 0'i32: - if line.len > 0: break - else: return false - if c == 10'i32: break # LF - if c == 13'i32: # CR - c = fgetc(f) # is the next char LF? - if c != 10'i32: ungetc(c, f) # no, put the character back - break - add line.string, chr(int(c)) - result = true +proc readLine(f: File, line: var TaintedString): bool = + template returnUntil(p: int) = + line.string[p] = '\0' + line.string.setLen(p) + return true + + var pos = 0 + # Use the currently reserved space for a first try + var space = cast[PGenericSeq](line.string).space + + while true: + if fgets(addr line.string[pos], space, f) == nil: + line.string.setLen(0) + return false + # This will cut the string short when it contains \0 + let last = pos + cstring(addr line.string[pos]).len-1 + if line.string[last] == '\l': + if last > 0 and line.string[last-1] == '\c': + returnUntil(last-1) + returnUntil(last) + pos = last+1 + space = 128 # Read in 128 bytes at a time + line.string.setLen(pos+space) proc readLine(f: File): TaintedString = result = TaintedString(newStringOfCap(80)) From 401189c3fe3f02ec90a9905f981667a1b9a1d40a Mon Sep 17 00:00:00 2001 From: def Date: Fri, 24 Jul 2015 01:12:32 +0200 Subject: [PATCH 2/3] No need to set trailing \0 --- lib/system/sysio.nim | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index dccb13303..e1c928e22 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -67,11 +67,6 @@ proc raiseEIO(msg: string) {.noinline, noreturn.} = sysFatal(IOError, msg) proc readLine(f: File, line: var TaintedString): bool = - template returnUntil(p: int) = - line.string[p] = '\0' - line.string.setLen(p) - return true - var pos = 0 # Use the currently reserved space for a first try var space = cast[PGenericSeq](line.string).space @@ -84,8 +79,10 @@ proc readLine(f: File, line: var TaintedString): bool = let last = pos + cstring(addr line.string[pos]).len-1 if line.string[last] == '\l': if last > 0 and line.string[last-1] == '\c': - returnUntil(last-1) - returnUntil(last) + line.string.setLen(last-1) + return true + line.string.setLen(last) + return true pos = last+1 space = 128 # Read in 128 bytes at a time line.string.setLen(pos+space) From a9b299b4ede7e96c35a6b558bbca946692d1e99d Mon Sep 17 00:00:00 2001 From: def Date: Sat, 25 Jul 2015 20:25:58 +0200 Subject: [PATCH 3/3] Improve readLine to work with strings containing \0 again. This decreases performance slightly. --- lib/system/sysio.nim | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index e1c928e22..74d844145 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -39,6 +39,10 @@ proc fseek(f: File, offset: clong, whence: int): int {. proc ftell(f: File): int {.importc: "ftell", header: "", tags: [].} proc setvbuf(stream: File, buf: pointer, typ, size: cint): cint {. importc, header: "", tags: [].} +proc memchr(s: pointer, c: cint, n: csize): pointer {. + importc: "memchr", header: "", tags: [].} +proc memset(s: pointer, c: cint, n: csize) {. + header: "", importc: "memset", tags: [].} {.push stackTrace:off, profiler:off.} proc write(f: File, c: cstring) = fputs(c, f) @@ -70,21 +74,27 @@ proc readLine(f: File, line: var TaintedString): bool = var pos = 0 # Use the currently reserved space for a first try var space = cast[PGenericSeq](line.string).space + line.string.setLen(space) while true: + # memset to \l so that we can tell how far fgets wrote, even on EOF, where + # fgets doesn't append an \l + memset(addr line.string[pos], '\l'.ord, space) if fgets(addr line.string[pos], space, f) == nil: line.string.setLen(0) return false - # This will cut the string short when it contains \0 - let last = pos + cstring(addr line.string[pos]).len-1 - if line.string[last] == '\l': + let m = memchr(addr line.string[pos], '\l'.ord, space) + if m != nil: + # \l found: Could be our own or the one by fgets, in any case, we're done + let last = cast[ByteAddress](m) - cast[ByteAddress](addr line.string[0]) if last > 0 and line.string[last-1] == '\c': line.string.setLen(last-1) return true line.string.setLen(last) return true - pos = last+1 - space = 128 # Read in 128 bytes at a time + # No \l found: Increase buffer and read more + inc pos, space + space = 128 # read in 128 bytes at a time line.string.setLen(pos+space) proc readLine(f: File): TaintedString =