Merge pull request #79 from Tass/master

readAll

lib/system.nim

@@ -1677,10 +1677,13 @@ when not defined(EcmaScript) and not defined(NimrodVM):
   proc FlushFile*(f: TFile) {.importc: "fflush", noDecl.}
     ## Flushes `f`'s buffer.
 
+  proc readAll*(file: TFile): TaintedString
+    ## Reads all data from the stream `file`. Raises an IO exception
+    ## in case of an error
+  
   proc readFile*(filename: string): TaintedString
-    ## Opens a file named `filename` for reading. Then reads the
+    ## Opens a file named `filename` for reading. Then calls `readAll`
-    ## file's content completely into a string and
+    ## and closes the file afterwards. Returns the string. 
-    ## closes the file afterwards. Returns the string. 
     ## Raises an IO exception in case of an error.
 
   proc writeFile*(filename, content: string)

lib/system/sysio.nim

@@ -24,6 +24,13 @@ proc putc(c: Char, stream: TFile) {.importc: "putc", nodecl.}
 proc fprintf(f: TFile, frmt: CString) {.importc: "fprintf", nodecl, varargs.}
 proc strlen(c: cstring): int {.importc: "strlen", nodecl.}
 
+
+# C routine that is used here:
+proc fread(buf: Pointer, size, n: int, f: TFile): int {.
+  importc: "fread", noDecl.}
+proc fseek(f: TFile, offset: clong, whence: int): int {.
+  importc: "fseek", noDecl.}
+proc ftell(f: TFile): int {.importc: "ftell", noDecl.}
 proc setvbuf(stream: TFile, buf: pointer, typ, size: cint): cint {.
   importc, nodecl.}
 
@@ -36,6 +43,9 @@ var
   IOFBF {.importc: "_IOFBF", nodecl.}: cint
   IONBF {.importc: "_IONBF", nodecl.}: cint
 
+const
+  buf_size = 4000
+
 proc raiseEIO(msg: string) {.noinline, noreturn.} =
   raise newException(EIO, msg)
 
@@ -82,21 +92,49 @@ proc write(f: TFile, c: Char) = putc(c, f)
 proc write(f: TFile, a: openArray[string]) =
   for x in items(a): write(f, x)
 
+proc readAllBuffer(file: TFile): string = 
+  # This proc is for TFile we want to read but don't know how many
+  # bytes we need to read before the buffer is empty.
+  result = ""
+  var buffer = newString(buf_size)
+  var bytesRead = buf_size
+  while bytesRead == buf_size:
+    bytesRead = readBuffer(file, addr(buffer[0]), buf_size)
+    result.add(buffer)
+  
+proc rawFileSize(file: TFile): int = 
+  # this does not raise an error opposed to `getFileSize`
+  var oldPos = ftell(file)
+  discard fseek(file, 0, 2) # seek the end of the file
+  result = ftell(file)
+  discard fseek(file, clong(oldPos), 0)
+
+proc readAllFile(file: TFile, len: int): string =
+  # We aquire the filesize beforehand and hope it doesn't change.
+  # Speeds things up.
+  if len >= high(int):
+    raiseEIO("file too big to fit in memory")
+  result = newString(int(len))
+  if readBuffer(file, addr(result[0]), int(len)) != len:
+    raiseEIO("error while reading from file")
+
+proc readAllFile(file: TFile): string =
+  var len = rawFileSize(file)
+  result = readAllFile(file, len)
+  
+proc readAll(file: TFile): TaintedString = 
+  # Separate handling needed because we need to buffer when we
+  # don't know the overall length of the TFile.
+  var len = rawFileSize(file)
+  if len >= 0:
+    result = readAllFile(file, len).TaintedSTring
+  else:
+    result = readAllBuffer(file).TaintedString
+  
 proc readFile(filename: string): TaintedString =
   var f = open(filename)
   try:
-    var len = getFileSize(f)
+    result = readAllFile(f).TaintedString
-    if len < high(int):
-      when taintMode:
-        result = newString(int(len)).TaintedString
-        if readBuffer(f, addr(string(result)[0]), int(len)) != len:
-          raiseEIO("error while reading from file")
-      else:
-        result = newString(int(len))
-        if readBuffer(f, addr(result[0]), int(len)) != len:
-          raiseEIO("error while reading from file")
-    else:
-      raiseEIO("file too big to fit in memory")
   finally:
     close(f)
 
@@ -157,13 +195,6 @@ proc open(f: var TFile, filehandle: TFileHandle, mode: TFileMode): bool =
   f = fdopen(filehandle, FormatOpen[mode])
   result = f != nil
 
-# C routine that is used here:
-proc fread(buf: Pointer, size, n: int, f: TFile): int {.
-  importc: "fread", noDecl.}
-proc fseek(f: TFile, offset: clong, whence: int): int {.
-  importc: "fseek", noDecl.}
-proc ftell(f: TFile): int {.importc: "ftell", noDecl.}
-
 proc fwrite(buf: Pointer, size, n: int, f: TFile): int {.
   importc: "fwrite", noDecl.}
 

tests/specials.nim

@@ -128,12 +128,19 @@ proc rejectThreadTests(r: var TResults, options: string) =
   rejectSingleTest(r, "tests/threads/tthreadanalysis3", options)
   rejectSingleTest(r, "tests/threads/tthreadheapviolation1", options)
 
+# ------------------------- IO tests -----------------------------------
+
+proc runIOTests(r: var TResults, options: string) =
+  compileSingleTest(r, "tests/system/helpers/readall_echo.nim", options)
+  runSingleTest(r, "tests/system/io", options)
+
 # ------------------------- register special tests here -----------------------
 proc runSpecialTests(r: var TResults, options: string) =
   runRodFiles(r, options)
   runDLLTests(r, options)
   runGCTests(r, options)
   runThreadTests(r, options & " --threads:on")
+  runIOTests(r, options)
 
 proc rejectSpecialTests(r: var TResults, options: string) =
   rejectThreadTests(r, options)

tests/system/helpers/readall_echo.nim

@@ -0,0 +1 @@
+echo(stdin.readAll)

tests/system/io.nim

@@ -0,0 +1,29 @@
+discard """output: '''[OK] stdin
+
+[OK] file'''"""
+
+import
+  unittest, osproc, streams, os, readall_echo
+const STRING_DATA = """Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet."""
+const TEST_FILE = "tests/testdata/string"
+
+proc echoLoop(str: string): string =
+  result = ""
+  var process = startProcess(os.addFileExt("tests/system/helpers/readall_echo", ExeExt))
+  var input = process.inputStream
+  input.write(str)
+  input.close()
+  var output = process.outputStream
+  discard process.waitForExit
+  while not output.atEnd:
+    result.add(output.readLine)
+    
+suite "io":
+  suite "readAll":
+    test "stdin":
+      check:
+        echoLoop(STRING_DATA) == STRING_DATA
+        echoLoop(STRING_DATA[0..3999]) == STRING_DATA[0..3999]
+    test "file":
+      check:
+        readFile(TEST_FILE) == STRING_DATA