Address review comments; Add documentation; Shared buffering mechanism for input and output streams

faststreams/inputs.nim

@@ -0,0 +1,577 @@
+import
+  os, memfiles, options,
+  stew/[ptrops, ranges/ptr_arith],
+  async_backend, buffers
+
+export
+  options, CloseBehavior
+
+type
+  InputStream* = ref object of RootObj
+    vtable: ptr InputStreamVTable # This is nil for unsafe memory inputs
+    buffers: PageBuffers          # This is nil for unsafe memory inputs
+    span: PageSpan
+    spanEndPos: Natural
+    closeFut: Future[void]        # This is nil before `close` is called
+
+  LayeredInputStream* = ref object of InputStream
+    subStream*: InputStream
+
+  InputStreamHandle* = object
+    s*: InputStream
+
+  AsyncInputStream* {.borrow: `.`.} = distinct InputStream
+
+  ReadSyncProc* = proc (s: InputStream)
+                       {.nimcall, gcsafe, raises: [IOError, Defect].}
+
+  ReadAsyncProc* = proc (s: InputStream): Future[void]
+                        {.nimcall, gcsafe, raises: [IOError, Defect].}
+
+  CloseSyncProc* = proc (s: InputStream)
+                        {.nimcall, gcsafe, raises: [IOError, Defect].}
+
+  CloseAsyncProc* = proc (s: InputStream): Future[void]
+                         {.nimcall, gcsafe, raises: [IOError, Defect].}
+
+  GetLenSyncProc* = proc (s: InputStream): Natural
+                         {.nimcall, gcsafe, raises: [IOError, Defect].}
+
+  InputStreamVTable* = object
+    readSync*: ReadSyncProc
+    readAsync*: ReadAsyncProc
+    closeSync*: CloseSyncProc
+    closeAsync*: CloseAsyncProc
+    getLenSync*: GetLenSyncProc
+
+  MemFileInputStream = ref object of InputStream
+    file: MemFile
+
+  FileInputStream = ref object of InputStream
+    file: File
+
+proc disconnectInputDevice(s: InputStream) =
+  # TODO
+  # Document the behavior that closeAsync is preferred
+  if s.vtable != nil:
+    if s.vtable.closeAsync != nil:
+      s.closeFut = s.vtable.closeAsync(s)
+    elif s.vtable.closeSync != nil:
+      s.vtable.closeSync(s)
+    s.vtable = nil
+
+template disconnectInputDevice(s: AsyncInputStream) =
+  disconnectInputDevice InputStream(s)
+
+proc preventFurtherReading(s: InputStream) =
+  s.vtable = nil
+  s.span = default(PageSpan)
+
+template preventFurtherReading(s: AsyncInputStream) =
+  preventFurtherReading InputStream(s)
+
+template makeHandle*(sp: InputStream): InputStreamHandle =
+  let s = sp
+  InputStreamHandle(s: s)
+
+proc close*(s: InputStream,
+            behavior = dontWaitAsyncClose)
+           {.raises: [IOError, Defect].} =
+  ## Closes the stream. Any resources associated with the stream
+  ## will be released and no further reading will be possible.
+  ##
+  ## If the underlying input device requires asynchronous closing
+  ## and `behavior` is set to `waitAsyncClose`, this proc will use
+  ## `waitFor` to block until the async operation completes.
+  s.disconnectInputDevice()
+  s.preventFurtherReading()
+  if s.closeFut != nil:
+    fsTranslateErrors "Stream closing failed":
+      if behavior == waitAsyncClose:
+        waitFor s.closeFut
+      else:
+        asyncCheck s.closeFut
+
+proc close*(s: AsyncInputStream): Future[void]
+           {.raises: [IOError, Defect].} =
+  ## Starts the asychronous closing of the stream and returns a future that
+  ## tracks the closing operation.
+  s.disconnectInputDevice()
+  s.preventFurtherReading()
+  result = InputStream(s).closeFut
+  doAssert result != nil
+
+template closeNoWait*(sp: AsyncInputStream|InputStream) =
+  ## Close the stream without waiting even if's async.
+  ## This operation will use `asyncCheck` internally to detect unhandled
+  ## errors from the closing operation.
+  close(InputStream(s), dontWaitAsyncClose)
+
+# TODO
+# The destructors are currently disabled because they seem to cause
+# mysterious segmentation faults related to corrupted GC internal
+# data structures.
+#[
+proc `=destroy`*(h: var InputStreamHandle) {.raises: [Defect].} =
+  if h.s != nil:
+    if h.s.vtable != nil and h.s.vtable.closeSync != nil:
+      try:
+        h.s.vtable.closeSync(h.s)
+      except IOError:
+        # Since this is a destructor, there is not much we can do here.
+        # If the user wanted to handle the error, they would have called
+        # `close` manually.
+        discard # TODO
+    # TODO ATTENTION!
+    # Uncommenting the following line will lead to a GC heap corruption.
+    # Most likely this leads to Nim collecting some object prematurely.
+    # h.s = nil
+    # We work-around the problem through more indirect incapacitatation
+    # of the stream object:
+    h.s.preventFurtherReading()
+]#
+
+converter implicitDeref*(h: InputStreamHandle): InputStream =
+  ## Any `InputStreamHandle` value can be implicitly converted to an
+  ## `InputStream` or an `AsyncInputStream` value.
+  h.s
+
+template vtableAddr*(vtable: InputStreamVTable): ptr InputStreamVTable =
+  # This is a simple work-around for the somewhat broken side
+  # effects analysis of Nim - reading from global let variables
+  # is considered a side-effect.
+  {.noSideEffect.}:
+    unsafeAddr vtable
+
+let MemFileInputVTable = InputStreamVTable(
+  closeSync: proc (s: InputStream)
+                  {.nimcall, gcsafe, raises: [IOError, Defect].} =
+    try:
+      close MemFileInputStream(s).file
+    except OSError as err:
+      raise newException(IOError, "Failed to close file", err)
+  ,
+  getLenSync: proc (s: InputStream): Natural
+                   {.nimcall, gcsafe, raises: [IOError, Defect].} =
+    s.span.len
+)
+
+proc memFileInput*(filename: string, mappedSize = -1, offset = 0): InputStreamHandle
+                  {.raises: [IOError, OSError].} =
+  ## Creates an input stream for reading the contents of a memory-mapped file.
+  ##
+  ## Using this API will provide better performance than `fileInput`,
+  ## but this comes at a cost of higher address space usage which may
+  ## be problematic when working with extremely large files.
+  ##
+  ## All parameters are forwarded to Nim's memfiles.open function:
+  ##
+  ## ``filename``
+  ##  The name of the file to read.
+  ##
+  ## ``mappedSize`` and ``offset``
+  ##  can be used to map only a slice of the file.
+  ##
+  ## ``offset`` must be multiples of the PAGE SIZE of your OS
+  ##  (usually 4K or 8K, but is unique to your OS)
+
+  # Nim's memfiles module will fail to map an empty file,
+  # but we don't consider this a problem. The stream will
+  # be in non-readable state from the start.
+  let fileSize = getFileSize(filename)
+  if fileSize == 0:
+    return makeHandle InputStream()
+
+  let
+    memFile = memfiles.open(filename,
+                            mode = fmRead,
+                            mappedSize = mappedSize,
+                            offset = offset)
+    head = cast[ptr byte](memFile.mem)
+    mappedSize = memFile.size
+
+  makeHandle MemFileInputStream(
+    vtable: vtableAddr MemFileInputVTable,
+    span: PageSpan(
+      startAddr: head,
+      endAddr: offset(head, mappedSize)),
+    file: memFile)
+
+proc readableNow*(s: InputStream): bool =
+  (not s.span.atEnd) or (s.buffers != nil and s.buffers.len > 1)
+
+template readableNow*(s: AsyncInputStream): bool =
+  readableNow InputStream(s)
+
+func totalUnconsumedBytes*(s: InputStream): Natural =
+  ## Returns the number of bytes that are currently sitting within the stream
+  ## buffers and that can be consumed with `read` or `advance`.
+  result = s.span.len
+  if s.buffers != nil:
+    result += s.buffers.totalBytesRead - s.spanEndPos
+
+template totalUnconsumedBytes*(s: AsyncInputStream): Natural =
+  ## Alias for InputStream.totalUnconsumedBytes
+  totalUnconsumedBytes InputStream(s)
+
+let FileInputVTable = InputStreamVTable(
+  readSync: proc (s: InputStream)
+                 {.nimcall, gcsafe, raises: [IOError, Defect].} =
+    let file = FileInputStream(s).file
+    s.buffers.writeToSpan(span):
+      file.readBuffer(span.startAddr, span.len)
+  ,
+  getLenSync: proc (s: InputStream): Natural
+                   {.nimcall, gcsafe, raises: [IOError, Defect].} =
+    let
+      s = FileInputStream(s)
+      runway = s.totalUnconsumedBytes
+
+    let preservedPos = getFilePos(s.file)
+    setFilePos(s.file, 0, fspEnd)
+    let endPos = getFilePos(s.file)
+    setFilePos(s.file, preservedPos)
+
+    endPos - preservedPos + runway
+  ,
+  closeSync: proc (s: InputStream)
+                  {.nimcall, gcsafe, raises: [IOError, Defect].} =
+    try:
+      close FileInputStream(s).file
+    except OSError as err:
+      raise newException(IOError, "Failed to close file", err)
+)
+
+proc fileInput*(filename: string,
+                offset = 0,
+                pageSize = defaultPageSize): InputStreamHandle
+               {.raises: [IOError, OSError].} =
+  ## Creates an input stream for reading the contents of a file
+  ## through Nim's `io` module.
+  ##
+  ## Parameters:
+  ##
+  ## ``filename``
+  ##  The name of the file to read.
+  ##
+  ## ``offset``
+  ##  Initial position in the file where reading should start.
+  ##
+  let file = system.open(filename, fmRead)
+
+  if offset != 0:
+    setFilePos(file, offset)
+
+  makeHandle FileInputStream(
+    vtable: vtableAddr FileInputVTable,
+    buffers: initPageBuffers(pageSize),
+    file: file)
+
+proc unsafeMemoryInput*(mem: openarray[byte]): InputStreamHandle =
+  let head = unsafeAddr mem[0]
+
+  makeHandle InputStream(
+    span: PageSpan(
+      startAddr: head,
+      endAddr: offset(head, mem.len)),
+    spanEndPos: mem.len)
+
+proc unsafeMemoryInput*(str: string): InputStreamHandle =
+  unsafeMemoryInput str.toOpenArrayByte(0, str.len - 1)
+
+proc len*(s: InputStream): Option[Natural] {.raises: [Defect, IOError].} =
+  if s.vtable == nil:
+    some s.span.len
+  elif s.vtable.getLenSync != nil:
+    some s.vtable.getLenSync(s)
+  else:
+    none Natural
+
+template len*(s: AsyncInputStream): int =
+  len InputStream(s)
+
+proc flipPage(s: InputStream) =
+  doAssert s.buffers.len > 1
+  discard s.buffers.popFirst
+  s.span = s.buffers[0].span
+  s.spanEndPos += s.span.len
+
+proc continueAfterRead(s: InputStream): bool =
+  # Please note that this is extracted into a proc only to reduce the code
+  # that ends up inlined into async procs by `bufferMoreDataImpl`.
+  # The inlining itself is required to support the await-free operation of
+  # the `readable` APIs.
+  let firstReadPage = s.buffers[0]
+
+  s.span = firstReadPage.span
+  let bytesRead = s.span.len
+  s.spanEndPos += bytesRead
+
+  # The read might have been incomplete which signals the EOF of the stream.
+  # If this is the case, we disconnect the input device which prevents any
+  # further attempts to read from it:
+  if wasEofReached(s.buffers):
+    s.disconnectInputDevice()
+
+  # If we read some bytes anyway, we tell the user code that our buffers
+  # contain some unconsumed data:
+  bytesRead > 0
+
+template bufferMoreDataImpl(s, awaiter, readOp: untyped): bool =
+  # This template is always called when the current page has been
+  # completely exhausted. It should produce `true` if more data was
+  # successfully buffered, so reading can continue.
+  #
+  # The vtable will be `nil` for a memory stream and `vtable.readOp`
+  # will be `nil` for a memFile. If we've reached here, this is the
+  # end of the memory buffer, so we can signal EOF:
+  if s.buffers == nil or s.vtable == nil or s.vtable.readOp == nil:
+    false
+  else:
+    # There might be additional pages in our buffer queue. If so, we
+    # just jump to the next one:
+    if s.buffers.len > 1:
+      flipPage s
+      true
+    else:
+      # We ask our input device to populate our page queue with newly
+      # read pages. The state of the queue afterwards will tell us if
+      # the read was successful. In `continueAfterRead`, we examine if
+      # EOF was reached, but please note that some data might have been
+      # read anyway:
+      awaiter s.vtable.readOp(s)
+      continueAfterRead(s)
+
+proc bufferMoreDataSync(s: InputStream): bool =
+  # This proc exists only to avoid inlining of the code of
+  # `bufferMoreDataImpl` into `readable` (which in turn is
+  # a template inlined in the user code).
+  bufferMoreDataImpl(s, noAwait, readSync)
+
+template readable*(sp: InputStream): bool =
+  ## Checks whether reading more data from the stream is possible.
+  ##
+  ## If there is any unconsumed data in the stream buffers, the
+  ## operation returns `true` immediately. You can call `read`
+  ## or `peek` afterwards to consume or examine the next byte
+  ## in the stream.
+  ##
+  ## If the stream buffers are empty, the operation may block
+  ## until more data becomes available. The end of the stream
+  ## may be reached at this point, which will be indicated by
+  ## a `false` return value. Any attempt to call `read` or
+  ## `peek` afterwards is considered a `Defect`.
+  ##
+  ## Please note that this API is intended for stream consumers
+  ## who need to consume the data one byte at a time. A typical
+  ## usage will be the following:
+  ##
+  ## ```nim
+  ## while stream.readable:
+  ##   case stream.peek.char
+  ##   of '"':
+  ##     parseString(stream)
+  ##   of '0'..'9':
+  ##     parseNumber(stream)
+  ##   of '\':
+  ##     discard stream.read # skip the slash
+  ##     let escapedChar = stream.read
+  ## ```
+  ##
+  ## Even though the user code consumes the data one byte at a time,
+  ## in the majority of cases this consist of simply incrementing a
+  ## pointer within the stream buffers. Only when the stream buffers
+  ## are exhausted, a new read operation will be executed throught
+  ## the stream input device which may repopulate the buffers with
+  ## fresh data. See `Stream Pages` for futher discussion of this.
+
+  # This is a template, because we want the pointer check to be
+  # inlined at the call sites. Only if it fails, we call into the
+  # larger non-inlined proc:
+  let s = sp
+  hasRunway(s.span) or bufferMoreDataSync(s)
+
+template readable*(sp: AsyncInputStream): bool =
+  ## Async version of `readable`.
+  ## The intended API usage is the same. Instead of blocking, an async
+  ## stream will use `await` while waiting for more data.
+  let s = sp
+  if hasRunway(s.span):
+    true
+  else:
+    bufferMoreDataImpl(s, fsAsync, readAsync)
+
+template readableNImpl(s, n, awaiter, readOp: untyped): bool =
+  let runway = s.totalUnconsumedBytes
+
+  if runway >= n:
+    true
+  elif s.buffers == nil or s.vtable == nil or s.vtable.readOp == nil:
+    false
+  else:
+    var
+      bytesDeficit = n - runway
+      targetBytesRead = s.buffers.totalBytesRead + bytesDeficit
+      res = false
+
+    while true:
+      awaiter s.vtable.readOp(s)
+
+      if wasEofReached(s.buffers):
+        s.disconnectInputDevice()
+        res = s.buffers.totalBytesRead >= targetBytesRead
+        break
+
+      if s.buffers.totalBytesRead >= targetBytesRead:
+        res = true
+        break
+
+    res
+
+proc readable*(s: InputStream, n: int): bool =
+  ## Checks whether reading `n` bytes from the input stream is possible.
+  ##
+  ## If there is enough unconsumed data in the stream buffers, the
+  ## operation will return `true` immediately. You can use `read`,
+  ## `peek`, `read(n)` or `peek(n)` afterwards to consume up to the
+  ## number of verified bytes. Please note that consuming more bytes
+  ## will be considered a `Defect`.
+  ##
+  ## If the stream buffers do not contain enough data, the operation
+  ## may block until more data becomes available. The end of the stream
+  ## may be reached at this point, which will be indicated by a `false`
+  ## return value. Please note that the stream might still contain some
+  ## unconsumed bytes after `readable(n)` returned false. You can use
+  ## `totalUnconsumedBytes` or a combination of `readable` and `read`
+  ## to consume the remaining bytes if desired.
+  ##
+  ## If possible, prefer consuming the data one byte at a time. This
+  ## ensures the most optimal usage of the stream buffers. Even after
+  ## calling `readable(n)`, it's still preferrable to continue with
+  ## `read` instead of `read(n)` because the later may require the
+  ## resulting bytes to be copied to a freshly allocated sequence.
+  ##
+  ## In the situation where the consumed bytes need to be copied to
+  ## an existing external buffer, `readInto` will provide the best
+  ## performance instead.
+  ##
+  ## Just like `readable`, this operation will invoke reads on the
+  ## stream input device only when necessary. See `Stream Pages`
+  ## for futher discussion of this.
+  readableNImpl(s, n, noAwait, readSync)
+
+template readable*(sp: AsyncInputStream, np: int): bool =
+  ## Async version of `readable(n)`.
+  ## The intended API usage is the same. Instead of blocking, an async
+  ## stream will use `await` while waiting for more data.
+  let
+    s = sp
+    n = np
+
+  readableNImpl(s, n, fsAwait, readAsync)
+
+proc peek*(s: InputStream): byte {.inline.} =
+  doAssert hasRunway(s.span)
+  return s.span.startAddr[]
+
+template peek*(s: AsyncInputStream): byte =
+  peek InputStream(s)
+
+proc peekAt*(s: InputStream, pos: int): byte {.inline.} =
+  # TODO implement page flipping
+  let peekHead = offset(s.span.startAddr, pos)
+  doAssert cast[uint](peekHead) < cast[uint](s.span.endAddr)
+  return peekHead[]
+
+template peekAt*(s: AsyncInputStream, pos: int): byte =
+  peekAt InputStream(s)
+
+proc advance*(s: InputStream) =
+  if hasRunway(s.span):
+    bumpPointer s.span
+  elif s.buffers != nil and s.buffers.len > 1:
+    flipPage s
+
+template advance*(s: AsyncInputStream) =
+  advance InputStream(s)
+
+proc read*(s: InputStream): byte =
+  result = s.peek()
+  advance s
+
+template read*(s: AsyncInputStream): byte =
+  read InputStream(s)
+
+proc readIntoEx*(s: InputStream, target: var openarray[byte]): int =
+  ## Read data into the destination buffer.
+  ##
+  ## Returns the number of bytes that were successfully
+  ## written to the buffer. The function will return a
+  ## number smaller than the buffer length only if EOF
+  ## was reached before the buffer was fully populated.
+  discard
+
+proc readInto*(s: InputStream, target: var openarray[byte]): bool =
+  ## Read data into the destination buffer.
+  ##
+  ## Returns `false` if EOF was reached before the buffer
+  ## was fully populated. if you need precise information
+  ## regarding the number of bytes read, see `readIntoEx`.
+  s.readIntoEx(target) == target.len
+
+template readInto*(s: AsyncInputStream, target: var openarray[byte]): bool =
+  ## Asynchronously read data into the destination buffer.
+  ##
+  ## Returns `false` if EOF was reached before the buffer
+  ## was fully populated. if you need precise information
+  ## regarding the number of bytes read, see `readIntoEx`.
+  ##
+  ## If there are enough bytes already buffered by the stream,
+  ## the expression will complete immediately.
+  ## Otherwise, it will await more bytes to become available.
+  discard
+
+proc checkReadAhead(s: InputStream, n: Natural): ptr byte =
+  # TODO: handle multi-page
+  result = s.span.startAddr
+  doAssert s.span.len >= n
+  bumpPointer s.span, n
+
+template read*(s: InputStream, n: Natural): auto =
+  makeOpenArray(checkReadAhead(s, n), n)
+
+proc next*(s: InputStream): Option[byte] =
+  if readable(s):
+    result = some read(s)
+
+template next*(sp: AsyncInputStream): Option[byte] =
+  let s = sp
+  if readable(s):
+    some read(s)
+  else:
+    none byte
+
+proc pos*(s: InputStream): int {.inline.} =
+  s.spanEndPos - s.span.len
+
+template pos*(s: AsyncInputStream): int =
+  pos InputStream(s)
+
+when false:
+  # Obsolete APIs for removal
+  proc bufferPos(s: InputStream, pos: int): ptr byte =
+    let offsetFromEnd = pos - s.spanEndPos
+    doAssert offsetFromEnd < 0
+    result = offset(s.span.endAddr, offsetFromEnd)
+    doAssert result >= s.bufferStart
+
+  proc `[]`*(s: InputStream, pos: int): byte {.inline.} =
+    s.bufferPos(pos)[]
+
+  proc rewind*(s: InputStream, delta: int) =
+    s.head = offset(s.head, -delta)
+    doAssert s.head >= s.bufferStart
+
+  proc rewindTo*(s: InputStream, pos: int) {.inline.} =
+    s.head = s.bufferPos(pos)
+