Don't import/export chronos by default (#20)

Chronos support is optional and should not have to be imported in order
to use faststreams for non-async use cases, as doing so pollutes the
global namespace and slows down compilation.

`async` support must now explicitly be enabled with
-d:async_backend=chronos|asyncdispatch

faststreams.nimble

@@ -1,7 +1,7 @@
 mode = ScriptMode.Verbose
 
 packageName   = "faststreams"
-version       = "0.2.0"
+version       = "0.3.0"
 author        = "Status Research & Development GmbH"
 description   = "Nearly zero-overhead input/output streams for Nim"
 license       = "Apache License 2.0"
@@ -21,7 +21,12 @@ proc test(env, path: string) =
     lang = getEnv"TEST_LANG"
 
   exec "nim " & lang & " " & env &
-    " -r --hints:off --skipParentCfg " & path
+    " -d:async_backend=none -r --hints:off --skipParentCfg " & path
+  exec "nim " & lang & " " & env &
+    " -d:async_backend=chronos -r --hints:off --skipParentCfg " & path
+  # TODO std backend is broken / untested
+  # exec "nim " & lang & " " & env &
+  #  " -d:async_backend=chronos -r --hints:off --skipParentCfg " & path
 
 task test, "Run all tests":
   test "-d:debug   --threads:on", "tests/all_tests"

faststreams/async_backend.nim

@@ -1,14 +1,25 @@
 const
-  faststreams_async_backend {.strdefine.} = "chronos"
+  # To compile with async support, use `-d:async_backend=chronos|asyncdispatch`
+  async_backend {.strdefine.} = "none"
+
+const
+  faststreams_async_backend {.strdefine.} = ""
+
+when faststreams_async_backend != "":
+  {.fatal: "use `-d:async_backend` instead".}
 
 type
   CloseBehavior* = enum
     waitAsyncClose
     dontWaitAsyncClose
 
-const debugHelpers* = defined(debugHelpers)
+const
+  debugHelpers* = defined(debugHelpers)
+  fsAsyncSupport* = async_backend != "none"
 
-when faststreams_async_backend == "chronos":
+when async_backend == "none":
+  discard
+elif async_backend == "chronos":
   import
     chronos
 
@@ -18,7 +29,7 @@ when faststreams_async_backend == "chronos":
   template fsAwait*(f: Future): untyped =
     await f
 
-elif faststreams_async_backend in ["std", "asyncdispatch"]:
+elif async_backend in ["std", "asyncdispatch"]:
   import
     std/asyncdispatch
 
@@ -36,7 +47,7 @@ elif faststreams_async_backend in ["std", "asyncdispatch"]:
   type Duration* = int
 
 else:
-  {.fatal: "Unrecognized network backend: " & faststreams_async_backend.}
+  {.fatal: "Unrecognized network backend: " & async_backend.}
 
 when defined(danger):
   template fsAssert*(x) = discard

faststreams/buffers.nim

@@ -22,6 +22,7 @@ type
     maxBufferedBytes*: Natural
 
     queue*: Deque[PageRef]
+    when fsAsyncSupport:
       waitingReader*: Future[void]
       waitingWriter*: Future[void]
 

faststreams/inputs.nim

@@ -6,7 +6,9 @@ import
 export
   options, CloseBehavior
 
-type
+when fsAsyncSupport:
+  # Circular type refs prevent more targeted `when`
+  type
     InputStream* = ref object of RootObj
       vtable*: ptr InputStreamVTable # This is nil for unsafe memory inputs
       buffers*: PageBuffers          # This is nil for unsafe memory inputs
@@ -16,12 +18,6 @@ type
       when debugHelpers:
         name*: string
 
-  LayeredInputStream* = ref object of InputStream
-    source*: InputStream
-    allowWaitFor*: bool
-
-  InputStreamHandle* = object
-    s*: InputStream
 
     AsyncInputStream* {.borrow: `.`.} = distinct InputStream
 
@@ -42,10 +38,47 @@ type
 
     InputStreamVTable* = object
       readSync*: ReadSyncProc
-    readAsync*: ReadAsyncProc
       closeSync*: CloseSyncProc
-    closeAsync*: CloseAsyncProc
       getLenSync*: GetLenSyncProc
+      readAsync*: ReadAsyncProc
+      closeAsync*: CloseAsyncProc
+
+    MaybeAsyncInputStream* = InputStream | AsyncInputStream
+
+else:
+  type
+    InputStream* = ref object of RootObj
+      vtable*: ptr InputStreamVTable # This is nil for unsafe memory inputs
+      buffers*: PageBuffers          # This is nil for unsafe memory inputs
+      span*: PageSpan
+      spanEndPos*: Natural
+      when debugHelpers:
+        name*: string
+
+
+    ReadSyncProc* = proc (s: InputStream, dst: pointer, dstLen: Natural): Natural
+                        {.nimcall, gcsafe, raises: [IOError, Defect].}
+
+    CloseSyncProc* = proc (s: InputStream)
+                          {.nimcall, gcsafe, raises: [IOError, Defect].}
+
+    GetLenSyncProc* = proc (s: InputStream): Option[Natural]
+                          {.nimcall, gcsafe, raises: [IOError, Defect].}
+
+    InputStreamVTable* = object
+      readSync*: ReadSyncProc
+      closeSync*: CloseSyncProc
+      getLenSync*: GetLenSyncProc
+
+    MaybeAsyncInputStream* = InputStream
+
+type
+  LayeredInputStream* = ref object of InputStream
+    source*: InputStream
+    allowWaitFor*: bool
+
+  InputStreamHandle* = object
+    s*: InputStream
 
   MemFileInputStream = ref object of InputStream
     file: MemFile
@@ -54,29 +87,37 @@ type
     file: File
 
 template Sync*(s: InputStream): InputStream = s
-template Async*(s: InputStream): AsyncInputStream = AsyncInputStream(s)
 
-template Sync*(s: AsyncInputStream): InputStream = InputStream(s)
+when fsAsyncSupport:
-template Async*(s: AsyncInputStream): AsyncInputStream = s
+  template Async*(s: InputStream): AsyncInputStream = AsyncInputStream(s)
+
+  template Sync*(s: AsyncInputStream): InputStream = InputStream(s)
+  template Async*(s: AsyncInputStream): AsyncInputStream = s
 
 proc disconnectInputDevice(s: InputStream) =
   # TODO
   # Document the behavior that closeAsync is preferred
   if s.vtable != nil:
+    when fsAsyncSupport:
       if s.vtable.closeAsync != nil:
           s.closeFut = s.vtable.closeAsync(s)
       elif s.vtable.closeSync != nil:
         s.vtable.closeSync(s)
+    else:
+      if s.vtable.closeSync != nil:
+        s.vtable.closeSync(s)
     s.vtable = nil
 
-template disconnectInputDevice(s: AsyncInputStream) =
+when fsAsyncSupport:
+  template disconnectInputDevice(s: AsyncInputStream) =
     disconnectInputDevice InputStream(s)
 
 proc preventFurtherReading(s: InputStream) =
   s.vtable = nil
   s.span = default(PageSpan)
 
-template preventFurtherReading(s: AsyncInputStream) =
+when fsAsyncSupport:
+  template preventFurtherReading(s: AsyncInputStream) =
     preventFurtherReading InputStream(s)
 
 template makeHandle*(sp: InputStream): InputStreamHandle =
@@ -94,6 +135,7 @@ proc close*(s: InputStream,
   ## `waitFor` to block until the async operation completes.
   s.disconnectInputDevice()
   s.preventFurtherReading()
+  when fsAsyncSupport:
     if s.closeFut != nil:
       fsTranslateErrors "Stream closing failed":
         if behavior == waitAsyncClose:
@@ -101,7 +143,8 @@ proc close*(s: InputStream,
         else:
           asyncCheck s.closeFut
 
-template close*(sp: AsyncInputStream) =
+when fsAsyncSupport:
+  template close*(sp: AsyncInputStream) =
     ## Starts the asychronous closing of the stream and returns a future that
     ## tracks the closing operation.
     let s = InputStream sp
@@ -110,7 +153,7 @@ template close*(sp: AsyncInputStream) =
     if s.closeFut != nil:
       fsAwait s.closeFut
 
-template closeNoWait*(sp: AsyncInputStream|InputStream) =
+template closeNoWait*(sp: MaybeAsyncInputStream) =
   ## Close the stream without waiting even if's async.
   ## This operation will use `asyncCheck` internally to detect unhandled
   ## errors from the closing operation.
@@ -224,17 +267,11 @@ proc readableNow*(s: InputStream): bool =
   getNewSpan s
   s.span.hasRunway
 
-template readableNow*(s: AsyncInputStream): bool =
+when fsAsyncSupport:
+  template readableNow*(s: AsyncInputStream): bool =
     readableNow InputStream(s)
 
-# TODO: The pure async interface should be moved in a separate module
+  proc readOnce*(sp: AsyncInputStream): Future[Natural] {.async.} =
-#       to make FastStreams more light-weight when the async back-end
-#       is not used (e.g. in Confutils)
-#
-#       The problem is that the `async` macro will pull the entire
-#       event loop right now.
-
-proc readOnce*(sp: AsyncInputStream): Future[Natural] {.async.} =
     let s = InputStream(sp)
     fsAssert s.buffers != nil and s.vtable != nil
 
@@ -246,7 +283,7 @@ proc readOnce*(sp: AsyncInputStream): Future[Natural] {.async.} =
     if result > 0 and s.span.len == 0:
       getNewSpan s
 
-proc timeoutToNextByteImpl(s: AsyncInputStream,
+  proc timeoutToNextByteImpl(s: AsyncInputStream,
                             deadline: Future): Future[bool] {.async.} =
     let readFut = s.readOnce
     fsAwait readFut or deadline
@@ -256,24 +293,26 @@ proc timeoutToNextByteImpl(s: AsyncInputStream,
     else:
       return false
 
-template timeoutToNextByte*(sp: AsyncInputStream, deadline: Future): bool =
+  template timeoutToNextByte*(sp: AsyncInputStream, deadline: Future): bool =
     let s = sp
     if readableNow(s):
       true
     else:
       fsAwait timeoutToNextByteImpl(s, deadline)
 
-template timeoutToNextByte*(sp: AsyncInputStream, timeout: Duration): bool =
+  template timeoutToNextByte*(sp: AsyncInputStream, timeout: Duration): bool =
     let s = sp
     if readableNow(s):
       true
     else:
       fsAwait timeoutToNextByteImpl(s, sleepAsync(timeout))
 
-proc closeAsync*(s: AsyncInputStream) {.async.} =
+  proc closeAsync*(s: AsyncInputStream) {.async.} =
     close s
 
-# TODO: End of purely async interface
+  template totalUnconsumedBytes*(s: AsyncInputStream): Natural =
+    ## Alias for InputStream.totalUnconsumedBytes
+    totalUnconsumedBytes InputStream(s)
 
 func getBestContiguousRunway(s: InputStream): Natural =
   result = s.span.len
@@ -298,10 +337,6 @@ func totalUnconsumedBytes*(s: InputStream): Natural =
 
   localRunway + runwayInBuffers
 
-template totalUnconsumedBytes*(s: AsyncInputStream): Natural =
-  ## Alias for InputStream.totalUnconsumedBytes
-  totalUnconsumedBytes InputStream(s)
-
 proc limitReadableRange(s: InputStream, rangeLen: Natural): Natural =
   s.vtable = nil
 
@@ -317,7 +352,7 @@ proc limitReadableRange(s: InputStream, rangeLen: Natural): Natural =
       s.buffers.queue.peekFirst.consumedTo -= bytesToUnconsume
       return s.buffers.setFauxEof(s.spanEndPos)
 
-template withReadableRange*(sp: InputStream|AsyncInputStream,
+template withReadableRange*(sp: MaybeAsyncInputStream,
                             rangeLen: Natural,
                             rangeStreamVarName, blk: untyped) =
   let
@@ -409,7 +444,8 @@ proc len*(s: InputStream): Option[Natural] {.raises: [Defect, IOError].} =
   else:
     none Natural
 
-template len*(s: AsyncInputStream): Option[Natural] =
+when fsAsyncSupport:
+  template len*(s: AsyncInputStream): Option[Natural] =
     len InputStream(s)
 
 func memoryInput*(buffers: PageBuffers): InputStreamHandle =
@@ -541,7 +577,8 @@ template readable*(sp: InputStream): bool =
   let s = sp
   hasRunway(s.span) or bufferMoreDataSync(s)
 
-template readable*(sp: AsyncInputStream): bool =
+when fsAsyncSupport:
+  template readable*(sp: AsyncInputStream): bool =
     ## Async version of `readable`.
     ## The intended API usage is the same. Instead of blocking, an async
     ## stream will use `await` while waiting for more data.
@@ -615,7 +652,8 @@ proc readable*(s: InputStream, n: int): bool =
   ## for futher discussion of this.
   readableNImpl(s, n, noAwait, readSync)
 
-template readable*(sp: AsyncInputStream, np: int): bool =
+when fsAsyncSupport:
+  template readable*(sp: AsyncInputStream, np: int): bool =
     ## Async version of `readable(n)`.
     ## The intended API usage is the same. Instead of blocking, an async
     ## stream will use `await` while waiting for more data.
@@ -633,7 +671,8 @@ template peek*(sp: InputStream): byte =
     getNewSpanOrDieTrying s
     s.span.startAddr[]
 
-template peek*(s: AsyncInputStream): byte =
+when fsAsyncSupport:
+  template peek*(s: AsyncInputStream): byte =
     peek InputStream(s)
 
 func readFromNewSpan(s: InputStream): byte =
@@ -650,7 +689,8 @@ template read*(sp: InputStream): byte =
   else:
     readFromNewSpan s
 
-template read*(s: AsyncInputStream): byte =
+when fsAsyncSupport:
+  template read*(s: AsyncInputStream): byte =
     read InputStream(s)
 
 proc peekAt*(s: InputStream, pos: int): byte {.inline.} =
@@ -659,7 +699,8 @@ proc peekAt*(s: InputStream, pos: int): byte {.inline.} =
   fsAssert cast[uint](peekHead) < cast[uint](s.span.endAddr)
   return peekHead[]
 
-template peekAt*(s: AsyncInputStream, pos: int): byte =
+when fsAsyncSupport:
+  template peekAt*(s: AsyncInputStream, pos: int): byte =
     peekAt InputStream(s), pos
 
 proc advance*(s: InputStream) =
@@ -673,10 +714,11 @@ proc advance*(s: InputStream, n: Natural) =
   for i in 0 ..< n:
     advance s
 
-template advance*(s: AsyncInputStream) =
+when fsAsyncSupport:
+  template advance*(s: AsyncInputStream) =
     advance InputStream(s)
 
-template advance*(s: AsyncInputStream, n: Natural) =
+  template advance*(s: AsyncInputStream, n: Natural) =
     advance InputStream(s), n
 
 proc drainBuffersInto*(s: InputStream, dstAddr: ptr byte, dstLen: Natural): Natural =
@@ -789,14 +831,15 @@ proc readInto*(s: InputStream, target: var openarray[byte]): bool =
   ## regarding the number of bytes read, see `readIntoEx`.
   s.readIntoEx(target) == target.len
 
-template readIntoEx*(sp: AsyncInputStream, dst: var openarray[byte]): int =
+when fsAsyncSupport:
+  template readIntoEx*(sp: AsyncInputStream, dst: var openarray[byte]): int =
     let s = InputStream(sp)
     # BEWARE! `openArrayToPair` here is needed to avoid
     # double evaluation of the `dst` expression:
     let (dstAddr, dstLen) = openArrayToPair(dst)
     readIntoExImpl(s, dstAddr, dstLen, fsAwait, readAsync)
 
-template readInto*(sp: AsyncInputStream, dst: var openarray[byte]): bool =
+  template readInto*(sp: AsyncInputStream, dst: var openarray[byte]): bool =
     ## Asynchronously read data into the destination buffer.
     ##
     ## Returns `false` if EOF was reached before the buffer
@@ -869,7 +912,8 @@ template read*(sp: InputStream, np: static Natural): openarray[byte] =
 template read*(s: InputStream, n: Natural): openarray[byte] =
   readNImpl(s, n, useHeapMem)
 
-template read*(s: AsyncInputStream, n: Natural): openarray[byte] =
+when fsAsyncSupport:
+  template read*(s: AsyncInputStream, n: Natural): openarray[byte] =
     read InputStream(s), n
 
 proc lookAheadMatch*(s: InputStream, data: openarray[byte]): bool =
@@ -879,14 +923,16 @@ proc lookAheadMatch*(s: InputStream, data: openarray[byte]): bool =
 
   return true
 
-template lookAheadMatch*(s: AsyncInputStream, data: openarray[byte]): bool =
+when fsAsyncSupport:
+  template lookAheadMatch*(s: AsyncInputStream, data: openarray[byte]): bool =
     lookAheadMatch InputStream(s)
 
 proc next*(s: InputStream): Option[byte] =
   if readable(s):
     result = some read(s)
 
-template next*(sp: AsyncInputStream): Option[byte] =
+when fsAsyncSupport:
+  template next*(sp: AsyncInputStream): Option[byte] =
     let s = sp
     if readable(s):
       some read(s)
@@ -896,6 +942,7 @@ template next*(sp: AsyncInputStream): Option[byte] =
 proc pos*(s: InputStream): int {.inline.} =
   s.spanEndPos - s.span.len
 
-template pos*(s: AsyncInputStream): int =
+when fsAsyncSupport:
+  template pos*(s: AsyncInputStream): int =
     pos InputStream(s)
 

faststreams/multisync.nim

@@ -1,11 +1,15 @@
 import
-  stew/shims/macros,
+  async_backend
-  async_backend, inputs, outputs
 
 export
   async_backend
 
-macro fsMultiSync*(body: untyped) =
+when fsAsyncSupport:
+  import
+    stew/shims/macros,
+    "."/[inputs, outputs]
+
+  macro fsMultiSync*(body: untyped) =
     # We will produce an identical copy of the annotated proc,
     # but taking async parameters and having the async pragma.
     var
@@ -32,4 +36,5 @@ macro fsMultiSync*(body: untyped) =
     result = newStmtList(body, asyncProcBody)
     when defined(debugSupportAsync):
       echo result.repr
-
+else:
+  macro fsMultiSync*(body: untyped) = body

faststreams/outputs.nim

@@ -14,7 +14,9 @@ import
 export
   initPageBuffers, CloseBehavior
 
-type
+when fsAsyncSupport:
+  # Circular type refs prevent more targeted `when`
+  type
     OutputStream* = ref object of RootObj
       vtable*: ptr OutputStreamVTable # This is nil for any memory output
       buffers*: PageBuffers           # This is nil for unsafe memory outputs
@@ -25,17 +27,6 @@ type
       when debugHelpers:
         name*: string
 
-  WriteCursor* = object
-    span: PageSpan
-    stream: OutputStream
-
-  LayeredOutputStream* = ref object of OutputStream
-    destination*: OutputStream
-    allowWaitFor*: bool
-
-  OutputStreamHandle* = object
-    s*: OutputStream
-
     AsyncOutputStream* {.borrow: `.`.} = distinct OutputStream
 
     WriteSyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural)
@@ -64,26 +55,76 @@ type
       closeSync*: CloseSyncProc
       closeAsync*: CloseAsyncProc
 
+    MaybeAsyncOutputStream* = OutputStream | AsyncOutputStream
+
+else:
+  type
+    OutputStream* = ref object of RootObj
+      vtable*: ptr OutputStreamVTable # This is nil for any memory output
+      buffers*: PageBuffers           # This is nil for unsafe memory outputs
+      span*: PageSpan
+      spanEndPos*: Natural
+      extCursorsCount: int
+      when debugHelpers:
+        name*: string
+
+
+    WriteSyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural)
+                          {.nimcall, gcsafe, raises: [IOError, Defect].}
+
+    FlushSyncProc* = proc (s: OutputStream)
+                          {.nimcall, gcsafe, raises: [IOError, Defect].}
+
+    CloseSyncProc* = proc (s: OutputStream)
+                          {.nimcall, gcsafe, raises: [IOError, Defect].}
+
+    OutputStreamVTable* = object
+      writeSync*: WriteSyncProc
+      flushSync*: FlushSyncProc
+      closeSync*: CloseSyncProc
+
+    MaybeAsyncOutputStream* = OutputStream
+
+type
+  WriteCursor* = object
+    span: PageSpan
+    stream: OutputStream
+
+  LayeredOutputStream* = ref object of OutputStream
+    destination*: OutputStream
+    allowWaitFor*: bool
+
+  OutputStreamHandle* = object
+    s*: OutputStream
+
   VarSizeWriteCursor* = distinct WriteCursor
 
   FileOutputStream = ref object of OutputStream
     file: File
 
 template Sync*(s: OutputStream): OutputStream = s
-template Async*(s: OutputStream): AsyncOutputStream = AsyncOutputStream(s)
 
-template Sync*(s: AsyncOutputStream): OutputStream = OutputStream(s)
+when fsAsyncSupport:
-template Async*(s: AsyncOutputStream): AsyncOutputStream = s
+  template Async*(s: OutputStream): AsyncOutputStream = AsyncOutputStream(s)
+
+  template Sync*(s: AsyncOutputStream): OutputStream = OutputStream(s)
+  template Async*(s: AsyncOutputStream): AsyncOutputStream = s
 
 proc disconnectOutputDevice(s: OutputStream) =
   if s.vtable != nil:
+    when fsAsyncSupport:
       if s.vtable.closeAsync != nil:
         s.closeFut = s.vtable.closeAsync(s)
       elif s.vtable.closeSync != nil:
         s.vtable.closeSync(s)
+    else:
+      if s.vtable.closeSync != nil:
+        s.vtable.closeSync(s)
+
     s.vtable = nil
 
-template disconnectOutputDevice(s: AsyncOutputStream) =
+when fsAsyncSupport:
+  template disconnectOutputDevice(s: AsyncOutputStream) =
     disconnectOutputDevice OutputStream(s)
 
 template flushImpl(s: OutputStream, awaiter, writeOp, flushOp: untyped) =
@@ -99,18 +140,12 @@ template flushImpl(s: OutputStream, awaiter, writeOp, flushOp: untyped) =
 proc flush*(s: OutputStream) =
   flushImpl(s, noAwait, writeSync, flushSync)
 
-template flush*(sp: AsyncOutputStream) =
-  let s = OutputStream sp
-  flushImpl(s, fsAwait, writeAsync, flushAsync)
-
-proc flushAsync*(s: AsyncOutputStream) {.async.} =
-  flush s
-
 proc close*(s: OutputStream,
             behavior = dontWaitAsyncClose)
            {.raises: [IOError, Defect].} =
   flush s
   disconnectOutputDevice(s)
+  when fsAsyncSupport:
     if s.closeFut != nil:
       fsTranslateErrors "Stream closing failed":
         if behavior == waitAsyncClose:
@@ -118,17 +153,7 @@ proc close*(s: OutputStream,
         else:
           asyncCheck s.closeFut
 
-template close*(sp: AsyncOutputStream) =
+template closeNoWait*(sp: MaybeAsyncOutputStream) =
-  let s = OutputStream sp
-  flush(Async s)
-  disconnectOutputDevice(s)
-  if s.closeFut != nil:
-    fsAwait s.closeFut
-
-proc closeAsync*(s: AsyncOutputStream) {.async.} =
-  close s
-
-template closeNoWait*(sp: AsyncOutputStream|OutputStream) =
   ## Close the stream without waiting even if's async.
   ## This operation will use `asyncCheck` internally to detect unhandled
   ## errors from the closing operation.
@@ -196,7 +221,8 @@ proc ensureRunway*(s: OutputStream, neededRunway: Natural) =
     s.buffers.ensureRunway(s.span, neededRunway)
     s.spanEndPos += (s.span.len - runway)
 
-template ensureRunway*(s: AsyncOutputStream, neededRunway: Natural) =
+when fsAsyncSupport:
+  template ensureRunway*(s: AsyncOutputStream, neededRunway: Natural) =
     ensureRunway OutputStream(s), neededRunway
 
 template implementWrites*(buffersParam: PageBuffers,
@@ -266,7 +292,8 @@ proc fileOutput*(filename: string,
 proc pos*(s: OutputStream): int =
   s.spanEndPos - s.span.len
 
-template pos*(s: AsyncOutputStream): int =
+when fsAsyncSupport:
+  template pos*(s: AsyncOutputStream): int =
     pos OutputStream(s)
 
 proc getBuffers*(s: OutputStream): PageBuffers =
@@ -313,7 +340,8 @@ proc drainAllBuffersSync(s: OutputStream, buf: pointer, bufSize: Natural) =
     s.span = s.buffers.getWritableSpan()
     s.spanEndPos += s.span.len
 
-proc drainAllBuffersAsync(s: OutputStream, buf: pointer, bufSize: Natural) {.async.} =
+when fsAsyncSupport:
+  proc drainAllBuffersAsync(s: OutputStream, buf: pointer, bufSize: Natural) {.async.} =
     fsAwait s.vtable.writeAsync(s, buf, bufSize)
     s.span = s.buffers.getWritableSpan()
     s.spanEndPos += s.span.len
@@ -533,20 +561,21 @@ template write*(sp: OutputStream, b: byte) =
   else:
     writeToNewSpan(s, b)
 
-proc write*(sp: AsyncOutputStream, b: byte) =
+when fsAsyncSupport:
+  proc write*(sp: AsyncOutputStream, b: byte) =
     let s = OutputStream sp
     if atEnd(s.span):
       addPage(s)
     writeByte(s.span, b)
 
-template writeAndWait*(sp: AsyncOutputStream, b: byte) =
+  template writeAndWait*(sp: AsyncOutputStream, b: byte) =
     let s = OutputStream sp
     if hasRunway(s.span):
       writeByte(s.span, b)
     else:
       writeToNewSpanImpl(s, b, fsAwait, writeAsync, drainAllBuffersAsync)
 
-template write*(s: AsyncOutputStream, x: char) =
+  template write*(s: AsyncOutputStream, x: char) =
     write s, byte(x)
 
 template write*(s: OutputStream|var WriteCursor, x: char) =
@@ -606,7 +635,7 @@ proc write*(s: OutputStream, bytes: openArray[byte]) =
 proc write*(s: OutputStream, chars: openArray[char]) =
   write s, charsToBytes(chars)
 
-proc write*(s: OutputStream|AsyncOutputStream, value: string) {.inline.} =
+proc write*(s: MaybeAsyncOutputStream, value: string) {.inline.} =
   write s, value.toOpenArrayByte(0, value.len - 1)
 
 template memCopyToBytes(value: auto): untyped =
@@ -618,24 +647,26 @@ template memCopyToBytes(value: auto): untyped =
 proc writeMemCopy*(s: OutputStream, value: auto) =
   write s, memCopyToBytes(value)
 
-proc writeBytesAsyncImpl(sp: OutputStream,
+when fsAsyncSupport:
+  proc writeBytesAsyncImpl(sp: OutputStream,
                           bytes: openarray[byte]): Future[void] =
     let s = sp
     writeBytesImpl(s, bytes):
       return s.vtable.writeAsync(s, unsafeAddr bytes[0], bytes.len)
 
-proc writeBytesAsyncImpl(s: OutputStream,
+  proc writeBytesAsyncImpl(s: OutputStream,
                           chars: openarray[char]): Future[void] =
     writeBytesAsyncImpl s, charsToBytes(chars)
 
-proc writeBytesAsyncImpl(s: OutputStream,
+  proc writeBytesAsyncImpl(s: OutputStream,
                           str: string): Future[void] =
     writeBytesAsyncImpl s, toOpenArray(str, 0, str.len - 1)
 
 template writeAndWait*(s: OutputStream, value: untyped) =
   write s, value
 
-template writeAndWait*(sp: AsyncOutputStream, value: untyped) =
+when fsAsyncSupport:
+  template writeAndWait*(sp: AsyncOutputStream, value: untyped) =
     bind writeBytesAsyncImpl
 
     let
@@ -647,7 +678,7 @@ template writeAndWait*(sp: AsyncOutputStream, value: untyped) =
       s.span = getWritableSpan s.buffers
       s.spanEndPos += s.span.len
 
-template writeMemCopyAndWait*(sp: AsyncOutputStream, value: auto) =
+  template writeMemCopyAndWait*(sp: AsyncOutputStream, value: auto) =
     writeAndWait(sp, memCopyToBytes(value))
 
 proc writeBytesToCursor(c: var WriteCursor, bytes: openarray[byte]) =
@@ -795,9 +826,28 @@ template getOutput*(s: OutputStream, T: type seq[byte]): seq[byte] =
 template getOutput*(s: OutputStream): seq[byte] =
   cast[seq[byte]](s.getOutput(string))
 
-template getOutput*(s: AsyncOutputStream): seq[byte] =
+when fsAsyncSupport:
+  template getOutput*(s: AsyncOutputStream): seq[byte] =
     getOutput OutputStream(s)
 
-template getOutput*(s: AsyncOutputStream, T: type): untyped =
+  template getOutput*(s: AsyncOutputStream, T: type): untyped =
     getOutput OutputStream(s), T
 
+  when fsAsyncSupport:
+    template flush*(sp: AsyncOutputStream) =
+      let s = OutputStream sp
+      flushImpl(s, fsAwait, writeAsync, flushAsync)
+
+    proc flushAsync*(s: AsyncOutputStream) {.async.} =
+      flush s
+
+    template close*(sp: AsyncOutputStream) =
+      let s = OutputStream sp
+      flush(Async s)
+      disconnectOutputDevice(s)
+      if s.closeFut != nil:
+        fsAwait s.closeFut
+
+    proc closeAsync*(s: AsyncOutputStream) {.async.} =
+      close s
+

faststreams/pipelines.nim

@@ -1,29 +1,33 @@
 import
-  macros,
+  "."/[inputs, outputs, async_backend]
-  inputs, outputs, buffers, async_backend
 
 export
   inputs, outputs, async_backend
 
-type
+when fsAsyncSupport:
+  import
+    std/macros,
+    ./buffers
+
+  type
     Pipe* = ref object
       # TODO: Make these stream handles
       input*: AsyncInputStream
       output*: AsyncOutputStream
       buffers*: PageBuffers
 
-template enterWait(fut: var Future, context: static string) =
+  template enterWait(fut: var Future, context: static string) =
     let wait = newFuture[void](context)
     fut = wait
     try: fsAwait wait
     finally: fut = nil
 
-template awake(fp: Future) =
+  template awake(fp: Future) =
     let f = fp
     if f != nil and not finished(f):
       complete f
 
-proc pipeRead(s: LayeredInputStream,
+  proc pipeRead(s: LayeredInputStream,
                 dst: pointer, dstLen: Natural): Future[Natural] {.async.} =
     let buffers = s.buffers
     if buffers.eofReached: return 0
@@ -49,7 +53,7 @@ proc pipeRead(s: LayeredInputStream,
 
     return bytesInBuffersNow - bytesInBuffersAtStart
 
-proc pipeWrite(s: LayeredOutputStream, src: pointer, srcLen: Natural) {.async.} =
+  proc pipeWrite(s: LayeredOutputStream, src: pointer, srcLen: Natural) {.async.} =
     let buffers = s.buffers
     while buffers.canAcceptWrite(srcLen) == false:
       buffers.waitingWriter.enterWait "waiting for reader to drain the buffers"
@@ -60,12 +64,12 @@ proc pipeWrite(s: LayeredOutputStream, src: pointer, srcLen: Natural) {.async.}
     awake buffers.waitingReader
     describeBuffers "pipeWrite", buffers
 
-template completedFuture(name: static string): untyped =
+  template completedFuture(name: static string): untyped =
     let fut = newFuture[void](name)
     complete fut
     fut
 
-let pipeInputVTable = InputStreamVTable(
+  let pipeInputVTable = InputStreamVTable(
     readSync: proc (s: InputStream, dst: pointer, dstLen: Natural): Natural
                   {.nimcall, gcsafe, raises: [IOError, Defect].} =
       fsTranslateErrors "Failed to read from pipe":
@@ -99,9 +103,9 @@ let pipeInputVTable = InputStreamVTable(
           return closeAsync(Async source)
         else:
           return completedFuture("pipeInput.closeAsync")
-)
+  )
 
-let pipeOutputVTable = OutputStreamVTable(
+  let pipeOutputVTable = OutputStreamVTable(
     writeSync: proc (s: OutputStream, src: pointer, srcLen: Natural)
                     {.nimcall, gcsafe, raises: [IOError, Defect].} =
       fsTranslateErrors "Failed to write all bytes to pipe":
@@ -156,9 +160,9 @@ let pipeOutputVTable = OutputStreamVTable(
           return closeAsync(Async destination)
         else:
           return completedFuture("pipeOutput.closeAsync")
-)
+  )
 
-func pipeInput*(source: InputStream,
+  func pipeInput*(source: InputStream,
                   pageSize = defaultPageSize,
                   allowWaitFor = false): AsyncInputStream =
     fsAssert pageSize > 0
@@ -169,7 +173,7 @@ func pipeInput*(source: InputStream,
       allowWaitFor: allowWaitFor,
       source: source)
 
-func pipeInput*(buffers: PageBuffers,
+  func pipeInput*(buffers: PageBuffers,
                   allowWaitFor = false,
                   source: InputStream = nil): AsyncInputStream =
     var spanEndPos = Natural 0
@@ -184,7 +188,7 @@ func pipeInput*(buffers: PageBuffers,
       allowWaitFor: allowWaitFor,
       source: source)
 
-proc pipeOutput*(destination: OutputStream,
+  proc pipeOutput*(destination: OutputStream,
                   pageSize = defaultPageSize,
                   maxBufferedBytes = defaultPageSize * 4,
                   allowWaitFor = false): AsyncOutputStream =
@@ -202,7 +206,7 @@ proc pipeOutput*(destination: OutputStream,
       allowWaitFor: allowWaitFor,
       destination: destination)
 
-proc pipeOutput*(buffers: PageBuffers,
+  proc pipeOutput*(buffers: PageBuffers,
                   allowWaitFor = false,
                   destination: OutputStream = nil): AsyncOutputStream =
     var span = buffers.getWritableSpan()
@@ -218,27 +222,27 @@ proc pipeOutput*(buffers: PageBuffers,
       allowWaitFor: allowWaitFor,
       destination: destination)
 
-func asyncPipe*(pageSize = defaultPageSize,
+  func asyncPipe*(pageSize = defaultPageSize,
                   maxBufferedBytes = defaultPageSize * 4): Pipe =
     fsAssert pageSize > 0
     Pipe(buffers: initPageBuffers(pageSize, maxBufferedBytes))
 
-func initReader*(pipe: Pipe): AsyncInputStream =
+  func initReader*(pipe: Pipe): AsyncInputStream =
     result = pipeInput(pipe.buffers)
     pipe.input = result
 
-func initWriter*(pipe: Pipe): AsyncOutputStream =
+  func initWriter*(pipe: Pipe): AsyncOutputStream =
     result = pipeOutput(pipe.buffers)
     pipe.output = result
 
-proc exchangeBuffersAfterPipilineStep(input: InputStream, output: OutputStream) =
+  proc exchangeBuffersAfterPipilineStep(input: InputStream, output: OutputStream) =
     let formerInputBuffers = input.buffers
     let formerOutputBuffers = output.getBuffers
 
     input.resetBuffers formerOutputBuffers
     output.recycleBuffers formerInputBuffers
 
-macro executePipeline*(start: InputStream, steps: varargs[untyped]): untyped =
+  macro executePipeline*(start: InputStream, steps: varargs[untyped]): untyped =
     result = newTree(nnkStmtListExpr)
 
     var
@@ -278,7 +282,7 @@ macro executePipeline*(start: InputStream, steps: varargs[untyped]): untyped =
     if defined(debugMacros) or defined(debugPipelines):
       echo result.repr
 
-macro executePipeline*(start: AsyncInputStream, steps: varargs[untyped]): untyped =
+  macro executePipeline*(start: AsyncInputStream, steps: varargs[untyped]): untyped =
     var
       stream = ident "stream"
       pipelineSteps = ident "pipelineSteps"

tests/test_inputs.nim

@@ -2,15 +2,11 @@
 
 import
   os, unittest2, strutils, random,
-  stew/ranges/ptr_arith, testutils,
+  testutils,
   ../faststreams, ../faststreams/textio
 
 setCurrentDir getAppDir()
 
-proc bytes(s: string): seq[byte] =
-  result = newSeqOfCap[byte](s.len)
-  for c in s: result.add byte(c)
-
 proc str(bytes: openarray[byte]): string =
   result = newStringOfCap(bytes.len)
   for b in items(bytes):
@@ -28,7 +24,7 @@ const
   asciiTableFile = "files" / "ascii_table.txt"
   asciiTableContents = slurp(asciiTableFile)
 
-procSuite "input stream":
+suite "input stream":
   template emptyInputTests(suiteName, setupCode: untyped) =
     suite suiteName & " empty inputs":
       setup setupCode

tests/test_outputs.nim

@@ -154,6 +154,15 @@ suite "output stream":
 
     checkOutputsMatch()
 
+  test "memcpy":
+    var x = 0x42'u8
+
+    nimSeq.add x
+    memStream.writeMemCopy x
+    let memStreamRes = memStream.getOutput
+
+    check memStreamRes == nimSeq
+
   template undelayedOutput(content: seq[byte]) {.dirty.} =
     nimSeq.add content
     streamWritingToExistingBuffer.write content

tests/test_pipelines.nim

@@ -1,24 +1,29 @@
 {.used.}
 
 import
+  testutils/unittests,
+
+  # FastStreams modules:
+  ../faststreams/[pipelines, multisync]
+
+when fsAsyncSupport:
+  import
     # Std lib:
     std/[strutils, random, base64, terminal],
-  # Other packages:
-  testutils/unittests,
     # FastStreams modules:
     ../faststreams/[pipelines, multisync],
     # Testing modules:
     ./base64 as fsBase64
 
-include system/timers
+  include system/timers
 
-type
+  type
     TestTimes = object
       fsPipeline: Nanos
       fsAsyncPipeline: Nanos
       stdFunctionCalls: Nanos
 
-proc upcaseAllCharacters(i: InputStream, o: OutputStream) {.fsMultiSync.} =
+  proc upcaseAllCharacters(i: InputStream, o: OutputStream) {.fsMultiSync.} =
     let inputLen = i.len
     if inputLen.isSome:
       o.ensureRunway inputLen.get
@@ -28,17 +33,17 @@ proc upcaseAllCharacters(i: InputStream, o: OutputStream) {.fsMultiSync.} =
 
     close o
 
-proc printTimes(t: TestTimes) =
+  proc printTimes(t: TestTimes) =
     styledEcho "  cpu time [FS Sync  ]: ", styleBright, $t.fsPipeline, "ms"
     styledEcho "  cpu time [FS Async ]: ", styleBright, $t.fsAsyncPipeline, "ms"
     styledEcho "  cpu time [Std Lib  ]: ", styleBright, $t.stdFunctionCalls, "ms"
 
-template timeit(timerVar: var Nanos, code: untyped) =
+  template timeit(timerVar: var Nanos, code: untyped) =
     let t0 = getTicks()
     code
     timerVar = int(getTicks() - t0) div 1000000
 
-proc getOutput(sp: AsyncInputStream, T: type string): Future[string] {.async.} =
+  proc getOutput(sp: AsyncInputStream, T: type string): Future[string] {.async.} =
     # this proc is a quick hack to let the test pass
     # do not use it in production code
     let size = sp.totalUnconsumedBytes()
@@ -47,8 +52,8 @@ proc getOutput(sp: AsyncInputStream, T: type string): Future[string] {.async.} =
       discard sp.readinto(data)
       result = cast[string](data)
 
-procSuite "pipelines":
+  suite "pipelines":
-  let loremIpsum = """
+    const loremIpsum = """
       Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod
       tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
       veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex
@@ -126,3 +131,6 @@ procSuite "pipelines":
       let fsAsyncres = await f
 
       check fsAsyncRes == toUpperAscii(inputText)
+else:
+  test "pipelines":
+    skip