Don't import/export chronos by default (#20)

Chronos support is optional and should not have to be imported in order
to use faststreams for non-async use cases, as doing so pollutes the
global namespace and slows down compilation.

`async` support must now explicitly be enabled with
-d:async_backend=chronos|asyncdispatch
This commit is contained in:
Jacek Sieka 2021-08-20 13:10:26 +02:00 • committed by GitHub
commit 3a0ab42573
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 841 additions and 705 deletions

View file

@ -1,7 +1,7 @@
mode = ScriptMode.Verbose
packageName = "faststreams"
version = "0.2.0"
version = "0.3.0"
author = "Status Research & Development GmbH"
description = "Nearly zero-overhead input/output streams for Nim"
license = "Apache License 2.0"
@ -21,7 +21,12 @@ proc test(env, path: string) =
lang = getEnv"TEST_LANG"
exec "nim " & lang & " " & env &
" -r --hints:off --skipParentCfg " & path
" -d:async_backend=none -r --hints:off --skipParentCfg " & path
exec "nim " & lang & " " & env &
" -d:async_backend=chronos -r --hints:off --skipParentCfg " & path
# TODO std backend is broken / untested
# exec "nim " & lang & " " & env &
# " -d:async_backend=chronos -r --hints:off --skipParentCfg " & path
task test, "Run all tests":
test "-d:debug --threads:on", "tests/all_tests"

View file

@ -1,14 +1,25 @@
const
faststreams_async_backend {.strdefine.} = "chronos"
# To compile with async support, use `-d:async_backend=chronos|asyncdispatch`
async_backend {.strdefine.} = "none"
const
faststreams_async_backend {.strdefine.} = ""
when faststreams_async_backend != "":
{.fatal: "use `-d:async_backend` instead".}
type
CloseBehavior* = enum
waitAsyncClose
dontWaitAsyncClose
const debugHelpers* = defined(debugHelpers)
const
debugHelpers* = defined(debugHelpers)
fsAsyncSupport* = async_backend != "none"
when faststreams_async_backend == "chronos":
when async_backend == "none":
discard
elif async_backend == "chronos":
import
chronos
@ -18,7 +29,7 @@ when faststreams_async_backend == "chronos":
template fsAwait*(f: Future): untyped =
await f
elif faststreams_async_backend in ["std", "asyncdispatch"]:
elif async_backend in ["std", "asyncdispatch"]:
import
std/asyncdispatch
@ -36,7 +47,7 @@ elif faststreams_async_backend in ["std", "asyncdispatch"]:
type Duration* = int
else:
{.fatal: "Unrecognized network backend: " & faststreams_async_backend.}
{.fatal: "Unrecognized network backend: " & async_backend.}
when defined(danger):
template fsAssert*(x) = discard

View file

@ -22,6 +22,7 @@ type
maxBufferedBytes*: Natural
queue*: Deque[PageRef]
when fsAsyncSupport:
waitingReader*: Future[void]
waitingWriter*: Future[void]

View file

@ -6,7 +6,9 @@ import
export
options, CloseBehavior
type
when fsAsyncSupport:
# Circular type refs prevent more targeted `when`
type
InputStream* = ref object of RootObj
vtable*: ptr InputStreamVTable # This is nil for unsafe memory inputs
buffers*: PageBuffers # This is nil for unsafe memory inputs
@ -16,12 +18,6 @@ type
when debugHelpers:
name*: string
LayeredInputStream* = ref object of InputStream
source*: InputStream
allowWaitFor*: bool
InputStreamHandle* = object
s*: InputStream
AsyncInputStream* {.borrow: `.`.} = distinct InputStream
@ -42,10 +38,47 @@ type
InputStreamVTable* = object
readSync*: ReadSyncProc
readAsync*: ReadAsyncProc
closeSync*: CloseSyncProc
closeAsync*: CloseAsyncProc
getLenSync*: GetLenSyncProc
readAsync*: ReadAsyncProc
closeAsync*: CloseAsyncProc
MaybeAsyncInputStream* = InputStream | AsyncInputStream
else:
type
InputStream* = ref object of RootObj
vtable*: ptr InputStreamVTable # This is nil for unsafe memory inputs
buffers*: PageBuffers # This is nil for unsafe memory inputs
span*: PageSpan
spanEndPos*: Natural
when debugHelpers:
name*: string
ReadSyncProc* = proc (s: InputStream, dst: pointer, dstLen: Natural): Natural
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseSyncProc* = proc (s: InputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
GetLenSyncProc* = proc (s: InputStream): Option[Natural]
{.nimcall, gcsafe, raises: [IOError, Defect].}
InputStreamVTable* = object
readSync*: ReadSyncProc
closeSync*: CloseSyncProc
getLenSync*: GetLenSyncProc
MaybeAsyncInputStream* = InputStream
type
LayeredInputStream* = ref object of InputStream
source*: InputStream
allowWaitFor*: bool
InputStreamHandle* = object
s*: InputStream
MemFileInputStream = ref object of InputStream
file: MemFile
@ -54,29 +87,37 @@ type
file: File
template Sync*(s: InputStream): InputStream = s
template Async*(s: InputStream): AsyncInputStream = AsyncInputStream(s)
template Sync*(s: AsyncInputStream): InputStream = InputStream(s)
template Async*(s: AsyncInputStream): AsyncInputStream = s
when fsAsyncSupport:
template Async*(s: InputStream): AsyncInputStream = AsyncInputStream(s)
template Sync*(s: AsyncInputStream): InputStream = InputStream(s)
template Async*(s: AsyncInputStream): AsyncInputStream = s
proc disconnectInputDevice(s: InputStream) =
# TODO
# Document the behavior that closeAsync is preferred
if s.vtable != nil:
when fsAsyncSupport:
if s.vtable.closeAsync != nil:
s.closeFut = s.vtable.closeAsync(s)
elif s.vtable.closeSync != nil:
s.vtable.closeSync(s)
else:
if s.vtable.closeSync != nil:
s.vtable.closeSync(s)
s.vtable = nil
template disconnectInputDevice(s: AsyncInputStream) =
when fsAsyncSupport:
template disconnectInputDevice(s: AsyncInputStream) =
disconnectInputDevice InputStream(s)
proc preventFurtherReading(s: InputStream) =
s.vtable = nil
s.span = default(PageSpan)
template preventFurtherReading(s: AsyncInputStream) =
when fsAsyncSupport:
template preventFurtherReading(s: AsyncInputStream) =
preventFurtherReading InputStream(s)
template makeHandle*(sp: InputStream): InputStreamHandle =
@ -94,6 +135,7 @@ proc close*(s: InputStream,
## `waitFor` to block until the async operation completes.
s.disconnectInputDevice()
s.preventFurtherReading()
when fsAsyncSupport:
if s.closeFut != nil:
fsTranslateErrors "Stream closing failed":
if behavior == waitAsyncClose:
@ -101,7 +143,8 @@ proc close*(s: InputStream,
else:
asyncCheck s.closeFut
template close*(sp: AsyncInputStream) =
when fsAsyncSupport:
template close*(sp: AsyncInputStream) =
## Starts the asychronous closing of the stream and returns a future that
## tracks the closing operation.
let s = InputStream sp
@ -110,7 +153,7 @@ template close*(sp: AsyncInputStream) =
if s.closeFut != nil:
fsAwait s.closeFut
template closeNoWait*(sp: AsyncInputStream|InputStream) =
template closeNoWait*(sp: MaybeAsyncInputStream) =
## Close the stream without waiting even if's async.
## This operation will use `asyncCheck` internally to detect unhandled
## errors from the closing operation.
@ -224,17 +267,11 @@ proc readableNow*(s: InputStream): bool =
getNewSpan s
s.span.hasRunway
template readableNow*(s: AsyncInputStream): bool =
when fsAsyncSupport:
template readableNow*(s: AsyncInputStream): bool =
readableNow InputStream(s)
# TODO: The pure async interface should be moved in a separate module
# to make FastStreams more light-weight when the async back-end
# is not used (e.g. in Confutils)
#
# The problem is that the `async` macro will pull the entire
# event loop right now.
proc readOnce*(sp: AsyncInputStream): Future[Natural] {.async.} =
proc readOnce*(sp: AsyncInputStream): Future[Natural] {.async.} =
let s = InputStream(sp)
fsAssert s.buffers != nil and s.vtable != nil
@ -246,7 +283,7 @@ proc readOnce*(sp: AsyncInputStream): Future[Natural] {.async.} =
if result > 0 and s.span.len == 0:
getNewSpan s
proc timeoutToNextByteImpl(s: AsyncInputStream,
proc timeoutToNextByteImpl(s: AsyncInputStream,
deadline: Future): Future[bool] {.async.} =
let readFut = s.readOnce
fsAwait readFut or deadline
@ -256,24 +293,26 @@ proc timeoutToNextByteImpl(s: AsyncInputStream,
else:
return false
template timeoutToNextByte*(sp: AsyncInputStream, deadline: Future): bool =
template timeoutToNextByte*(sp: AsyncInputStream, deadline: Future): bool =
let s = sp
if readableNow(s):
true
else:
fsAwait timeoutToNextByteImpl(s, deadline)
template timeoutToNextByte*(sp: AsyncInputStream, timeout: Duration): bool =
template timeoutToNextByte*(sp: AsyncInputStream, timeout: Duration): bool =
let s = sp
if readableNow(s):
true
else:
fsAwait timeoutToNextByteImpl(s, sleepAsync(timeout))
proc closeAsync*(s: AsyncInputStream) {.async.} =
proc closeAsync*(s: AsyncInputStream) {.async.} =
close s
# TODO: End of purely async interface
template totalUnconsumedBytes*(s: AsyncInputStream): Natural =
## Alias for InputStream.totalUnconsumedBytes
totalUnconsumedBytes InputStream(s)
func getBestContiguousRunway(s: InputStream): Natural =
result = s.span.len
@ -298,10 +337,6 @@ func totalUnconsumedBytes*(s: InputStream): Natural =
localRunway + runwayInBuffers
template totalUnconsumedBytes*(s: AsyncInputStream): Natural =
## Alias for InputStream.totalUnconsumedBytes
totalUnconsumedBytes InputStream(s)
proc limitReadableRange(s: InputStream, rangeLen: Natural): Natural =
s.vtable = nil
@ -317,7 +352,7 @@ proc limitReadableRange(s: InputStream, rangeLen: Natural): Natural =
s.buffers.queue.peekFirst.consumedTo -= bytesToUnconsume
return s.buffers.setFauxEof(s.spanEndPos)
template withReadableRange*(sp: InputStream|AsyncInputStream,
template withReadableRange*(sp: MaybeAsyncInputStream,
rangeLen: Natural,
rangeStreamVarName, blk: untyped) =
let
@ -409,7 +444,8 @@ proc len*(s: InputStream): Option[Natural] {.raises: [Defect, IOError].} =
else:
none Natural
template len*(s: AsyncInputStream): Option[Natural] =
when fsAsyncSupport:
template len*(s: AsyncInputStream): Option[Natural] =
len InputStream(s)
func memoryInput*(buffers: PageBuffers): InputStreamHandle =
@ -541,7 +577,8 @@ template readable*(sp: InputStream): bool =
let s = sp
hasRunway(s.span) or bufferMoreDataSync(s)
template readable*(sp: AsyncInputStream): bool =
when fsAsyncSupport:
template readable*(sp: AsyncInputStream): bool =
## Async version of `readable`.
## The intended API usage is the same. Instead of blocking, an async
## stream will use `await` while waiting for more data.
@ -615,7 +652,8 @@ proc readable*(s: InputStream, n: int): bool =
## for futher discussion of this.
readableNImpl(s, n, noAwait, readSync)
template readable*(sp: AsyncInputStream, np: int): bool =
when fsAsyncSupport:
template readable*(sp: AsyncInputStream, np: int): bool =
## Async version of `readable(n)`.
## The intended API usage is the same. Instead of blocking, an async
## stream will use `await` while waiting for more data.
@ -633,7 +671,8 @@ template peek*(sp: InputStream): byte =
getNewSpanOrDieTrying s
s.span.startAddr[]
template peek*(s: AsyncInputStream): byte =
when fsAsyncSupport:
template peek*(s: AsyncInputStream): byte =
peek InputStream(s)
func readFromNewSpan(s: InputStream): byte =
@ -650,7 +689,8 @@ template read*(sp: InputStream): byte =
else:
readFromNewSpan s
template read*(s: AsyncInputStream): byte =
when fsAsyncSupport:
template read*(s: AsyncInputStream): byte =
read InputStream(s)
proc peekAt*(s: InputStream, pos: int): byte {.inline.} =
@ -659,7 +699,8 @@ proc peekAt*(s: InputStream, pos: int): byte {.inline.} =
fsAssert cast[uint](peekHead) < cast[uint](s.span.endAddr)
return peekHead[]
template peekAt*(s: AsyncInputStream, pos: int): byte =
when fsAsyncSupport:
template peekAt*(s: AsyncInputStream, pos: int): byte =
peekAt InputStream(s), pos
proc advance*(s: InputStream) =
@ -673,10 +714,11 @@ proc advance*(s: InputStream, n: Natural) =
for i in 0 ..< n:
advance s
template advance*(s: AsyncInputStream) =
when fsAsyncSupport:
template advance*(s: AsyncInputStream) =
advance InputStream(s)
template advance*(s: AsyncInputStream, n: Natural) =
template advance*(s: AsyncInputStream, n: Natural) =
advance InputStream(s), n
proc drainBuffersInto*(s: InputStream, dstAddr: ptr byte, dstLen: Natural): Natural =
@ -789,14 +831,15 @@ proc readInto*(s: InputStream, target: var openarray[byte]): bool =
## regarding the number of bytes read, see `readIntoEx`.
s.readIntoEx(target) == target.len
template readIntoEx*(sp: AsyncInputStream, dst: var openarray[byte]): int =
when fsAsyncSupport:
template readIntoEx*(sp: AsyncInputStream, dst: var openarray[byte]): int =
let s = InputStream(sp)
# BEWARE! `openArrayToPair` here is needed to avoid
# double evaluation of the `dst` expression:
let (dstAddr, dstLen) = openArrayToPair(dst)
readIntoExImpl(s, dstAddr, dstLen, fsAwait, readAsync)
template readInto*(sp: AsyncInputStream, dst: var openarray[byte]): bool =
template readInto*(sp: AsyncInputStream, dst: var openarray[byte]): bool =
## Asynchronously read data into the destination buffer.
##
## Returns `false` if EOF was reached before the buffer
@ -869,7 +912,8 @@ template read*(sp: InputStream, np: static Natural): openarray[byte] =
template read*(s: InputStream, n: Natural): openarray[byte] =
readNImpl(s, n, useHeapMem)
template read*(s: AsyncInputStream, n: Natural): openarray[byte] =
when fsAsyncSupport:
template read*(s: AsyncInputStream, n: Natural): openarray[byte] =
read InputStream(s), n
proc lookAheadMatch*(s: InputStream, data: openarray[byte]): bool =
@ -879,14 +923,16 @@ proc lookAheadMatch*(s: InputStream, data: openarray[byte]): bool =
return true
template lookAheadMatch*(s: AsyncInputStream, data: openarray[byte]): bool =
when fsAsyncSupport:
template lookAheadMatch*(s: AsyncInputStream, data: openarray[byte]): bool =
lookAheadMatch InputStream(s)
proc next*(s: InputStream): Option[byte] =
if readable(s):
result = some read(s)
template next*(sp: AsyncInputStream): Option[byte] =
when fsAsyncSupport:
template next*(sp: AsyncInputStream): Option[byte] =
let s = sp
if readable(s):
some read(s)
@ -896,6 +942,7 @@ template next*(sp: AsyncInputStream): Option[byte] =
proc pos*(s: InputStream): int {.inline.} =
s.spanEndPos - s.span.len
template pos*(s: AsyncInputStream): int =
when fsAsyncSupport:
template pos*(s: AsyncInputStream): int =
pos InputStream(s)

View file

@ -1,11 +1,15 @@
import
stew/shims/macros,
async_backend, inputs, outputs
async_backend
export
async_backend
macro fsMultiSync*(body: untyped) =
when fsAsyncSupport:
import
stew/shims/macros,
"."/[inputs, outputs]
macro fsMultiSync*(body: untyped) =
# We will produce an identical copy of the annotated proc,
# but taking async parameters and having the async pragma.
var
@ -32,4 +36,5 @@ macro fsMultiSync*(body: untyped) =
result = newStmtList(body, asyncProcBody)
when defined(debugSupportAsync):
echo result.repr
else:
macro fsMultiSync*(body: untyped) = body

View file

@ -14,7 +14,9 @@ import
export
initPageBuffers, CloseBehavior
type
when fsAsyncSupport:
# Circular type refs prevent more targeted `when`
type
OutputStream* = ref object of RootObj
vtable*: ptr OutputStreamVTable # This is nil for any memory output
buffers*: PageBuffers # This is nil for unsafe memory outputs
@ -25,17 +27,6 @@ type
when debugHelpers:
name*: string
WriteCursor* = object
span: PageSpan
stream: OutputStream
LayeredOutputStream* = ref object of OutputStream
destination*: OutputStream
allowWaitFor*: bool
OutputStreamHandle* = object
s*: OutputStream
AsyncOutputStream* {.borrow: `.`.} = distinct OutputStream
WriteSyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural)
@ -64,26 +55,76 @@ type
closeSync*: CloseSyncProc
closeAsync*: CloseAsyncProc
MaybeAsyncOutputStream* = OutputStream | AsyncOutputStream
else:
type
OutputStream* = ref object of RootObj
vtable*: ptr OutputStreamVTable # This is nil for any memory output
buffers*: PageBuffers # This is nil for unsafe memory outputs
span*: PageSpan
spanEndPos*: Natural
extCursorsCount: int
when debugHelpers:
name*: string
WriteSyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural)
{.nimcall, gcsafe, raises: [IOError, Defect].}
FlushSyncProc* = proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseSyncProc* = proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
OutputStreamVTable* = object
writeSync*: WriteSyncProc
flushSync*: FlushSyncProc
closeSync*: CloseSyncProc
MaybeAsyncOutputStream* = OutputStream
type
WriteCursor* = object
span: PageSpan
stream: OutputStream
LayeredOutputStream* = ref object of OutputStream
destination*: OutputStream
allowWaitFor*: bool
OutputStreamHandle* = object
s*: OutputStream
VarSizeWriteCursor* = distinct WriteCursor
FileOutputStream = ref object of OutputStream
file: File
template Sync*(s: OutputStream): OutputStream = s
template Async*(s: OutputStream): AsyncOutputStream = AsyncOutputStream(s)
template Sync*(s: AsyncOutputStream): OutputStream = OutputStream(s)
template Async*(s: AsyncOutputStream): AsyncOutputStream = s
when fsAsyncSupport:
template Async*(s: OutputStream): AsyncOutputStream = AsyncOutputStream(s)
template Sync*(s: AsyncOutputStream): OutputStream = OutputStream(s)
template Async*(s: AsyncOutputStream): AsyncOutputStream = s
proc disconnectOutputDevice(s: OutputStream) =
if s.vtable != nil:
when fsAsyncSupport:
if s.vtable.closeAsync != nil:
s.closeFut = s.vtable.closeAsync(s)
elif s.vtable.closeSync != nil:
s.vtable.closeSync(s)
else:
if s.vtable.closeSync != nil:
s.vtable.closeSync(s)
s.vtable = nil
template disconnectOutputDevice(s: AsyncOutputStream) =
when fsAsyncSupport:
template disconnectOutputDevice(s: AsyncOutputStream) =
disconnectOutputDevice OutputStream(s)
template flushImpl(s: OutputStream, awaiter, writeOp, flushOp: untyped) =
@ -99,18 +140,12 @@ template flushImpl(s: OutputStream, awaiter, writeOp, flushOp: untyped) =
proc flush*(s: OutputStream) =
flushImpl(s, noAwait, writeSync, flushSync)
template flush*(sp: AsyncOutputStream) =
let s = OutputStream sp
flushImpl(s, fsAwait, writeAsync, flushAsync)
proc flushAsync*(s: AsyncOutputStream) {.async.} =
flush s
proc close*(s: OutputStream,
behavior = dontWaitAsyncClose)
{.raises: [IOError, Defect].} =
flush s
disconnectOutputDevice(s)
when fsAsyncSupport:
if s.closeFut != nil:
fsTranslateErrors "Stream closing failed":
if behavior == waitAsyncClose:
@ -118,17 +153,7 @@ proc close*(s: OutputStream,
else:
asyncCheck s.closeFut
template close*(sp: AsyncOutputStream) =
let s = OutputStream sp
flush(Async s)
disconnectOutputDevice(s)
if s.closeFut != nil:
fsAwait s.closeFut
proc closeAsync*(s: AsyncOutputStream) {.async.} =
close s
template closeNoWait*(sp: AsyncOutputStream|OutputStream) =
template closeNoWait*(sp: MaybeAsyncOutputStream) =
## Close the stream without waiting even if's async.
## This operation will use `asyncCheck` internally to detect unhandled
## errors from the closing operation.
@ -196,7 +221,8 @@ proc ensureRunway*(s: OutputStream, neededRunway: Natural) =
s.buffers.ensureRunway(s.span, neededRunway)
s.spanEndPos += (s.span.len - runway)
template ensureRunway*(s: AsyncOutputStream, neededRunway: Natural) =
when fsAsyncSupport:
template ensureRunway*(s: AsyncOutputStream, neededRunway: Natural) =
ensureRunway OutputStream(s), neededRunway
template implementWrites*(buffersParam: PageBuffers,
@ -266,7 +292,8 @@ proc fileOutput*(filename: string,
proc pos*(s: OutputStream): int =
s.spanEndPos - s.span.len
template pos*(s: AsyncOutputStream): int =
when fsAsyncSupport:
template pos*(s: AsyncOutputStream): int =
pos OutputStream(s)
proc getBuffers*(s: OutputStream): PageBuffers =
@ -313,7 +340,8 @@ proc drainAllBuffersSync(s: OutputStream, buf: pointer, bufSize: Natural) =
s.span = s.buffers.getWritableSpan()
s.spanEndPos += s.span.len
proc drainAllBuffersAsync(s: OutputStream, buf: pointer, bufSize: Natural) {.async.} =
when fsAsyncSupport:
proc drainAllBuffersAsync(s: OutputStream, buf: pointer, bufSize: Natural) {.async.} =
fsAwait s.vtable.writeAsync(s, buf, bufSize)
s.span = s.buffers.getWritableSpan()
s.spanEndPos += s.span.len
@ -533,20 +561,21 @@ template write*(sp: OutputStream, b: byte) =
else:
writeToNewSpan(s, b)
proc write*(sp: AsyncOutputStream, b: byte) =
when fsAsyncSupport:
proc write*(sp: AsyncOutputStream, b: byte) =
let s = OutputStream sp
if atEnd(s.span):
addPage(s)
writeByte(s.span, b)
template writeAndWait*(sp: AsyncOutputStream, b: byte) =
template writeAndWait*(sp: AsyncOutputStream, b: byte) =
let s = OutputStream sp
if hasRunway(s.span):
writeByte(s.span, b)
else:
writeToNewSpanImpl(s, b, fsAwait, writeAsync, drainAllBuffersAsync)
template write*(s: AsyncOutputStream, x: char) =
template write*(s: AsyncOutputStream, x: char) =
write s, byte(x)
template write*(s: OutputStream|var WriteCursor, x: char) =
@ -606,7 +635,7 @@ proc write*(s: OutputStream, bytes: openArray[byte]) =
proc write*(s: OutputStream, chars: openArray[char]) =
write s, charsToBytes(chars)
proc write*(s: OutputStream|AsyncOutputStream, value: string) {.inline.} =
proc write*(s: MaybeAsyncOutputStream, value: string) {.inline.} =
write s, value.toOpenArrayByte(0, value.len - 1)
template memCopyToBytes(value: auto): untyped =
@ -618,24 +647,26 @@ template memCopyToBytes(value: auto): untyped =
proc writeMemCopy*(s: OutputStream, value: auto) =
write s, memCopyToBytes(value)
proc writeBytesAsyncImpl(sp: OutputStream,
when fsAsyncSupport:
proc writeBytesAsyncImpl(sp: OutputStream,
bytes: openarray[byte]): Future[void] =
let s = sp
writeBytesImpl(s, bytes):
return s.vtable.writeAsync(s, unsafeAddr bytes[0], bytes.len)
proc writeBytesAsyncImpl(s: OutputStream,
proc writeBytesAsyncImpl(s: OutputStream,
chars: openarray[char]): Future[void] =
writeBytesAsyncImpl s, charsToBytes(chars)
proc writeBytesAsyncImpl(s: OutputStream,
proc writeBytesAsyncImpl(s: OutputStream,
str: string): Future[void] =
writeBytesAsyncImpl s, toOpenArray(str, 0, str.len - 1)
template writeAndWait*(s: OutputStream, value: untyped) =
write s, value
template writeAndWait*(sp: AsyncOutputStream, value: untyped) =
when fsAsyncSupport:
template writeAndWait*(sp: AsyncOutputStream, value: untyped) =
bind writeBytesAsyncImpl
let
@ -647,7 +678,7 @@ template writeAndWait*(sp: AsyncOutputStream, value: untyped) =
s.span = getWritableSpan s.buffers
s.spanEndPos += s.span.len
template writeMemCopyAndWait*(sp: AsyncOutputStream, value: auto) =
template writeMemCopyAndWait*(sp: AsyncOutputStream, value: auto) =
writeAndWait(sp, memCopyToBytes(value))
proc writeBytesToCursor(c: var WriteCursor, bytes: openarray[byte]) =
@ -795,9 +826,28 @@ template getOutput*(s: OutputStream, T: type seq[byte]): seq[byte] =
template getOutput*(s: OutputStream): seq[byte] =
cast[seq[byte]](s.getOutput(string))
template getOutput*(s: AsyncOutputStream): seq[byte] =
when fsAsyncSupport:
template getOutput*(s: AsyncOutputStream): seq[byte] =
getOutput OutputStream(s)
template getOutput*(s: AsyncOutputStream, T: type): untyped =
template getOutput*(s: AsyncOutputStream, T: type): untyped =
getOutput OutputStream(s), T
when fsAsyncSupport:
template flush*(sp: AsyncOutputStream) =
let s = OutputStream sp
flushImpl(s, fsAwait, writeAsync, flushAsync)
proc flushAsync*(s: AsyncOutputStream) {.async.} =
flush s
template close*(sp: AsyncOutputStream) =
let s = OutputStream sp
flush(Async s)
disconnectOutputDevice(s)
if s.closeFut != nil:
fsAwait s.closeFut
proc closeAsync*(s: AsyncOutputStream) {.async.} =
close s

View file

@ -1,29 +1,33 @@
import
macros,
inputs, outputs, buffers, async_backend
"."/[inputs, outputs, async_backend]
export
inputs, outputs, async_backend
type
when fsAsyncSupport:
import
std/macros,
./buffers
type
Pipe* = ref object
# TODO: Make these stream handles
input*: AsyncInputStream
output*: AsyncOutputStream
buffers*: PageBuffers
template enterWait(fut: var Future, context: static string) =
template enterWait(fut: var Future, context: static string) =
let wait = newFuture[void](context)
fut = wait
try: fsAwait wait
finally: fut = nil
template awake(fp: Future) =
template awake(fp: Future) =
let f = fp
if f != nil and not finished(f):
complete f
proc pipeRead(s: LayeredInputStream,
proc pipeRead(s: LayeredInputStream,
dst: pointer, dstLen: Natural): Future[Natural] {.async.} =
let buffers = s.buffers
if buffers.eofReached: return 0
@ -49,7 +53,7 @@ proc pipeRead(s: LayeredInputStream,
return bytesInBuffersNow - bytesInBuffersAtStart
proc pipeWrite(s: LayeredOutputStream, src: pointer, srcLen: Natural) {.async.} =
proc pipeWrite(s: LayeredOutputStream, src: pointer, srcLen: Natural) {.async.} =
let buffers = s.buffers
while buffers.canAcceptWrite(srcLen) == false:
buffers.waitingWriter.enterWait "waiting for reader to drain the buffers"
@ -60,12 +64,12 @@ proc pipeWrite(s: LayeredOutputStream, src: pointer, srcLen: Natural) {.async.}
awake buffers.waitingReader
describeBuffers "pipeWrite", buffers
template completedFuture(name: static string): untyped =
template completedFuture(name: static string): untyped =
let fut = newFuture[void](name)
complete fut
fut
let pipeInputVTable = InputStreamVTable(
let pipeInputVTable = InputStreamVTable(
readSync: proc (s: InputStream, dst: pointer, dstLen: Natural): Natural
{.nimcall, gcsafe, raises: [IOError, Defect].} =
fsTranslateErrors "Failed to read from pipe":
@ -99,9 +103,9 @@ let pipeInputVTable = InputStreamVTable(
return closeAsync(Async source)
else:
return completedFuture("pipeInput.closeAsync")
)
)
let pipeOutputVTable = OutputStreamVTable(
let pipeOutputVTable = OutputStreamVTable(
writeSync: proc (s: OutputStream, src: pointer, srcLen: Natural)
{.nimcall, gcsafe, raises: [IOError, Defect].} =
fsTranslateErrors "Failed to write all bytes to pipe":
@ -156,9 +160,9 @@ let pipeOutputVTable = OutputStreamVTable(
return closeAsync(Async destination)
else:
return completedFuture("pipeOutput.closeAsync")
)
)
func pipeInput*(source: InputStream,
func pipeInput*(source: InputStream,
pageSize = defaultPageSize,
allowWaitFor = false): AsyncInputStream =
fsAssert pageSize > 0
@ -169,7 +173,7 @@ func pipeInput*(source: InputStream,
allowWaitFor: allowWaitFor,
source: source)
func pipeInput*(buffers: PageBuffers,
func pipeInput*(buffers: PageBuffers,
allowWaitFor = false,
source: InputStream = nil): AsyncInputStream =
var spanEndPos = Natural 0
@ -184,7 +188,7 @@ func pipeInput*(buffers: PageBuffers,
allowWaitFor: allowWaitFor,
source: source)
proc pipeOutput*(destination: OutputStream,
proc pipeOutput*(destination: OutputStream,
pageSize = defaultPageSize,
maxBufferedBytes = defaultPageSize * 4,
allowWaitFor = false): AsyncOutputStream =
@ -202,7 +206,7 @@ proc pipeOutput*(destination: OutputStream,
allowWaitFor: allowWaitFor,
destination: destination)
proc pipeOutput*(buffers: PageBuffers,
proc pipeOutput*(buffers: PageBuffers,
allowWaitFor = false,
destination: OutputStream = nil): AsyncOutputStream =
var span = buffers.getWritableSpan()
@ -218,27 +222,27 @@ proc pipeOutput*(buffers: PageBuffers,
allowWaitFor: allowWaitFor,
destination: destination)
func asyncPipe*(pageSize = defaultPageSize,
func asyncPipe*(pageSize = defaultPageSize,
maxBufferedBytes = defaultPageSize * 4): Pipe =
fsAssert pageSize > 0
Pipe(buffers: initPageBuffers(pageSize, maxBufferedBytes))
func initReader*(pipe: Pipe): AsyncInputStream =
func initReader*(pipe: Pipe): AsyncInputStream =
result = pipeInput(pipe.buffers)
pipe.input = result
func initWriter*(pipe: Pipe): AsyncOutputStream =
func initWriter*(pipe: Pipe): AsyncOutputStream =
result = pipeOutput(pipe.buffers)
pipe.output = result
proc exchangeBuffersAfterPipilineStep(input: InputStream, output: OutputStream) =
proc exchangeBuffersAfterPipilineStep(input: InputStream, output: OutputStream) =
let formerInputBuffers = input.buffers
let formerOutputBuffers = output.getBuffers
input.resetBuffers formerOutputBuffers
output.recycleBuffers formerInputBuffers
macro executePipeline*(start: InputStream, steps: varargs[untyped]): untyped =
macro executePipeline*(start: InputStream, steps: varargs[untyped]): untyped =
result = newTree(nnkStmtListExpr)
var
@ -278,7 +282,7 @@ macro executePipeline*(start: InputStream, steps: varargs[untyped]): untyped =
if defined(debugMacros) or defined(debugPipelines):
echo result.repr
macro executePipeline*(start: AsyncInputStream, steps: varargs[untyped]): untyped =
macro executePipeline*(start: AsyncInputStream, steps: varargs[untyped]): untyped =
var
stream = ident "stream"
pipelineSteps = ident "pipelineSteps"

View file

@ -2,15 +2,11 @@
import
os, unittest2, strutils, random,
stew/ranges/ptr_arith, testutils,
testutils,
../faststreams, ../faststreams/textio
setCurrentDir getAppDir()
proc bytes(s: string): seq[byte] =
result = newSeqOfCap[byte](s.len)
for c in s: result.add byte(c)
proc str(bytes: openarray[byte]): string =
result = newStringOfCap(bytes.len)
for b in items(bytes):
@ -28,7 +24,7 @@ const
asciiTableFile = "files" / "ascii_table.txt"
asciiTableContents = slurp(asciiTableFile)
procSuite "input stream":
suite "input stream":
template emptyInputTests(suiteName, setupCode: untyped) =
suite suiteName & " empty inputs":
setup setupCode

View file

@ -154,6 +154,15 @@ suite "output stream":
checkOutputsMatch()
test "memcpy":
var x = 0x42'u8
nimSeq.add x
memStream.writeMemCopy x
let memStreamRes = memStream.getOutput
check memStreamRes == nimSeq
template undelayedOutput(content: seq[byte]) {.dirty.} =
nimSeq.add content
streamWritingToExistingBuffer.write content

View file

@ -1,24 +1,29 @@
{.used.}
import
testutils/unittests,
# FastStreams modules:
../faststreams/[pipelines, multisync]
when fsAsyncSupport:
import
# Std lib:
std/[strutils, random, base64, terminal],
# Other packages:
testutils/unittests,
# FastStreams modules:
../faststreams/[pipelines, multisync],
# Testing modules:
./base64 as fsBase64
include system/timers
include system/timers
type
type
TestTimes = object
fsPipeline: Nanos
fsAsyncPipeline: Nanos
stdFunctionCalls: Nanos
proc upcaseAllCharacters(i: InputStream, o: OutputStream) {.fsMultiSync.} =
proc upcaseAllCharacters(i: InputStream, o: OutputStream) {.fsMultiSync.} =
let inputLen = i.len
if inputLen.isSome:
o.ensureRunway inputLen.get
@ -28,17 +33,17 @@ proc upcaseAllCharacters(i: InputStream, o: OutputStream) {.fsMultiSync.} =
close o
proc printTimes(t: TestTimes) =
proc printTimes(t: TestTimes) =
styledEcho " cpu time [FS Sync ]: ", styleBright, $t.fsPipeline, "ms"
styledEcho " cpu time [FS Async ]: ", styleBright, $t.fsAsyncPipeline, "ms"
styledEcho " cpu time [Std Lib ]: ", styleBright, $t.stdFunctionCalls, "ms"
template timeit(timerVar: var Nanos, code: untyped) =
template timeit(timerVar: var Nanos, code: untyped) =
let t0 = getTicks()
code
timerVar = int(getTicks() - t0) div 1000000
proc getOutput(sp: AsyncInputStream, T: type string): Future[string] {.async.} =
proc getOutput(sp: AsyncInputStream, T: type string): Future[string] {.async.} =
# this proc is a quick hack to let the test pass
# do not use it in production code
let size = sp.totalUnconsumedBytes()
@ -47,8 +52,8 @@ proc getOutput(sp: AsyncInputStream, T: type string): Future[string] {.async.} =
discard sp.readinto(data)
result = cast[string](data)
procSuite "pipelines":
let loremIpsum = """
suite "pipelines":
const loremIpsum = """
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex
@ -126,3 +131,6 @@ procSuite "pipelines":
let fsAsyncres = await f
check fsAsyncRes == toUpperAscii(inputText)
else:
test "pipelines":
skip