jyapayne/nim-faststreams
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Don't import/export chronos by default (#20)

Browse source 
Chronos support is optional and should not have to be imported in order
to use faststreams for non-async use cases, as doing so pollutes the
global namespace and slows down compilation.

`async` support must now explicitly be enabled with
-d:async_backend=chronos|asyncdispatch
This commit is contained in:
Jacek Sieka 2021-08-20 13:10:26 +02:00 • committed by GitHub
commit 3a0ab42573
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 841 additions and 705 deletions
Download patch file Download diff file Expand all files Collapse all files

9
faststreams.nimble
View file

@ -1,7 +1,7 @@
mode = ScriptMode.Verbose mode = ScriptMode.Verbose
packageName = "faststreams" packageName = "faststreams"
version = "0.2.0" version = "0.3.0"
author = "Status Research & Development GmbH" author = "Status Research & Development GmbH"
description = "Nearly zero-overhead input/output streams for Nim" description = "Nearly zero-overhead input/output streams for Nim"
license = "Apache License 2.0" license = "Apache License 2.0"
@ -21,7 +21,12 @@ proc test(env, path: string) =
lang = getEnv"TEST_LANG" lang = getEnv"TEST_LANG"
exec "nim " & lang & " " & env & exec "nim " & lang & " " & env &
" -r --hints:off --skipParentCfg " & path " -d:async_backend=none -r --hints:off --skipParentCfg " & path
exec "nim " & lang & " " & env &
" -d:async_backend=chronos -r --hints:off --skipParentCfg " & path
# TODO std backend is broken / untested
# exec "nim " & lang & " " & env &
# " -d:async_backend=chronos -r --hints:off --skipParentCfg " & path
task test, "Run all tests": task test, "Run all tests":
test "-d:debug --threads:on", "tests/all_tests" test "-d:debug --threads:on", "tests/all_tests"

21
faststreams/async_backend.nim
View file

@ -1,14 +1,25 @@
const const
faststreams_async_backend {.strdefine.} = "chronos" # To compile with async support, use `-d:async_backend=chronos|asyncdispatch`
async_backend {.strdefine.} = "none"
const
faststreams_async_backend {.strdefine.} = ""
when faststreams_async_backend != "":
{.fatal: "use `-d:async_backend` instead".}
type type
CloseBehavior* = enum CloseBehavior* = enum
waitAsyncClose waitAsyncClose
dontWaitAsyncClose dontWaitAsyncClose
const debugHelpers* = defined(debugHelpers) const
debugHelpers* = defined(debugHelpers)
fsAsyncSupport* = async_backend != "none"
when faststreams_async_backend == "chronos": when async_backend == "none":
discard
elif async_backend == "chronos":
import import
chronos chronos
@ -18,7 +29,7 @@ when faststreams_async_backend == "chronos":
template fsAwait*(f: Future): untyped = template fsAwait*(f: Future): untyped =
await f await f
elif faststreams_async_backend in ["std", "asyncdispatch"]: elif async_backend in ["std", "asyncdispatch"]:
import import
std/asyncdispatch std/asyncdispatch
@ -36,7 +47,7 @@ elif faststreams_async_backend in ["std", "asyncdispatch"]:
type Duration* = int type Duration* = int
else: else:
{.fatal: "Unrecognized network backend: " & faststreams_async_backend.} {.fatal: "Unrecognized network backend: " & async_backend.}
when defined(danger): when defined(danger):
template fsAssert*(x) = discard template fsAssert*(x) = discard

5
faststreams/buffers.nim
View file

@ -22,8 +22,9 @@ type
maxBufferedBytes*: Natural maxBufferedBytes*: Natural
queue*: Deque[PageRef] queue*: Deque[PageRef]
waitingReader*: Future[void] when fsAsyncSupport:
waitingWriter*: Future[void] waitingReader*: Future[void]
waitingWriter*: Future[void]
eofReached*: bool eofReached*: bool
fauxEofPos*: Natural fauxEofPos*: Natural

381
faststreams/inputs.nim
View file

@ -6,16 +6,73 @@ import
export export
options, CloseBehavior options, CloseBehavior
type when fsAsyncSupport:
InputStream* = ref object of RootObj # Circular type refs prevent more targeted `when`
vtable*: ptr InputStreamVTable # This is nil for unsafe memory inputs type
buffers*: PageBuffers # This is nil for unsafe memory inputs InputStream* = ref object of RootObj
span*: PageSpan vtable*: ptr InputStreamVTable # This is nil for unsafe memory inputs
spanEndPos*: Natural buffers*: PageBuffers # This is nil for unsafe memory inputs
closeFut*: Future[void] # This is nil before `close` is called span*: PageSpan
when debugHelpers: spanEndPos*: Natural
name*: string closeFut*: Future[void] # This is nil before `close` is called
when debugHelpers:
name*: string
AsyncInputStream* {.borrow: `.`.} = distinct InputStream
ReadSyncProc* = proc (s: InputStream, dst: pointer, dstLen: Natural): Natural
{.nimcall, gcsafe, raises: [IOError, Defect].}
ReadAsyncProc* = proc (s: InputStream, dst: pointer, dstLen: Natural): Future[Natural]
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseSyncProc* = proc (s: InputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseAsyncProc* = proc (s: InputStream): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].}
GetLenSyncProc* = proc (s: InputStream): Option[Natural]
{.nimcall, gcsafe, raises: [IOError, Defect].}
InputStreamVTable* = object
readSync*: ReadSyncProc
closeSync*: CloseSyncProc
getLenSync*: GetLenSyncProc
readAsync*: ReadAsyncProc
closeAsync*: CloseAsyncProc
MaybeAsyncInputStream* = InputStream | AsyncInputStream
else:
type
InputStream* = ref object of RootObj
vtable*: ptr InputStreamVTable # This is nil for unsafe memory inputs
buffers*: PageBuffers # This is nil for unsafe memory inputs
span*: PageSpan
spanEndPos*: Natural
when debugHelpers:
name*: string
ReadSyncProc* = proc (s: InputStream, dst: pointer, dstLen: Natural): Natural
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseSyncProc* = proc (s: InputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
GetLenSyncProc* = proc (s: InputStream): Option[Natural]
{.nimcall, gcsafe, raises: [IOError, Defect].}
InputStreamVTable* = object
readSync*: ReadSyncProc
closeSync*: CloseSyncProc
getLenSync*: GetLenSyncProc
MaybeAsyncInputStream* = InputStream
type
LayeredInputStream* = ref object of InputStream LayeredInputStream* = ref object of InputStream
source*: InputStream source*: InputStream
allowWaitFor*: bool allowWaitFor*: bool
@ -23,30 +80,6 @@ type
InputStreamHandle* = object InputStreamHandle* = object
s*: InputStream s*: InputStream
AsyncInputStream* {.borrow: `.`.} = distinct InputStream
ReadSyncProc* = proc (s: InputStream, dst: pointer, dstLen: Natural): Natural
{.nimcall, gcsafe, raises: [IOError, Defect].}
ReadAsyncProc* = proc (s: InputStream, dst: pointer, dstLen: Natural): Future[Natural]
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseSyncProc* = proc (s: InputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseAsyncProc* = proc (s: InputStream): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].}
GetLenSyncProc* = proc (s: InputStream): Option[Natural]
{.nimcall, gcsafe, raises: [IOError, Defect].}
InputStreamVTable* = object
readSync*: ReadSyncProc
readAsync*: ReadAsyncProc
closeSync*: CloseSyncProc
closeAsync*: CloseAsyncProc
getLenSync*: GetLenSyncProc
MemFileInputStream = ref object of InputStream MemFileInputStream = ref object of InputStream
file: MemFile file: MemFile
@ -54,30 +87,38 @@ type
file: File file: File
template Sync*(s: InputStream): InputStream = s template Sync*(s: InputStream): InputStream = s
template Async*(s: InputStream): AsyncInputStream = AsyncInputStream(s)
template Sync*(s: AsyncInputStream): InputStream = InputStream(s) when fsAsyncSupport:
template Async*(s: AsyncInputStream): AsyncInputStream = s template Async*(s: InputStream): AsyncInputStream = AsyncInputStream(s)
template Sync*(s: AsyncInputStream): InputStream = InputStream(s)
template Async*(s: AsyncInputStream): AsyncInputStream = s
proc disconnectInputDevice(s: InputStream) = proc disconnectInputDevice(s: InputStream) =
# TODO # TODO
# Document the behavior that closeAsync is preferred # Document the behavior that closeAsync is preferred
if s.vtable != nil: if s.vtable != nil:
if s.vtable.closeAsync != nil: when fsAsyncSupport:
s.closeFut = s.vtable.closeAsync(s) if s.vtable.closeAsync != nil:
elif s.vtable.closeSync != nil: s.closeFut = s.vtable.closeAsync(s)
s.vtable.closeSync(s) elif s.vtable.closeSync != nil:
s.vtable.closeSync(s)
else:
if s.vtable.closeSync != nil:
s.vtable.closeSync(s)
s.vtable = nil s.vtable = nil
template disconnectInputDevice(s: AsyncInputStream) = when fsAsyncSupport:
disconnectInputDevice InputStream(s) template disconnectInputDevice(s: AsyncInputStream) =
disconnectInputDevice InputStream(s)
proc preventFurtherReading(s: InputStream) = proc preventFurtherReading(s: InputStream) =
s.vtable = nil s.vtable = nil
s.span = default(PageSpan) s.span = default(PageSpan)
template preventFurtherReading(s: AsyncInputStream) = when fsAsyncSupport:
preventFurtherReading InputStream(s) template preventFurtherReading(s: AsyncInputStream) =
preventFurtherReading InputStream(s)
template makeHandle*(sp: InputStream): InputStreamHandle = template makeHandle*(sp: InputStream): InputStreamHandle =
let s = sp let s = sp
@ -94,23 +135,25 @@ proc close*(s: InputStream,
## `waitFor` to block until the async operation completes. ## `waitFor` to block until the async operation completes.
s.disconnectInputDevice() s.disconnectInputDevice()
s.preventFurtherReading() s.preventFurtherReading()
if s.closeFut != nil: when fsAsyncSupport:
fsTranslateErrors "Stream closing failed": if s.closeFut != nil:
if behavior == waitAsyncClose: fsTranslateErrors "Stream closing failed":
waitFor s.closeFut if behavior == waitAsyncClose:
else: waitFor s.closeFut
asyncCheck s.closeFut else:
asyncCheck s.closeFut
template close*(sp: AsyncInputStream) = when fsAsyncSupport:
## Starts the asychronous closing of the stream and returns a future that template close*(sp: AsyncInputStream) =
## tracks the closing operation. ## Starts the asychronous closing of the stream and returns a future that
let s = InputStream sp ## tracks the closing operation.
disconnectInputDevice(s) let s = InputStream sp
preventFurtherReading(s) disconnectInputDevice(s)
if s.closeFut != nil: preventFurtherReading(s)
fsAwait s.closeFut if s.closeFut != nil:
fsAwait s.closeFut
template closeNoWait*(sp: AsyncInputStream|InputStream) = template closeNoWait*(sp: MaybeAsyncInputStream) =
## Close the stream without waiting even if's async. ## Close the stream without waiting even if's async.
## This operation will use `asyncCheck` internally to detect unhandled ## This operation will use `asyncCheck` internally to detect unhandled
## errors from the closing operation. ## errors from the closing operation.
@ -224,56 +267,52 @@ proc readableNow*(s: InputStream): bool =
getNewSpan s getNewSpan s
s.span.hasRunway s.span.hasRunway
template readableNow*(s: AsyncInputStream): bool = when fsAsyncSupport:
readableNow InputStream(s) template readableNow*(s: AsyncInputStream): bool =
readableNow InputStream(s)
# TODO: The pure async interface should be moved in a separate module proc readOnce*(sp: AsyncInputStream): Future[Natural] {.async.} =
# to make FastStreams more light-weight when the async back-end let s = InputStream(sp)
# is not used (e.g. in Confutils) fsAssert s.buffers != nil and s.vtable != nil
#
# The problem is that the `async` macro will pull the entire
# event loop right now.
proc readOnce*(sp: AsyncInputStream): Future[Natural] {.async.} = result = fsAwait s.vtable.readAsync(s, nil, 0)
let s = InputStream(sp)
fsAssert s.buffers != nil and s.vtable != nil
result = fsAwait s.vtable.readAsync(s, nil, 0) if s.buffers.eofReached:
disconnectInputDevice(s)
if s.buffers.eofReached: if result > 0 and s.span.len == 0:
disconnectInputDevice(s) getNewSpan s
if result > 0 and s.span.len == 0: proc timeoutToNextByteImpl(s: AsyncInputStream,
getNewSpan s deadline: Future): Future[bool] {.async.} =
let readFut = s.readOnce
fsAwait readFut or deadline
if not readFut.finished:
readFut.cancel()
return true
else:
return false
proc timeoutToNextByteImpl(s: AsyncInputStream, template timeoutToNextByte*(sp: AsyncInputStream, deadline: Future): bool =
deadline: Future): Future[bool] {.async.} = let s = sp
let readFut = s.readOnce if readableNow(s):
fsAwait readFut or deadline true
if not readFut.finished: else:
readFut.cancel() fsAwait timeoutToNextByteImpl(s, deadline)
return true
else:
return false
template timeoutToNextByte*(sp: AsyncInputStream, deadline: Future): bool = template timeoutToNextByte*(sp: AsyncInputStream, timeout: Duration): bool =
let s = sp let s = sp
if readableNow(s): if readableNow(s):
true true
else: else:
fsAwait timeoutToNextByteImpl(s, deadline) fsAwait timeoutToNextByteImpl(s, sleepAsync(timeout))
template timeoutToNextByte*(sp: AsyncInputStream, timeout: Duration): bool = proc closeAsync*(s: AsyncInputStream) {.async.} =
let s = sp close s
if readableNow(s):
true
else:
fsAwait timeoutToNextByteImpl(s, sleepAsync(timeout))
proc closeAsync*(s: AsyncInputStream) {.async.} = template totalUnconsumedBytes*(s: AsyncInputStream): Natural =
close s ## Alias for InputStream.totalUnconsumedBytes
totalUnconsumedBytes InputStream(s)
# TODO: End of purely async interface
func getBestContiguousRunway(s: InputStream): Natural = func getBestContiguousRunway(s: InputStream): Natural =
result = s.span.len result = s.span.len
@ -298,10 +337,6 @@ func totalUnconsumedBytes*(s: InputStream): Natural =
localRunway + runwayInBuffers localRunway + runwayInBuffers
template totalUnconsumedBytes*(s: AsyncInputStream): Natural =
## Alias for InputStream.totalUnconsumedBytes
totalUnconsumedBytes InputStream(s)
proc limitReadableRange(s: InputStream, rangeLen: Natural): Natural = proc limitReadableRange(s: InputStream, rangeLen: Natural): Natural =
s.vtable = nil s.vtable = nil
@ -317,7 +352,7 @@ proc limitReadableRange(s: InputStream, rangeLen: Natural): Natural =
s.buffers.queue.peekFirst.consumedTo -= bytesToUnconsume s.buffers.queue.peekFirst.consumedTo -= bytesToUnconsume
return s.buffers.setFauxEof(s.spanEndPos) return s.buffers.setFauxEof(s.spanEndPos)
template withReadableRange*(sp: InputStream|AsyncInputStream, template withReadableRange*(sp: MaybeAsyncInputStream,
rangeLen: Natural, rangeLen: Natural,
rangeStreamVarName, blk: untyped) = rangeStreamVarName, blk: untyped) =
let let
@ -409,8 +444,9 @@ proc len*(s: InputStream): Option[Natural] {.raises: [Defect, IOError].} =
else: else:
none Natural none Natural
template len*(s: AsyncInputStream): Option[Natural] = when fsAsyncSupport:
len InputStream(s) template len*(s: AsyncInputStream): Option[Natural] =
len InputStream(s)
func memoryInput*(buffers: PageBuffers): InputStreamHandle = func memoryInput*(buffers: PageBuffers): InputStreamHandle =
var spanEndPos = Natural 0 var spanEndPos = Natural 0
@ -541,15 +577,16 @@ template readable*(sp: InputStream): bool =
let s = sp let s = sp
hasRunway(s.span) or bufferMoreDataSync(s) hasRunway(s.span) or bufferMoreDataSync(s)
template readable*(sp: AsyncInputStream): bool = when fsAsyncSupport:
## Async version of `readable`. template readable*(sp: AsyncInputStream): bool =
## The intended API usage is the same. Instead of blocking, an async ## Async version of `readable`.
## stream will use `await` while waiting for more data. ## The intended API usage is the same. Instead of blocking, an async
let s = InputStream sp ## stream will use `await` while waiting for more data.
if hasRunway(s.span): let s = InputStream sp
true if hasRunway(s.span):
else: true
bufferMoreDataImpl(s, fsAwait, readAsync) else:
bufferMoreDataImpl(s, fsAwait, readAsync)
func continueAfterReadN(s: InputStream, func continueAfterReadN(s: InputStream,
runwayBeforeRead, bytesRead: Natural) = runwayBeforeRead, bytesRead: Natural) =
@ -615,15 +652,16 @@ proc readable*(s: InputStream, n: int): bool =
## for futher discussion of this. ## for futher discussion of this.
readableNImpl(s, n, noAwait, readSync) readableNImpl(s, n, noAwait, readSync)
template readable*(sp: AsyncInputStream, np: int): bool = when fsAsyncSupport:
## Async version of `readable(n)`. template readable*(sp: AsyncInputStream, np: int): bool =
## The intended API usage is the same. Instead of blocking, an async ## Async version of `readable(n)`.
## stream will use `await` while waiting for more data. ## The intended API usage is the same. Instead of blocking, an async
let ## stream will use `await` while waiting for more data.
s = InputStream sp let
n = np s = InputStream sp
n = np
readableNImpl(s, n, fsAwait, readAsync) readableNImpl(s, n, fsAwait, readAsync)
template peek*(sp: InputStream): byte = template peek*(sp: InputStream): byte =
let s = sp let s = sp
@ -633,8 +671,9 @@ template peek*(sp: InputStream): byte =
getNewSpanOrDieTrying s getNewSpanOrDieTrying s
s.span.startAddr[] s.span.startAddr[]
template peek*(s: AsyncInputStream): byte = when fsAsyncSupport:
peek InputStream(s) template peek*(s: AsyncInputStream): byte =
peek InputStream(s)
func readFromNewSpan(s: InputStream): byte = func readFromNewSpan(s: InputStream): byte =
getNewSpanOrDieTrying s getNewSpanOrDieTrying s
@ -650,8 +689,9 @@ template read*(sp: InputStream): byte =
else: else:
readFromNewSpan s readFromNewSpan s
template read*(s: AsyncInputStream): byte = when fsAsyncSupport:
read InputStream(s) template read*(s: AsyncInputStream): byte =
read InputStream(s)
proc peekAt*(s: InputStream, pos: int): byte {.inline.} = proc peekAt*(s: InputStream, pos: int): byte {.inline.} =
# TODO implement page flipping # TODO implement page flipping
@ -659,8 +699,9 @@ proc peekAt*(s: InputStream, pos: int): byte {.inline.} =
fsAssert cast[uint](peekHead) < cast[uint](s.span.endAddr) fsAssert cast[uint](peekHead) < cast[uint](s.span.endAddr)
return peekHead[] return peekHead[]
template peekAt*(s: AsyncInputStream, pos: int): byte = when fsAsyncSupport:
peekAt InputStream(s), pos template peekAt*(s: AsyncInputStream, pos: int): byte =
peekAt InputStream(s), pos
proc advance*(s: InputStream) = proc advance*(s: InputStream) =
if hasRunway(s.span): if hasRunway(s.span):
@ -673,11 +714,12 @@ proc advance*(s: InputStream, n: Natural) =
for i in 0 ..< n: for i in 0 ..< n:
advance s advance s
template advance*(s: AsyncInputStream) = when fsAsyncSupport:
advance InputStream(s) template advance*(s: AsyncInputStream) =
advance InputStream(s)
template advance*(s: AsyncInputStream, n: Natural) = template advance*(s: AsyncInputStream, n: Natural) =
advance InputStream(s), n advance InputStream(s), n
proc drainBuffersInto*(s: InputStream, dstAddr: ptr byte, dstLen: Natural): Natural = proc drainBuffersInto*(s: InputStream, dstAddr: ptr byte, dstLen: Natural): Natural =
var var
@ -789,29 +831,30 @@ proc readInto*(s: InputStream, target: var openarray[byte]): bool =
## regarding the number of bytes read, see `readIntoEx`. ## regarding the number of bytes read, see `readIntoEx`.
s.readIntoEx(target) == target.len s.readIntoEx(target) == target.len
template readIntoEx*(sp: AsyncInputStream, dst: var openarray[byte]): int = when fsAsyncSupport:
let s = InputStream(sp) template readIntoEx*(sp: AsyncInputStream, dst: var openarray[byte]): int =
# BEWARE! `openArrayToPair` here is needed to avoid let s = InputStream(sp)
# double evaluation of the `dst` expression: # BEWARE! `openArrayToPair` here is needed to avoid
let (dstAddr, dstLen) = openArrayToPair(dst) # double evaluation of the `dst` expression:
readIntoExImpl(s, dstAddr, dstLen, fsAwait, readAsync) let (dstAddr, dstLen) = openArrayToPair(dst)
readIntoExImpl(s, dstAddr, dstLen, fsAwait, readAsync)
template readInto*(sp: AsyncInputStream, dst: var openarray[byte]): bool = template readInto*(sp: AsyncInputStream, dst: var openarray[byte]): bool =
## Asynchronously read data into the destination buffer. ## Asynchronously read data into the destination buffer.
## ##
## Returns `false` if EOF was reached before the buffer ## Returns `false` if EOF was reached before the buffer
## was fully populated. if you need precise information ## was fully populated. if you need precise information
## regarding the number of bytes read, see `readIntoEx`. ## regarding the number of bytes read, see `readIntoEx`.
## ##
## If there are enough bytes already buffered by the stream, ## If there are enough bytes already buffered by the stream,
## the expression will complete immediately. ## the expression will complete immediately.
## Otherwise, it will await more bytes to become available. ## Otherwise, it will await more bytes to become available.
let s = InputStream(sp) let s = InputStream(sp)
# BEWARE! `openArrayToPair` here is needed to avoid # BEWARE! `openArrayToPair` here is needed to avoid
# double evaluation of the `dst` expression: # double evaluation of the `dst` expression:
let (dstAddr, dstLen) = openArrayToPair(dst) let (dstAddr, dstLen) = openArrayToPair(dst)
readIntoExImpl(s, dstAddr, dstLen, fsAwait, readAsync) == dstLen readIntoExImpl(s, dstAddr, dstLen, fsAwait, readAsync) == dstLen
template useHeapMem(_: Natural) = template useHeapMem(_: Natural) =
var buffer: seq[byte] var buffer: seq[byte]
@ -869,8 +912,9 @@ template read*(sp: InputStream, np: static Natural): openarray[byte] =
template read*(s: InputStream, n: Natural): openarray[byte] = template read*(s: InputStream, n: Natural): openarray[byte] =
readNImpl(s, n, useHeapMem) readNImpl(s, n, useHeapMem)
template read*(s: AsyncInputStream, n: Natural): openarray[byte] = when fsAsyncSupport:
read InputStream(s), n template read*(s: AsyncInputStream, n: Natural): openarray[byte] =
read InputStream(s), n
proc lookAheadMatch*(s: InputStream, data: openarray[byte]): bool = proc lookAheadMatch*(s: InputStream, data: openarray[byte]): bool =
for i in 0 ..< data.len: for i in 0 ..< data.len:
@ -879,23 +923,26 @@ proc lookAheadMatch*(s: InputStream, data: openarray[byte]): bool =
return true return true
template lookAheadMatch*(s: AsyncInputStream, data: openarray[byte]): bool = when fsAsyncSupport:
lookAheadMatch InputStream(s) template lookAheadMatch*(s: AsyncInputStream, data: openarray[byte]): bool =
lookAheadMatch InputStream(s)
proc next*(s: InputStream): Option[byte] = proc next*(s: InputStream): Option[byte] =
if readable(s): if readable(s):
result = some read(s) result = some read(s)
template next*(sp: AsyncInputStream): Option[byte] = when fsAsyncSupport:
let s = sp template next*(sp: AsyncInputStream): Option[byte] =
if readable(s): let s = sp
some read(s) if readable(s):
else: some read(s)
none byte else:
none byte
proc pos*(s: InputStream): int {.inline.} = proc pos*(s: InputStream): int {.inline.} =
s.spanEndPos - s.span.len s.spanEndPos - s.span.len
template pos*(s: AsyncInputStream): int = when fsAsyncSupport:
pos InputStream(s) template pos*(s: AsyncInputStream): int =
pos InputStream(s)

55
faststreams/multisync.nim
View file

@ -1,35 +1,40 @@
import import
stew/shims/macros, async_backend
async_backend, inputs, outputs
export export
async_backend async_backend
macro fsMultiSync*(body: untyped) = when fsAsyncSupport:
# We will produce an identical copy of the annotated proc, import
# but taking async parameters and having the async pragma. stew/shims/macros,
var "."/[inputs, outputs]
asyncProcBody = copy body
asyncProcParams = asyncProcBody[3]
asyncProcBody.addPragma(bindSym"async") macro fsMultiSync*(body: untyped) =
# We will produce an identical copy of the annotated proc,
# but taking async parameters and having the async pragma.
var
asyncProcBody = copy body
asyncProcParams = asyncProcBody[3]
# The return types becomes Future[T] asyncProcBody.addPragma(bindSym"async")
if asyncProcParams[0].kind == nnkEmpty:
asyncProcParams[0] = newTree(nnkBracketExpr, bindSym"Future", ident"void")
else:
asyncProcParams[0] = newTree(nnkBracketExpr, bindSym"Future", asyncProcParams[0])
# We replace all stream inputs with their async counterparts # The return types becomes Future[T]
for i in 1 ..< asyncProcParams.len: if asyncProcParams[0].kind == nnkEmpty:
let paramsDef = asyncProcParams[i] asyncProcParams[0] = newTree(nnkBracketExpr, bindSym"Future", ident"void")
let typ = paramsDef[^2] else:
if eqIdent(typ, "InputStream"): asyncProcParams[0] = newTree(nnkBracketExpr, bindSym"Future", asyncProcParams[0])
paramsDef[^2] = bindSym "AsyncInputStream"
elif eqIdent(typ, "OutputStream"):
paramsDef[^2] = bindSym "AsyncOutputStream"
result = newStmtList(body, asyncProcBody) # We replace all stream inputs with their async counterparts
when defined(debugSupportAsync): for i in 1 ..< asyncProcParams.len:
echo result.repr let paramsDef = asyncProcParams[i]
let typ = paramsDef[^2]
if eqIdent(typ, "InputStream"):
paramsDef[^2] = bindSym "AsyncInputStream"
elif eqIdent(typ, "OutputStream"):
paramsDef[^2] = bindSym "AsyncOutputStream"
result = newStmtList(body, asyncProcBody)
when defined(debugSupportAsync):
echo result.repr
else:
macro fsMultiSync*(body: untyped) = body

288
faststreams/outputs.nim
View file

@ -14,17 +14,78 @@ import
export export
initPageBuffers, CloseBehavior initPageBuffers, CloseBehavior
type when fsAsyncSupport:
OutputStream* = ref object of RootObj # Circular type refs prevent more targeted `when`
vtable*: ptr OutputStreamVTable # This is nil for any memory output type
buffers*: PageBuffers # This is nil for unsafe memory outputs OutputStream* = ref object of RootObj
span*: PageSpan vtable*: ptr OutputStreamVTable # This is nil for any memory output
spanEndPos*: Natural buffers*: PageBuffers # This is nil for unsafe memory outputs
extCursorsCount: int span*: PageSpan
closeFut: Future[void] # This is nil before `close` is called spanEndPos*: Natural
when debugHelpers: extCursorsCount: int
name*: string closeFut: Future[void] # This is nil before `close` is called
when debugHelpers:
name*: string
AsyncOutputStream* {.borrow: `.`.} = distinct OutputStream
WriteSyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural)
{.nimcall, gcsafe, raises: [IOError, Defect].}
WriteAsyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].}
FlushSyncProc* = proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
FlushAsyncProc* = proc (s: OutputStream): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseSyncProc* = proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseAsyncProc* = proc (s: OutputStream): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].}
OutputStreamVTable* = object
writeSync*: WriteSyncProc
writeAsync*: WriteAsyncProc
flushSync*: FlushSyncProc
flushAsync*: FlushAsyncProc
closeSync*: CloseSyncProc
closeAsync*: CloseAsyncProc
MaybeAsyncOutputStream* = OutputStream | AsyncOutputStream
else:
type
OutputStream* = ref object of RootObj
vtable*: ptr OutputStreamVTable # This is nil for any memory output
buffers*: PageBuffers # This is nil for unsafe memory outputs
span*: PageSpan
spanEndPos*: Natural
extCursorsCount: int
when debugHelpers:
name*: string
WriteSyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural)
{.nimcall, gcsafe, raises: [IOError, Defect].}
FlushSyncProc* = proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseSyncProc* = proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
OutputStreamVTable* = object
writeSync*: WriteSyncProc
flushSync*: FlushSyncProc
closeSync*: CloseSyncProc
MaybeAsyncOutputStream* = OutputStream
type
WriteCursor* = object WriteCursor* = object
span: PageSpan span: PageSpan
stream: OutputStream stream: OutputStream
@ -36,55 +97,35 @@ type
OutputStreamHandle* = object OutputStreamHandle* = object
s*: OutputStream s*: OutputStream
AsyncOutputStream* {.borrow: `.`.} = distinct OutputStream
WriteSyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural)
{.nimcall, gcsafe, raises: [IOError, Defect].}
WriteAsyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].}
FlushSyncProc* = proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
FlushAsyncProc* = proc (s: OutputStream): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseSyncProc* = proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseAsyncProc* = proc (s: OutputStream): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].}
OutputStreamVTable* = object
writeSync*: WriteSyncProc
writeAsync*: WriteAsyncProc
flushSync*: FlushSyncProc
flushAsync*: FlushAsyncProc
closeSync*: CloseSyncProc
closeAsync*: CloseAsyncProc
VarSizeWriteCursor* = distinct WriteCursor VarSizeWriteCursor* = distinct WriteCursor
FileOutputStream = ref object of OutputStream FileOutputStream = ref object of OutputStream
file: File file: File
template Sync*(s: OutputStream): OutputStream = s template Sync*(s: OutputStream): OutputStream = s
template Async*(s: OutputStream): AsyncOutputStream = AsyncOutputStream(s)
template Sync*(s: AsyncOutputStream): OutputStream = OutputStream(s) when fsAsyncSupport:
template Async*(s: AsyncOutputStream): AsyncOutputStream = s template Async*(s: OutputStream): AsyncOutputStream = AsyncOutputStream(s)
template Sync*(s: AsyncOutputStream): OutputStream = OutputStream(s)
template Async*(s: AsyncOutputStream): AsyncOutputStream = s
proc disconnectOutputDevice(s: OutputStream) = proc disconnectOutputDevice(s: OutputStream) =
if s.vtable != nil: if s.vtable != nil:
if s.vtable.closeAsync != nil: when fsAsyncSupport:
s.closeFut = s.vtable.closeAsync(s) if s.vtable.closeAsync != nil:
elif s.vtable.closeSync != nil: s.closeFut = s.vtable.closeAsync(s)
s.vtable.closeSync(s) elif s.vtable.closeSync != nil:
s.vtable.closeSync(s)
else:
if s.vtable.closeSync != nil:
s.vtable.closeSync(s)
s.vtable = nil s.vtable = nil
template disconnectOutputDevice(s: AsyncOutputStream) = when fsAsyncSupport:
disconnectOutputDevice OutputStream(s) template disconnectOutputDevice(s: AsyncOutputStream) =
disconnectOutputDevice OutputStream(s)
template flushImpl(s: OutputStream, awaiter, writeOp, flushOp: untyped) = template flushImpl(s: OutputStream, awaiter, writeOp, flushOp: untyped) =
fsAssert s.extCursorsCount == 0 fsAssert s.extCursorsCount == 0
@ -99,36 +140,20 @@ template flushImpl(s: OutputStream, awaiter, writeOp, flushOp: untyped) =
proc flush*(s: OutputStream) = proc flush*(s: OutputStream) =
flushImpl(s, noAwait, writeSync, flushSync) flushImpl(s, noAwait, writeSync, flushSync)
template flush*(sp: AsyncOutputStream) =
let s = OutputStream sp
flushImpl(s, fsAwait, writeAsync, flushAsync)
proc flushAsync*(s: AsyncOutputStream) {.async.} =
flush s
proc close*(s: OutputStream, proc close*(s: OutputStream,
behavior = dontWaitAsyncClose) behavior = dontWaitAsyncClose)
{.raises: [IOError, Defect].} = {.raises: [IOError, Defect].} =
flush s flush s
disconnectOutputDevice(s) disconnectOutputDevice(s)
if s.closeFut != nil: when fsAsyncSupport:
fsTranslateErrors "Stream closing failed": if s.closeFut != nil:
if behavior == waitAsyncClose: fsTranslateErrors "Stream closing failed":
waitFor s.closeFut if behavior == waitAsyncClose:
else: waitFor s.closeFut
asyncCheck s.closeFut else:
asyncCheck s.closeFut
template close*(sp: AsyncOutputStream) = template closeNoWait*(sp: MaybeAsyncOutputStream) =
let s = OutputStream sp
flush(Async s)
disconnectOutputDevice(s)
if s.closeFut != nil:
fsAwait s.closeFut
proc closeAsync*(s: AsyncOutputStream) {.async.} =
close s
template closeNoWait*(sp: AsyncOutputStream|OutputStream) =
## Close the stream without waiting even if's async. ## Close the stream without waiting even if's async.
## This operation will use `asyncCheck` internally to detect unhandled ## This operation will use `asyncCheck` internally to detect unhandled
## errors from the closing operation. ## errors from the closing operation.
@ -196,8 +221,9 @@ proc ensureRunway*(s: OutputStream, neededRunway: Natural) =
s.buffers.ensureRunway(s.span, neededRunway) s.buffers.ensureRunway(s.span, neededRunway)
s.spanEndPos += (s.span.len - runway) s.spanEndPos += (s.span.len - runway)
template ensureRunway*(s: AsyncOutputStream, neededRunway: Natural) = when fsAsyncSupport:
ensureRunway OutputStream(s), neededRunway template ensureRunway*(s: AsyncOutputStream, neededRunway: Natural) =
ensureRunway OutputStream(s), neededRunway
template implementWrites*(buffersParam: PageBuffers, template implementWrites*(buffersParam: PageBuffers,
srcParam: pointer, srcParam: pointer,
@ -266,8 +292,9 @@ proc fileOutput*(filename: string,
proc pos*(s: OutputStream): int = proc pos*(s: OutputStream): int =
s.spanEndPos - s.span.len s.spanEndPos - s.span.len
template pos*(s: AsyncOutputStream): int = when fsAsyncSupport:
pos OutputStream(s) template pos*(s: AsyncOutputStream): int =
pos OutputStream(s)
proc getBuffers*(s: OutputStream): PageBuffers = proc getBuffers*(s: OutputStream): PageBuffers =
fsAssert s.buffers != nil fsAssert s.buffers != nil
@ -313,10 +340,11 @@ proc drainAllBuffersSync(s: OutputStream, buf: pointer, bufSize: Natural) =
s.span = s.buffers.getWritableSpan() s.span = s.buffers.getWritableSpan()
s.spanEndPos += s.span.len s.spanEndPos += s.span.len
proc drainAllBuffersAsync(s: OutputStream, buf: pointer, bufSize: Natural) {.async.} = when fsAsyncSupport:
fsAwait s.vtable.writeAsync(s, buf, bufSize) proc drainAllBuffersAsync(s: OutputStream, buf: pointer, bufSize: Natural) {.async.} =
s.span = s.buffers.getWritableSpan() fsAwait s.vtable.writeAsync(s, buf, bufSize)
s.spanEndPos += s.span.len s.span = s.buffers.getWritableSpan()
s.spanEndPos += s.span.len
proc createCursor(s: OutputStream, size: int): WriteCursor = proc createCursor(s: OutputStream, size: int): WriteCursor =
inc s.extCursorsCount inc s.extCursorsCount
@ -533,21 +561,22 @@ template write*(sp: OutputStream, b: byte) =
else: else:
writeToNewSpan(s, b) writeToNewSpan(s, b)
proc write*(sp: AsyncOutputStream, b: byte) = when fsAsyncSupport:
let s = OutputStream sp proc write*(sp: AsyncOutputStream, b: byte) =
if atEnd(s.span): let s = OutputStream sp
addPage(s) if atEnd(s.span):
writeByte(s.span, b) addPage(s)
template writeAndWait*(sp: AsyncOutputStream, b: byte) =
let s = OutputStream sp
if hasRunway(s.span):
writeByte(s.span, b) writeByte(s.span, b)
else:
writeToNewSpanImpl(s, b, fsAwait, writeAsync, drainAllBuffersAsync)
template write*(s: AsyncOutputStream, x: char) = template writeAndWait*(sp: AsyncOutputStream, b: byte) =
write s, byte(x) let s = OutputStream sp
if hasRunway(s.span):
writeByte(s.span, b)
else:
writeToNewSpanImpl(s, b, fsAwait, writeAsync, drainAllBuffersAsync)
template write*(s: AsyncOutputStream, x: char) =
write s, byte(x)
template write*(s: OutputStream|var WriteCursor, x: char) = template write*(s: OutputStream|var WriteCursor, x: char) =
bind write bind write
@ -606,7 +635,7 @@ proc write*(s: OutputStream, bytes: openArray[byte]) =
proc write*(s: OutputStream, chars: openArray[char]) = proc write*(s: OutputStream, chars: openArray[char]) =
write s, charsToBytes(chars) write s, charsToBytes(chars)
proc write*(s: OutputStream|AsyncOutputStream, value: string) {.inline.} = proc write*(s: MaybeAsyncOutputStream, value: string) {.inline.} =
write s, value.toOpenArrayByte(0, value.len - 1) write s, value.toOpenArrayByte(0, value.len - 1)
template memCopyToBytes(value: auto): untyped = template memCopyToBytes(value: auto): untyped =
@ -618,37 +647,39 @@ template memCopyToBytes(value: auto): untyped =
proc writeMemCopy*(s: OutputStream, value: auto) = proc writeMemCopy*(s: OutputStream, value: auto) =
write s, memCopyToBytes(value) write s, memCopyToBytes(value)
proc writeBytesAsyncImpl(sp: OutputStream, when fsAsyncSupport:
bytes: openarray[byte]): Future[void] = proc writeBytesAsyncImpl(sp: OutputStream,
let s = sp bytes: openarray[byte]): Future[void] =
writeBytesImpl(s, bytes): let s = sp
return s.vtable.writeAsync(s, unsafeAddr bytes[0], bytes.len) writeBytesImpl(s, bytes):
return s.vtable.writeAsync(s, unsafeAddr bytes[0], bytes.len)
proc writeBytesAsyncImpl(s: OutputStream, proc writeBytesAsyncImpl(s: OutputStream,
chars: openarray[char]): Future[void] = chars: openarray[char]): Future[void] =
writeBytesAsyncImpl s, charsToBytes(chars) writeBytesAsyncImpl s, charsToBytes(chars)
proc writeBytesAsyncImpl(s: OutputStream, proc writeBytesAsyncImpl(s: OutputStream,
str: string): Future[void] = str: string): Future[void] =
writeBytesAsyncImpl s, toOpenArray(str, 0, str.len - 1) writeBytesAsyncImpl s, toOpenArray(str, 0, str.len - 1)
template writeAndWait*(s: OutputStream, value: untyped) = template writeAndWait*(s: OutputStream, value: untyped) =
write s, value write s, value
template writeAndWait*(sp: AsyncOutputStream, value: untyped) = when fsAsyncSupport:
bind writeBytesAsyncImpl template writeAndWait*(sp: AsyncOutputStream, value: untyped) =
bind writeBytesAsyncImpl
let let
s = OutputStream sp s = OutputStream sp
f = writeBytesAsyncImpl(s, value) f = writeBytesAsyncImpl(s, value)
if f != nil: if f != nil:
fsAwait(f) fsAwait(f)
s.span = getWritableSpan s.buffers s.span = getWritableSpan s.buffers
s.spanEndPos += s.span.len s.spanEndPos += s.span.len
template writeMemCopyAndWait*(sp: AsyncOutputStream, value: auto) = template writeMemCopyAndWait*(sp: AsyncOutputStream, value: auto) =
writeAndWait(sp, memCopyToBytes(value)) writeAndWait(sp, memCopyToBytes(value))
proc writeBytesToCursor(c: var WriteCursor, bytes: openarray[byte]) = proc writeBytesToCursor(c: var WriteCursor, bytes: openarray[byte]) =
var var
@ -795,9 +826,28 @@ template getOutput*(s: OutputStream, T: type seq[byte]): seq[byte] =
template getOutput*(s: OutputStream): seq[byte] = template getOutput*(s: OutputStream): seq[byte] =
cast[seq[byte]](s.getOutput(string)) cast[seq[byte]](s.getOutput(string))
template getOutput*(s: AsyncOutputStream): seq[byte] = when fsAsyncSupport:
getOutput OutputStream(s) template getOutput*(s: AsyncOutputStream): seq[byte] =
getOutput OutputStream(s)
template getOutput*(s: AsyncOutputStream, T: type): untyped = template getOutput*(s: AsyncOutputStream, T: type): untyped =
getOutput OutputStream(s), T getOutput OutputStream(s), T
when fsAsyncSupport:
template flush*(sp: AsyncOutputStream) =
let s = OutputStream sp
flushImpl(s, fsAwait, writeAsync, flushAsync)
proc flushAsync*(s: AsyncOutputStream) {.async.} =
flush s
template close*(sp: AsyncOutputStream) =
let s = OutputStream sp
flush(Async s)
disconnectOutputDevice(s)
if s.closeFut != nil:
fsAwait s.closeFut
proc closeAsync*(s: AsyncOutputStream) {.async.} =
close s

554
faststreams/pipelines.nim
View file

@ -1,337 +1,341 @@
import import
macros, "."/[inputs, outputs, async_backend]
inputs, outputs, buffers, async_backend
export export
inputs, outputs, async_backend inputs, outputs, async_backend
type when fsAsyncSupport:
Pipe* = ref object import
# TODO: Make these stream handles std/macros,
input*: AsyncInputStream ./buffers
output*: AsyncOutputStream
buffers*: PageBuffers
template enterWait(fut: var Future, context: static string) = type
let wait = newFuture[void](context) Pipe* = ref object
fut = wait # TODO: Make these stream handles
try: fsAwait wait input*: AsyncInputStream
finally: fut = nil output*: AsyncOutputStream
buffers*: PageBuffers
template awake(fp: Future) = template enterWait(fut: var Future, context: static string) =
let f = fp let wait = newFuture[void](context)
if f != nil and not finished(f): fut = wait
complete f try: fsAwait wait
finally: fut = nil
proc pipeRead(s: LayeredInputStream, template awake(fp: Future) =
dst: pointer, dstLen: Natural): Future[Natural] {.async.} = let f = fp
let buffers = s.buffers if f != nil and not finished(f):
if buffers.eofReached: return 0 complete f
var proc pipeRead(s: LayeredInputStream,
bytesInBuffersAtStart = buffers.totalBufferedBytes dst: pointer, dstLen: Natural): Future[Natural] {.async.} =
minBytesExpected = max(1, dstLen) let buffers = s.buffers
bytesInBuffersNow = bytesInBuffersAtStart if buffers.eofReached: return 0
var
bytesInBuffersAtStart = buffers.totalBufferedBytes
minBytesExpected = max(1, dstLen)
bytesInBuffersNow = bytesInBuffersAtStart
while bytesInBuffersNow < minBytesExpected:
awake buffers.waitingWriter
buffers.waitingReader.enterWait "waiting for writer to buffer more data"
bytesInBuffersNow = buffers.totalBufferedBytes
if buffers.eofReached:
return bytesInBuffersNow - bytesInBuffersAtStart
if dst != nil:
let drained {.used.} = drainBuffersInto(s, cast[ptr byte](dst), dstLen)
fsAssert drained == dstLen
while bytesInBuffersNow < minBytesExpected:
awake buffers.waitingWriter awake buffers.waitingWriter
buffers.waitingReader.enterWait "waiting for writer to buffer more data"
bytesInBuffersNow = buffers.totalBufferedBytes return bytesInBuffersNow - bytesInBuffersAtStart
if buffers.eofReached:
return bytesInBuffersNow - bytesInBuffersAtStart
if dst != nil: proc pipeWrite(s: LayeredOutputStream, src: pointer, srcLen: Natural) {.async.} =
let drained {.used.} = drainBuffersInto(s, cast[ptr byte](dst), dstLen) let buffers = s.buffers
fsAssert drained == dstLen while buffers.canAcceptWrite(srcLen) == false:
buffers.waitingWriter.enterWait "waiting for reader to drain the buffers"
awake buffers.waitingWriter if src != nil:
buffers.appendUnbufferedWrite(src, srcLen)
return bytesInBuffersNow - bytesInBuffersAtStart awake buffers.waitingReader
describeBuffers "pipeWrite", buffers
proc pipeWrite(s: LayeredOutputStream, src: pointer, srcLen: Natural) {.async.} = template completedFuture(name: static string): untyped =
let buffers = s.buffers let fut = newFuture[void](name)
while buffers.canAcceptWrite(srcLen) == false: complete fut
buffers.waitingWriter.enterWait "waiting for reader to drain the buffers" fut
if src != nil: let pipeInputVTable = InputStreamVTable(
buffers.appendUnbufferedWrite(src, srcLen) readSync: proc (s: InputStream, dst: pointer, dstLen: Natural): Natural
awake buffers.waitingReader
describeBuffers "pipeWrite", buffers
template completedFuture(name: static string): untyped =
let fut = newFuture[void](name)
complete fut
fut
let pipeInputVTable = InputStreamVTable(
readSync: proc (s: InputStream, dst: pointer, dstLen: Natural): Natural
{.nimcall, gcsafe, raises: [IOError, Defect].} =
fsTranslateErrors "Failed to read from pipe":
let ls = LayeredInputStream(s)
fsAssert ls.allowWaitFor
return waitFor pipeRead(ls, dst, dstLen)
,
readAsync: proc (s: InputStream, dst: pointer, dstLen: Natural): Future[Natural]
{.nimcall, gcsafe, raises: [IOError, Defect].} = {.nimcall, gcsafe, raises: [IOError, Defect].} =
fsTranslateErrors "Unexpected error from the async macro": fsTranslateErrors "Failed to read from pipe":
let ls = LayeredInputStream(s) let ls = LayeredInputStream(s)
return pipeRead(ls, dst, dstLen) fsAssert ls.allowWaitFor
, return waitFor pipeRead(ls, dst, dstLen)
getLenSync: proc (s: InputStream): Option[Natural] ,
{.nimcall, gcsafe, raises: [IOError, Defect].} = readAsync: proc (s: InputStream, dst: pointer, dstLen: Natural): Future[Natural]
let source = LayeredInputStream(s).source {.nimcall, gcsafe, raises: [IOError, Defect].} =
if source != nil: fsTranslateErrors "Unexpected error from the async macro":
return source.len let ls = LayeredInputStream(s)
, return pipeRead(ls, dst, dstLen)
closeSync: proc (s: InputStream) ,
{.nimcall, gcsafe, raises: [IOError, Defect].} = getLenSync: proc (s: InputStream): Option[Natural]
let source = LayeredInputStream(s).source {.nimcall, gcsafe, raises: [IOError, Defect].} =
if source != nil:
close source
,
closeAsync: proc (s: InputStream): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].} =
fsTranslateErrors "Unexpected error from the async macro":
let source = LayeredInputStream(s).source let source = LayeredInputStream(s).source
if source != nil: if source != nil:
return closeAsync(Async source) return source.len
else: ,
return completedFuture("pipeInput.closeAsync") closeSync: proc (s: InputStream)
) {.nimcall, gcsafe, raises: [IOError, Defect].} =
let source = LayeredInputStream(s).source
if source != nil:
close source
,
closeAsync: proc (s: InputStream): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].} =
fsTranslateErrors "Unexpected error from the async macro":
let source = LayeredInputStream(s).source
if source != nil:
return closeAsync(Async source)
else:
return completedFuture("pipeInput.closeAsync")
)
let pipeOutputVTable = OutputStreamVTable( let pipeOutputVTable = OutputStreamVTable(
writeSync: proc (s: OutputStream, src: pointer, srcLen: Natural) writeSync: proc (s: OutputStream, src: pointer, srcLen: Natural)
{.nimcall, gcsafe, raises: [IOError, Defect].} = {.nimcall, gcsafe, raises: [IOError, Defect].} =
fsTranslateErrors "Failed to write all bytes to pipe": fsTranslateErrors "Failed to write all bytes to pipe":
var ls = LayeredOutputStream(s) var ls = LayeredOutputStream(s)
fsAssert ls.allowWaitFor fsAssert ls.allowWaitFor
waitFor pipeWrite(ls, src, srcLen) waitFor pipeWrite(ls, src, srcLen)
, ,
writeAsync: proc (s: OutputStream, src: pointer, srcLen: Natural): Future[void] writeAsync: proc (s: OutputStream, src: pointer, srcLen: Natural): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].} = {.nimcall, gcsafe, raises: [IOError, Defect].} =
# TODO: The async macro is raising exceptions even when # TODO: The async macro is raising exceptions even when
# merely forwarding a future: # merely forwarding a future:
fsTranslateErrors "Unexpected error from the async macro": fsTranslateErrors "Unexpected error from the async macro":
return pipeWrite(LayeredOutputStream s, src, srcLen) return pipeWrite(LayeredOutputStream s, src, srcLen)
, ,
flushSync: proc (s: OutputStream) flushSync: proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].} = {.nimcall, gcsafe, raises: [IOError, Defect].} =
let destination = LayeredOutputStream(s).destination
if destination != nil:
flush destination
,
flushAsync: proc (s: OutputStream): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].} =
fsTranslateErrors "Unexpected error from the async macro":
let destination = LayeredOutputStream(s).destination let destination = LayeredOutputStream(s).destination
if destination != nil: if destination != nil:
return flushAsync(Async destination) flush destination
else: ,
return completedFuture("pipeOutput.flushAsync") flushAsync: proc (s: OutputStream): Future[void]
, {.nimcall, gcsafe, raises: [IOError, Defect].} =
closeSync: proc (s: OutputStream) fsTranslateErrors "Unexpected error from the async macro":
{.nimcall, gcsafe, raises: [IOError, Defect].} = let destination = LayeredOutputStream(s).destination
if destination != nil:
return flushAsync(Async destination)
else:
return completedFuture("pipeOutput.flushAsync")
,
closeSync: proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].} =
s.buffers.eofReached = true s.buffers.eofReached = true
fsTranslateErrors "Unexpected error from Future.complete": fsTranslateErrors "Unexpected error from Future.complete":
awake s.buffers.waitingReader awake s.buffers.waitingReader
let destination = LayeredOutputStream(s).destination
if destination != nil:
close destination
,
closeAsync: proc (s: OutputStream): Future[void]
{.nimcall, gcsafe, raises: [IOError, Defect].} =
s.buffers.eofReached = true
fsTranslateErrors "Unexpected error from Future.complete":
awake s.buffers.waitingReader
fsTranslateErrors "Unexpected error from the async macro":
let destination = LayeredOutputStream(s).destination let destination = LayeredOutputStream(s).destination
if destination != nil: if destination != nil:
return closeAsync(Async destination) close destination
else: ,
return completedFuture("pipeOutput.closeAsync") closeAsync: proc (s: OutputStream): Future[void]
) {.nimcall, gcsafe, raises: [IOError, Defect].} =
s.buffers.eofReached = true
func pipeInput*(source: InputStream, fsTranslateErrors "Unexpected error from Future.complete":
pageSize = defaultPageSize, awake s.buffers.waitingReader
allowWaitFor = false): AsyncInputStream =
fsAssert pageSize > 0
AsyncInputStream LayeredInputStream( fsTranslateErrors "Unexpected error from the async macro":
vtable: vtableAddr pipeInputVTable, let destination = LayeredOutputStream(s).destination
buffers: initPageBuffers pageSize, if destination != nil:
allowWaitFor: allowWaitFor, return closeAsync(Async destination)
source: source) else:
return completedFuture("pipeOutput.closeAsync")
)
func pipeInput*(buffers: PageBuffers, func pipeInput*(source: InputStream,
allowWaitFor = false, pageSize = defaultPageSize,
source: InputStream = nil): AsyncInputStream = allowWaitFor = false): AsyncInputStream =
var spanEndPos = Natural 0 fsAssert pageSize > 0
var span = if buffers.len == 0: default(PageSpan)
else: buffers.obtainReadableSpan(spanEndPos)
AsyncInputStream LayeredInputStream( AsyncInputStream LayeredInputStream(
vtable: vtableAddr pipeInputVTable, vtable: vtableAddr pipeInputVTable,
buffers: buffers, buffers: initPageBuffers pageSize,
span: span, allowWaitFor: allowWaitFor,
spanEndPos: span.len, source: source)
allowWaitFor: allowWaitFor,
source: source)
proc pipeOutput*(destination: OutputStream, func pipeInput*(buffers: PageBuffers,
pageSize = defaultPageSize, allowWaitFor = false,
maxBufferedBytes = defaultPageSize * 4, source: InputStream = nil): AsyncInputStream =
allowWaitFor = false): AsyncOutputStream = var spanEndPos = Natural 0
fsAssert pageSize > 0 var span = if buffers.len == 0: default(PageSpan)
else: buffers.obtainReadableSpan(spanEndPos)
var AsyncInputStream LayeredInputStream(
buffers = initPageBuffers pageSize vtable: vtableAddr pipeInputVTable,
span = buffers.getWritableSpan() buffers: buffers,
span: span,
spanEndPos: span.len,
allowWaitFor: allowWaitFor,
source: source)
AsyncOutputStream LayeredOutputStream( proc pipeOutput*(destination: OutputStream,
vtable: vtableAddr pipeOutputVTable, pageSize = defaultPageSize,
buffers: buffers, maxBufferedBytes = defaultPageSize * 4,
span: span, allowWaitFor = false): AsyncOutputStream =
spanEndPos: span.len, fsAssert pageSize > 0
allowWaitFor: allowWaitFor,
destination: destination)
proc pipeOutput*(buffers: PageBuffers,
allowWaitFor = false,
destination: OutputStream = nil): AsyncOutputStream =
var span = buffers.getWritableSpan()
AsyncOutputStream LayeredOutputStream(
vtable: vtableAddr pipeOutputVTable,
buffers: buffers,
span: span,
# TODO What if the buffers are partially populated?
# Should we adjust the spanEndPos? This would
# need the old buffers.totalBytesWritten var.
spanEndPos: span.len,
allowWaitFor: allowWaitFor,
destination: destination)
func asyncPipe*(pageSize = defaultPageSize,
maxBufferedBytes = defaultPageSize * 4): Pipe =
fsAssert pageSize > 0
Pipe(buffers: initPageBuffers(pageSize, maxBufferedBytes))
func initReader*(pipe: Pipe): AsyncInputStream =
result = pipeInput(pipe.buffers)
pipe.input = result
func initWriter*(pipe: Pipe): AsyncOutputStream =
result = pipeOutput(pipe.buffers)
pipe.output = result
proc exchangeBuffersAfterPipilineStep(input: InputStream, output: OutputStream) =
let formerInputBuffers = input.buffers
let formerOutputBuffers = output.getBuffers
input.resetBuffers formerOutputBuffers
output.recycleBuffers formerInputBuffers
macro executePipeline*(start: InputStream, steps: varargs[untyped]): untyped =
result = newTree(nnkStmtListExpr)
var
inputVal = start
outputVal = newCall(bindSym"memoryOutput")
inputVar = genSym(nskVar, "input")
outputVar = genSym(nskVar, "output")
step0 = steps[0]
result.add quote do:
var var
`inputVar` = `inputVal` buffers = initPageBuffers pageSize
`outputVar` = OutputStream `outputVal` span = buffers.getWritableSpan()
`step0`(`inputVar`, `outputVar`) AsyncOutputStream LayeredOutputStream(
vtable: vtableAddr pipeOutputVTable,
buffers: buffers,
span: span,
spanEndPos: span.len,
allowWaitFor: allowWaitFor,
destination: destination)
proc pipeOutput*(buffers: PageBuffers,
allowWaitFor = false,
destination: OutputStream = nil): AsyncOutputStream =
var span = buffers.getWritableSpan()
AsyncOutputStream LayeredOutputStream(
vtable: vtableAddr pipeOutputVTable,
buffers: buffers,
span: span,
# TODO What if the buffers are partially populated?
# Should we adjust the spanEndPos? This would
# need the old buffers.totalBytesWritten var.
spanEndPos: span.len,
allowWaitFor: allowWaitFor,
destination: destination)
func asyncPipe*(pageSize = defaultPageSize,
maxBufferedBytes = defaultPageSize * 4): Pipe =
fsAssert pageSize > 0
Pipe(buffers: initPageBuffers(pageSize, maxBufferedBytes))
func initReader*(pipe: Pipe): AsyncInputStream =
result = pipeInput(pipe.buffers)
pipe.input = result
func initWriter*(pipe: Pipe): AsyncOutputStream =
result = pipeOutput(pipe.buffers)
pipe.output = result
proc exchangeBuffersAfterPipilineStep(input: InputStream, output: OutputStream) =
let formerInputBuffers = input.buffers
let formerOutputBuffers = output.getBuffers
input.resetBuffers formerOutputBuffers
output.recycleBuffers formerInputBuffers
macro executePipeline*(start: InputStream, steps: varargs[untyped]): untyped =
result = newTree(nnkStmtListExpr)
var
inputVal = start
outputVal = newCall(bindSym"memoryOutput")
inputVar = genSym(nskVar, "input")
outputVar = genSym(nskVar, "output")
step0 = steps[0]
if steps.len > 2:
let step1 = steps[1]
result.add quote do: result.add quote do:
let formerInputBuffers = `inputVar`.buffers var
`inputVar` = memoryInput(getBuffers `outputVar`) `inputVar` = `inputVal`
recycleBuffers(`outputVar`, formerInputBuffers) `outputVar` = OutputStream `outputVal`
`step1`(`inputVar`, `outputVar`)
for i in 2 .. steps.len - 2: `step0`(`inputVar`, `outputVar`)
let step = steps[i]
result.add quote do:
exchangeBuffersAfterPipilineStep(`inputVar`, `outputVar`)
`step`(`inputVar`, `outputVar`)
var closingCall = steps[^1] if steps.len > 2:
closingCall.insert(1, outputVar) let step1 = steps[1]
result.add closingCall result.add quote do:
let formerInputBuffers = `inputVar`.buffers
`inputVar` = memoryInput(getBuffers `outputVar`)
recycleBuffers(`outputVar`, formerInputBuffers)
`step1`(`inputVar`, `outputVar`)
if defined(debugMacros) or defined(debugPipelines): for i in 2 .. steps.len - 2:
echo result.repr let step = steps[i]
result.add quote do:
exchangeBuffersAfterPipilineStep(`inputVar`, `outputVar`)
`step`(`inputVar`, `outputVar`)
macro executePipeline*(start: AsyncInputStream, steps: varargs[untyped]): untyped = var closingCall = steps[^1]
var closingCall.insert(1, outputVar)
stream = ident "stream" result.add closingCall
pipelineSteps = ident "pipelineSteps"
pipelineBody = newTree(nnkStmtList)
step0 = steps[0] if defined(debugMacros) or defined(debugPipelines):
stepOutput = genSym(nskVar, "pipe") echo result.repr
pipelineBody.add quote do: macro executePipeline*(start: AsyncInputStream, steps: varargs[untyped]): untyped =
var `pipelineSteps` = newSeq[Future[void]]() var
var `stepOutput` = asyncPipe() stream = ident "stream"
add `pipelineSteps`, `step0`(`stream`, initWriter(`stepOutput`)) pipelineSteps = ident "pipelineSteps"
pipelineBody = newTree(nnkStmtList)
var step0 = steps[0]
stepInput = stepOutput stepOutput = genSym(nskVar, "pipe")
for i in 1 .. steps.len - 2:
var step = steps[i]
stepOutput = genSym(nskVar, "pipe")
pipelineBody.add quote do: pipelineBody.add quote do:
var `pipelineSteps` = newSeq[Future[void]]()
var `stepOutput` = asyncPipe() var `stepOutput` = asyncPipe()
add `pipelineSteps`, `step`(initReader(`stepInput`), initWriter(`stepOutput`)) add `pipelineSteps`, `step0`(`stream`, initWriter(`stepOutput`))
stepInput = stepOutput var
stepInput = stepOutput
var RetTypeExpr = copy steps[^1] for i in 1 .. steps.len - 2:
RetTypeExpr.insert(1, newCall("default", ident"AsyncInputStream")) var step = steps[i]
stepOutput = genSym(nskVar, "pipe")
var closingCall = steps[^1] pipelineBody.add quote do:
closingCall.insert(1, newCall(bindSym"initReader", stepInput)) var `stepOutput` = asyncPipe()
add `pipelineSteps`, `step`(initReader(`stepInput`), initWriter(`stepOutput`))
pipelineBody.add quote do: stepInput = stepOutput
fsAwait allFutures(`pipelineSteps`)
`closingCall`
result = quote do: var RetTypeExpr = copy steps[^1]
type UserOpRetType = type(`RetTypeExpr`) RetTypeExpr.insert(1, newCall("default", ident"AsyncInputStream"))
when UserOpRetType is Future: var closingCall = steps[^1]
type RetType = type(default(UserOpRetType).read) closingCall.insert(1, newCall(bindSym"initReader", stepInput))
else:
type RetType = UserOpRetType pipelineBody.add quote do:
fsAwait allFutures(`pipelineSteps`)
`closingCall`
result = quote do:
type UserOpRetType = type(`RetTypeExpr`)
proc pipelineProc(`stream`: AsyncInputStream): Future[RetType] {.async.} =
when UserOpRetType is Future: when UserOpRetType is Future:
var f = `pipelineBody` type RetType = type(default(UserOpRetType).read)
return fsAwait(f)
else: else:
return `pipelineBody` type RetType = UserOpRetType
pipelineProc(`start`) proc pipelineProc(`stream`: AsyncInputStream): Future[RetType] {.async.} =
when UserOpRetType is Future:
var f = `pipelineBody`
return fsAwait(f)
else:
return `pipelineBody`
when defined(debugMacros): pipelineProc(`start`)
echo result.repr
when defined(debugMacros):
echo result.repr

8
tests/test_inputs.nim
View file

@ -2,15 +2,11 @@
import import
os, unittest2, strutils, random, os, unittest2, strutils, random,
stew/ranges/ptr_arith, testutils, testutils,
../faststreams, ../faststreams/textio ../faststreams, ../faststreams/textio
setCurrentDir getAppDir() setCurrentDir getAppDir()
proc bytes(s: string): seq[byte] =
result = newSeqOfCap[byte](s.len)
for c in s: result.add byte(c)
proc str(bytes: openarray[byte]): string = proc str(bytes: openarray[byte]): string =
result = newStringOfCap(bytes.len) result = newStringOfCap(bytes.len)
for b in items(bytes): for b in items(bytes):
@ -28,7 +24,7 @@ const
asciiTableFile = "files" / "ascii_table.txt" asciiTableFile = "files" / "ascii_table.txt"
asciiTableContents = slurp(asciiTableFile) asciiTableContents = slurp(asciiTableFile)
procSuite "input stream": suite "input stream":
template emptyInputTests(suiteName, setupCode: untyped) = template emptyInputTests(suiteName, setupCode: untyped) =
suite suiteName & " empty inputs": suite suiteName & " empty inputs":
setup setupCode setup setupCode

9
tests/test_outputs.nim
View file

@ -154,6 +154,15 @@ suite "output stream":
checkOutputsMatch() checkOutputsMatch()
test "memcpy":
var x = 0x42'u8
nimSeq.add x
memStream.writeMemCopy x
let memStreamRes = memStream.getOutput
check memStreamRes == nimSeq
template undelayedOutput(content: seq[byte]) {.dirty.} = template undelayedOutput(content: seq[byte]) {.dirty.} =
nimSeq.add content nimSeq.add content
streamWritingToExistingBuffer.write content streamWritingToExistingBuffer.write content

214
tests/test_pipelines.nim
View file

@ -1,128 +1,136 @@
{.used.} {.used.}
import import
# Std lib:
std/[strutils, random, base64, terminal],
# Other packages:
testutils/unittests, testutils/unittests,
# FastStreams modules: # FastStreams modules:
../faststreams/[pipelines, multisync], ../faststreams/[pipelines, multisync]
# Testing modules:
./base64 as fsBase64
include system/timers when fsAsyncSupport:
import
# Std lib:
std/[strutils, random, base64, terminal],
# FastStreams modules:
../faststreams/[pipelines, multisync],
# Testing modules:
./base64 as fsBase64
type include system/timers
TestTimes = object
fsPipeline: Nanos
fsAsyncPipeline: Nanos
stdFunctionCalls: Nanos
proc upcaseAllCharacters(i: InputStream, o: OutputStream) {.fsMultiSync.} = type
let inputLen = i.len TestTimes = object
if inputLen.isSome: fsPipeline: Nanos
o.ensureRunway inputLen.get fsAsyncPipeline: Nanos
stdFunctionCalls: Nanos
while i.readable: proc upcaseAllCharacters(i: InputStream, o: OutputStream) {.fsMultiSync.} =
o.write toUpperAscii(i.read.char) let inputLen = i.len
if inputLen.isSome:
o.ensureRunway inputLen.get
close o while i.readable:
o.write toUpperAscii(i.read.char)
proc printTimes(t: TestTimes) = close o
styledEcho " cpu time [FS Sync ]: ", styleBright, $t.fsPipeline, "ms"
styledEcho " cpu time [FS Async ]: ", styleBright, $t.fsAsyncPipeline, "ms"
styledEcho " cpu time [Std Lib ]: ", styleBright, $t.stdFunctionCalls, "ms"
template timeit(timerVar: var Nanos, code: untyped) = proc printTimes(t: TestTimes) =
let t0 = getTicks() styledEcho " cpu time [FS Sync ]: ", styleBright, $t.fsPipeline, "ms"
code styledEcho " cpu time [FS Async ]: ", styleBright, $t.fsAsyncPipeline, "ms"
timerVar = int(getTicks() - t0) div 1000000 styledEcho " cpu time [Std Lib ]: ", styleBright, $t.stdFunctionCalls, "ms"
proc getOutput(sp: AsyncInputStream, T: type string): Future[string] {.async.} = template timeit(timerVar: var Nanos, code: untyped) =
# this proc is a quick hack to let the test pass let t0 = getTicks()
# do not use it in production code code
let size = sp.totalUnconsumedBytes() timerVar = int(getTicks() - t0) div 1000000
if size > 0:
var data = newSeq[byte](size)
discard sp.readinto(data)
result = cast[string](data)
procSuite "pipelines": proc getOutput(sp: AsyncInputStream, T: type string): Future[string] {.async.} =
let loremIpsum = """ # this proc is a quick hack to let the test pass
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod # do not use it in production code
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim let size = sp.totalUnconsumedBytes()
veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex if size > 0:
ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate var data = newSeq[byte](size)
velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat discard sp.readinto(data)
cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id result = cast[string](data)
est laborum.
""" suite "pipelines":
const loremIpsum = """
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex
ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate
velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat
cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id
est laborum.
test "upper-case/base64 pipeline benchmark": """
var
times: TestTimes
stdRes: string
fsRes: string
fsAsyncRes: string
let inputText = loremIpsum.repeat(5000) test "upper-case/base64 pipeline benchmark":
var
times: TestTimes
stdRes: string
fsRes: string
fsAsyncRes: string
timeIt times.stdFunctionCalls: let inputText = loremIpsum.repeat(5000)
stdRes = base64.decode(base64.encode(toUpperAscii(inputText)))
timeIt times.fsPipeline: timeIt times.stdFunctionCalls:
fsRes = executePipeline(unsafeMemoryInput(inputText), stdRes = base64.decode(base64.encode(toUpperAscii(inputText)))
timeIt times.fsPipeline:
fsRes = executePipeline(unsafeMemoryInput(inputText),
upcaseAllCharacters,
base64encode,
base64decode,
getOutput string)
timeIt times.fsAsyncPipeline:
fsAsyncRes = waitFor executePipeline(Async unsafeMemoryInput(inputText),
upcaseAllCharacters,
base64encode,
base64decode,
getOutput string)
check fsAsyncRes == stdRes
check fsRes == stdRes
printTimes times
asyncTest "upper-case/base64 async pipeline":
let pipe = asyncPipe()
let inputText = repeat(loremIpsum, 100)
proc pipeFeeder(s: AsyncOutputStream) {.gcsafe, async.} =
randomize 1234
var pos = 0
while pos != inputText.len:
let bytesToWrite = rand(15)
if bytesToWrite == 0:
s.write inputText[pos]
inc pos
else:
let endPos = min(pos + bytesToWrite, inputText.len)
s.writeAndWait inputText[pos ..< endPos]
pos = endPos
let sleep = rand(50) - 45
if sleep > 0:
await sleepAsync(sleep.milliseconds)
close s
asyncCheck pipeFeeder(pipe.initWriter)
let f = executePipeline(pipe.initReader,
upcaseAllCharacters, upcaseAllCharacters,
base64encode, base64encode,
base64decode, base64decode,
getOutput string) getOutput string)
timeIt times.fsAsyncPipeline: let fsAsyncres = await f
fsAsyncRes = waitFor executePipeline(Async unsafeMemoryInput(inputText),
upcaseAllCharacters,
base64encode,
base64decode,
getOutput string)
check fsAsyncRes == stdRes check fsAsyncRes == toUpperAscii(inputText)
check fsRes == stdRes else:
test "pipelines":
printTimes times skip
asyncTest "upper-case/base64 async pipeline":
let pipe = asyncPipe()
let inputText = repeat(loremIpsum, 100)
proc pipeFeeder(s: AsyncOutputStream) {.gcsafe, async.} =
randomize 1234
var pos = 0
while pos != inputText.len:
let bytesToWrite = rand(15)
if bytesToWrite == 0:
s.write inputText[pos]
inc pos
else:
let endPos = min(pos + bytesToWrite, inputText.len)
s.writeAndWait inputText[pos ..< endPos]
pos = endPos
let sleep = rand(50) - 45
if sleep > 0:
await sleepAsync(sleep.milliseconds)
close s
asyncCheck pipeFeeder(pipe.initWriter)
let f = executePipeline(pipe.initReader,
upcaseAllCharacters,
base64encode,
base64decode,
getOutput string)
let fsAsyncres = await f
check fsAsyncRes == toUpperAscii(inputText)