Don't import/export chronos by default (#20)
Chronos support is optional and should not have to be imported in order to use faststreams for non-async use cases, as doing so pollutes the global namespace and slows down compilation. `async` support must now explicitly be enabled with -d:async_backend=chronos|asyncdispatch
This commit is contained in:
parent
5eb7fd0c90
commit
3a0ab42573
10 changed files with 841 additions and 705 deletions
|
|
@ -1,7 +1,7 @@
|
||||||
mode = ScriptMode.Verbose
|
mode = ScriptMode.Verbose
|
||||||
|
|
||||||
packageName = "faststreams"
|
packageName = "faststreams"
|
||||||
version = "0.2.0"
|
version = "0.3.0"
|
||||||
author = "Status Research & Development GmbH"
|
author = "Status Research & Development GmbH"
|
||||||
description = "Nearly zero-overhead input/output streams for Nim"
|
description = "Nearly zero-overhead input/output streams for Nim"
|
||||||
license = "Apache License 2.0"
|
license = "Apache License 2.0"
|
||||||
|
|
@ -21,7 +21,12 @@ proc test(env, path: string) =
|
||||||
lang = getEnv"TEST_LANG"
|
lang = getEnv"TEST_LANG"
|
||||||
|
|
||||||
exec "nim " & lang & " " & env &
|
exec "nim " & lang & " " & env &
|
||||||
" -r --hints:off --skipParentCfg " & path
|
" -d:async_backend=none -r --hints:off --skipParentCfg " & path
|
||||||
|
exec "nim " & lang & " " & env &
|
||||||
|
" -d:async_backend=chronos -r --hints:off --skipParentCfg " & path
|
||||||
|
# TODO std backend is broken / untested
|
||||||
|
# exec "nim " & lang & " " & env &
|
||||||
|
# " -d:async_backend=chronos -r --hints:off --skipParentCfg " & path
|
||||||
|
|
||||||
task test, "Run all tests":
|
task test, "Run all tests":
|
||||||
test "-d:debug --threads:on", "tests/all_tests"
|
test "-d:debug --threads:on", "tests/all_tests"
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,25 @@
|
||||||
const
|
const
|
||||||
faststreams_async_backend {.strdefine.} = "chronos"
|
# To compile with async support, use `-d:async_backend=chronos|asyncdispatch`
|
||||||
|
async_backend {.strdefine.} = "none"
|
||||||
|
|
||||||
|
const
|
||||||
|
faststreams_async_backend {.strdefine.} = ""
|
||||||
|
|
||||||
|
when faststreams_async_backend != "":
|
||||||
|
{.fatal: "use `-d:async_backend` instead".}
|
||||||
|
|
||||||
type
|
type
|
||||||
CloseBehavior* = enum
|
CloseBehavior* = enum
|
||||||
waitAsyncClose
|
waitAsyncClose
|
||||||
dontWaitAsyncClose
|
dontWaitAsyncClose
|
||||||
|
|
||||||
const debugHelpers* = defined(debugHelpers)
|
const
|
||||||
|
debugHelpers* = defined(debugHelpers)
|
||||||
|
fsAsyncSupport* = async_backend != "none"
|
||||||
|
|
||||||
when faststreams_async_backend == "chronos":
|
when async_backend == "none":
|
||||||
|
discard
|
||||||
|
elif async_backend == "chronos":
|
||||||
import
|
import
|
||||||
chronos
|
chronos
|
||||||
|
|
||||||
|
|
@ -18,7 +29,7 @@ when faststreams_async_backend == "chronos":
|
||||||
template fsAwait*(f: Future): untyped =
|
template fsAwait*(f: Future): untyped =
|
||||||
await f
|
await f
|
||||||
|
|
||||||
elif faststreams_async_backend in ["std", "asyncdispatch"]:
|
elif async_backend in ["std", "asyncdispatch"]:
|
||||||
import
|
import
|
||||||
std/asyncdispatch
|
std/asyncdispatch
|
||||||
|
|
||||||
|
|
@ -36,7 +47,7 @@ elif faststreams_async_backend in ["std", "asyncdispatch"]:
|
||||||
type Duration* = int
|
type Duration* = int
|
||||||
|
|
||||||
else:
|
else:
|
||||||
{.fatal: "Unrecognized network backend: " & faststreams_async_backend.}
|
{.fatal: "Unrecognized network backend: " & async_backend.}
|
||||||
|
|
||||||
when defined(danger):
|
when defined(danger):
|
||||||
template fsAssert*(x) = discard
|
template fsAssert*(x) = discard
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,7 @@ type
|
||||||
maxBufferedBytes*: Natural
|
maxBufferedBytes*: Natural
|
||||||
|
|
||||||
queue*: Deque[PageRef]
|
queue*: Deque[PageRef]
|
||||||
|
when fsAsyncSupport:
|
||||||
waitingReader*: Future[void]
|
waitingReader*: Future[void]
|
||||||
waitingWriter*: Future[void]
|
waitingWriter*: Future[void]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,8 @@ import
|
||||||
export
|
export
|
||||||
options, CloseBehavior
|
options, CloseBehavior
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
|
# Circular type refs prevent more targeted `when`
|
||||||
type
|
type
|
||||||
InputStream* = ref object of RootObj
|
InputStream* = ref object of RootObj
|
||||||
vtable*: ptr InputStreamVTable # This is nil for unsafe memory inputs
|
vtable*: ptr InputStreamVTable # This is nil for unsafe memory inputs
|
||||||
|
|
@ -16,12 +18,6 @@ type
|
||||||
when debugHelpers:
|
when debugHelpers:
|
||||||
name*: string
|
name*: string
|
||||||
|
|
||||||
LayeredInputStream* = ref object of InputStream
|
|
||||||
source*: InputStream
|
|
||||||
allowWaitFor*: bool
|
|
||||||
|
|
||||||
InputStreamHandle* = object
|
|
||||||
s*: InputStream
|
|
||||||
|
|
||||||
AsyncInputStream* {.borrow: `.`.} = distinct InputStream
|
AsyncInputStream* {.borrow: `.`.} = distinct InputStream
|
||||||
|
|
||||||
|
|
@ -42,10 +38,47 @@ type
|
||||||
|
|
||||||
InputStreamVTable* = object
|
InputStreamVTable* = object
|
||||||
readSync*: ReadSyncProc
|
readSync*: ReadSyncProc
|
||||||
readAsync*: ReadAsyncProc
|
|
||||||
closeSync*: CloseSyncProc
|
closeSync*: CloseSyncProc
|
||||||
closeAsync*: CloseAsyncProc
|
|
||||||
getLenSync*: GetLenSyncProc
|
getLenSync*: GetLenSyncProc
|
||||||
|
readAsync*: ReadAsyncProc
|
||||||
|
closeAsync*: CloseAsyncProc
|
||||||
|
|
||||||
|
MaybeAsyncInputStream* = InputStream | AsyncInputStream
|
||||||
|
|
||||||
|
else:
|
||||||
|
type
|
||||||
|
InputStream* = ref object of RootObj
|
||||||
|
vtable*: ptr InputStreamVTable # This is nil for unsafe memory inputs
|
||||||
|
buffers*: PageBuffers # This is nil for unsafe memory inputs
|
||||||
|
span*: PageSpan
|
||||||
|
spanEndPos*: Natural
|
||||||
|
when debugHelpers:
|
||||||
|
name*: string
|
||||||
|
|
||||||
|
|
||||||
|
ReadSyncProc* = proc (s: InputStream, dst: pointer, dstLen: Natural): Natural
|
||||||
|
{.nimcall, gcsafe, raises: [IOError, Defect].}
|
||||||
|
|
||||||
|
CloseSyncProc* = proc (s: InputStream)
|
||||||
|
{.nimcall, gcsafe, raises: [IOError, Defect].}
|
||||||
|
|
||||||
|
GetLenSyncProc* = proc (s: InputStream): Option[Natural]
|
||||||
|
{.nimcall, gcsafe, raises: [IOError, Defect].}
|
||||||
|
|
||||||
|
InputStreamVTable* = object
|
||||||
|
readSync*: ReadSyncProc
|
||||||
|
closeSync*: CloseSyncProc
|
||||||
|
getLenSync*: GetLenSyncProc
|
||||||
|
|
||||||
|
MaybeAsyncInputStream* = InputStream
|
||||||
|
|
||||||
|
type
|
||||||
|
LayeredInputStream* = ref object of InputStream
|
||||||
|
source*: InputStream
|
||||||
|
allowWaitFor*: bool
|
||||||
|
|
||||||
|
InputStreamHandle* = object
|
||||||
|
s*: InputStream
|
||||||
|
|
||||||
MemFileInputStream = ref object of InputStream
|
MemFileInputStream = ref object of InputStream
|
||||||
file: MemFile
|
file: MemFile
|
||||||
|
|
@ -54,6 +87,8 @@ type
|
||||||
file: File
|
file: File
|
||||||
|
|
||||||
template Sync*(s: InputStream): InputStream = s
|
template Sync*(s: InputStream): InputStream = s
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template Async*(s: InputStream): AsyncInputStream = AsyncInputStream(s)
|
template Async*(s: InputStream): AsyncInputStream = AsyncInputStream(s)
|
||||||
|
|
||||||
template Sync*(s: AsyncInputStream): InputStream = InputStream(s)
|
template Sync*(s: AsyncInputStream): InputStream = InputStream(s)
|
||||||
|
|
@ -63,12 +98,17 @@ proc disconnectInputDevice(s: InputStream) =
|
||||||
# TODO
|
# TODO
|
||||||
# Document the behavior that closeAsync is preferred
|
# Document the behavior that closeAsync is preferred
|
||||||
if s.vtable != nil:
|
if s.vtable != nil:
|
||||||
|
when fsAsyncSupport:
|
||||||
if s.vtable.closeAsync != nil:
|
if s.vtable.closeAsync != nil:
|
||||||
s.closeFut = s.vtable.closeAsync(s)
|
s.closeFut = s.vtable.closeAsync(s)
|
||||||
elif s.vtable.closeSync != nil:
|
elif s.vtable.closeSync != nil:
|
||||||
s.vtable.closeSync(s)
|
s.vtable.closeSync(s)
|
||||||
|
else:
|
||||||
|
if s.vtable.closeSync != nil:
|
||||||
|
s.vtable.closeSync(s)
|
||||||
s.vtable = nil
|
s.vtable = nil
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template disconnectInputDevice(s: AsyncInputStream) =
|
template disconnectInputDevice(s: AsyncInputStream) =
|
||||||
disconnectInputDevice InputStream(s)
|
disconnectInputDevice InputStream(s)
|
||||||
|
|
||||||
|
|
@ -76,6 +116,7 @@ proc preventFurtherReading(s: InputStream) =
|
||||||
s.vtable = nil
|
s.vtable = nil
|
||||||
s.span = default(PageSpan)
|
s.span = default(PageSpan)
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template preventFurtherReading(s: AsyncInputStream) =
|
template preventFurtherReading(s: AsyncInputStream) =
|
||||||
preventFurtherReading InputStream(s)
|
preventFurtherReading InputStream(s)
|
||||||
|
|
||||||
|
|
@ -94,6 +135,7 @@ proc close*(s: InputStream,
|
||||||
## `waitFor` to block until the async operation completes.
|
## `waitFor` to block until the async operation completes.
|
||||||
s.disconnectInputDevice()
|
s.disconnectInputDevice()
|
||||||
s.preventFurtherReading()
|
s.preventFurtherReading()
|
||||||
|
when fsAsyncSupport:
|
||||||
if s.closeFut != nil:
|
if s.closeFut != nil:
|
||||||
fsTranslateErrors "Stream closing failed":
|
fsTranslateErrors "Stream closing failed":
|
||||||
if behavior == waitAsyncClose:
|
if behavior == waitAsyncClose:
|
||||||
|
|
@ -101,6 +143,7 @@ proc close*(s: InputStream,
|
||||||
else:
|
else:
|
||||||
asyncCheck s.closeFut
|
asyncCheck s.closeFut
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template close*(sp: AsyncInputStream) =
|
template close*(sp: AsyncInputStream) =
|
||||||
## Starts the asychronous closing of the stream and returns a future that
|
## Starts the asychronous closing of the stream and returns a future that
|
||||||
## tracks the closing operation.
|
## tracks the closing operation.
|
||||||
|
|
@ -110,7 +153,7 @@ template close*(sp: AsyncInputStream) =
|
||||||
if s.closeFut != nil:
|
if s.closeFut != nil:
|
||||||
fsAwait s.closeFut
|
fsAwait s.closeFut
|
||||||
|
|
||||||
template closeNoWait*(sp: AsyncInputStream|InputStream) =
|
template closeNoWait*(sp: MaybeAsyncInputStream) =
|
||||||
## Close the stream without waiting even if's async.
|
## Close the stream without waiting even if's async.
|
||||||
## This operation will use `asyncCheck` internally to detect unhandled
|
## This operation will use `asyncCheck` internally to detect unhandled
|
||||||
## errors from the closing operation.
|
## errors from the closing operation.
|
||||||
|
|
@ -224,16 +267,10 @@ proc readableNow*(s: InputStream): bool =
|
||||||
getNewSpan s
|
getNewSpan s
|
||||||
s.span.hasRunway
|
s.span.hasRunway
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template readableNow*(s: AsyncInputStream): bool =
|
template readableNow*(s: AsyncInputStream): bool =
|
||||||
readableNow InputStream(s)
|
readableNow InputStream(s)
|
||||||
|
|
||||||
# TODO: The pure async interface should be moved in a separate module
|
|
||||||
# to make FastStreams more light-weight when the async back-end
|
|
||||||
# is not used (e.g. in Confutils)
|
|
||||||
#
|
|
||||||
# The problem is that the `async` macro will pull the entire
|
|
||||||
# event loop right now.
|
|
||||||
|
|
||||||
proc readOnce*(sp: AsyncInputStream): Future[Natural] {.async.} =
|
proc readOnce*(sp: AsyncInputStream): Future[Natural] {.async.} =
|
||||||
let s = InputStream(sp)
|
let s = InputStream(sp)
|
||||||
fsAssert s.buffers != nil and s.vtable != nil
|
fsAssert s.buffers != nil and s.vtable != nil
|
||||||
|
|
@ -273,7 +310,9 @@ template timeoutToNextByte*(sp: AsyncInputStream, timeout: Duration): bool =
|
||||||
proc closeAsync*(s: AsyncInputStream) {.async.} =
|
proc closeAsync*(s: AsyncInputStream) {.async.} =
|
||||||
close s
|
close s
|
||||||
|
|
||||||
# TODO: End of purely async interface
|
template totalUnconsumedBytes*(s: AsyncInputStream): Natural =
|
||||||
|
## Alias for InputStream.totalUnconsumedBytes
|
||||||
|
totalUnconsumedBytes InputStream(s)
|
||||||
|
|
||||||
func getBestContiguousRunway(s: InputStream): Natural =
|
func getBestContiguousRunway(s: InputStream): Natural =
|
||||||
result = s.span.len
|
result = s.span.len
|
||||||
|
|
@ -298,10 +337,6 @@ func totalUnconsumedBytes*(s: InputStream): Natural =
|
||||||
|
|
||||||
localRunway + runwayInBuffers
|
localRunway + runwayInBuffers
|
||||||
|
|
||||||
template totalUnconsumedBytes*(s: AsyncInputStream): Natural =
|
|
||||||
## Alias for InputStream.totalUnconsumedBytes
|
|
||||||
totalUnconsumedBytes InputStream(s)
|
|
||||||
|
|
||||||
proc limitReadableRange(s: InputStream, rangeLen: Natural): Natural =
|
proc limitReadableRange(s: InputStream, rangeLen: Natural): Natural =
|
||||||
s.vtable = nil
|
s.vtable = nil
|
||||||
|
|
||||||
|
|
@ -317,7 +352,7 @@ proc limitReadableRange(s: InputStream, rangeLen: Natural): Natural =
|
||||||
s.buffers.queue.peekFirst.consumedTo -= bytesToUnconsume
|
s.buffers.queue.peekFirst.consumedTo -= bytesToUnconsume
|
||||||
return s.buffers.setFauxEof(s.spanEndPos)
|
return s.buffers.setFauxEof(s.spanEndPos)
|
||||||
|
|
||||||
template withReadableRange*(sp: InputStream|AsyncInputStream,
|
template withReadableRange*(sp: MaybeAsyncInputStream,
|
||||||
rangeLen: Natural,
|
rangeLen: Natural,
|
||||||
rangeStreamVarName, blk: untyped) =
|
rangeStreamVarName, blk: untyped) =
|
||||||
let
|
let
|
||||||
|
|
@ -409,6 +444,7 @@ proc len*(s: InputStream): Option[Natural] {.raises: [Defect, IOError].} =
|
||||||
else:
|
else:
|
||||||
none Natural
|
none Natural
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template len*(s: AsyncInputStream): Option[Natural] =
|
template len*(s: AsyncInputStream): Option[Natural] =
|
||||||
len InputStream(s)
|
len InputStream(s)
|
||||||
|
|
||||||
|
|
@ -541,6 +577,7 @@ template readable*(sp: InputStream): bool =
|
||||||
let s = sp
|
let s = sp
|
||||||
hasRunway(s.span) or bufferMoreDataSync(s)
|
hasRunway(s.span) or bufferMoreDataSync(s)
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template readable*(sp: AsyncInputStream): bool =
|
template readable*(sp: AsyncInputStream): bool =
|
||||||
## Async version of `readable`.
|
## Async version of `readable`.
|
||||||
## The intended API usage is the same. Instead of blocking, an async
|
## The intended API usage is the same. Instead of blocking, an async
|
||||||
|
|
@ -615,6 +652,7 @@ proc readable*(s: InputStream, n: int): bool =
|
||||||
## for futher discussion of this.
|
## for futher discussion of this.
|
||||||
readableNImpl(s, n, noAwait, readSync)
|
readableNImpl(s, n, noAwait, readSync)
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template readable*(sp: AsyncInputStream, np: int): bool =
|
template readable*(sp: AsyncInputStream, np: int): bool =
|
||||||
## Async version of `readable(n)`.
|
## Async version of `readable(n)`.
|
||||||
## The intended API usage is the same. Instead of blocking, an async
|
## The intended API usage is the same. Instead of blocking, an async
|
||||||
|
|
@ -633,6 +671,7 @@ template peek*(sp: InputStream): byte =
|
||||||
getNewSpanOrDieTrying s
|
getNewSpanOrDieTrying s
|
||||||
s.span.startAddr[]
|
s.span.startAddr[]
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template peek*(s: AsyncInputStream): byte =
|
template peek*(s: AsyncInputStream): byte =
|
||||||
peek InputStream(s)
|
peek InputStream(s)
|
||||||
|
|
||||||
|
|
@ -650,6 +689,7 @@ template read*(sp: InputStream): byte =
|
||||||
else:
|
else:
|
||||||
readFromNewSpan s
|
readFromNewSpan s
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template read*(s: AsyncInputStream): byte =
|
template read*(s: AsyncInputStream): byte =
|
||||||
read InputStream(s)
|
read InputStream(s)
|
||||||
|
|
||||||
|
|
@ -659,6 +699,7 @@ proc peekAt*(s: InputStream, pos: int): byte {.inline.} =
|
||||||
fsAssert cast[uint](peekHead) < cast[uint](s.span.endAddr)
|
fsAssert cast[uint](peekHead) < cast[uint](s.span.endAddr)
|
||||||
return peekHead[]
|
return peekHead[]
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template peekAt*(s: AsyncInputStream, pos: int): byte =
|
template peekAt*(s: AsyncInputStream, pos: int): byte =
|
||||||
peekAt InputStream(s), pos
|
peekAt InputStream(s), pos
|
||||||
|
|
||||||
|
|
@ -673,6 +714,7 @@ proc advance*(s: InputStream, n: Natural) =
|
||||||
for i in 0 ..< n:
|
for i in 0 ..< n:
|
||||||
advance s
|
advance s
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template advance*(s: AsyncInputStream) =
|
template advance*(s: AsyncInputStream) =
|
||||||
advance InputStream(s)
|
advance InputStream(s)
|
||||||
|
|
||||||
|
|
@ -789,6 +831,7 @@ proc readInto*(s: InputStream, target: var openarray[byte]): bool =
|
||||||
## regarding the number of bytes read, see `readIntoEx`.
|
## regarding the number of bytes read, see `readIntoEx`.
|
||||||
s.readIntoEx(target) == target.len
|
s.readIntoEx(target) == target.len
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template readIntoEx*(sp: AsyncInputStream, dst: var openarray[byte]): int =
|
template readIntoEx*(sp: AsyncInputStream, dst: var openarray[byte]): int =
|
||||||
let s = InputStream(sp)
|
let s = InputStream(sp)
|
||||||
# BEWARE! `openArrayToPair` here is needed to avoid
|
# BEWARE! `openArrayToPair` here is needed to avoid
|
||||||
|
|
@ -869,6 +912,7 @@ template read*(sp: InputStream, np: static Natural): openarray[byte] =
|
||||||
template read*(s: InputStream, n: Natural): openarray[byte] =
|
template read*(s: InputStream, n: Natural): openarray[byte] =
|
||||||
readNImpl(s, n, useHeapMem)
|
readNImpl(s, n, useHeapMem)
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template read*(s: AsyncInputStream, n: Natural): openarray[byte] =
|
template read*(s: AsyncInputStream, n: Natural): openarray[byte] =
|
||||||
read InputStream(s), n
|
read InputStream(s), n
|
||||||
|
|
||||||
|
|
@ -879,6 +923,7 @@ proc lookAheadMatch*(s: InputStream, data: openarray[byte]): bool =
|
||||||
|
|
||||||
return true
|
return true
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template lookAheadMatch*(s: AsyncInputStream, data: openarray[byte]): bool =
|
template lookAheadMatch*(s: AsyncInputStream, data: openarray[byte]): bool =
|
||||||
lookAheadMatch InputStream(s)
|
lookAheadMatch InputStream(s)
|
||||||
|
|
||||||
|
|
@ -886,6 +931,7 @@ proc next*(s: InputStream): Option[byte] =
|
||||||
if readable(s):
|
if readable(s):
|
||||||
result = some read(s)
|
result = some read(s)
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template next*(sp: AsyncInputStream): Option[byte] =
|
template next*(sp: AsyncInputStream): Option[byte] =
|
||||||
let s = sp
|
let s = sp
|
||||||
if readable(s):
|
if readable(s):
|
||||||
|
|
@ -896,6 +942,7 @@ template next*(sp: AsyncInputStream): Option[byte] =
|
||||||
proc pos*(s: InputStream): int {.inline.} =
|
proc pos*(s: InputStream): int {.inline.} =
|
||||||
s.spanEndPos - s.span.len
|
s.spanEndPos - s.span.len
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template pos*(s: AsyncInputStream): int =
|
template pos*(s: AsyncInputStream): int =
|
||||||
pos InputStream(s)
|
pos InputStream(s)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,14 @@
|
||||||
import
|
import
|
||||||
stew/shims/macros,
|
async_backend
|
||||||
async_backend, inputs, outputs
|
|
||||||
|
|
||||||
export
|
export
|
||||||
async_backend
|
async_backend
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
|
import
|
||||||
|
stew/shims/macros,
|
||||||
|
"."/[inputs, outputs]
|
||||||
|
|
||||||
macro fsMultiSync*(body: untyped) =
|
macro fsMultiSync*(body: untyped) =
|
||||||
# We will produce an identical copy of the annotated proc,
|
# We will produce an identical copy of the annotated proc,
|
||||||
# but taking async parameters and having the async pragma.
|
# but taking async parameters and having the async pragma.
|
||||||
|
|
@ -32,4 +36,5 @@ macro fsMultiSync*(body: untyped) =
|
||||||
result = newStmtList(body, asyncProcBody)
|
result = newStmtList(body, asyncProcBody)
|
||||||
when defined(debugSupportAsync):
|
when defined(debugSupportAsync):
|
||||||
echo result.repr
|
echo result.repr
|
||||||
|
else:
|
||||||
|
macro fsMultiSync*(body: untyped) = body
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,8 @@ import
|
||||||
export
|
export
|
||||||
initPageBuffers, CloseBehavior
|
initPageBuffers, CloseBehavior
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
|
# Circular type refs prevent more targeted `when`
|
||||||
type
|
type
|
||||||
OutputStream* = ref object of RootObj
|
OutputStream* = ref object of RootObj
|
||||||
vtable*: ptr OutputStreamVTable # This is nil for any memory output
|
vtable*: ptr OutputStreamVTable # This is nil for any memory output
|
||||||
|
|
@ -25,17 +27,6 @@ type
|
||||||
when debugHelpers:
|
when debugHelpers:
|
||||||
name*: string
|
name*: string
|
||||||
|
|
||||||
WriteCursor* = object
|
|
||||||
span: PageSpan
|
|
||||||
stream: OutputStream
|
|
||||||
|
|
||||||
LayeredOutputStream* = ref object of OutputStream
|
|
||||||
destination*: OutputStream
|
|
||||||
allowWaitFor*: bool
|
|
||||||
|
|
||||||
OutputStreamHandle* = object
|
|
||||||
s*: OutputStream
|
|
||||||
|
|
||||||
AsyncOutputStream* {.borrow: `.`.} = distinct OutputStream
|
AsyncOutputStream* {.borrow: `.`.} = distinct OutputStream
|
||||||
|
|
||||||
WriteSyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural)
|
WriteSyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural)
|
||||||
|
|
@ -64,12 +55,56 @@ type
|
||||||
closeSync*: CloseSyncProc
|
closeSync*: CloseSyncProc
|
||||||
closeAsync*: CloseAsyncProc
|
closeAsync*: CloseAsyncProc
|
||||||
|
|
||||||
|
MaybeAsyncOutputStream* = OutputStream | AsyncOutputStream
|
||||||
|
|
||||||
|
else:
|
||||||
|
type
|
||||||
|
OutputStream* = ref object of RootObj
|
||||||
|
vtable*: ptr OutputStreamVTable # This is nil for any memory output
|
||||||
|
buffers*: PageBuffers # This is nil for unsafe memory outputs
|
||||||
|
span*: PageSpan
|
||||||
|
spanEndPos*: Natural
|
||||||
|
extCursorsCount: int
|
||||||
|
when debugHelpers:
|
||||||
|
name*: string
|
||||||
|
|
||||||
|
|
||||||
|
WriteSyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural)
|
||||||
|
{.nimcall, gcsafe, raises: [IOError, Defect].}
|
||||||
|
|
||||||
|
FlushSyncProc* = proc (s: OutputStream)
|
||||||
|
{.nimcall, gcsafe, raises: [IOError, Defect].}
|
||||||
|
|
||||||
|
CloseSyncProc* = proc (s: OutputStream)
|
||||||
|
{.nimcall, gcsafe, raises: [IOError, Defect].}
|
||||||
|
|
||||||
|
OutputStreamVTable* = object
|
||||||
|
writeSync*: WriteSyncProc
|
||||||
|
flushSync*: FlushSyncProc
|
||||||
|
closeSync*: CloseSyncProc
|
||||||
|
|
||||||
|
MaybeAsyncOutputStream* = OutputStream
|
||||||
|
|
||||||
|
type
|
||||||
|
WriteCursor* = object
|
||||||
|
span: PageSpan
|
||||||
|
stream: OutputStream
|
||||||
|
|
||||||
|
LayeredOutputStream* = ref object of OutputStream
|
||||||
|
destination*: OutputStream
|
||||||
|
allowWaitFor*: bool
|
||||||
|
|
||||||
|
OutputStreamHandle* = object
|
||||||
|
s*: OutputStream
|
||||||
|
|
||||||
VarSizeWriteCursor* = distinct WriteCursor
|
VarSizeWriteCursor* = distinct WriteCursor
|
||||||
|
|
||||||
FileOutputStream = ref object of OutputStream
|
FileOutputStream = ref object of OutputStream
|
||||||
file: File
|
file: File
|
||||||
|
|
||||||
template Sync*(s: OutputStream): OutputStream = s
|
template Sync*(s: OutputStream): OutputStream = s
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template Async*(s: OutputStream): AsyncOutputStream = AsyncOutputStream(s)
|
template Async*(s: OutputStream): AsyncOutputStream = AsyncOutputStream(s)
|
||||||
|
|
||||||
template Sync*(s: AsyncOutputStream): OutputStream = OutputStream(s)
|
template Sync*(s: AsyncOutputStream): OutputStream = OutputStream(s)
|
||||||
|
|
@ -77,12 +112,18 @@ template Async*(s: AsyncOutputStream): AsyncOutputStream = s
|
||||||
|
|
||||||
proc disconnectOutputDevice(s: OutputStream) =
|
proc disconnectOutputDevice(s: OutputStream) =
|
||||||
if s.vtable != nil:
|
if s.vtable != nil:
|
||||||
|
when fsAsyncSupport:
|
||||||
if s.vtable.closeAsync != nil:
|
if s.vtable.closeAsync != nil:
|
||||||
s.closeFut = s.vtable.closeAsync(s)
|
s.closeFut = s.vtable.closeAsync(s)
|
||||||
elif s.vtable.closeSync != nil:
|
elif s.vtable.closeSync != nil:
|
||||||
s.vtable.closeSync(s)
|
s.vtable.closeSync(s)
|
||||||
|
else:
|
||||||
|
if s.vtable.closeSync != nil:
|
||||||
|
s.vtable.closeSync(s)
|
||||||
|
|
||||||
s.vtable = nil
|
s.vtable = nil
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template disconnectOutputDevice(s: AsyncOutputStream) =
|
template disconnectOutputDevice(s: AsyncOutputStream) =
|
||||||
disconnectOutputDevice OutputStream(s)
|
disconnectOutputDevice OutputStream(s)
|
||||||
|
|
||||||
|
|
@ -99,18 +140,12 @@ template flushImpl(s: OutputStream, awaiter, writeOp, flushOp: untyped) =
|
||||||
proc flush*(s: OutputStream) =
|
proc flush*(s: OutputStream) =
|
||||||
flushImpl(s, noAwait, writeSync, flushSync)
|
flushImpl(s, noAwait, writeSync, flushSync)
|
||||||
|
|
||||||
template flush*(sp: AsyncOutputStream) =
|
|
||||||
let s = OutputStream sp
|
|
||||||
flushImpl(s, fsAwait, writeAsync, flushAsync)
|
|
||||||
|
|
||||||
proc flushAsync*(s: AsyncOutputStream) {.async.} =
|
|
||||||
flush s
|
|
||||||
|
|
||||||
proc close*(s: OutputStream,
|
proc close*(s: OutputStream,
|
||||||
behavior = dontWaitAsyncClose)
|
behavior = dontWaitAsyncClose)
|
||||||
{.raises: [IOError, Defect].} =
|
{.raises: [IOError, Defect].} =
|
||||||
flush s
|
flush s
|
||||||
disconnectOutputDevice(s)
|
disconnectOutputDevice(s)
|
||||||
|
when fsAsyncSupport:
|
||||||
if s.closeFut != nil:
|
if s.closeFut != nil:
|
||||||
fsTranslateErrors "Stream closing failed":
|
fsTranslateErrors "Stream closing failed":
|
||||||
if behavior == waitAsyncClose:
|
if behavior == waitAsyncClose:
|
||||||
|
|
@ -118,17 +153,7 @@ proc close*(s: OutputStream,
|
||||||
else:
|
else:
|
||||||
asyncCheck s.closeFut
|
asyncCheck s.closeFut
|
||||||
|
|
||||||
template close*(sp: AsyncOutputStream) =
|
template closeNoWait*(sp: MaybeAsyncOutputStream) =
|
||||||
let s = OutputStream sp
|
|
||||||
flush(Async s)
|
|
||||||
disconnectOutputDevice(s)
|
|
||||||
if s.closeFut != nil:
|
|
||||||
fsAwait s.closeFut
|
|
||||||
|
|
||||||
proc closeAsync*(s: AsyncOutputStream) {.async.} =
|
|
||||||
close s
|
|
||||||
|
|
||||||
template closeNoWait*(sp: AsyncOutputStream|OutputStream) =
|
|
||||||
## Close the stream without waiting even if's async.
|
## Close the stream without waiting even if's async.
|
||||||
## This operation will use `asyncCheck` internally to detect unhandled
|
## This operation will use `asyncCheck` internally to detect unhandled
|
||||||
## errors from the closing operation.
|
## errors from the closing operation.
|
||||||
|
|
@ -196,6 +221,7 @@ proc ensureRunway*(s: OutputStream, neededRunway: Natural) =
|
||||||
s.buffers.ensureRunway(s.span, neededRunway)
|
s.buffers.ensureRunway(s.span, neededRunway)
|
||||||
s.spanEndPos += (s.span.len - runway)
|
s.spanEndPos += (s.span.len - runway)
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template ensureRunway*(s: AsyncOutputStream, neededRunway: Natural) =
|
template ensureRunway*(s: AsyncOutputStream, neededRunway: Natural) =
|
||||||
ensureRunway OutputStream(s), neededRunway
|
ensureRunway OutputStream(s), neededRunway
|
||||||
|
|
||||||
|
|
@ -266,6 +292,7 @@ proc fileOutput*(filename: string,
|
||||||
proc pos*(s: OutputStream): int =
|
proc pos*(s: OutputStream): int =
|
||||||
s.spanEndPos - s.span.len
|
s.spanEndPos - s.span.len
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template pos*(s: AsyncOutputStream): int =
|
template pos*(s: AsyncOutputStream): int =
|
||||||
pos OutputStream(s)
|
pos OutputStream(s)
|
||||||
|
|
||||||
|
|
@ -313,6 +340,7 @@ proc drainAllBuffersSync(s: OutputStream, buf: pointer, bufSize: Natural) =
|
||||||
s.span = s.buffers.getWritableSpan()
|
s.span = s.buffers.getWritableSpan()
|
||||||
s.spanEndPos += s.span.len
|
s.spanEndPos += s.span.len
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
proc drainAllBuffersAsync(s: OutputStream, buf: pointer, bufSize: Natural) {.async.} =
|
proc drainAllBuffersAsync(s: OutputStream, buf: pointer, bufSize: Natural) {.async.} =
|
||||||
fsAwait s.vtable.writeAsync(s, buf, bufSize)
|
fsAwait s.vtable.writeAsync(s, buf, bufSize)
|
||||||
s.span = s.buffers.getWritableSpan()
|
s.span = s.buffers.getWritableSpan()
|
||||||
|
|
@ -533,6 +561,7 @@ template write*(sp: OutputStream, b: byte) =
|
||||||
else:
|
else:
|
||||||
writeToNewSpan(s, b)
|
writeToNewSpan(s, b)
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
proc write*(sp: AsyncOutputStream, b: byte) =
|
proc write*(sp: AsyncOutputStream, b: byte) =
|
||||||
let s = OutputStream sp
|
let s = OutputStream sp
|
||||||
if atEnd(s.span):
|
if atEnd(s.span):
|
||||||
|
|
@ -606,7 +635,7 @@ proc write*(s: OutputStream, bytes: openArray[byte]) =
|
||||||
proc write*(s: OutputStream, chars: openArray[char]) =
|
proc write*(s: OutputStream, chars: openArray[char]) =
|
||||||
write s, charsToBytes(chars)
|
write s, charsToBytes(chars)
|
||||||
|
|
||||||
proc write*(s: OutputStream|AsyncOutputStream, value: string) {.inline.} =
|
proc write*(s: MaybeAsyncOutputStream, value: string) {.inline.} =
|
||||||
write s, value.toOpenArrayByte(0, value.len - 1)
|
write s, value.toOpenArrayByte(0, value.len - 1)
|
||||||
|
|
||||||
template memCopyToBytes(value: auto): untyped =
|
template memCopyToBytes(value: auto): untyped =
|
||||||
|
|
@ -618,6 +647,7 @@ template memCopyToBytes(value: auto): untyped =
|
||||||
proc writeMemCopy*(s: OutputStream, value: auto) =
|
proc writeMemCopy*(s: OutputStream, value: auto) =
|
||||||
write s, memCopyToBytes(value)
|
write s, memCopyToBytes(value)
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
proc writeBytesAsyncImpl(sp: OutputStream,
|
proc writeBytesAsyncImpl(sp: OutputStream,
|
||||||
bytes: openarray[byte]): Future[void] =
|
bytes: openarray[byte]): Future[void] =
|
||||||
let s = sp
|
let s = sp
|
||||||
|
|
@ -635,6 +665,7 @@ proc writeBytesAsyncImpl(s: OutputStream,
|
||||||
template writeAndWait*(s: OutputStream, value: untyped) =
|
template writeAndWait*(s: OutputStream, value: untyped) =
|
||||||
write s, value
|
write s, value
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template writeAndWait*(sp: AsyncOutputStream, value: untyped) =
|
template writeAndWait*(sp: AsyncOutputStream, value: untyped) =
|
||||||
bind writeBytesAsyncImpl
|
bind writeBytesAsyncImpl
|
||||||
|
|
||||||
|
|
@ -795,9 +826,28 @@ template getOutput*(s: OutputStream, T: type seq[byte]): seq[byte] =
|
||||||
template getOutput*(s: OutputStream): seq[byte] =
|
template getOutput*(s: OutputStream): seq[byte] =
|
||||||
cast[seq[byte]](s.getOutput(string))
|
cast[seq[byte]](s.getOutput(string))
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
template getOutput*(s: AsyncOutputStream): seq[byte] =
|
template getOutput*(s: AsyncOutputStream): seq[byte] =
|
||||||
getOutput OutputStream(s)
|
getOutput OutputStream(s)
|
||||||
|
|
||||||
template getOutput*(s: AsyncOutputStream, T: type): untyped =
|
template getOutput*(s: AsyncOutputStream, T: type): untyped =
|
||||||
getOutput OutputStream(s), T
|
getOutput OutputStream(s), T
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
|
template flush*(sp: AsyncOutputStream) =
|
||||||
|
let s = OutputStream sp
|
||||||
|
flushImpl(s, fsAwait, writeAsync, flushAsync)
|
||||||
|
|
||||||
|
proc flushAsync*(s: AsyncOutputStream) {.async.} =
|
||||||
|
flush s
|
||||||
|
|
||||||
|
template close*(sp: AsyncOutputStream) =
|
||||||
|
let s = OutputStream sp
|
||||||
|
flush(Async s)
|
||||||
|
disconnectOutputDevice(s)
|
||||||
|
if s.closeFut != nil:
|
||||||
|
fsAwait s.closeFut
|
||||||
|
|
||||||
|
proc closeAsync*(s: AsyncOutputStream) {.async.} =
|
||||||
|
close s
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,14 @@
|
||||||
import
|
import
|
||||||
macros,
|
"."/[inputs, outputs, async_backend]
|
||||||
inputs, outputs, buffers, async_backend
|
|
||||||
|
|
||||||
export
|
export
|
||||||
inputs, outputs, async_backend
|
inputs, outputs, async_backend
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
|
import
|
||||||
|
std/macros,
|
||||||
|
./buffers
|
||||||
|
|
||||||
type
|
type
|
||||||
Pipe* = ref object
|
Pipe* = ref object
|
||||||
# TODO: Make these stream handles
|
# TODO: Make these stream handles
|
||||||
|
|
|
||||||
|
|
@ -2,15 +2,11 @@
|
||||||
|
|
||||||
import
|
import
|
||||||
os, unittest2, strutils, random,
|
os, unittest2, strutils, random,
|
||||||
stew/ranges/ptr_arith, testutils,
|
testutils,
|
||||||
../faststreams, ../faststreams/textio
|
../faststreams, ../faststreams/textio
|
||||||
|
|
||||||
setCurrentDir getAppDir()
|
setCurrentDir getAppDir()
|
||||||
|
|
||||||
proc bytes(s: string): seq[byte] =
|
|
||||||
result = newSeqOfCap[byte](s.len)
|
|
||||||
for c in s: result.add byte(c)
|
|
||||||
|
|
||||||
proc str(bytes: openarray[byte]): string =
|
proc str(bytes: openarray[byte]): string =
|
||||||
result = newStringOfCap(bytes.len)
|
result = newStringOfCap(bytes.len)
|
||||||
for b in items(bytes):
|
for b in items(bytes):
|
||||||
|
|
@ -28,7 +24,7 @@ const
|
||||||
asciiTableFile = "files" / "ascii_table.txt"
|
asciiTableFile = "files" / "ascii_table.txt"
|
||||||
asciiTableContents = slurp(asciiTableFile)
|
asciiTableContents = slurp(asciiTableFile)
|
||||||
|
|
||||||
procSuite "input stream":
|
suite "input stream":
|
||||||
template emptyInputTests(suiteName, setupCode: untyped) =
|
template emptyInputTests(suiteName, setupCode: untyped) =
|
||||||
suite suiteName & " empty inputs":
|
suite suiteName & " empty inputs":
|
||||||
setup setupCode
|
setup setupCode
|
||||||
|
|
|
||||||
|
|
@ -154,6 +154,15 @@ suite "output stream":
|
||||||
|
|
||||||
checkOutputsMatch()
|
checkOutputsMatch()
|
||||||
|
|
||||||
|
test "memcpy":
|
||||||
|
var x = 0x42'u8
|
||||||
|
|
||||||
|
nimSeq.add x
|
||||||
|
memStream.writeMemCopy x
|
||||||
|
let memStreamRes = memStream.getOutput
|
||||||
|
|
||||||
|
check memStreamRes == nimSeq
|
||||||
|
|
||||||
template undelayedOutput(content: seq[byte]) {.dirty.} =
|
template undelayedOutput(content: seq[byte]) {.dirty.} =
|
||||||
nimSeq.add content
|
nimSeq.add content
|
||||||
streamWritingToExistingBuffer.write content
|
streamWritingToExistingBuffer.write content
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,15 @@
|
||||||
{.used.}
|
{.used.}
|
||||||
|
|
||||||
|
import
|
||||||
|
testutils/unittests,
|
||||||
|
|
||||||
|
# FastStreams modules:
|
||||||
|
../faststreams/[pipelines, multisync]
|
||||||
|
|
||||||
|
when fsAsyncSupport:
|
||||||
import
|
import
|
||||||
# Std lib:
|
# Std lib:
|
||||||
std/[strutils, random, base64, terminal],
|
std/[strutils, random, base64, terminal],
|
||||||
# Other packages:
|
|
||||||
testutils/unittests,
|
|
||||||
# FastStreams modules:
|
# FastStreams modules:
|
||||||
../faststreams/[pipelines, multisync],
|
../faststreams/[pipelines, multisync],
|
||||||
# Testing modules:
|
# Testing modules:
|
||||||
|
|
@ -47,8 +52,8 @@ proc getOutput(sp: AsyncInputStream, T: type string): Future[string] {.async.} =
|
||||||
discard sp.readinto(data)
|
discard sp.readinto(data)
|
||||||
result = cast[string](data)
|
result = cast[string](data)
|
||||||
|
|
||||||
procSuite "pipelines":
|
suite "pipelines":
|
||||||
let loremIpsum = """
|
const loremIpsum = """
|
||||||
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod
|
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod
|
||||||
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
|
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
|
||||||
veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex
|
veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex
|
||||||
|
|
@ -126,3 +131,6 @@ procSuite "pipelines":
|
||||||
let fsAsyncres = await f
|
let fsAsyncres = await f
|
||||||
|
|
||||||
check fsAsyncRes == toUpperAscii(inputText)
|
check fsAsyncRes == toUpperAscii(inputText)
|
||||||
|
else:
|
||||||
|
test "pipelines":
|
||||||
|
skip
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue