jyapayne/nim-faststreams
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Expand the documentation; Clean up debugging code; Enable all tests

Browse source
This commit is contained in:
Zahary Karadjov 2020-05-05 19:20:25 +03:00
commit 18f9488bee
No known key found for this signature in database
GPG key ID: C8936F8A3073D609
11 changed files with 205 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

7
faststreams/async_backend.nim
View file

@ -36,6 +36,13 @@ elif faststreams_async_backend in ["std", "asyncdispatch"]:
else:
{.fatal: "Unrecognized network backend: " & faststreams_async_backend.}
when defined(danger):
template fsAssert*(x) = discard
template fsAssert*(x, msg) = discard
else:
template fsAssert*(x) = doAssert(x)
template fsAssert*(x, msg) = doAssert(x, msg)
template fsTranslateErrors*(errMsg: string, body: untyped) =
try:
body

12
faststreams/buffers.nim
View file

@ -142,16 +142,16 @@ func nextReadableSpan*(buffers: PageBuffers, span: var PageSpan) =
pageReadableEnd = firstPage.readableEnd
if span.endAddr == nil:
doAssert buffers.queue.len > 0
fsAssert buffers.queue.len > 0
span = obtainReadableSpan buffers.queue[0]
elif span.endAddr != pageReadableEnd:
# Check whether the span points within the current page:
doAssert distance(firstPage.allocationStart, span.endAddr) >= 0 and
fsAssert distance(firstPage.allocationStart, span.endAddr) >= 0 and
distance(span.endAddr, pageReadableEnd) >= 0
span.endAddr = pageReadableEnd
firstPage.consumedTo = firstPage.writtenTo
else:
doAssert buffers.queue.len > 1
fsAssert buffers.queue.len > 1
discard buffers.queue.popFirst
span = obtainReadableSpan buffers.queue[0]
@ -209,7 +209,7 @@ func ensureRunway*(buffers: PageBuffers,
# This is a more complicated path that should almost never
# trigger in practice in a typically implemented code that
# calls `ensureRunway` at the beggining of a transformation.
doAssert buffers.queue.len > 0
fsAssert buffers.queue.len > 0
let currPage = buffers.queue.peekLast
if currPage.hasDelayedWritesAtPageStart:
@ -266,7 +266,7 @@ func splitLastPageAt*(buffers: PageBuffers, address: ptr byte) =
buffers.queue.addLast newPage
iterator consumePages*(buffers: PageBuffers): PageRef =
doAssert buffers != nil
fsAssert buffers != nil
var recycledPage: PageRef
while buffers.queue.len > 0:
@ -337,7 +337,7 @@ template implementWrites*(buffersParam: PageBuffers,
if bytesWritten != writeLenVar: raiseError()
if srcLen > 0:
doAssert src != nil
fsAssert src != nil
let bytesWritten = writeBlock
if bytesWritten != writeLenVar: raiseError()

4
faststreams/chronos_adapters.nim
View file

@ -45,7 +45,7 @@ let chronosInputVTable = InputStreamVTable(
readSync: proc (s: InputStream, dst: pointer, dstLen: Natural): Natural
{.nimcall, gcsafe, raises: [IOError, Defect].} =
var cs = ChronosInputStream(s)
doAssert cs.allowWaitFor
fsAssert cs.allowWaitFor
waitFor chronosReadOnce(cs, dst, dstLen)
,
readAsync: proc (s: InputStream, dst: pointer, dstLen: Natural): Future[Natural]
@ -74,7 +74,7 @@ let chronosOutputVTable = OutputStreamVTable(
writeSync: proc (s: OutputStream, src: pointer, srcLen: Natural)
{.nimcall, gcsafe, raises: [IOError, Defect].} =
var cs = ChronosOutputStream(s)
doAssert cs.allowWaitFor
fsAssert cs.allowWaitFor
waitFor chronosWrites(cs, src, srcLen)
,
writeAsync: proc (s: OutputStream, src: pointer, srcLen: Natural): Future[void]

58
faststreams/inputs.nim
View file

@ -216,7 +216,7 @@ template readableNow*(s: AsyncInputStream): bool =
readableNow InputStream(s)
func flipPage(s: InputStream) =
doAssert s.buffers.len > 1
fsAssert s.buffers != nil and s.buffers.len > 1
discard s.buffers.popFirst
s.span = obtainReadableSpan s.buffers[0]
s.spanEndPos += s.span.len
@ -348,7 +348,7 @@ func memoryInput*(data: openarray[char]): InputStreamHandle =
proc resetBuffers*(s: InputStream, buffers: PageBuffers) =
# This should be used only on safe memory input streams
doAssert s.vtable == nil and s.buffers != nil and buffers.len > 0
fsAssert s.vtable == nil and s.buffers != nil and buffers.len > 0
s.buffers = buffers
s.span = obtainReadableSpan buffers.queue[0]
s.spanEndPos = s.span.len
@ -530,17 +530,43 @@ template readable*(sp: AsyncInputStream, np: int): bool =
readableNImpl(s, n, fsAwait, readAsync)
proc peek*(s: InputStream): byte {.inline.} =
doAssert hasRunway(s.span)
return s.span.startAddr[]
when false:
func flipPagePeek(s: InputStream): byte =
flipPage s
result = s.span.startAddr[]
func flipPageRead(s: InputStream): byte =
flipPage s
result = s.span.startAddr[]
bumpPointer s.span
template peek*(sp: InputStream): byte =
let s = sp
if hasRunway(s.span):
s.span.startAddr[]
else:
flipPage s
s.span.startAddr[]
template peek*(s: AsyncInputStream): byte =
peek InputStream(s)
template read*(sp: InputStream): byte =
let s = sp
if hasRunway(s.span):
let res = s.span.startAddr[]
bumpPointer(s.span)
res
else:
flipPageRead s
template read*(s: AsyncInputStream): byte =
read InputStream(s)
proc peekAt*(s: InputStream, pos: int): byte {.inline.} =
# TODO implement page flipping
let peekHead = offset(s.span.startAddr, pos)
doAssert cast[uint](peekHead) < cast[uint](s.span.endAddr)
fsAssert cast[uint](peekHead) < cast[uint](s.span.endAddr)
return peekHead[]
template peekAt*(s: AsyncInputStream, pos: int): byte =
@ -549,19 +575,12 @@ template peekAt*(s: AsyncInputStream, pos: int): byte =
proc advance*(s: InputStream) =
if hasRunway(s.span):
bumpPointer s.span
elif s.buffers != nil and s.buffers.len > 1:
else:
flipPage s
template advance*(s: AsyncInputStream) =
advance InputStream(s)
proc read*(s: InputStream): byte =
result = s.peek()
advance s
template read*(s: AsyncInputStream): byte =
read InputStream(s)
proc drainBuffersInto*(s: InputStream, dstAddr: ptr byte, dstLen: Natural): Natural =
var
dst = dstAddr
@ -691,7 +710,7 @@ template readInto*(sp: AsyncInputStream, dst: var openarray[byte]): bool =
proc readOnce*(sp: AsyncInputStream): Future[Natural] =
let s = InputStream(sp)
doAssert s.buffers != nil and s.vtable != nil
fsAssert s.buffers != nil and s.vtable != nil
s.vtable.readAsync(s, nil, 0)
when defined(windows):
@ -724,7 +743,8 @@ template readNImpl(sp: InputStream,
if n > runway:
startAddr = allocMem(tmpSeq, n, np)
doAssert drainBuffersInto(s, startAddr, n) == n
let drained {.used.} = drainBuffersInto(s, startAddr, n)
fsAssert drained == n
else:
startAddr = s.span.startAddr
bumpPointer s.span, n
@ -775,16 +795,16 @@ when false:
# Obsolete APIs for removal
proc bufferPos(s: InputStream, pos: int): ptr byte =
let offsetFromEnd = pos - s.spanEndPos
doAssert offsetFromEnd < 0
fsAssert offsetFromEnd < 0
result = offset(s.span.endAddr, offsetFromEnd)
doAssert result >= s.bufferStart
fsAssert result >= s.bufferStart
proc `[]`*(s: InputStream, pos: int): byte {.inline.} =
s.bufferPos(pos)[]
proc rewind*(s: InputStream, delta: int) =
s.head = offset(s.head, -delta)
doAssert s.head >= s.bufferStart
fsAssert s.head >= s.bufferStart
proc rewindTo*(s: InputStream, pos: int) {.inline.} =
s.head = s.bufferPos(pos)

36
faststreams/outputs.nim
View file

@ -87,7 +87,7 @@ template disconnectOutputDevice(s: AsyncOutputStream) =
disconnectOutputDevice OutputStream(s)
template flushImpl(s: OutputStream, awaiter, writeOp, flushOp: untyped) =
doAssert s.extCursorsCount == 0
fsAssert s.extCursorsCount == 0
if s.vtable != nil:
if s.buffers != nil:
trackWrittenTo(s.buffers, s.span.startAddr)
@ -159,10 +159,6 @@ template canExtendOutput(s: OutputStream): bool =
# Streams writing to pre-allocated existing buffers cannot be grown
s != nil and s.buffers != nil
template isExternalCursor(c: var WriteCursor): bool =
# Is this the original stream cursor or is it one created by a "delayed write"
addr(c) != addr(c.stream.cursor)
proc addPage(s: OutputStream) =
let
nextPageSize = s.buffers.pageSize
@ -175,7 +171,7 @@ template makeHandle*(sp: OutputStream): OutputStreamHandle =
OutputStreamHandle(s: s)
proc memoryOutput*(pageSize = defaultPageSize): OutputStreamHandle =
doAssert pageSize > 0
fsAssert pageSize > 0
# We are not creating an initial output page, because `ensureRunway`
# can determine the most appropriate size.
makeHandle OutputStream(buffers: initPageBuffers(pageSize))
@ -196,7 +192,7 @@ proc ensureRunway*(s: OutputStream, neededRunway: Natural) =
# If you use an unsafe memory output, you must ensure that
# it will have a large enough size to hold the data you are
# feeding to it.
doAssert s.buffers != nil, "Unsafe memory output of insufficient size"
fsAssert s.buffers != nil, "Unsafe memory output of insufficient size"
s.buffers.ensureRunway(s.span, neededRunway)
s.spanEndPos += (s.span.len - runway)
@ -246,7 +242,7 @@ template pos*(s: AsyncOutputStream): int =
pos OutputStream(s)
proc getBuffers*(s: OutputStream): PageBuffers =
doAssert s.buffers != nil
fsAssert s.buffers != nil
s.buffers.trackWrittenTo s.span.startAddr
return s.buffers
@ -334,7 +330,7 @@ proc delayFixedSizeWrite*(s: OutputStream, size: Natural): WriteCursor =
proc delayVarSizeWrite*(s: OutputStream, maxSize: Natural): VarSizeWriteCursor =
## Please note that using variable sized writes are not supported
## for unbuffered streams and unsafe memory inputs.
doAssert s.buffers != nil
fsAssert s.buffers != nil
let runway = s.span.len
if maxSize <= runway:
@ -368,11 +364,11 @@ proc delayVarSizeWrite*(s: OutputStream, maxSize: Natural): VarSizeWriteCursor =
s.spanEndPos += nextPageSize
proc finalize*(cursor: var WriteCursor) =
doAssert cursor.stream.extCursorsCount > 0
fsAssert cursor.stream.extCursorsCount > 0
dec cursor.stream.extCursorsCount
proc finalWrite*(cursor: var WriteCursor, data: openArray[byte]) =
doAssert data.len == cursor.span.len
fsAssert data.len == cursor.span.len
copyMem(cursor.span.startAddr, unsafeAddr data[0], data.len)
finalize cursor
@ -380,7 +376,7 @@ proc finalWrite*(c: var VarSizeWriteCursor, data: openArray[byte]) =
template cursor: auto = WriteCursor(c)
let overestimatedBytes = cursor.span.len - data.len
doAssert overestimatedBytes >= 0
fsAssert overestimatedBytes >= 0
for page in items(cursor.stream.buffers.queue):
let baseAddr = page.allocationStart
@ -398,7 +394,7 @@ proc finalWrite*(c: var VarSizeWriteCursor, data: openArray[byte]) =
finalize cursor
return
doAssert false
fsAssert false
proc tryMovingToNextPage(c: var WriteCursor) =
# A split cursor is a fixed-size cursor that ended up on page boundary.
@ -447,13 +443,13 @@ proc tryMovingToNextPage(c: var WriteCursor) =
# We didn't find any page that this cursor was ending, so this is not
# a split cursor. This means that the user just tried to write past the
# pre-allocated cursor span, which is considered a Defect (a range error)
doAssert false, "Attempt to write past the end of a cursor"
fsAssert false, "Attempt to write past the end of a cursor"
template writeByteImpl(s: OutputStream, b: byte, awaiter, writeOp, drainOp: untyped) =
if atEnd(s.span):
# Unsafe memory outputs don't use pages at all, so if our cursor
# reached here, this is a range violation defect:
doAssert canExtendOutput(s)
fsAssert canExtendOutput(s)
if s.vtable == nil or s.extCursorsCount > 0:
# This is the main cursor of a stream, but we are either not
@ -512,7 +508,7 @@ proc writeToANewPage(s: OutputStream, bytes: openArray[byte]) =
copyMem(s.span.startAddr, inputPos, runway)
reduceInput runway
doAssert s.buffers != nil
fsAssert s.buffers != nil
let nextPageSize = nextAlignedSize(inputLen, s.buffers.pageSize)
let nextPage = s.buffers.addWritablePage(nextPageSize)
@ -618,7 +614,7 @@ proc writeBytesToCursor(c: var WriteCursor, bytes: openarray[byte]) =
# On the next page, we have a new runway
runway = c.span.len
# The write shouldn't go past the end of the new runway
doAssert inputLen <= runway
fsAssert inputLen <= runway
copyMem(c.span.startAddr, inputPos, inputLen)
c.span.startAddr = offset(c.span.startAddr, inputLen)
@ -644,7 +640,7 @@ template consumeOutputs*(sp: OutputStream, bytesVar, body: untyped) =
## Before consuming the outputs, all outstanding delayed writes must
## be finalized.
let s = sp
doAssert s.extCursorsCount == 0 and s.buffers != nil
fsAssert s.extCursorsCount == 0 and s.buffers != nil
for pageReadableStart, pageLen in consumePageBuffers(s.buffers):
template bytesVar: untyped =
@ -673,7 +669,7 @@ template consumeContiguousOutput*(sp: OutputStream, bytesVar, body: untyped) =
bytesPtr: ptr byte
bytesLen: int
doAssert s.extCursorsCount == 0 and s.buffers != nil
fsAssert s.extCursorsCount == 0 and s.buffers != nil
if s.buffers.queue.len == 1:
let page = s.buffers.queue[0]
@ -702,7 +698,7 @@ proc getOutput*(s: OutputStream, T: type string): string =
##
## Before consuming the output, all outstanding delayed writes must be finalized.
##
doAssert s.extCursorsCount == 0 and s.buffers != nil
fsAssert s.extCursorsCount == 0 and s.buffers != nil
s.buffers.trackWrittenTo s.span.startAddr
if s.buffers.queue.len == 1:

26
faststreams/pipelines.nim
View file

@ -5,11 +5,6 @@ import
export
inputs, outputs, async_backend
template clearAndWait(ep: AsyncEvent) =
let e = ep
clear e
await e.wait()
type
FsAsyncPipe* = ref object
# TODO: Make these stream handles
@ -38,22 +33,17 @@ proc pipeRead(s: LayeredInputStream,
minBytesExpected = max(1, dstLen)
bytesInBuffersNow = bytesInBuffersAtStart
describeBuffers "at start", buffers
while bytesInBuffersNow < minBytesExpected:
awake buffers.waitingWriter
echo "About to wait for writer"
buffers.waitingReader.enterWait "waiting for writer to buffer more data"
echo "Awaken from wait"
bytesInBuffersNow = buffers.totalBufferedBytes
if buffers.eofReached:
echo "read bytes ", bytesInBuffersNow - bytesInBuffersAtStart
describeBuffers "at end", buffers
return bytesInBuffersNow - bytesInBuffersAtStart
if dst != nil:
doAssert drainBuffersInto(s, cast[ptr byte](dst), dstLen) == dstLen
let drained {.used.} = drainBuffersInto(s, cast[ptr byte](dst), dstLen)
fsAssert drained == dstLen
awake buffers.waitingWriter
@ -61,7 +51,6 @@ proc pipeRead(s: LayeredInputStream,
proc pipeWrite(s: LayeredOutputStream, src: pointer, srcLen: Natural) {.async.} =
let buffers = s.buffers
echo "pipe write"
while buffers.canAcceptWrite(srcLen) == false:
buffers.waitingWriter.enterWait "waiting for reader to drain the buffers"
@ -81,7 +70,7 @@ let pipeInputVTable = InputStreamVTable(
{.nimcall, gcsafe, raises: [IOError, Defect].} =
fsTranslateErrors "Failed to read from pipe":
let ls = LayeredInputStream(s)
doAssert ls.allowWaitFor
fsAssert ls.allowWaitFor
return waitFor pipeRead(ls, dst, dstLen)
,
readAsync: proc (s: InputStream, dst: pointer, dstLen: Natural): Future[Natural]
@ -117,7 +106,7 @@ let pipeOutputVTable = OutputStreamVTable(
{.nimcall, gcsafe, raises: [IOError, Defect].} =
fsTranslateErrors "Failed to write all bytes to pipe":
var ls = LayeredOutputStream(s)
doAssert ls.allowWaitFor
fsAssert ls.allowWaitFor
waitFor pipeWrite(ls, src, srcLen)
,
writeAsync: proc (s: OutputStream, src: pointer, srcLen: Natural): Future[void]
@ -146,7 +135,6 @@ let pipeOutputVTable = OutputStreamVTable(
{.nimcall, gcsafe, raises: [IOError, Defect].} =
s.buffers.eofReached = true
echo "writer closes the stream"
fsTranslateErrors "Unexpected error from Future.complete":
awake s.buffers.waitingReader
@ -173,7 +161,7 @@ let pipeOutputVTable = OutputStreamVTable(
func pipeInput*(source: InputStream,
pageSize = defaultPageSize,
allowWaitFor = false): AsyncInputStream =
doAssert pageSize > 0
fsAssert pageSize > 0
AsyncInputStream LayeredInputStream(
vtable: vtableAddr pipeInputVTable,
@ -199,7 +187,7 @@ proc pipeOutput*(destination: OutputStream,
pageSize = defaultPageSize,
maxBufferedBytes = defaultPageSize * 4,
allowWaitFor = false): AsyncOutputStream =
doAssert pageSize > 0
fsAssert pageSize > 0
var
buffers = initPageBuffers pageSize
@ -231,7 +219,7 @@ proc pipeOutput*(buffers: PageBuffers,
func asyncPipe*(pageSize = defaultPageSize,
maxBufferedBytes = defaultPageSize * 4): FsAsyncPipe =
doAssert pageSize > 0
fsAssert pageSize > 0
FsAsyncPipe(buffers: initPageBuffers(pageSize, maxBufferedBytes))
func initReader*(pipe: FsAsyncPipe): AsyncInputStream =

8
faststreams/textio.nim
View file

@ -1,6 +1,6 @@
import
stew/ptrops,
inputs, outputs, buffers, multisync
inputs, outputs, buffers, async_backend, multisync
template matchingIntType(T: type int64): type = uint64
template matchingIntType(T: type int32): type = uint32
@ -112,7 +112,7 @@ const
Digits* = {'0'..'9'}
proc readLine*(s: InputStream, keepEol = false): TaintedString =
doAssert readableNow(s)
fsAssert readableNow(s)
while s.readable:
let c = s.peek.char
@ -131,7 +131,7 @@ proc readLine*(s: InputStream, keepEol = false): TaintedString =
proc readUntil*(s: InputStream,
sep: openarray[char]): Option[TaintedString] =
doAssert readableNow(s)
fsAssert readableNow(s)
var res = ""
while s.readable(sep.len):
if s.lookAheadMatch(charsToBytes(sep)):
@ -150,7 +150,7 @@ iterator lines*(s: InputStream, keepEol = false): TaintedString =
yield readLine(s, keepEol)
proc readUnsignedInt*(s: InputStream, T: type[CompiledUIntTypes]): T =
doAssert s.readable and s.peek.char in Digits
fsAssert s.readable and s.peek.char in Digits
template eatDigitAndPeek: char =
advance s