jyapayne/nim-faststreams
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Harmonize the APIs and the coding practices in input_stream and output_stream

Browse source
This commit is contained in:
Zahary Karadjov 2020-04-10 15:30:58 +03:00
commit 3efad6f7f2
No known key found for this signature in database
GPG key ID: C8936F8A3073D609
4 changed files with 186 additions and 155 deletions
Download patch file Download diff file Expand all files Collapse all files

231
faststreams/input_stream.nim
View file

@ -1,192 +1,223 @@
import import
memfiles, options, stew/[ptrops, ranges/ptr_arith] memfiles, options,
stew/[ptrops, ranges/ptr_arith]
const
# pageSize = 4096
debugHelpers = false
type type
StreamReader = proc (bufferStart: ptr byte, bufferSize: int): int {.gcsafe.} # We inherit from RootObj because in layered streams
# TODO: use openarray once it's supported # the stream itself is often used as an `outputDevice`
AsyncStreamReader = StreamReader # proc (bufferStart: ptr byte, bufferSize: int): Future[int] InputStreamObj = object of RootObj
CloseStreamProc = proc ()
ByteStream* = object of RootObj
head*: ptr byte head*: ptr byte
bufferSize: int bufferSize: int
bufferStart, bufferEnd: ptr byte bufferStart, bufferEnd: ptr byte
cursorsList: ptr StreamCursor
reader: StreamReader
asyncReader: AsyncStreamReader
closeStream: CloseStreamProc
bufferEndPos: int bufferEndPos: int
inputDevice*: RootRef
vtable*: ptr InputStreamVTable
StreamCursor* = object InputStream* = ref InputStreamObj
head, bufferEnd: ptr byte
nextCursor: ptr StreamCursor
BufferedStream*[size: static int] = object of ByteStream AsciiInputStream* = distinct InputStream
buffer: array[size, byte] Utf8InputStream* = distinct InputStream
AsciiStream* = distinct ByteStream ReadSyncProc* = proc (s: InputStream, buffer: ptr byte, bufSize: int): int
UnicodeStream* = distinct ByteStream {.nimcall, gcsafe, raises: [IOError, Defect].}
ObjectStream*[T] = distinct ByteStream
# TODO: ByteStreamVar should become a `var` short-lived pointer once this is supported ReadAsyncCallback* = proc (s: InputStream, bytesRead: int)
ByteStreamVar* = ref ByteStream {.nimcall, gcsafe, raises: [Defect].}
AsciiStreamVar* = ref AsciiStream
InputStreamVar* = ref ByteStream ReadAsyncProc* = proc (s: InputStream,
buffer: ptr byte, bufSize: int,
cb: ReadAsyncCallback)
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseStreamProc* = proc (s: InputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
InputStreamVTable* = object
readSync*: ReadSyncProc
readAsync*: ReadAsyncProc
closeStream*: CloseStreamProc
hasKnownLen*: bool
FileInput = ref object of RootObj
file: MemFile
const
debugHelpers = false
nimAllocatorMetadataSize* = 0
# TODO: Get this from Nim's allocator.
# The goal is to make perfect page-aligned allocations
defaultPageSize = 4096 - nimAllocatorMetadataSize
let FileStreamVTable = InputStreamVTable(
readSync: nil,
readAsync: nil,
closeStream: proc (s: InputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].} =
try:
close FileInput(s.inputDevice).file
except OSError as err:
raise newException(IOError, "Failed to close file", err)
,
hasKnownLen: true
)
template vtableAddr*(vtable: InputStreamVTable): ptr InputStreamVTable =
## This is a simple work-around for the somewhat broken side
## effects analysis of Nim - reading from global let variables
## is considered a side-effect.
{.noSideEffect.}:
unsafeAddr vtable
proc fileInput*(filename: string): InputStream =
let
inputDevice = FileInput(file: memfiles.open(filename))
head = cast[ptr byte](inputDevice.file.mem)
fileSize = inputDevice.file.size
result = InputStream(
head: head,
bufferEnd: offset(head, fileSize),
bufferEndPos: fileSize,
inputDevice: inputDevice,
vtable: vtableAddr FileStreamVTable)
proc openFile*(filename: string): ByteStreamVar =
new result
var memFile = memfiles.open(filename)
result.head = cast[ptr byte](memFile.mem)
when debugHelpers: when debugHelpers:
result.bufferStart = result.head result.bufferStart = result.head
result.bufferEnd = offset(result.head, memFile.size)
result.bufferEndPos = memFile.size
result.closeStream = proc = close memFile
proc init*(T: type ByteStream, proc implementInputStream*(vtable: ptr InputStreamVTable,
mem: openarray[byte], inputDevice: RootRef,
reader = StreamReader(nil), pageSize = defaultPageSize): InputStream =
asyncReader = AsyncStreamReader(nil)): ByteStream = # TODO: We need to allocate memory here and start reading
# TODO: the result should use `from mem` once it's supported InputStream(inputDevice: inputDevice,
result.head = unsafeAddr mem[0] vtable: vtable)
result.bufferEnd = offset(result.head, mem.len)
result.bufferEndPos = mem.len
result.reader = reader
result.asyncReader = asyncReader
proc memoryStream*(mem: openarray[byte]): ByteStreamVar = # TODO: mark the result with `from mem` proc memoryInput*(mem: openarray[byte]): InputStream =
new result let head = unsafeAddr mem[0]
result[] = ByteStream.init(mem) InputStream(
head: head,
bufferEnd: offset(head, mem.len),
bufferEndPos: mem.len)
proc memoryStream*(str: string): ByteStreamVar = # TODO: mark the result with `from str` proc memoryInput*(str: string): InputStream =
new result memoryInput str.toOpenArrayByte(0, str.len - 1)
result[] = init(ByteStream, str.toOpenArrayByte(0, str.len - 1))
proc init*(T: type BufferedStream, proc endPos*(s: InputStream): int =
reader = StreamReader(nil), doAssert s.vtable == nil or s.vtable.hasKnownLen
asyncReader = AsyncStreamReader(nil)): BufferedStream =
result.init result.buffer, reader, asyncReader
proc endPos*(s: InputStreamVar): int =
# TODO This needs to use a VTable for `system.File` based streams
return s.bufferEndPos return s.bufferEndPos
proc syncRead(s: var ByteStream): bool = proc syncRead(s: InputStream): bool =
let bytesRead = s.reader(s.bufferStart, s.bufferSize) let bytesRead = s.vtable.readSync(s, s.bufferStart, s.bufferSize)
if bytesRead == 0: if bytesRead == 0:
s.reader = nil # TODO close the input device
s.inputDevice = nil
return true return true
else: else:
s.bufferEnd = offset(s.bufferStart, bytesRead) s.bufferEnd = offset(s.bufferStart, bytesRead)
s.bufferEndPos += bytesRead s.bufferEndPos += bytesRead
return false return false
proc ensureBytes*(s: var ByteStream, n: int): bool = proc ensureBytes*(s: InputStream, n: int): bool =
if distance(s.head, s.bufferEnd) >= n: if distance(s.head, s.bufferEnd) >= n:
return true return true
if s.reader == nil: if s.inputDevice == nil:
return false return false
# TODO
doAssert false, "Multi-buffer reading will be implemented later" doAssert false, "Multi-buffer reading will be implemented later"
proc eof*(s: var ByteStream): bool = proc eof*(s: InputStream): bool =
if s.head != s.bufferEnd: if s.head != s.bufferEnd:
return false return false
if s.reader == nil: if s.inputDevice == nil:
return true return true
return s.syncRead() return s.syncRead()
proc eob*(s: ByteStream): bool {.inline.} = #proc eob*(s: InputStream): bool {.inline.} =
s.head != s.bufferEnd # s.head != s.bufferEnd
proc peek*(s: ByteStream): byte {.inline.} = proc peek*(s: InputStream): byte {.inline.} =
doAssert s.head != s.bufferEnd doAssert s.head != s.bufferEnd
return s.head[] return s.head[]
when debugHelpers: when debugHelpers:
proc showPosition*(s: ByteStream) = proc showPosition*(s: InputStream) =
echo "head at ", distance(s.bufferStart, s.head), "/", echo "head at ", distance(s.bufferStart, s.head), "/",
distance(s.bufferStart, s.bufferEnd) distance(s.bufferStart, s.bufferEnd)
proc advance*(s: var ByteStream) = proc advance*(s: InputStream) =
if s.head != s.bufferEnd: if s.head != s.bufferEnd:
s.head = offset(s.head, 1) s.head = offset(s.head, 1)
elif s.reader != nil: elif s.inputDevice != nil:
discard s.syncRead() discard s.syncRead()
proc read*(s: var ByteStream): byte = proc read*(s: InputStream): byte =
result = s.peek() result = s.peek()
advance s advance s
proc checkReadAhead(s: ByteStreamVar, n: int): ptr byte = proc checkReadAhead(s: InputStream, n: int): ptr byte =
result = s.head result = s.head
doAssert distance(s.head, s.bufferEnd) >= n doAssert distance(s.head, s.bufferEnd) >= n
s.head = offset(s.head, n) s.head = offset(s.head, n)
template readBytes*(s: ByteStreamVar, n: int): auto = template readBytes*(s: InputStream, n: int): auto =
makeOpenArray(checkReadAhead(s, n), n) makeOpenArray(checkReadAhead(s, n), n)
proc next*(s: var ByteStream): Option[byte] = proc next*(s: InputStream): Option[byte] =
if not s.eof: if not s.eof:
result = some s.read() result = some s.read()
proc bufferPos(s: ByteStream, pos: int): ptr byte = proc bufferPos(s: InputStream, pos: int): ptr byte =
let offsetFromEnd = pos - s.bufferEndPos let offsetFromEnd = pos - s.bufferEndPos
doAssert offsetFromEnd < 0 doAssert offsetFromEnd < 0
result = offset(s.bufferEnd, offsetFromEnd) result = offset(s.bufferEnd, offsetFromEnd)
doAssert result >= s.bufferStart doAssert result >= s.bufferStart
proc pos*(s: ByteStream): int {.inline.} = proc pos*(s: InputStream): int {.inline.} =
s.bufferEndPos - distance(s.head, s.bufferEnd) s.bufferEndPos - distance(s.head, s.bufferEnd)
proc firstAccessiblePos*(s: ByteStream): int {.inline.} = proc firstAccessiblePos*(s: InputStream): int {.inline.} =
s.bufferEndPos - distance(s.bufferStart, s.bufferEnd) s.bufferEndPos - distance(s.bufferStart, s.bufferEnd)
proc `[]`*(s: ByteStream, pos: int): byte {.inline.} = proc `[]`*(s: InputStream, pos: int): byte {.inline.} =
s.bufferPos(pos)[] s.bufferPos(pos)[]
proc rewind*(s: var ByteStream, delta: int) = proc rewind*(s: InputStream, delta: int) =
s.head = offset(s.head, -delta) s.head = offset(s.head, -delta)
doAssert s.head >= s.bufferStart doAssert s.head >= s.bufferStart
proc rewindTo*(s: var ByteStream, pos: int) {.inline.} = proc rewindTo*(s: InputStream, pos: int) {.inline.} =
s.head = s.bufferPos(pos) s.head = s.bufferPos(pos)
# TODO: use a destructor once we migrate to Nim 0.20 # TODO: use a destructor once we migrate to Nim 0.20
proc close*(s: var ByteStream) = # TODO: It's not appropriate for this to raise
if s.closeStream != nil: proc close*(s: InputStream) {.raises: [IOError, Defect].} =
s.closeStream() if s.vtable != nil:
s.vtable.closeStream(s)
# TODO: Use `distrinct with` once it's supported template pos*(s: AsciiInputStream|Utf8InputStream): int =
template pos*(s: AsciiStream|UnicodeStream|ObjectStream): int = InputStream(s).pos
ByteStream(s).pos
template eof*(s: var (AsciiStream|UnicodeStream|ObjectStream)): bool = template eof*(s: AsciiInputStream|Utf8InputStream): bool =
ByteStream(s).eof InputStream(s).eof
template eob*(s: var (AsciiStream|UnicodeStream|ObjectStream)): bool = #template eob*(s: AsciiInputStream|Utf8InputStream): bool =
ByteStream(s).eob # InputStream(s).eob
template close*(s: AsciiStream|UnicodeStream|ObjectStream) = template close*(s: AsciiInputStream|Utf8InputStream) =
close ByteStream(s) close InputStream(s)
template advance*(s: var AsciiStream) = template advance*(s: AsciiInputStream) =
advance ByteStream(s) advance InputStream(s)
template peek*(s: AsciiStream): char = template peek*(s: AsciiInputStream): char =
char ByteStream(s).peek() char InputStream(s).peek()
template read*(s: var AsciiStream): char = template read*(s: var AsciiInputStream): char =
char ByteStream(s).read() char InputStream(s).read()
template next*(s: var AsciiStream): Option[char] = template next*(s: var AsciiInputStream): Option[char] =
cast[Option[char]](ByteStream(s).next()) cast[Option[char]](InputStream(s).next())

100
faststreams/output_stream.nim
View file

@ -13,15 +13,14 @@ type
cursor*: WriteCursor cursor*: WriteCursor
pages: Deque[OutputPage] pages: Deque[OutputPage]
endPos: int endPos: int
vtable*: ptr OutputStreamVTable
outputDevice*: RootRef
extCursorsCount: int extCursorsCount: int
pageSize: int pageSize: int
maxWriteSize*: int maxWriteSize*: int
minWriteSize*: int minWriteSize*: int
outputDevice*: RootRef
vtable*: ptr OutputStreamVTable
OutputStream* = ref OutputStreamObj OutputStream* = ref OutputStreamObj
OutputStreamVar = OutputStream
WritePageProc* = proc (s: OutputStream, page: openarray[byte]) WritePageProc* = proc (s: OutputStream, page: openarray[byte])
{.nimcall, gcsafe, raises: [IOError, Defect].} {.nimcall, gcsafe, raises: [IOError, Defect].}
@ -35,7 +34,7 @@ type
WriteCursor* = object WriteCursor* = object
head, bufferEnd: ptr byte head, bufferEnd: ptr byte
stream: OutputStreamVar stream: OutputStream
VarSizeWriteCursor* = distinct WriteCursor VarSizeWriteCursor* = distinct WriteCursor
@ -43,15 +42,16 @@ type
file: File file: File
const const
allocatorMetadata = 0 # TODO: Get this from Nim's allocator. nimAllocatorMetadataSize* = 0
# TODO: Get this from Nim's allocator.
# The goal is to make perfect page-aligned allocations # The goal is to make perfect page-aligned allocations
defaultPageSize = 4096 - allocatorMetadata - 1 # 1 byte for the null terminator defaultPageSize = 4096 - nimAllocatorMetadataSize - 1 # 1 byte for the null terminator
proc createWriteCursor*[R, T](x: var array[R, T]): WriteCursor = proc createWriteCursor*[R, T](x: var array[R, T]): WriteCursor =
let startAddr = cast[ptr byte](addr x[0]) let startAddr = cast[ptr byte](addr x[0])
WriteCursor(head: startAddr, bufferEnd: offset(startAddr, sizeof x)) WriteCursor(head: startAddr, bufferEnd: offset(startAddr, sizeof x))
template canExtendOutput(s: OutputStreamVar): bool = template canExtendOutput(s: OutputStream): bool =
# Streams writing to pre-allocated existing buffers cannot be grown # Streams writing to pre-allocated existing buffers cannot be grown
s != nil and s.pageSize > 0 s != nil and s.pageSize > 0
@ -62,23 +62,23 @@ template isExternalCursor(c: var WriteCursor): bool =
func runway*(c: var WriteCursor): int {.inline.} = func runway*(c: var WriteCursor): int {.inline.} =
distance(c.head, c.bufferEnd) distance(c.head, c.bufferEnd)
proc flipPage(s: OutputStreamVar) = proc flipPage(s: OutputStream) =
s.cursor.head = cast[ptr byte](addr s.pages[s.pages.len - 1].buffer[0]) s.cursor.head = cast[ptr byte](addr s.pages[s.pages.len - 1].buffer[0])
# TODO: There is an assumption here and elsewhere that `s.pages[^1]` has # TODO: There is an assumption here and elsewhere that `s.pages[^1]` has
# a length equal to `s.pageSize` # a length equal to `s.pageSize`
s.cursor.bufferEnd = cast[ptr byte](offset(s.cursor.head, s.pageSize)) s.cursor.bufferEnd = cast[ptr byte](offset(s.cursor.head, s.pageSize))
s.endPos += s.pageSize s.endPos += s.pageSize
proc addPage(s: OutputStreamVar) = proc addPage(s: OutputStream) =
s.pages.addLast OutputPage(buffer: newString(s.pageSize), s.pages.addLast OutputPage(buffer: newString(s.pageSize),
startOffset: 0) startOffset: 0)
s.flipPage s.flipPage
proc implementOutputStream*(pageSize: int, proc implementOutputStream*(outputDevice: RootRef,
vtable: ptr OutputStreamVTable,
pageSize = defaultPageSize,
maxWriteSize = high(int), maxWriteSize = high(int),
minWriteSize = 1, minWriteSize = 1): OutputStream =
vtable: ptr OutputStreamVTable = nil,
outputDevice: RootRef = nil): OutputStream =
## This proc is intented for use by module that implement ## This proc is intented for use by module that implement
## their own flavours of OutputStream by providing a custom ## their own flavours of OutputStream by providing a custom
## VTable. Such modules should export easier to use high-level ## VTable. Such modules should export easier to use high-level
@ -94,18 +94,25 @@ proc implementOutputStream*(pageSize: int,
result.addPage result.addPage
result.cursor.stream = result result.cursor.stream = result
proc init*(T: type OutputStream, proc memoryOutput*(pageSize = defaultPageSize): OutputStream =
pageSize = defaultPageSize): OutputStream = implementOutputStream(outputDevice = nil, vtable = nil, pageSize = pageSize)
implementOutputStream pageSize
proc memoryOutput*(buffer: pointer, len: int): OutputStream =
result = OutputStream()
let buffer = cast[ptr byte](buffer)
result.cursor.head = buffer
result.cursor.bufferEnd = offset(buffer, len)
result.cursor.stream = result
result.endPos = len
let FileStreamVTable = OutputStreamVTable( let FileStreamVTable = OutputStreamVTable(
writePage: proc (s: OutputStreamVar, data: openarray[byte]) {.nimcall, gcsafe.} = writePage: proc (s: OutputStream, data: openarray[byte]) {.nimcall, gcsafe.} =
var output = FileOutput(s.outputDevice) var output = FileOutput(s.outputDevice)
var written = output.file.writeBuffer(unsafeAddr data[0], data.len) var written = output.file.writeBuffer(unsafeAddr data[0], data.len)
if written != data.len: if written != data.len:
raise newException(IOError, "Failed to write OutputStream page.") raise newException(IOError, "Failed to write OutputStream page.")
, ,
flush: proc (s: OutputStreamVar) {.nimcall, gcsafe.} = flush: proc (s: OutputStream) {.nimcall, gcsafe.} =
var output = FileOutput(s.outputDevice) var output = FileOutput(s.outputDevice)
flushFile output.file flushFile output.file
) )
@ -117,34 +124,27 @@ template vtableAddr*(vtable: OutputStreamVTable): ptr OutputStreamVTable =
{.noSideEffect.}: {.noSideEffect.}:
unsafeAddr vtable unsafeAddr vtable
proc init*(T: type OutputStream, proc fileOutput*(filename: string,
filename: string, fileMode: FileMode = fmWrite,
pageSize = defaultPageSize): OutputStream = pageSize = defaultPageSize): OutputStream {.
implementOutputStream pageSize, raises: [IOError, Defect]
outputDevice = FileOutput(file: open(filename, fmWrite)), .} =
vtable = vtableAddr FileStreamVTable implementOutputStream FileOutput(file: open(filename, fileMode)),
vtable = vtableAddr FileStreamVTable,
pageSize = pageSize
proc init*(T: type OutputStream, proc pos*(s: OutputStream): int =
buffer: pointer, len: int): OutputStream =
result = OutputStream()
let buffer = cast[ptr byte](buffer)
result.cursor.head = buffer
result.cursor.bufferEnd = offset(buffer, len)
result.cursor.stream = result
result.endPos = len
proc pos*(s: OutputStreamVar): int =
s.endPos - s.cursor.runway s.endPos - s.cursor.runway
proc safeWritePage(s: OutputStreamVar, data: openarray[byte]) {.inline.} = proc safeWritePage(s: OutputStream, data: openarray[byte]) {.inline.} =
if data.len > 0: s.vtable.writePage(s, data) if data.len > 0: s.vtable.writePage(s, data)
proc writePages(s: OutputStreamVar, skipLast = 0) = proc writePages(s: OutputStream, skipLast = 0) =
assert s.vtable != nil assert s.vtable != nil
for i in 0 ..< s.pages.len - skipLast: for i in 0 ..< s.pages.len - skipLast:
s.safeWritePage s.pages[i].buffer.toOpenArrayByte(0, s.pages[i].buffer.len - 1) s.safeWritePage s.pages[i].buffer.toOpenArrayByte(0, s.pages[i].buffer.len - 1)
proc writePartialPage(s: OutputStreamVar, page: var OutputPage) = proc writePartialPage(s: OutputStream, page: var OutputPage) =
assert s.vtable != nil assert s.vtable != nil
let let
unwrittenBytes = s.cursor.runway unwrittenBytes = s.cursor.runway
@ -157,7 +157,7 @@ proc writePartialPage(s: OutputStreamVar, page: var OutputPage) =
page.startOffset = 0 page.startOffset = 0
s.flipPage s.flipPage
proc flush*(s: OutputStreamVar) = proc flush*(s: OutputStream) =
doAssert s.extCursorsCount == 0 doAssert s.extCursorsCount == 0
if s.vtable != nil: if s.vtable != nil:
# We write all pages except the last one # We write all pages except the last one
@ -169,13 +169,13 @@ proc flush*(s: OutputStreamVar) =
# Finally, we flush # Finally, we flush
s.vtable.flush(s) s.vtable.flush(s)
proc writePendingPagesAndLeaveOne(s: OutputStreamVar) {.inline.} = proc writePendingPagesAndLeaveOne(s: OutputStream) {.inline.} =
s.writePages s.writePages
s.pages.shrink(fromFirst = s.pages.len - 1) s.pages.shrink(fromFirst = s.pages.len - 1)
s.pages[0].startOffset = 0 s.pages[0].startOffset = 0
s.flipPage s.flipPage
proc tryFlushing(s: OutputStreamVar) {.inline.} = proc tryFlushing(s: OutputStream) {.inline.} =
# Pre-conditions: # Pre-conditions:
# * The cursor has reached the current buffer end # * The cursor has reached the current buffer end
# #
@ -232,7 +232,7 @@ template append*(c: var WriteCursor, x: char) =
bind append bind append
c.append byte(x) c.append byte(x)
proc writeDataAsPages(s: OutputStreamVar, data: ptr byte, dataLen: int) = proc writeDataAsPages(s: OutputStream, data: ptr byte, dataLen: int) =
var var
data = data data = data
dataLen = dataLen dataLen = dataLen
@ -387,15 +387,15 @@ template append*(c: var WriteCursor, str: string) =
bind append bind append
c.append str.toOpenArrayByte(0, str.len - 1) c.append str.toOpenArrayByte(0, str.len - 1)
template append*(s: OutputStreamVar, value: auto) = template append*(s: OutputStream, value: auto) =
bind append bind append
s.cursor.append value s.cursor.append value
template appendMemCopy*(s: OutputStreamVar, value: auto) = template appendMemCopy*(s: OutputStream, value: auto) =
bind append bind append
s.cursor.append value s.cursor.append value
proc getOutput*(s: OutputStreamVar, T: type string): string = proc getOutput*(s: OutputStream, T: type string): string =
doAssert s.vtable == nil and s.extCursorsCount == 0 and s.pageSize > 0 doAssert s.vtable == nil and s.extCursorsCount == 0 and s.pageSize > 0
s.pages[s.pages.len - 1].buffer.setLen(s.pageSize - s.cursor.runway) s.pages[s.pages.len - 1].buffer.setLen(s.pageSize - s.cursor.runway)
@ -408,18 +408,18 @@ proc getOutput*(s: OutputStreamVar, T: type string): string =
result.add page.buffer.toOpenArray(page.startOffset.int, result.add page.buffer.toOpenArray(page.startOffset.int,
page.buffer.len - 1) page.buffer.len - 1)
template getOutput*(s: OutputStreamVar, T: type seq[byte]): seq[byte] = template getOutput*(s: OutputStream, T: type seq[byte]): seq[byte] =
cast[seq[byte]](s.getOutput(string)) cast[seq[byte]](s.getOutput(string))
template getOutput*(s: OutputStreamVar): seq[byte] = template getOutput*(s: OutputStream): seq[byte] =
cast[seq[byte]](s.getOutput(string)) cast[seq[byte]](s.getOutput(string))
proc finishPageEarly(s: OutputStreamVar, unwrittenBytes: int) {.inline.} = proc finishPageEarly(s: OutputStream, unwrittenBytes: int) {.inline.} =
s.pages[s.pages.len - 1].buffer.setLen(s.pageSize - unwrittenBytes) s.pages[s.pages.len - 1].buffer.setLen(s.pageSize - unwrittenBytes)
s.endPos -= unwrittenBytes s.endPos -= unwrittenBytes
s.tryFlushing() s.tryFlushing()
proc createCursor(s: OutputStreamVar, size: int): WriteCursor = proc createCursor(s: OutputStream, size: int): WriteCursor =
inc s.extCursorsCount inc s.extCursorsCount
result = WriteCursor(head: s.cursor.head, result = WriteCursor(head: s.cursor.head,
@ -428,7 +428,7 @@ proc createCursor(s: OutputStreamVar, size: int): WriteCursor =
s.cursor.head = result.bufferEnd s.cursor.head = result.bufferEnd
proc delayFixedSizeWrite*(s: OutputStreamVar, size: Natural): WriteCursor = proc delayFixedSizeWrite*(s: OutputStream, size: Natural): WriteCursor =
let remainingBytesInPage = s.cursor.runway let remainingBytesInPage = s.cursor.runway
if size <= remainingBytesInPage: if size <= remainingBytesInPage:
result = s.createCursor(size) result = s.createCursor(size)
@ -449,7 +449,7 @@ proc delayFixedSizeWrite*(s: OutputStreamVar, size: Natural): WriteCursor =
s.cursor.bufferEnd = pageEnd s.cursor.bufferEnd = pageEnd
s.endPos += (s.pageSize - size) s.endPos += (s.pageSize - size)
proc delayVarSizeWrite*(s: OutputStreamVar, maxSize: Natural): VarSizeWriteCursor = proc delayVarSizeWrite*(s: OutputStream, maxSize: Natural): VarSizeWriteCursor =
doAssert maxSize < s.pageSize doAssert maxSize < s.pageSize
s.finishPageEarly s.cursor.runway s.finishPageEarly s.cursor.runway
VarSizeWriteCursor s.createCursor(maxSize) VarSizeWriteCursor s.createCursor(maxSize)

2
tests/test_input_stream.nim
View file

@ -5,7 +5,7 @@ import
suite "input stream": suite "input stream":
test "string input": test "string input":
var input = repeat("1234 5678 90AB CDEF\n", 1000) var input = repeat("1234 5678 90AB CDEF\n", 1000)
var stream = memoryStream(input) var stream = memoryInput(input)
check: check:
(stream.readBytes(4) == "1234".toOpenArrayByte(0, 3)) (stream.readBytes(4) == "1234".toOpenArrayByte(0, 3))

6
tests/test_output_stream.nim
View file

@ -21,14 +21,14 @@ proc randomBytes(n: int): seq[byte] =
suite "output stream": suite "output stream":
setup: setup:
var memStream = OutputStream.init var memStream = memoryOutput()
var altOutput: seq[byte] = @[] var altOutput: seq[byte] = @[]
var tempFilePath = getTempDir() / "faststreams_testfile" var tempFilePath = getTempDir() / "faststreams_testfile"
var fileStream = OutputStream.init tempFilePath var fileStream = fileOutput(tempFilePath)
const bufferSize = 1000000 const bufferSize = 1000000
var buffer = alloc(bufferSize) var buffer = alloc(bufferSize)
var existingBufferStream = OutputStream.init(buffer, bufferSize) var existingBufferStream = memoryOutput(buffer, bufferSize)
teardown: teardown:
removeFile tempFilePath removeFile tempFilePath