jyapayne/nim-faststreams
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add destructors

Browse source
This commit is contained in:
Zahary Karadjov 2020-04-11 21:27:33 +03:00
commit 1f49f1179d
No known key found for this signature in database
GPG key ID: C8936F8A3073D609
3 changed files with 149 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

99
faststreams/input_stream.nim
View file

@ -3,17 +3,19 @@ import
stew/[ptrops, ranges/ptr_arith] stew/[ptrops, ranges/ptr_arith]
type type
# We inherit from RootObj because in layered streams InputStream* = ref object of RootObj
# the stream itself is often used as an `outputDevice` vtable*: ptr InputStreamVTable
InputStreamObj = object of RootObj
head*: ptr byte head*: ptr byte
bufferSize: int bufferSize: int
bufferStart, bufferEnd: ptr byte bufferStart, bufferEnd: ptr byte
bufferEndPos: int bufferEndPos: int
inputDevice*: RootRef inputDevice*: RootRef
vtable*: ptr InputStreamVTable
InputStream* = ref InputStreamObj LayeredInputStream* = ref object of InputStream
subStream*: InputStream
InputStreamHandle* = object
s*: InputStream
AsciiInputStream* = distinct InputStream AsciiInputStream* = distinct InputStream
Utf8InputStream* = distinct InputStream Utf8InputStream* = distinct InputStream
@ -29,16 +31,23 @@ type
cb: ReadAsyncCallback) cb: ReadAsyncCallback)
{.nimcall, gcsafe, raises: [IOError, Defect].} {.nimcall, gcsafe, raises: [IOError, Defect].}
CloseStreamProc* = proc (s: InputStream) CloseSyncProc* = proc (s: InputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].} {.nimcall, gcsafe, raises: [IOError, Defect].}
CloseAsyncCallback* = proc (s: InputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseAsyncProc* = proc (s: InputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
InputStreamVTable* = object InputStreamVTable* = object
readSync*: ReadSyncProc readSync*: ReadSyncProc
readAsync*: ReadAsyncProc readAsync*: ReadAsyncProc
closeStream*: CloseStreamProc closeSync*: CloseSyncProc
closeAsync*: CloseAsyncProc
hasKnownLen*: bool hasKnownLen*: bool
FileInput = ref object of RootObj FileInputStream = ref object of InputStream
file: MemFile file: MemFile
const const
@ -46,15 +55,34 @@ const
nimAllocatorMetadataSize* = 0 nimAllocatorMetadataSize* = 0
# TODO: Get this from Nim's allocator. # TODO: Get this from Nim's allocator.
# The goal is to make perfect page-aligned allocations # The goal is to make perfect page-aligned allocations
defaultPageSize = 4096 - nimAllocatorMetadataSize # defaultPageSize = 4096 - nimAllocatorMetadataSize
proc close*(s: var InputStream) {.raises: [IOError, Defect].} =
if s != nil:
if s.vtable != nil and s.vtable.closeSync != nil:
s.vtable.closeSync(s)
s = nil
proc `=destroy`*(h: var InputStreamHandle) {.raises: [Defect].} =
if h.s != nil:
if h.s.vtable != nil and h.s.vtable.closeSync != nil:
try:
h.s.vtable.closeSync(h.s)
except IOError:
# Since this is a destructor, there is not much we can do here.
# If the user wanted to handle the error, they would have called
# `close` manually.
discard # TODO
h.s = nil
converter implicitDeref*(h: InputStreamHandle): InputStream =
h.s
let FileStreamVTable = InputStreamVTable( let FileStreamVTable = InputStreamVTable(
readSync: nil, closeSync: proc (s: InputStream)
readAsync: nil, {.nimcall, gcsafe, raises: [IOError, Defect].} =
closeStream: proc (s: InputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].} =
try: try:
close FileInput(s.inputDevice).file close FileInputStream(s).file
except OSError as err: except OSError as err:
raise newException(IOError, "Failed to close file", err) raise newException(IOError, "Failed to close file", err)
, ,
@ -68,37 +96,32 @@ template vtableAddr*(vtable: InputStreamVTable): ptr InputStreamVTable =
{.noSideEffect.}: {.noSideEffect.}:
unsafeAddr vtable unsafeAddr vtable
proc fileInput*(filename: string): InputStream = proc fileInput*(filename: string): InputStreamHandle =
let let
inputDevice = FileInput(file: memfiles.open(filename)) memFile = memfiles.open(filename)
head = cast[ptr byte](inputDevice.file.mem) head = cast[ptr byte](memFile.mem)
fileSize = inputDevice.file.size fileSize = memFile.size
result = InputStream( var stream = FileInputStream(
vtable: vtableAddr FileStreamVTable,
head: head, head: head,
bufferEnd: offset(head, fileSize), bufferEnd: offset(head, fileSize),
bufferEndPos: fileSize, bufferEndPos: fileSize,
inputDevice: inputDevice, file: memFile)
vtable: vtableAddr FileStreamVTable)
when debugHelpers: when debugHelpers:
result.bufferStart = result.head stream.bufferStart = head
proc implementInputStream*(vtable: ptr InputStreamVTable, InputStreamHandle(s: stream)
inputDevice: RootRef,
pageSize = defaultPageSize): InputStream =
# TODO: We need to allocate memory here and start reading
InputStream(inputDevice: inputDevice,
vtable: vtable)
proc memoryInput*(mem: openarray[byte]): InputStream = proc memoryInput*(mem: openarray[byte]): InputStreamHandle =
let head = unsafeAddr mem[0] let head = unsafeAddr mem[0]
InputStream( InputStreamHandle(s: InputStream(
head: head, head: head,
bufferEnd: offset(head, mem.len), bufferEnd: offset(head, mem.len),
bufferEndPos: mem.len) bufferEndPos: mem.len))
proc memoryInput*(str: string): InputStream = proc memoryInput*(str: string): InputStreamHandle =
memoryInput str.toOpenArrayByte(0, str.len - 1) memoryInput str.toOpenArrayByte(0, str.len - 1)
proc endPos*(s: InputStream): int = proc endPos*(s: InputStream): int =
@ -188,17 +211,11 @@ proc rewind*(s: InputStream, delta: int) =
proc rewindTo*(s: InputStream, pos: int) {.inline.} = proc rewindTo*(s: InputStream, pos: int) {.inline.} =
s.head = s.bufferPos(pos) s.head = s.bufferPos(pos)
# TODO: use a destructor once we migrate to Nim 0.20
# TODO: It's not appropriate for this to raise
proc close*(s: InputStream) {.raises: [IOError, Defect].} =
if s.vtable != nil:
s.vtable.closeStream(s)
template pos*(s: AsciiInputStream|Utf8InputStream): int = template pos*(s: AsciiInputStream|Utf8InputStream): int =
InputStream(s).pos pos InputStream(s)
template eof*(s: AsciiInputStream|Utf8InputStream): bool = template eof*(s: AsciiInputStream|Utf8InputStream): bool =
InputStream(s).eof eof InputStream(s)
template close*(s: AsciiInputStream|Utf8InputStream) = template close*(s: AsciiInputStream|Utf8InputStream) =
close InputStream(s) close InputStream(s)

142
faststreams/output_stream.nim
View file

@ -7,9 +7,8 @@ type
buffer: string buffer: string
startOffset: int startOffset: int
# We inherit from RootObj because in layered streams OutputStream* = ref object of RootObj
# the stream itself is often used as an `outputDevice` vtable*: ptr OutputStreamVTable
OutputStreamObj = object of RootObj
cursor*: WriteCursor cursor*: WriteCursor
pages: Deque[OutputPage] pages: Deque[OutputPage]
endPos: int endPos: int
@ -17,10 +16,12 @@ type
pageSize: int pageSize: int
maxWriteSize*: int maxWriteSize*: int
minWriteSize*: int minWriteSize*: int
outputDevice*: RootRef
vtable*: ptr OutputStreamVTable
OutputStream* = ref OutputStreamObj LayeredOutputStream* = ref object of OutputStream
subStream*: OutputStream
OutputStreamHandle* = object
s*: OutputStream
WritePageProc* = proc (s: OutputStream, page: openarray[byte]) WritePageProc* = proc (s: OutputStream, page: openarray[byte])
{.nimcall, gcsafe, raises: [IOError, Defect].} {.nimcall, gcsafe, raises: [IOError, Defect].}
@ -28,9 +29,20 @@ type
FlushProc* = proc (s: OutputStream) FlushProc* = proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].} {.nimcall, gcsafe, raises: [IOError, Defect].}
CloseSyncProc* = proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseAsyncCallback* = proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
CloseAsyncProc* = proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].}
OutputStreamVTable* = object OutputStreamVTable* = object
writePage*: WritePageProc writePage*: WritePageProc
flush*: FlushProc flush*: FlushProc
closeSync*: CloseSyncProc
closeAsyncProc*: CloseAsyncProc
WriteCursor* = object WriteCursor* = object
head, bufferEnd: ptr byte head, bufferEnd: ptr byte
@ -38,7 +50,7 @@ type
VarSizeWriteCursor* = distinct WriteCursor VarSizeWriteCursor* = distinct WriteCursor
FileOutput = ref object of RootObj FileOutputStream = ref object of OutputStream
file: File file: File
const const
@ -47,9 +59,26 @@ const
# The goal is to make perfect page-aligned allocations # The goal is to make perfect page-aligned allocations
defaultPageSize = 4096 - nimAllocatorMetadataSize - 1 # 1 byte for the null terminator defaultPageSize = 4096 - nimAllocatorMetadataSize - 1 # 1 byte for the null terminator
proc createWriteCursor*[R, T](x: var array[R, T]): WriteCursor = proc close*(s: var OutputStream) {.raises: [IOError, Defect].} =
let startAddr = cast[ptr byte](addr x[0]) if s != nil:
WriteCursor(head: startAddr, bufferEnd: offset(startAddr, sizeof x)) if s.vtable != nil and s.vtable.closeSync != nil:
s.vtable.closeSync(s)
s = nil
proc `=destroy`*(h: var OutputStreamHandle) {.raises: [Defect].} =
if h.s != nil:
if h.s.vtable != nil and h.s.vtable.closeSync != nil:
try:
h.s.vtable.closeSync(h.s)
except IOError:
# Since this is a destructor, there is not much we can do here.
# If the user wanted to handle the error, they would have called
# `close` manually.
discard # TODO
h.s = nil
converter implicitDeref*(h: OutputStreamHandle): OutputStream =
h.s
template canExtendOutput(s: OutputStream): bool = template canExtendOutput(s: OutputStream): bool =
# Streams writing to pre-allocated existing buffers cannot be grown # Streams writing to pre-allocated existing buffers cannot be grown
@ -74,47 +103,48 @@ proc addPage(s: OutputStream) =
startOffset: 0) startOffset: 0)
s.flipPage s.flipPage
proc implementOutputStream*(outputDevice: RootRef, proc initWithSinglePage*(s: OutputStream) =
vtable: ptr OutputStreamVTable, s.addPage()
pageSize = defaultPageSize, s.cursor.stream = s
maxWriteSize = high(int),
minWriteSize = 1): OutputStream = proc memoryOutput*(pageSize = defaultPageSize): OutputStreamHandle =
## This proc is intented for use by module that implement var stream = OutputStream(
## their own flavours of OutputStream by providing a custom
## VTable. Such modules should export easier to use high-level
## constructors intended for end-users.
result = OutputStream(
pageSize: pageSize, pageSize: pageSize,
maxWriteSize: maxWriteSize, minWriteSize: 1,
minWriteSize: minWriteSize, maxWriteSize: high(int),
pages: initDeque[OutputPage](), pages: initDeque[OutputPage]())
vtable: vtable,
outputDevice: outputDevice)
result.addPage stream.initWithSinglePage()
result.cursor.stream = result
proc memoryOutput*(pageSize = defaultPageSize): OutputStream = OutputStreamHandle(s: stream)
implementOutputStream(outputDevice = nil, vtable = nil, pageSize = pageSize)
proc memoryOutput*(buffer: pointer, len: int): OutputStream = proc memoryOutput*(buffer: pointer, len: int): OutputStreamHandle =
result = OutputStream()
let buffer = cast[ptr byte](buffer) let buffer = cast[ptr byte](buffer)
result.cursor.head = buffer
result.cursor.bufferEnd = offset(buffer, len) var stream = OutputStream()
result.cursor.stream = result stream.cursor.head = buffer
result.endPos = len stream.cursor.bufferEnd = offset(buffer, len)
stream.cursor.stream = stream
stream.endPos = len
OutputStreamHandle(s: stream)
let FileStreamVTable = OutputStreamVTable( let FileStreamVTable = OutputStreamVTable(
writePage: proc (s: OutputStream, data: openarray[byte]) {.nimcall, gcsafe.} = writePage: proc (s: OutputStream,
var output = FileOutput(s.outputDevice) data: openarray[byte])
var written = output.file.writeBuffer(unsafeAddr data[0], data.len) {.nimcall, gcsafe, raises: [IOError, Defect].} =
var file = FileOutputStream(s).file
var written = file.writeBuffer(unsafeAddr data[0], data.len)
if written != data.len: if written != data.len:
raise newException(IOError, "Failed to write OutputStream page.") raise newException(IOError, "Failed to write OutputStream page.")
, ,
flush: proc (s: OutputStream) {.nimcall, gcsafe.} = flush: proc (s: OutputStream)
var output = FileOutput(s.outputDevice) {.nimcall, gcsafe, raises: [IOError, Defect].} =
flushFile output.file flushFile FileOutputStream(s).file
,
closeSync: proc (s: OutputStream)
{.nimcall, gcsafe, raises: [IOError, Defect].} =
close FileOutputStream(s).file
) )
template vtableAddr*(vtable: OutputStreamVTable): ptr OutputStreamVTable = template vtableAddr*(vtable: OutputStreamVTable): ptr OutputStreamVTable =
@ -126,12 +156,22 @@ template vtableAddr*(vtable: OutputStreamVTable): ptr OutputStreamVTable =
proc fileOutput*(filename: string, proc fileOutput*(filename: string,
fileMode: FileMode = fmWrite, fileMode: FileMode = fmWrite,
pageSize = defaultPageSize): OutputStream {. pageSize = defaultPageSize): OutputStreamHandle {.
raises: [IOError, Defect] raises: [IOError, Defect]
.} = .} =
implementOutputStream FileOutput(file: open(filename, fileMode)), let f = open(filename, fileMode)
vtable = vtableAddr FileStreamVTable,
pageSize = pageSize var stream = FileOutputStream(
vtable: vtableAddr FileStreamVTable,
pageSize: pageSize,
minWriteSize: 1,
maxWriteSize: high(int),
pages: initDeque[OutputPage](),
file: f)
stream.initWithSinglePage()
OutputStreamHandle(s: stream)
proc pos*(s: OutputStream): int = proc pos*(s: OutputStream): int =
s.endPos - s.cursor.runway s.endPos - s.cursor.runway
@ -387,15 +427,15 @@ template appendMemCopy*[T](c: var WriteCursor, value: T) =
template append*(c: var WriteCursor, str: string) = template append*(c: var WriteCursor, str: string) =
bind append bind append
c.append str.toOpenArrayByte(0, str.len - 1) append c, str.toOpenArrayByte(0, str.len - 1)
template append*(s: OutputStream, value: auto) = template append*(s: OutputStream, value: auto) =
bind append bind append
s.cursor.append value append s.cursor, value
template appendMemCopy*(s: OutputStream, value: auto) = template appendMemCopy*(s: OutputStream, value: auto) =
bind append bind append
s.cursor.append value append s.cursor, value
proc getOutput*(s: OutputStream, T: type string): string = proc getOutput*(s: OutputStream, T: type string): string =
doAssert s.vtable == nil and s.extCursorsCount == 0 and s.pageSize > 0 doAssert s.vtable == nil and s.extCursorsCount == 0 and s.pageSize > 0
@ -433,9 +473,9 @@ proc createCursor(s: OutputStream, size: int): WriteCursor =
proc delayFixedSizeWrite*(s: OutputStream, size: Natural): WriteCursor = proc delayFixedSizeWrite*(s: OutputStream, size: Natural): WriteCursor =
let remainingBytesInPage = s.cursor.runway let remainingBytesInPage = s.cursor.runway
if size <= remainingBytesInPage: if size <= remainingBytesInPage:
result = s.createCursor(size) result = createCursor(s, size)
else: else:
result = s.createCursor(remainingBytesInPage) result = createCursor(s, remainingBytesInPage)
var size = size - remainingBytesInPage var size = size - remainingBytesInPage
s.endPos += size s.endPos += size
while size > s.pageSize: while size > s.pageSize:
@ -454,7 +494,7 @@ proc delayFixedSizeWrite*(s: OutputStream, size: Natural): WriteCursor =
proc delayVarSizeWrite*(s: OutputStream, maxSize: Natural): VarSizeWriteCursor = proc delayVarSizeWrite*(s: OutputStream, maxSize: Natural): VarSizeWriteCursor =
doAssert maxSize < s.pageSize doAssert maxSize < s.pageSize
s.finishPageEarly s.cursor.runway s.finishPageEarly s.cursor.runway
VarSizeWriteCursor s.createCursor(maxSize) VarSizeWriteCursor createCursor(s, maxSize)
proc finalize*(cursor: var WriteCursor) = proc finalize*(cursor: var WriteCursor) =
doAssert cursor.stream.extCursorsCount > 0 doAssert cursor.stream.extCursorsCount > 0

11
tests/test_output_stream.nim
View file

@ -152,14 +152,3 @@ suite "output stream":
# The final outputs are the same # The final outputs are the same
check altOutput == memStream.getOutput check altOutput == memStream.getOutput
test "nim odd behavior":
# discovered in status-im/nim-snappy#2
proc encode(s: OutputStream) =
check true
proc abc(s: OutputStream) =
OutputStream(s.outputDevice).encode
check true
var os = new OutputStream
os.abc()