Implement readInto and read(n)
This commit is contained in:
Zahary Karadjov
parent
b24300bd3f
commit
bf930b6f1b
12 changed files with 636 additions and 220 deletions
|
|
@ -4,22 +4,13 @@ import
|
|||
async_backend
|
||||
|
||||
type
|
||||
PageKind* = enum
|
||||
userPage
|
||||
stringPage
|
||||
mallocPage
|
||||
|
||||
PageSpan* = object
|
||||
startAddr*, endAddr*: ptr byte
|
||||
|
||||
Page* = object
|
||||
startOffset*: Natural
|
||||
endOffset*: Natural
|
||||
case kind*: PageKind
|
||||
of userPage, mallocPage:
|
||||
bufferStart, bufferEnd: ptr byte
|
||||
of stringPage:
|
||||
data*: ref string
|
||||
data*: ref string
|
||||
|
||||
PageRef* = ref Page
|
||||
|
||||
|
|
@ -47,38 +38,30 @@ const
|
|||
defaultPageSize* = 4096 - (pageMetadataSize + nimAllocatorMetadataSize)
|
||||
maxStackUsage* = 16384
|
||||
|
||||
func pageBaseAddr*(page: PageRef): ptr byte =
|
||||
if page.kind == stringPage:
|
||||
cast[ptr byte](addr page.data[][0])
|
||||
else:
|
||||
page.bufferStart
|
||||
proc openArrayToPair*(a: var openarray[byte]): (ptr byte, Natural) =
|
||||
(addr a[0], Natural(a.len))
|
||||
|
||||
template pageBaseAddr*(page: PageRef): ptr byte =
|
||||
cast[ptr byte](addr page.data[][0])
|
||||
|
||||
func pageStartAddr*(page: PageRef): ptr byte =
|
||||
if page.kind == stringPage:
|
||||
offset(cast[ptr byte](addr page.data[][0]), page.startOffset)
|
||||
else:
|
||||
offset(page.bufferStart, page.startOffset)
|
||||
offset(cast[ptr byte](addr page.data[][0]), page.startOffset)
|
||||
|
||||
func pageEndAddr*(page: PageRef): ptr byte =
|
||||
if page.kind == stringPage:
|
||||
offset(cast[ptr byte](addr page.data[][0]), page.endOffset)
|
||||
else:
|
||||
offset(page.bufferStart, page.endOffset)
|
||||
offset(cast[ptr byte](addr page.data[][0]), page.endOffset)
|
||||
|
||||
func pageLen*(page: PageRef): Natural =
|
||||
page.endOffset - page.startOffset
|
||||
|
||||
template pageChars*(page: PageRef): untyped =
|
||||
let baseAddr = cast[ptr UncheckedArray[char]](pageBaseAddr(page))
|
||||
toOpenArray(baseAddr, page.startOffset, page.endOffset - 1)
|
||||
|
||||
func span*(page: PageRef, writable: static[bool] = false): PageSpan =
|
||||
if page.kind == stringPage:
|
||||
let baseAddr = cast[ptr byte](addr page.data[][0])
|
||||
PageSpan(startAddr: offset(baseAddr, page.startOffset),
|
||||
endAddr: offset(baseAddr, when writable: page.data[].len
|
||||
else: page.endOffset))
|
||||
else:
|
||||
PageSpan(startAddr: offset(page.bufferStart, page.startOffset),
|
||||
endAddr: when writable: page.bufferEnd
|
||||
else: offset(page.bufferStart, page.endOffset))
|
||||
let baseAddr = page.pageBaseAddr
|
||||
PageSpan(startAddr: offset(baseAddr, page.startOffset),
|
||||
endAddr: offset(baseAddr, when writable: page.data[].len
|
||||
else: page.endOffset))
|
||||
|
||||
template writableSpan*(page: PageRef): PageSpan =
|
||||
span(page, writable = true)
|
||||
|
|
@ -94,37 +77,46 @@ template allocRef[T: not ref](x: T): ref T =
|
|||
res[] = x
|
||||
res
|
||||
|
||||
func getWritablePage*(buffers: PageBuffers): PageRef =
|
||||
# TODO: The semantics of this func are quite unusual
|
||||
# I should find a more appropriate name
|
||||
if buffers.queue.len == 0:
|
||||
result = PageRef(kind: stringPage,
|
||||
data: allocRef newString(buffers.pageSize),
|
||||
endOffset: buffers.pageSize)
|
||||
buffers.queue.addLast result
|
||||
else:
|
||||
result = buffers.queue[0]
|
||||
|
||||
func addWritablePage*(buffers: PageBuffers, pageSize: Natural): PageRef =
|
||||
result = PageRef(kind: stringPage,
|
||||
data: allocRef newString(pageSize),
|
||||
result = PageRef(data: allocRef newString(pageSize),
|
||||
endOffset: pageSize)
|
||||
buffers.queue.addLast result
|
||||
|
||||
func getWritablePage*(buffers: PageBuffers,
|
||||
preferredSize: Natural): PageRef =
|
||||
if buffers.queue.len == 1:
|
||||
let recycledPage = buffers.queue.peekLast
|
||||
if recycledPage.endOffset == 0 and
|
||||
recycledPage.data[].len == preferredSize:
|
||||
recycledPage.endOffset = recycledPage.data[].len
|
||||
return recycledPage
|
||||
|
||||
return addWritablePage(buffers, preferredSize)
|
||||
|
||||
func addWritablePage*(buffers: PageBuffers): PageRef =
|
||||
buffers.addWritablePage(buffers.pageSize)
|
||||
|
||||
template getWritableSpan*(buffers: PageBuffers): PageSpan =
|
||||
getWritablePage(buffers).span(writable = true)
|
||||
getWritablePage(buffers, buffers.pageSize).span(writable = true)
|
||||
|
||||
proc getReadableSpan*(buffers: PageBuffers): PageSpan =
|
||||
if buffers.queue.len > 1:
|
||||
discard buffers.queue.popFirst
|
||||
|
||||
buffers.queue[0].span
|
||||
|
||||
func ensureRunway*(buffers: PageBuffers, neededRunway: Natural): PageSpan =
|
||||
doAssert buffers.queue.len == 0
|
||||
buffers.pageSize = neededRunway
|
||||
getWritableSpan(buffers)
|
||||
|
||||
template len*(buffers: PageBuffers): int =
|
||||
template len*(buffers: PageBuffers): Natural =
|
||||
buffers.queue.len
|
||||
|
||||
func totalBufferredBytes*(buffers: PageBuffers): Natural =
|
||||
for i in 1 ..< buffers.queue.len:
|
||||
result += buffers.queue[i].pageLen
|
||||
|
||||
template popFirst*(buffers: PageBuffers): PageRef =
|
||||
buffers.queue.popFirst
|
||||
|
||||
|
|
@ -148,54 +140,36 @@ func endLastPageAt*(buffers: PageBuffers, address: ptr byte) =
|
|||
var topPage = buffers.queue.peekLast
|
||||
topPage.endOffset = distance(topPage.pageBaseAddr, address)
|
||||
|
||||
func trackPageWrite*(page: PageRef, bytesWritten: Natural) {.inline.} =
|
||||
page.endOffset = page.startOffset + bytesWritten
|
||||
|
||||
template writeToSpan*(buffersParam: PageBuffers,
|
||||
spanVarName, writeExpr: untyped) =
|
||||
var
|
||||
buffers = buffersParam
|
||||
page = buffers.getWritablePage
|
||||
spanVarName = page.writableSpan
|
||||
|
||||
# TODO: what if we exit with an exception here?
|
||||
# Are the side-effects of `getWritablePage` above OK to keep?
|
||||
|
||||
let bytesWritten = writeExpr
|
||||
trackPageWrite(page, bytesWritten)
|
||||
|
||||
if bytesWritten == 0:
|
||||
buffers.eofReached = true
|
||||
|
||||
func nextAlignedSize*(minSize, pageSize: Natural): Natural =
|
||||
# TODO: This is not perfectly accurate. Revisit later
|
||||
((minSize div pageSize) + 1) * pageSize
|
||||
|
||||
template consumeAllPages*(buffersParam: PageBuffers,
|
||||
pageAddrVar, pageLenVar, body: untyped) =
|
||||
let buffers = buffersParam
|
||||
iterator consumePages*(buffers: PageBuffers): PageRef =
|
||||
doAssert buffers != nil
|
||||
|
||||
var recycledPage: PageRef
|
||||
for page in buffers.queue:
|
||||
let
|
||||
pageAddrVar = page.pageStartAddr
|
||||
pageLenVar = page.endOffset - page.startOffset
|
||||
|
||||
if page.kind == stringPage and page.data[].len == buffers.pageSize:
|
||||
recycledPage = page
|
||||
while buffers.queue.len > 0:
|
||||
var page = peekFirst(buffers.queue)
|
||||
|
||||
# TODO: what if the body throws an exception?
|
||||
# Should we do anything with the remaining pages?
|
||||
body
|
||||
# Should we do anything with the consumed page?
|
||||
yield page
|
||||
|
||||
buffers.queue.clear()
|
||||
if page.data[].len == buffers.pageSize:
|
||||
recycledPage = page
|
||||
|
||||
discard buffers.queue.popFirst
|
||||
|
||||
if recycledPage != nil:
|
||||
recycledPage.startOffset = 0
|
||||
recycledPage.endOffset = 0
|
||||
buffers.queue.addLast recycledPage
|
||||
|
||||
iterator consumePageBuffers*(buffers: PageBuffers): (ptr byte, Natural) =
|
||||
for page in consumePages(buffers):
|
||||
yield (page.pageStartAddr,
|
||||
Natural(page.endOffset - page.startOffset))
|
||||
|
||||
template wasEofReached*(buffers: PageBuffers): bool =
|
||||
buffers.eofReached
|
||||
|
||||
|
|
@ -224,3 +198,74 @@ template charsToBytes*(chars: openArray[char]): untyped =
|
|||
var charsStart = unsafeAddr chars[0]
|
||||
makeOpenArray(cast[ptr byte](charsStart), chars.len)
|
||||
|
||||
template implementWrites*(buffersParam: PageBuffers,
|
||||
srcParam: pointer,
|
||||
srcLenParam: Natural,
|
||||
dstDesc: static string,
|
||||
writeStartVar, writeLenVar,
|
||||
writeBlock: untyped) =
|
||||
let
|
||||
buffers = buffersParam
|
||||
writeStartVar = srcParam
|
||||
writeLenVar = srcLenParam
|
||||
|
||||
template raiseError =
|
||||
raise newException(IOError, "Failed to write all bytes to " & dstDesc)
|
||||
|
||||
if buffers != nil:
|
||||
for writeStartVar, writeLenVar in consumePageBuffers(s.buffers):
|
||||
let bytesWritten = writeBlock
|
||||
# TODO: Can we repair the buffers here?
|
||||
if bytesWritten != writeLenVar: raiseError()
|
||||
buffers.totalBytesWritten += bytesWritten
|
||||
|
||||
if srcLen > 0:
|
||||
doAssert src != nil
|
||||
let bytesWritten = writeBlock
|
||||
if bytesWritten != writeLenVar: raiseError()
|
||||
# TODO: Fix this after removing the unbuffered streams
|
||||
if buffers != nil:
|
||||
buffers.totalBytesWritten += bytesWritten
|
||||
|
||||
type
|
||||
ReadFlag* = enum
|
||||
partialReadIsEof
|
||||
zeroReadIsNotEof
|
||||
|
||||
ReadFlags* = set[ReadFlag]
|
||||
|
||||
template implementSingleRead*(buffersParam: PageBuffers,
|
||||
dstParam: pointer,
|
||||
dstLenParam: Natural,
|
||||
flags: static ReadFlags,
|
||||
readStartVar, readLenVar,
|
||||
readBlock: untyped): Natural =
|
||||
var
|
||||
buffers = buffersParam
|
||||
readStartVar = dstParam
|
||||
readLenVar = dstLenParam
|
||||
bytesRead: Natural
|
||||
|
||||
if readStartVar != nil:
|
||||
bytesRead = readBlock
|
||||
else:
|
||||
let
|
||||
bestPageSize = nextAlignedSize(readLenVar, buffers.pageSize)
|
||||
page = getWritablePage(buffers, bestPageSize)
|
||||
|
||||
readStartVar = page.pageStartAddr
|
||||
readLenVar = page.endOffset - page.startOffset
|
||||
|
||||
# TODO: what if we exit with an exception here?
|
||||
# Are the side-effects of `getWritablePage` above OK to keep?
|
||||
bytesRead = readBlock
|
||||
page.endOffset = page.startOffset + bytesRead
|
||||
|
||||
if (bytesRead == 0 and zeroReadIsNotEof notin flags) or
|
||||
(partialReadIsEof in flags and bytesRead < readLenVar):
|
||||
buffers.eofReached = true
|
||||
else:
|
||||
buffers.totalBytesRead += bytesRead
|
||||
|
||||
bytesRead
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue