jyapayne/nim-faststreams
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Enable --styleCheck:usages

Browse source
This commit is contained in:
Jacek Sieka 2021-12-05 16:11:54 +01:00 • committed by zah
commit 1f62235d4f
9 changed files with 39 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -34,7 +34,7 @@ in a way that allows the read and write operations to be handled without any
dynamic dispatch in the majority of cases. dynamic dispatch in the majority of cases.
In particular, reading from a `memoryInput` or writing to a `memoryOutput` In particular, reading from a `memoryInput` or writing to a `memoryOutput`
will have similar performance to a loop iterating over an `openarray` or will have similar performance to a loop iterating over an `openArray` or
another loop populating a pre-allocated `string`. `memFileInput` offers another loop populating a pre-allocated `string`. `memFileInput` offers
the same performance characteristics when working with files. The idiomatic the same performance characteristics when working with files. The idiomatic
use of the APIs with the rest of the stream types will result in a highly use of the APIs with the rest of the stream types will result in a highly

8
faststreams.nimble
View file

@ -20,13 +20,15 @@ proc test(env, path: string) =
if existsEnv"TEST_LANG": if existsEnv"TEST_LANG":
lang = getEnv"TEST_LANG" lang = getEnv"TEST_LANG"
let common_args = "-r -f --hints:off --skipParentCfg --styleCheck:usages --styleCheck:error"
exec "nim " & lang & " " & env & exec "nim " & lang & " " & env &
" -d:async_backend=none -r --hints:off --skipParentCfg " & path " -d:async_backend=none " & common_args & " " & path
exec "nim " & lang & " " & env & exec "nim " & lang & " " & env &
" -d:async_backend=chronos -r --hints:off --skipParentCfg " & path " -d:async_backend=chronos " & common_args & " " & path
# TODO std backend is broken / untested # TODO std backend is broken / untested
# exec "nim " & lang & " " & env & # exec "nim " & lang & " " & env &
# " -d:async_backend=chronos -r --hints:off --skipParentCfg " & path # " -d:async_backend=chronos " & common_args & " " & path
task test, "Run all tests": task test, "Run all tests":
test "-d:debug --threads:on", "tests/all_tests" test "-d:debug --threads:on", "tests/all_tests"

2
faststreams/buffers.nim
View file

@ -55,7 +55,7 @@ else:
template describeBuffers*(context: static string, buffers: PageBuffers) = template describeBuffers*(context: static string, buffers: PageBuffers) =
discard discard
func openArrayToPair*(a: var openarray[byte]): (ptr byte, Natural) = func openArrayToPair*(a: var openArray[byte]): (ptr byte, Natural) =
(addr a[0], Natural(a.len)) (addr a[0], Natural(a.len))
template allocationStart*(page: PageRef): ptr byte = template allocationStart*(page: PageRef): ptr byte =

28
faststreams/inputs.nim
View file

@ -424,7 +424,7 @@ proc fileInput*(filename: string,
buffers: initPageBuffers(pageSize), buffers: initPageBuffers(pageSize),
file: file) file: file)
proc unsafeMemoryInput*(mem: openarray[byte]): InputStreamHandle = proc unsafeMemoryInput*(mem: openArray[byte]): InputStreamHandle =
let head = unsafeAddr mem[0] let head = unsafeAddr mem[0]
makeHandle InputStream( makeHandle InputStream(
@ -457,7 +457,7 @@ func memoryInput*(buffers: PageBuffers): InputStreamHandle =
span: span, span: span,
spanEndPos: spanEndPos) spanEndPos: spanEndPos)
func memoryInput*(data: openarray[byte]): InputStreamHandle = func memoryInput*(data: openArray[byte]): InputStreamHandle =
let stream = if data.len > 0: let stream = if data.len > 0:
let let
buffers = initPageBuffers(data.len) buffers = initPageBuffers(data.len)
@ -474,7 +474,7 @@ func memoryInput*(data: openarray[byte]): InputStreamHandle =
makeHandle stream makeHandle stream
func memoryInput*(data: openarray[char]): InputStreamHandle = func memoryInput*(data: openArray[char]): InputStreamHandle =
memoryInput charsToBytes(data) memoryInput charsToBytes(data)
proc resetBuffers*(s: InputStream, buffers: PageBuffers) = proc resetBuffers*(s: InputStream, buffers: PageBuffers) =
@ -812,7 +812,7 @@ template readIntoExImpl(s: InputStream,
dstLen - bytesDeficit dstLen - bytesDeficit
proc readIntoEx*(s: InputStream, dst: var openarray[byte]): int = proc readIntoEx*(s: InputStream, dst: var openArray[byte]): int =
## Read data into the destination buffer. ## Read data into the destination buffer.
## ##
## Returns the number of bytes that were successfully ## Returns the number of bytes that were successfully
@ -823,7 +823,7 @@ proc readIntoEx*(s: InputStream, dst: var openarray[byte]): int =
let dstLen = dst.len let dstLen = dst.len
readIntoExImpl(s, dstAddr, dstLen, noAwait, readSync) readIntoExImpl(s, dstAddr, dstLen, noAwait, readSync)
proc readInto*(s: InputStream, target: var openarray[byte]): bool = proc readInto*(s: InputStream, target: var openArray[byte]): bool =
## Read data into the destination buffer. ## Read data into the destination buffer.
## ##
## Returns `false` if EOF was reached before the buffer ## Returns `false` if EOF was reached before the buffer
@ -832,14 +832,14 @@ proc readInto*(s: InputStream, target: var openarray[byte]): bool =
s.readIntoEx(target) == target.len s.readIntoEx(target) == target.len
when fsAsyncSupport: when fsAsyncSupport:
template readIntoEx*(sp: AsyncInputStream, dst: var openarray[byte]): int = template readIntoEx*(sp: AsyncInputStream, dst: var openArray[byte]): int =
let s = InputStream(sp) let s = InputStream(sp)
# BEWARE! `openArrayToPair` here is needed to avoid # BEWARE! `openArrayToPair` here is needed to avoid
# double evaluation of the `dst` expression: # double evaluation of the `dst` expression:
let (dstAddr, dstLen) = openArrayToPair(dst) let (dstAddr, dstLen) = openArrayToPair(dst)
readIntoExImpl(s, dstAddr, dstLen, fsAwait, readAsync) readIntoExImpl(s, dstAddr, dstLen, fsAwait, readAsync)
template readInto*(sp: AsyncInputStream, dst: var openarray[byte]): bool = template readInto*(sp: AsyncInputStream, dst: var openArray[byte]): bool =
## Asynchronously read data into the destination buffer. ## Asynchronously read data into the destination buffer.
## ##
## Returns `false` if EOF was reached before the buffer ## Returns `false` if EOF was reached before the buffer
@ -871,7 +871,7 @@ template useStackMem(n: static Natural) =
template readNImpl(sp: InputStream, template readNImpl(sp: InputStream,
np: Natural, np: Natural,
createAllocMemOp: untyped): openarray[byte] = createAllocMemOp: untyped): openArray[byte] =
let let
s = sp s = sp
n = np n = np
@ -881,7 +881,7 @@ template readNImpl(sp: InputStream,
# to appear in different branches of an if statement, the code must # to appear in different branches of an if statement, the code must
# be written in this branch-free linear fashion. The `dataCopy` seq # be written in this branch-free linear fashion. The `dataCopy` seq
# may remain empty in the case where we use stack memory or return # may remain empty in the case where we use stack memory or return
# an `openarray` from the existing span. # an `openArray` from the existing span.
var startAddr: ptr byte var startAddr: ptr byte
block: block:
@ -902,21 +902,21 @@ template readNImpl(sp: InputStream,
makeOpenArray(startAddr, n) makeOpenArray(startAddr, n)
template read*(sp: InputStream, np: static Natural): openarray[byte] = template read*(sp: InputStream, np: static Natural): openArray[byte] =
const n = np const n = np
when n < maxStackUsage: when n < maxStackUsage:
readNImpl(sp, n, useStackMem) readNImpl(sp, n, useStackMem)
else: else:
readNImpl(sp, n, useHeapMem) readNImpl(sp, n, useHeapMem)
template read*(s: InputStream, n: Natural): openarray[byte] = template read*(s: InputStream, n: Natural): openArray[byte] =
readNImpl(s, n, useHeapMem) readNImpl(s, n, useHeapMem)
when fsAsyncSupport: when fsAsyncSupport:
template read*(s: AsyncInputStream, n: Natural): openarray[byte] = template read*(s: AsyncInputStream, n: Natural): openArray[byte] =
read InputStream(s), n read InputStream(s), n
proc lookAheadMatch*(s: InputStream, data: openarray[byte]): bool = proc lookAheadMatch*(s: InputStream, data: openArray[byte]): bool =
for i in 0 ..< data.len: for i in 0 ..< data.len:
if s.peekAt(i) != data[i]: if s.peekAt(i) != data[i]:
return false return false
@ -924,7 +924,7 @@ proc lookAheadMatch*(s: InputStream, data: openarray[byte]): bool =
return true return true
when fsAsyncSupport: when fsAsyncSupport:
template lookAheadMatch*(s: AsyncInputStream, data: openarray[byte]): bool = template lookAheadMatch*(s: AsyncInputStream, data: openArray[byte]): bool =
lookAheadMatch InputStream(s) lookAheadMatch InputStream(s)
proc next*(s: InputStream): Option[byte] = proc next*(s: InputStream): Option[byte] =

14
faststreams/outputs.nim
View file

@ -652,13 +652,13 @@ proc writeMemCopy*(s: OutputStream, value: auto) =
when fsAsyncSupport: when fsAsyncSupport:
proc writeBytesAsyncImpl(sp: OutputStream, proc writeBytesAsyncImpl(sp: OutputStream,
bytes: openarray[byte]): Future[void] = bytes: openArray[byte]): Future[void] =
let s = sp let s = sp
writeBytesImpl(s, bytes): writeBytesImpl(s, bytes):
return s.vtable.writeAsync(s, unsafeAddr bytes[0], bytes.len) return s.vtable.writeAsync(s, unsafeAddr bytes[0], bytes.len)
proc writeBytesAsyncImpl(s: OutputStream, proc writeBytesAsyncImpl(s: OutputStream,
chars: openarray[char]): Future[void] = chars: openArray[char]): Future[void] =
writeBytesAsyncImpl s, charsToBytes(chars) writeBytesAsyncImpl s, charsToBytes(chars)
proc writeBytesAsyncImpl(s: OutputStream, proc writeBytesAsyncImpl(s: OutputStream,
@ -684,7 +684,7 @@ when fsAsyncSupport:
template writeMemCopyAndWait*(sp: AsyncOutputStream, value: auto) = template writeMemCopyAndWait*(sp: AsyncOutputStream, value: auto) =
writeAndWait(sp, memCopyToBytes(value)) writeAndWait(sp, memCopyToBytes(value))
proc writeBytesToCursor(c: var WriteCursor, bytes: openarray[byte]) = proc writeBytesToCursor(c: var WriteCursor, bytes: openArray[byte]) =
var var
runway = c.span.len runway = c.span.len
inputPos = unsafeAddr bytes[0] inputPos = unsafeAddr bytes[0]
@ -713,11 +713,11 @@ proc writeBytesToCursor(c: var WriteCursor, bytes: openarray[byte]) =
copyMem(c.span.startAddr, inputPos, inputLen) copyMem(c.span.startAddr, inputPos, inputLen)
c.span.startAddr = offset(c.span.startAddr, inputLen) c.span.startAddr = offset(c.span.startAddr, inputLen)
template write*(c: var WriteCursor, bytes: openarray[byte]) = template write*(c: var WriteCursor, bytes: openArray[byte]) =
bind writeBytesToCursor bind writeBytesToCursor
writeBytesToCursor(c, bytes) writeBytesToCursor(c, bytes)
proc write*(c: var WriteCursor, chars: openarray[char]) {.inline.} = proc write*(c: var WriteCursor, chars: openArray[char]) {.inline.} =
var charsStart = unsafeAddr chars[0] var charsStart = unsafeAddr chars[0]
writeBytesToCursor(c, makeOpenArray(cast[ptr byte](charsStart), chars.len)) writeBytesToCursor(c, makeOpenArray(cast[ptr byte](charsStart), chars.len))
@ -749,7 +749,7 @@ template consumeOutputs*(s: OutputStream, bytesVar, body: untyped) =
## ##
## Before consuming the outputs, all outstanding delayed writes must ## Before consuming the outputs, all outstanding delayed writes must
## be finalized. ## be finalized.
proc consumer(bytesVar: openarray[byte]) {.gcsafe, raises: [Defect].} = proc consumer(bytesVar: openArray[byte]) {.gcsafe, raises: [Defect].} =
body body
consumeOutputsImpl(s, consumer) consumeOutputsImpl(s, consumer)
@ -795,7 +795,7 @@ template consumeContiguousOutput*(s: OutputStream, bytesVar, body: untyped) =
## Before consuming the output, all outstanding delayed writes must ## Before consuming the output, all outstanding delayed writes must
## be finalized. ## be finalized.
## ##
proc consumer(bytesVar: openarray[byte]) {.gcsafe, raises: [Defect].} = proc consumer(bytesVar: openArray[byte]) {.gcsafe, raises: [Defect].} =
body body
consumeContiguousOutputImpl(s, consumer) consumeContiguousOutputImpl(s, consumer)

8
faststreams/textio.nim
View file

@ -93,7 +93,7 @@ proc writeText*(s: OutputStream, x: CompiledIntTypes) =
writeText(s, MatchingUInt(x)) writeText(s, MatchingUInt(x))
when defined(c): when defined(c):
proc writeText*(s: OutputStream, x: float64|float32|float) = proc writeText*(s: OutputStream, x: float64|float32|float) =
## Write the floating point number to the output stream. It has less overhead ## Write the floating point number to the output stream. It has less overhead
## than `$` because it is directly written to the stream without ## than `$` because it is directly written to the stream without
@ -112,14 +112,14 @@ template writeText*(s: OutputStream, str: string) =
template writeText*(s: OutputStream, val: auto) = template writeText*(s: OutputStream, val: auto) =
write s, $val write s, $val
proc writeHex*(s: OutputStream, bytes: openarray[byte]) = proc writeHex*(s: OutputStream, bytes: openArray[byte]) =
const hexChars = "0123456789abcdef" const hexChars = "0123456789abcdef"
for b in bytes: for b in bytes:
s.write hexChars[int b shr 4 and 0xF] s.write hexChars[int b shr 4 and 0xF]
s.write hexChars[int b and 0xF] s.write hexChars[int b and 0xF]
proc writeHex*(s: OutputStream, chars: openarray[char]) = proc writeHex*(s: OutputStream, chars: openArray[char]) =
writeHex s, charsToBytes(chars) writeHex s, charsToBytes(chars)
const const
@ -145,7 +145,7 @@ proc readLine*(s: InputStream, keepEol = false): TaintedString {.fsMultiSync.} =
result.add s.read.char result.add s.read.char
proc readUntil*(s: InputStream, proc readUntil*(s: InputStream,
sep: openarray[char]): Option[TaintedString] = sep: openArray[char]): Option[TaintedString] =
fsAssert readableNow(s) fsAssert readableNow(s)
var res = "" var res = ""
while s.readable(sep.len): while s.readable(sep.len):

2
tests/base64.nim
View file

@ -78,7 +78,7 @@ proc base64decode*(i: InputStream, o: OutputStream) {.fsMultiSync.} =
proc decodeSize(size: int): int = proc decodeSize(size: int): int =
return (size * 3 div 4) + 6 return (size * 3 div 4) + 6
proc raiseInvalidChar(c: byte, pos: int) {.noReturn.} = proc raiseInvalidChar(c: byte, pos: int) {.noreturn.} =
raise newException(ValueError, raise newException(ValueError,
"Invalid base64 format character `" & char(c) & "` at location " & $pos & ".") "Invalid base64 format character `" & char(c) & "` at location " & $pos & ".")

2
tests/test_inputs.nim
View file

@ -7,7 +7,7 @@ import
setCurrentDir getAppDir() setCurrentDir getAppDir()
proc str(bytes: openarray[byte]): string = proc str(bytes: openArray[byte]): string =
result = newStringOfCap(bytes.len) result = newStringOfCap(bytes.len)
for b in items(bytes): for b in items(bytes):
result.add b.char result.add b.char

10
tests/test_pipelines.nim
View file

@ -49,7 +49,7 @@ when fsAsyncSupport:
let size = sp.totalUnconsumedBytes() let size = sp.totalUnconsumedBytes()
if size > 0: if size > 0:
var data = newSeq[byte](size) var data = newSeq[byte](size)
discard sp.readinto(data) discard sp.readInto(data)
result = cast[string](data) result = cast[string](data)
suite "pipelines": suite "pipelines":
@ -73,17 +73,17 @@ when fsAsyncSupport:
let inputText = loremIpsum.repeat(5000) let inputText = loremIpsum.repeat(5000)
timeIt times.stdFunctionCalls: timeit times.stdFunctionCalls:
stdRes = base64.decode(base64.encode(toUpperAscii(inputText))) stdRes = base64.decode(base64.encode(toUpperAscii(inputText)))
timeIt times.fsPipeline: timeit times.fsPipeline:
fsRes = executePipeline(unsafeMemoryInput(inputText), fsRes = executePipeline(unsafeMemoryInput(inputText),
upcaseAllCharacters, upcaseAllCharacters,
base64encode, base64encode,
base64decode, base64decode,
getOutput string) getOutput string)
timeIt times.fsAsyncPipeline: timeit times.fsAsyncPipeline:
fsAsyncRes = waitFor executePipeline(Async unsafeMemoryInput(inputText), fsAsyncRes = waitFor executePipeline(Async unsafeMemoryInput(inputText),
upcaseAllCharacters, upcaseAllCharacters,
base64encode, base64encode,
@ -130,7 +130,7 @@ when fsAsyncSupport:
let fsAsyncres = await f let fsAsyncres = await f
check fsAsyncRes == toUpperAscii(inputText) check fsAsyncres == toUpperAscii(inputText)
else: else:
test "pipelines": test "pipelines":
skip skip