Address review comments; Add documentation; Shared buffering mechanism for input and output streams
This commit is contained in:
Zahary Karadjov
parent
caab5c917f
commit
b24300bd3f
24 changed files with 2396 additions and 1060 deletions
|
|
@ -1,5 +1,6 @@
|
|||
import
|
||||
test_input_stream,
|
||||
test_output_stream,
|
||||
test_pipelines
|
||||
test_inputs,
|
||||
test_outputs,
|
||||
test_pipelines,
|
||||
test_readme_examples
|
||||
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@ const
|
|||
invalidChar = 255
|
||||
paddingByte = byte('=')
|
||||
|
||||
template encodeSize(size: int): int = (size * 4 div 3) + 6
|
||||
|
||||
proc base64encode*(i: InputStream, o: OutputStream) =
|
||||
var
|
||||
n: uint32
|
||||
|
|
@ -25,7 +23,7 @@ proc base64encode*(i: InputStream, o: OutputStream) =
|
|||
n = exp
|
||||
|
||||
template outputChar(x: typed) =
|
||||
o.append cb64[x and 63]
|
||||
o.write cb64[x and 63]
|
||||
|
||||
while i.readable(3):
|
||||
inputByte(b shl 16)
|
||||
|
|
@ -43,12 +41,12 @@ proc base64encode*(i: InputStream, o: OutputStream) =
|
|||
outputChar(n shr 18)
|
||||
outputChar(n shr 12)
|
||||
outputChar(n shr 6)
|
||||
o.append paddingByte
|
||||
o.write paddingByte
|
||||
else:
|
||||
outputChar(n shr 18)
|
||||
outputChar(n shr 12)
|
||||
o.append paddingByte
|
||||
o.append paddingByte
|
||||
o.write paddingByte
|
||||
o.write paddingByte
|
||||
|
||||
proc initDecodeTable*(): array[256, char] =
|
||||
# computes a decode table at compile time
|
||||
|
|
@ -80,11 +78,11 @@ proc base64decode*(i: InputStream, o: OutputStream) =
|
|||
raiseInvalidChar(c, i.pos - 1)
|
||||
|
||||
template outputChar(x: untyped) =
|
||||
o.append char(x and 255)
|
||||
o.write char(x and 255)
|
||||
|
||||
let inputLen = i.len
|
||||
if inputLen != lengthUnknown:
|
||||
o.prepareRunway decodeSize(inputLen)
|
||||
if inputLen.isSome:
|
||||
o.ensureRunway decodeSize(inputLen.get)
|
||||
|
||||
# hot loop: read 4 characters at at time
|
||||
while i.readable(8):
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
import
|
||||
unittest, strutils, stew/ranges/ptr_arith,
|
||||
../faststreams
|
||||
|
||||
suite "input stream":
|
||||
test "string input":
|
||||
var input = repeat("1234 5678 90AB CDEF\n", 1000)
|
||||
var stream = memoryInput(input)
|
||||
|
||||
check:
|
||||
(stream.read(4) == "1234".toOpenArrayByte(0, 3))
|
||||
|
||||
36
tests/test_inputs.nim
Normal file
36
tests/test_inputs.nim
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
{.used.}
|
||||
|
||||
import
|
||||
os, unittest, strutils, stew/ranges/ptr_arith,
|
||||
../faststreams
|
||||
|
||||
suite "input stream":
|
||||
test "empty input":
|
||||
var str = ""
|
||||
var i = unsafeMemoryInput(str)
|
||||
|
||||
check:
|
||||
i.readable == false
|
||||
i.next.isNone
|
||||
|
||||
expect Defect:
|
||||
echo i.read
|
||||
|
||||
test "missing file input":
|
||||
const fileName = "there-is-no-such-faststreams-file-1"
|
||||
|
||||
check not fileExists(fileName)
|
||||
expect CatchableError: discard fileInput(fileName)
|
||||
|
||||
check not fileExists(fileName)
|
||||
expect CatchableError: discard memFileInput(fileName)
|
||||
|
||||
check not fileExists(fileName)
|
||||
|
||||
test "simple":
|
||||
var input = repeat("1234 5678 90AB CDEF\n", 1000)
|
||||
var stream = unsafeMemoryInput(input)
|
||||
|
||||
check:
|
||||
(stream.read(4) == "1234".toOpenArrayByte(0, 3))
|
||||
|
||||
|
|
@ -1,154 +0,0 @@
|
|||
import
|
||||
os, unittest, random,
|
||||
stew/ranges/ptr_arith,
|
||||
../faststreams
|
||||
|
||||
proc bytes(s: string): seq[byte] =
|
||||
result = newSeqOfCap[byte](s.len)
|
||||
for c in s: result.add byte(c)
|
||||
|
||||
template bytes(c: char): byte = byte(c)
|
||||
template bytes(b: seq[byte]): seq[byte] = b
|
||||
|
||||
proc repeat(b: byte, count: int): seq[byte] =
|
||||
result = newSeq[byte](count)
|
||||
for i in 0 ..< count: result[i] = b
|
||||
|
||||
proc randomBytes(n: int): seq[byte] =
|
||||
result.newSeq n
|
||||
for i in 0 ..< n:
|
||||
result[i] = byte(rand(255))
|
||||
|
||||
suite "output stream":
|
||||
setup:
|
||||
var memStream = memoryOutput()
|
||||
var altOutput: seq[byte] = @[]
|
||||
var tempFilePath = getTempDir() / "faststreams_testfile"
|
||||
var fileStream = fileOutput(tempFilePath)
|
||||
|
||||
const bufferSize = 1000000
|
||||
var buffer = alloc(bufferSize)
|
||||
var existingBufferStream = memoryOutput(buffer, bufferSize)
|
||||
|
||||
teardown:
|
||||
removeFile tempFilePath
|
||||
|
||||
template output(val: auto) {.dirty.} =
|
||||
altOutput.add bytes(val)
|
||||
|
||||
memStream.append val
|
||||
fileStream.append val
|
||||
existingBufferStream.append val
|
||||
|
||||
template checkOutputsMatch =
|
||||
fileStream.flush
|
||||
|
||||
let
|
||||
fileContents = readFile(tempFilePath).string.bytes
|
||||
memStreamContents = memStream.getOutput
|
||||
|
||||
let outputsMatch =
|
||||
altOutput == memStreamContents and
|
||||
altOutput == makeOpenArray(cast[ptr byte](buffer),
|
||||
existingBufferStream.pos)
|
||||
|
||||
check outputsMatch
|
||||
|
||||
test "no appends produce an empty output":
|
||||
checkOutputsMatch()
|
||||
|
||||
test "append zero length slice":
|
||||
output ""
|
||||
checkOutputsMatch()
|
||||
|
||||
test "string output":
|
||||
for i in 0 .. 1:
|
||||
output $i
|
||||
output " bottles on the wall"
|
||||
output '\n'
|
||||
|
||||
checkOutputsMatch()
|
||||
|
||||
test "delayed write":
|
||||
output "initial output\n"
|
||||
const delayedWriteContent = bytes "delayed write\n"
|
||||
|
||||
var cursor = memStream.delayFixedSizeWrite(delayedWriteContent.len)
|
||||
let cursorStart = memStream.pos
|
||||
altOutput.add delayedWriteContent
|
||||
|
||||
fileStream.append delayedWriteContent
|
||||
existingBufferStream.append delayedWriteContent
|
||||
|
||||
var totalBytesWritten = 0
|
||||
for i, count in [12, 342, 2121, 23, 1, 34012, 932]:
|
||||
output repeat(byte(i), count)
|
||||
totalBytesWritten += count
|
||||
check memStream.pos - cursorStart == totalBytesWritten
|
||||
|
||||
cursor.writeAndFinalize delayedWriteContent
|
||||
|
||||
checkOutputsMatch()
|
||||
|
||||
test "multi-page delayed writes":
|
||||
randomize(1000)
|
||||
|
||||
type
|
||||
DelayedWrite = object
|
||||
cursor: WriteCursor
|
||||
content: seq[byte]
|
||||
written: int
|
||||
|
||||
var delayedWrites = newSeq[DelayedWrite]()
|
||||
|
||||
for i in 0..50:
|
||||
let
|
||||
size = rand(8000) + 2000
|
||||
randomBytes = randomBytes(size)
|
||||
decision = rand(100)
|
||||
|
||||
if decision < 70:
|
||||
# Write at some random cursor
|
||||
if delayedWrites.len == 0:
|
||||
continue
|
||||
|
||||
let
|
||||
i = rand(delayedWrites.len - 1)
|
||||
written = delayedWrites[i].written
|
||||
remaining = delayedWrites[i].content.len - written
|
||||
toWrite = min(rand(remaining) + 10, remaining)
|
||||
|
||||
delayedWrites[i].cursor.append delayedWrites[i].content[written ..< written + toWrite]
|
||||
delayedWrites[i].written += toWrite
|
||||
|
||||
if remaining - toWrite == 0:
|
||||
finalize delayedWrites[i].cursor
|
||||
if i != delayedWrites.len - 1:
|
||||
swap(delayedWrites[i], delayedWrites[^1])
|
||||
delayedWrites.setLen(delayedWrites.len - 1)
|
||||
|
||||
elif decision < 90:
|
||||
# Normal write
|
||||
memStream.append randomBytes
|
||||
altOutput.add randomBytes
|
||||
|
||||
else:
|
||||
# Create cursor
|
||||
altOutput.add randomBytes
|
||||
delayedWrites.add DelayedWrite(
|
||||
cursor: memStream.delayFixedSizeWrite(randomBytes.len),
|
||||
content: randomBytes,
|
||||
written: 0)
|
||||
|
||||
# Check that the stream position is consistently tracked at every step
|
||||
check altOutput.len == memStream.pos
|
||||
|
||||
# Write all unwritten data to all outstanding cursors
|
||||
for dw in mitems(delayedWrites):
|
||||
let remaining = dw.content.len - dw.written
|
||||
dw.cursor.append dw.content[dw.written ..< dw.written + remaining]
|
||||
finalize dw.cursor
|
||||
|
||||
# The final outputs are the same
|
||||
check altOutput == memStream.getOutput
|
||||
|
||||
240
tests/test_outputs.nim
Normal file
240
tests/test_outputs.nim
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
{.used.}
|
||||
|
||||
import
|
||||
os, unittest, random,
|
||||
stew/ranges/ptr_arith,
|
||||
../faststreams, ../faststreams/textio
|
||||
|
||||
proc bytes(s: string): seq[byte] =
|
||||
result = newSeqOfCap[byte](s.len)
|
||||
for c in s: result.add byte(c)
|
||||
|
||||
template bytes(c: char): byte = byte(c)
|
||||
template bytes(b: seq[byte]): seq[byte] = b
|
||||
template bytes[N, T](b: array[N, T]): seq[byte] = @b
|
||||
|
||||
proc repeat(b: byte, count: int): seq[byte] =
|
||||
result = newSeq[byte](count)
|
||||
for i in 0 ..< count: result[i] = b
|
||||
|
||||
proc randomBytes(n: int): seq[byte] =
|
||||
result.newSeq n
|
||||
for i in 0 ..< n:
|
||||
result[i] = byte(rand(255))
|
||||
|
||||
proc readAllAndClose(s: InputStream): seq[byte] =
|
||||
while s.readable:
|
||||
result.add s.read
|
||||
|
||||
close(s)
|
||||
|
||||
import memfiles
|
||||
|
||||
suite "output stream":
|
||||
setup:
|
||||
var
|
||||
nimSeq: seq[byte] = @[]
|
||||
|
||||
memStream = memoryOutput()
|
||||
smallPageSizeStream = memoryOutput(pageSize = 10)
|
||||
largePageSizeStream = memoryOutput(pageSize = 1000000)
|
||||
|
||||
fileOutputPath = getTempDir() / "faststreams_testfile"
|
||||
unbufferedFileOutputPath = getTempDir() / "faststreams_testfile_unbuffered"
|
||||
|
||||
fileStream = fileOutput(fileOutputPath)
|
||||
unbufferedFileStream = fileOutput(unbufferedFileOutputPath, pageSize = 0)
|
||||
|
||||
bufferSize = 1000000
|
||||
buffer = alloc(bufferSize)
|
||||
streamWritingToExistingBuffer = unsafeMemoryOutput(buffer, bufferSize)
|
||||
|
||||
teardown:
|
||||
removeFile fileOutputPath
|
||||
removeFile unbufferedFileOutputPath
|
||||
dealloc buffer
|
||||
|
||||
template output(val: auto) {.dirty.} =
|
||||
nimSeq.add bytes(val)
|
||||
|
||||
memStream.write val
|
||||
smallPageSizeStream.write val
|
||||
largePageSizeStream.write val
|
||||
|
||||
fileStream.write val
|
||||
unbufferedFileStream.write val
|
||||
|
||||
streamWritingToExistingBuffer.write val
|
||||
|
||||
template outputText(val: auto) =
|
||||
let valAsStr = $val
|
||||
nimSeq.add valAsStr.toOpenArrayByte(0, valAsStr.len - 1)
|
||||
|
||||
memStream.writeText val
|
||||
smallPageSizeStream.writeText val
|
||||
largePageSizeStream.writeText val
|
||||
|
||||
fileStream.writeText val
|
||||
unbufferedFileStream.writeText val
|
||||
|
||||
streamWritingToExistingBuffer.writeText val
|
||||
|
||||
template checkOutputsMatch(showResults = false,
|
||||
skipUnbufferedFile = false) =
|
||||
flush fileStream
|
||||
close fileStream
|
||||
|
||||
flush unbufferedFileStream
|
||||
close unbufferedFileStream
|
||||
|
||||
check fileExists(fileOutputPath) and
|
||||
fileExists(unbufferedFileOutputPath)
|
||||
|
||||
let
|
||||
memStreamRes = memStream.getOutput
|
||||
readFileRes = readFile(fileOutputPath).string.bytes
|
||||
fileInputRes = fileInput(fileOutputPath).readAllAndClose
|
||||
memFileInputRes = memFileInput(fileOutputPath).readAllAndClose
|
||||
fileInputWithSmallPagesRes = fileInput(fileOutputPath, pageSize = 10).readAllAndClose
|
||||
|
||||
when showResults:
|
||||
checkpoint "Nim seq result"
|
||||
checkpoint $nimSeq
|
||||
|
||||
checkpoint "Writes to existing buffer result"
|
||||
checkpoint $makeOpenArray(cast[ptr byte](buffer),
|
||||
streamWritingToExistingBuffer.pos)
|
||||
|
||||
checkpoint "mem stream result"
|
||||
checkpoint $memStreamRes
|
||||
|
||||
checkpoint "readFile result"
|
||||
checkpoint $readFileRes
|
||||
|
||||
checkpoint "fileInput result"
|
||||
checkpoint $fileInputRes
|
||||
|
||||
checkpoint "memFileInput result"
|
||||
checkpoint $memFileInputRes
|
||||
|
||||
checkpoint "fileInput with small pageSize result"
|
||||
checkpoint $fileInputWithSmallPagesRes
|
||||
|
||||
let outputsMatch =
|
||||
nimSeq == makeOpenArray(cast[ptr byte](buffer),
|
||||
streamWritingToExistingBuffer.pos) and
|
||||
nimSeq == memStreamRes and
|
||||
nimSeq == readFileRes and
|
||||
nimSeq == fileInputRes and
|
||||
nimSeq == memFileInputRes and
|
||||
nimSeq == fileInputWithSmallPagesRes
|
||||
|
||||
check outputsMatch
|
||||
|
||||
when not skipUnbufferedFile:
|
||||
let unbufferedFileRes = readFile(unbufferedFileOutputPath).string.bytes
|
||||
check nimSeq == unbufferedFileRes
|
||||
|
||||
test "no appends produce an empty output":
|
||||
checkOutputsMatch()
|
||||
|
||||
test "write zero length slices":
|
||||
output ""
|
||||
output newSeq[byte]()
|
||||
var arr: array[0, byte]
|
||||
output arr
|
||||
|
||||
check nimSeq.len == 0
|
||||
checkOutputsMatch()
|
||||
|
||||
test "text output":
|
||||
for i in 1 .. 100:
|
||||
outputText i
|
||||
outputText " bottles on the wall"
|
||||
outputText '\n'
|
||||
|
||||
checkOutputsMatch()
|
||||
|
||||
test "delayed write":
|
||||
output "initial output\n"
|
||||
const delayedWriteContent = bytes "delayed write\n"
|
||||
|
||||
var cursor = memStream.delayFixedSizeWrite(delayedWriteContent.len)
|
||||
let cursorStart = memStream.pos
|
||||
|
||||
nimSeq.add delayedWriteContent
|
||||
fileStream.write delayedWriteContent
|
||||
streamWritingToExistingBuffer.write delayedWriteContent
|
||||
|
||||
var totalBytesWritten = 0
|
||||
for i, count in [2]: # 12, 342, 2121, 23, 1, 34012, 932]:
|
||||
output repeat(byte(i), count)
|
||||
totalBytesWritten += count
|
||||
check memStream.pos - cursorStart == totalBytesWritten
|
||||
|
||||
cursor.finalWrite delayedWriteContent
|
||||
|
||||
checkOutputsMatch(skipUnbufferedFile = true)
|
||||
|
||||
test "multi-page delayed writes":
|
||||
randomize(1000)
|
||||
|
||||
type
|
||||
DelayedWrite = object
|
||||
cursor: WriteCursor
|
||||
content: seq[byte]
|
||||
written: int
|
||||
|
||||
var delayedWrites = newSeq[DelayedWrite]()
|
||||
|
||||
for i in 0..50:
|
||||
let
|
||||
size = rand(8000) + 2000
|
||||
randomBytes = randomBytes(size)
|
||||
decision = rand(100)
|
||||
|
||||
if decision < 70:
|
||||
# Write at some random cursor
|
||||
if delayedWrites.len == 0:
|
||||
continue
|
||||
|
||||
let
|
||||
i = rand(delayedWrites.len - 1)
|
||||
written = delayedWrites[i].written
|
||||
remaining = delayedWrites[i].content.len - written
|
||||
toWrite = min(rand(remaining) + 10, remaining)
|
||||
|
||||
delayedWrites[i].cursor.write delayedWrites[i].content[written ..< written + toWrite]
|
||||
delayedWrites[i].written += toWrite
|
||||
|
||||
if remaining - toWrite == 0:
|
||||
finalize delayedWrites[i].cursor
|
||||
if i != delayedWrites.len - 1:
|
||||
swap(delayedWrites[i], delayedWrites[^1])
|
||||
delayedWrites.setLen(delayedWrites.len - 1)
|
||||
|
||||
elif decision < 90:
|
||||
# Normal write
|
||||
memStream.write randomBytes
|
||||
nimSeq.add randomBytes
|
||||
|
||||
else:
|
||||
# Create cursor
|
||||
nimSeq.add randomBytes
|
||||
delayedWrites.add DelayedWrite(
|
||||
cursor: memStream.delayFixedSizeWrite(randomBytes.len),
|
||||
content: randomBytes,
|
||||
written: 0)
|
||||
|
||||
# Check that the stream position is consistently tracked at every step
|
||||
check nimSeq.len == memStream.pos
|
||||
|
||||
# Write all unwritten data to all outstanding cursors
|
||||
for dw in mitems(delayedWrites):
|
||||
let remaining = dw.content.len - dw.written
|
||||
dw.cursor.write dw.content[dw.written ..< dw.written + remaining]
|
||||
finalize dw.cursor
|
||||
|
||||
# The final outputs are the same
|
||||
check nimSeq == memStream.getOutput
|
||||
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
{.used.}
|
||||
|
||||
import
|
||||
std/[unittest, strutils, base64],
|
||||
../faststreams/pipelines,
|
||||
|
|
@ -13,7 +15,7 @@ type
|
|||
|
||||
proc upcaseAllCharacters(i: InputStream, o: OutputStream) =
|
||||
while i.readable:
|
||||
o.append toUpperAscii(char i.read())
|
||||
o.write toUpperAscii(char i.read())
|
||||
|
||||
template timeit(timerVar: var Nanos, code: untyped) =
|
||||
let t0 = getTicks()
|
||||
|
|
@ -40,7 +42,7 @@ suite "pipelines":
|
|||
|
||||
timeIt times.fsPipeline:
|
||||
var memOut = memoryOutput()
|
||||
executePipeline(memoryInput(loremIpsum),
|
||||
executePipeline(unsafeMemoryInput(loremIpsum),
|
||||
upcaseAllCharacters,
|
||||
base64encode,
|
||||
base64decode,
|
||||
|
|
|
|||
53
tests/test_readme_examples.nim
Normal file
53
tests/test_readme_examples.nim
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
{.used.}
|
||||
|
||||
import
|
||||
typetraits, ../faststreams
|
||||
|
||||
proc writeNimRepr*(stream: OutputStream, str: string) =
|
||||
stream.write '"'
|
||||
|
||||
for c in str:
|
||||
if c == '"':
|
||||
stream.write ['\'', '"']
|
||||
else:
|
||||
stream.write c
|
||||
|
||||
stream.write '"'
|
||||
|
||||
proc writeNimRepr*(stream: OutputStream, x: char) =
|
||||
stream.write ['\'', x, '\'']
|
||||
|
||||
proc writeNimRepr*(stream: OutputStream, x: int) =
|
||||
stream.write $x # Making this more optimal has been left
|
||||
# as an exercise for the reader
|
||||
|
||||
proc writeNimRepr*[T](stream: OutputStream, obj: T) =
|
||||
stream.write typetraits.name(T)
|
||||
stream.write '('
|
||||
|
||||
var firstField = true
|
||||
for name, val in fieldPairs(obj):
|
||||
if not firstField:
|
||||
stream.write ", "
|
||||
|
||||
stream.write name
|
||||
stream.write ": "
|
||||
stream.writeNimRepr val
|
||||
|
||||
firstField = false
|
||||
|
||||
stream.write ')'
|
||||
|
||||
type
|
||||
ABC = object
|
||||
a: int
|
||||
b: char
|
||||
c: string
|
||||
|
||||
block:
|
||||
var stream = memoryOutput()
|
||||
stream.writeNimRepr(ABC(a: 1, b: 'b', c: "str"))
|
||||
var repr = stream.getOutput(string)
|
||||
|
||||
doAssert repr == "ABC(a: 1, b: 'b', c: \"str\")"
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue