Initial version (extracted from nim-serialization)

faststreams/input_stream.nim

@@ -0,0 +1,149 @@
+import
+  memfiles, options, ranges/ptr_arith
+
+const
+  pageSize = 4096
+
+type
+  StreamReader = proc (bufferStart: ptr byte, bufferSize: int): int
+  # TODO: use openarray once it's supported
+  AsyncStreamReader = StreamReader # proc (bufferStart: ptr byte, bufferSize: int): Future[int]
+  CloseStreamProc = proc ()
+
+  ByteStream* = object of RootObj
+    head: ptr byte
+    bufferSize: int
+    bufferStart, bufferEnd: ptr byte
+    reader: StreamReader
+    asyncReader: AsyncStreamReader
+    closeStream: CloseStreamProc
+    bufferEndPos: int
+
+  BufferedStream*[size: static int] = object of ByteStream
+    buffer: array[size, byte]
+
+  AsciiStream* = distinct ByteStream
+  UnicodeStream* = distinct ByteStream
+  ObjectStream*[T] = distinct ByteStream
+
+proc openFile*(filename: string): ByteStream =
+  var memFile = memfiles.open(filename)
+  result.head = cast[ptr byte](memFile.mem)
+  result.bufferEnd = result.head.shift memFile.size
+  result.bufferEndPos = memFile.size
+  result.closeStream = proc = close memFile
+
+proc init*(T: type ByteStream,
+           mem: openarray[byte],
+           reader = StreamReader(nil),
+           asyncReader = AsyncStreamReader(nil)): ByteStream =
+  # TODO: the result should use `from mem` once it's supported
+  result.head = unsafeAddr mem[0]
+  result.bufferEnd = result.head.shift mem.len
+  result.reader = reader
+  result.asyncReader = asyncReader
+
+template memoryStream*(mem: openarray[byte]): ByteStream =
+  ByteStream.init(mem)
+
+template memoryStream*(str: string): ByteStream =
+  bind init
+  init(ByteStream, str.toOpenArrayByte(0, str.len - 1))
+
+proc init*(T: type BufferedStream,
+           reader = StreamReader(nil),
+           asyncReader = AsyncStreamReader(nil)): BufferedStream =
+  result.init result.buffer, reader, asyncReader
+
+proc syncRead(s: var ByteStream): bool =
+  let bytesRead = s.reader(s.bufferStart, s.bufferSize)
+  if bytesRead == 0:
+    s.reader = nil
+    return true
+  else:
+    s.bufferEnd = s.bufferStart.shift bytesRead
+    s.bufferEndPos += bytesRead
+    return false
+
+proc eof*(s: var ByteStream): bool =
+  if s.head != s.bufferEnd:
+    return false
+
+  if s.reader == nil:
+    return true
+
+  return s.syncRead()
+
+proc eob*(s: ByteStream): bool {.inline.} =
+  s.head != s.bufferEnd
+
+proc peek*(s: ByteStream): byte {.inline.} =
+  assert s.head != s.bufferEnd
+  return s.head[]
+
+proc advance*(s: var ByteStream) =
+  if s.head != s.bufferEnd:
+    s.head = s.head.shift 1
+  elif s.reader != nil:
+    discard s.syncRead()
+
+proc read*(s: var ByteStream): byte =
+  result = s.peek()
+  advance s
+
+proc next*(s: var ByteStream): Option[byte] =
+  if not s.eof:
+    result = some s.read()
+
+proc bufferPos(s: ByteStream, pos: int): ptr byte =
+  let shiftFromEnd = pos - s.bufferEndPos
+  assert shiftFromEnd < 0
+  result = s.bufferEnd.shift shiftFromEnd
+  assert result >= s.bufferStart
+
+proc pos*(s: ByteStream): int {.inline.} =
+  s.bufferEndPos - distance(s.head, s.bufferEnd)
+
+proc firstAccessiblePos*(s: ByteStream): int {.inline.} =
+  s.bufferEndPos - distance(s.bufferStart, s.bufferEnd)
+
+proc `[]`*(s: ByteStream, pos: int): byte {.inline.} =
+  s.bufferPos(pos)[]
+
+proc rewind*(s: var ByteStream, delta: int) =
+  s.head = s.head.shift(-delta)
+  assert s.head >= s.bufferStart
+
+proc rewindTo*(s: var ByteStream, pos: int) {.inline.} =
+  s.head = s.bufferPos(pos)
+
+# TODO: use a destructor once we migrate to Nim 0.20
+proc close*(s: var ByteStream) =
+  if s.closeStream != nil:
+    s.closeStream()
+
+# TODO: Use `distrinct with` once it's supported
+template pos*(s: AsciiStream|UnicodeStream|ObjectStream): int =
+  ByteStream(s).pos
+
+template eof*(s: var (AsciiStream|UnicodeStream|ObjectStream)): bool =
+  ByteStream(s).eof
+
+template eob*(s: var (AsciiStream|UnicodeStream|ObjectStream)): bool =
+  ByteStream(s).eob
+
+template close*(s: AsciiStream|UnicodeStream|ObjectStream) =
+  close ByteStream(s)
+
+template advance*(s: var AsciiStream) =
+  advance ByteStream(s)
+
+template peek*(s: AsciiStream): char =
+  char ByteStream(s).peek()
+
+template read*(s: var AsciiStream): char =
+  char ByteStream(s).read()
+
+template next*(s: var AsciiStream): Option[char] =
+  cast[Option[char]](ByteStream(s).next())
+

faststreams/output_stream.nim

@@ -0,0 +1,91 @@
+import
+  ranges/ptr_arith
+
+type
+  OutputStreamVTable = tuple
+    prepareOutput: proc (s: ptr OutputStream, size: int) {.nimcall.}
+    finish: proc (s: ptr OutputStream) {.nimcall.}
+
+  OutputStream* = object {.inheritable.}
+    head, bufferEnd: ptr byte
+    vtable: ptr OutputStreamVTable
+
+  MemoryOutputStream*[T] = object of OutputStream
+    output: T
+
+  StringOutputStream* = MemoryOutputStream[string]
+  BytesOutputStream* = MemoryOutputStream[seq[byte]]
+
+  FileOutputStream* = object of OutputStream
+    file: File
+
+const
+  pageSize = 4096
+
+proc prepareOutputImpl[T](s: ptr OutputStream, size: int) =
+  let s = cast[ptr T](s)
+  let currentSize = s.output.len
+  s.output.setLen(s.output.len + pageSize)
+  s.head = cast[ptr byte](addr s.output[currentSize])
+  s.bufferEnd = cast[ptr byte](shift(addr s.output[0], s.output.len))
+
+proc finishImpl[T](s: ptr OutputStream) =
+  let s = cast[ptr T](s)
+  s.output.setLen(s.output.len - distance(s.head, s.bufferEnd))
+
+proc init*(T: type MemoryOutputStream): T =
+  var vtable {.global.} = (prepareOutputImpl[T], finishImpl[T])
+  result.vtable = addr vtable
+  result.vtable.prepareOutput(addr result, pageSize)
+
+proc append*(s: var OutputStream, b: byte) =
+  if s.head == s.bufferEnd:
+    s.vtable.prepareOutput(addr s, pageSize)
+
+  s.head[] = b
+  s.head = shift(s.head, 1)
+
+template append*(s: var OutputStream, c: char) =
+  s.append byte(c)
+
+proc flush*(s: var OutputStream) =
+  s.vtable.finish(addr s)
+
+proc getOutput*(s: var MemoryOutputStream): auto =
+  flush s
+  shallow s.output
+  return s.output
+
+proc append*(s: var OutputStream, bytes: openarray[byte]) =
+  # TODO: this can use copyMem
+  for b in bytes:
+    s.append b
+
+proc append*(s: var OutputStream, chars: openarray[char]) =
+  # TODO: this can use copyMem
+  for c in chars:
+    s.append byte(c)
+
+template append*(s: var OutputStream, str: string) =
+  s.append str.toOpenArrayByte(0, str.len - 1)
+
+# Any stream
+
+proc appendNumberImpl(s: var OutputStream, number: BiggestInt) =
+  # TODO: don't allocate
+  s.append $number
+
+proc appendNumberImpl(s: var OutputStream, number: BiggestUInt) =
+  # TODO: don't allocate
+  s.append $number
+
+template toBiggestRepr(i: SomeUnsignedInt): BiggestUInt =
+  BiggestUInt(i)
+
+template toBiggestRepr(i: SomeSignedInt): BiggestInt =
+  BiggestInt(i)
+
+template appendNumber*(s: var OutputStream, i: SomeInteger) =
+  # TODO: specify radix/base
+  appendNumberImpl(s, toBiggestRepr(i))
+