From cb54254b77a44fda6c1b30321e3a37fa8749943c Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 17 Dec 2018 23:48:33 +0200 Subject: [PATCH] Initial version (extracted from nim-serialization) --- LICENSE-APACHEv2 | 201 ++++++++++++++++++++++++++++++++++ LICENSE-MIT | 21 ++++ faststreams.nim | 6 + faststreams.nimble | 12 ++ faststreams/input_stream.nim | 149 +++++++++++++++++++++++++ faststreams/output_stream.nim | 91 +++++++++++++++ tests/test_output_stream.nim | 21 ++++ 7 files changed, 501 insertions(+) create mode 100644 LICENSE-APACHEv2 create mode 100644 LICENSE-MIT create mode 100644 faststreams.nim create mode 100644 faststreams.nimble create mode 100644 faststreams/input_stream.nim create mode 100644 faststreams/output_stream.nim create mode 100644 tests/test_output_stream.nim diff --git a/LICENSE-APACHEv2 b/LICENSE-APACHEv2 new file mode 100644 index 0000000..782d1bf --- /dev/null +++ b/LICENSE-APACHEv2 @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Status Research & Development GmbH + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..8766e65 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Status Research & Development GmbH + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/faststreams.nim b/faststreams.nim new file mode 100644 index 0000000..8a8d538 --- /dev/null +++ b/faststreams.nim @@ -0,0 +1,6 @@ +import + faststreams/[input_stream, output_stream] + +export + input_stream, output_stream + diff --git a/faststreams.nimble b/faststreams.nimble new file mode 100644 index 0000000..d3268bd --- /dev/null +++ b/faststreams.nimble @@ -0,0 +1,12 @@ +mode = ScriptMode.Verbose + +packageName = "faststreams" +version = "0.1.0" +author = "Status Research & Development GmbH" +description = "Nearly zero-overhead input/output streams for Nim" +license = "Apache License 2.0" +skipDirs = @["tests"] + +requires "nim >= 0.17.0", + "ranges" + diff --git a/faststreams/input_stream.nim b/faststreams/input_stream.nim new file mode 100644 index 0000000..e3665b6 --- /dev/null +++ b/faststreams/input_stream.nim @@ -0,0 +1,149 @@ +import + memfiles, options, ranges/ptr_arith + +const + pageSize = 4096 + +type + StreamReader = proc (bufferStart: ptr byte, bufferSize: int): int + # TODO: use openarray once it's supported + AsyncStreamReader = StreamReader # proc (bufferStart: ptr byte, bufferSize: int): Future[int] + CloseStreamProc = proc () + + ByteStream* = object of RootObj + head: ptr byte + bufferSize: int + bufferStart, bufferEnd: ptr byte + reader: StreamReader + asyncReader: AsyncStreamReader + closeStream: CloseStreamProc + bufferEndPos: int + + BufferedStream*[size: static int] = object of ByteStream + buffer: array[size, byte] + + AsciiStream* = distinct ByteStream + UnicodeStream* = distinct ByteStream + ObjectStream*[T] = distinct ByteStream + +proc openFile*(filename: string): ByteStream = + var memFile = memfiles.open(filename) + result.head = cast[ptr byte](memFile.mem) + result.bufferEnd = result.head.shift memFile.size + result.bufferEndPos = memFile.size + result.closeStream = proc = close memFile + +proc init*(T: type ByteStream, + mem: openarray[byte], + reader = StreamReader(nil), + asyncReader = AsyncStreamReader(nil)): ByteStream = + # TODO: the result should use `from mem` once it's supported + result.head = unsafeAddr mem[0] + result.bufferEnd = result.head.shift mem.len + result.reader = reader + result.asyncReader = asyncReader + +template memoryStream*(mem: openarray[byte]): ByteStream = + ByteStream.init(mem) + +template memoryStream*(str: string): ByteStream = + bind init + init(ByteStream, str.toOpenArrayByte(0, str.len - 1)) + +proc init*(T: type BufferedStream, + reader = StreamReader(nil), + asyncReader = AsyncStreamReader(nil)): BufferedStream = + result.init result.buffer, reader, asyncReader + +proc syncRead(s: var ByteStream): bool = + let bytesRead = s.reader(s.bufferStart, s.bufferSize) + if bytesRead == 0: + s.reader = nil + return true + else: + s.bufferEnd = s.bufferStart.shift bytesRead + s.bufferEndPos += bytesRead + return false + +proc eof*(s: var ByteStream): bool = + if s.head != s.bufferEnd: + return false + + if s.reader == nil: + return true + + return s.syncRead() + +proc eob*(s: ByteStream): bool {.inline.} = + s.head != s.bufferEnd + +proc peek*(s: ByteStream): byte {.inline.} = + assert s.head != s.bufferEnd + return s.head[] + +proc advance*(s: var ByteStream) = + if s.head != s.bufferEnd: + s.head = s.head.shift 1 + elif s.reader != nil: + discard s.syncRead() + +proc read*(s: var ByteStream): byte = + result = s.peek() + advance s + +proc next*(s: var ByteStream): Option[byte] = + if not s.eof: + result = some s.read() + +proc bufferPos(s: ByteStream, pos: int): ptr byte = + let shiftFromEnd = pos - s.bufferEndPos + assert shiftFromEnd < 0 + result = s.bufferEnd.shift shiftFromEnd + assert result >= s.bufferStart + +proc pos*(s: ByteStream): int {.inline.} = + s.bufferEndPos - distance(s.head, s.bufferEnd) + +proc firstAccessiblePos*(s: ByteStream): int {.inline.} = + s.bufferEndPos - distance(s.bufferStart, s.bufferEnd) + +proc `[]`*(s: ByteStream, pos: int): byte {.inline.} = + s.bufferPos(pos)[] + +proc rewind*(s: var ByteStream, delta: int) = + s.head = s.head.shift(-delta) + assert s.head >= s.bufferStart + +proc rewindTo*(s: var ByteStream, pos: int) {.inline.} = + s.head = s.bufferPos(pos) + +# TODO: use a destructor once we migrate to Nim 0.20 +proc close*(s: var ByteStream) = + if s.closeStream != nil: + s.closeStream() + +# TODO: Use `distrinct with` once it's supported +template pos*(s: AsciiStream|UnicodeStream|ObjectStream): int = + ByteStream(s).pos + +template eof*(s: var (AsciiStream|UnicodeStream|ObjectStream)): bool = + ByteStream(s).eof + +template eob*(s: var (AsciiStream|UnicodeStream|ObjectStream)): bool = + ByteStream(s).eob + +template close*(s: AsciiStream|UnicodeStream|ObjectStream) = + close ByteStream(s) + +template advance*(s: var AsciiStream) = + advance ByteStream(s) + +template peek*(s: AsciiStream): char = + char ByteStream(s).peek() + +template read*(s: var AsciiStream): char = + char ByteStream(s).read() + +template next*(s: var AsciiStream): Option[char] = + cast[Option[char]](ByteStream(s).next()) + diff --git a/faststreams/output_stream.nim b/faststreams/output_stream.nim new file mode 100644 index 0000000..69e7006 --- /dev/null +++ b/faststreams/output_stream.nim @@ -0,0 +1,91 @@ +import + ranges/ptr_arith + +type + OutputStreamVTable = tuple + prepareOutput: proc (s: ptr OutputStream, size: int) {.nimcall.} + finish: proc (s: ptr OutputStream) {.nimcall.} + + OutputStream* = object {.inheritable.} + head, bufferEnd: ptr byte + vtable: ptr OutputStreamVTable + + MemoryOutputStream*[T] = object of OutputStream + output: T + + StringOutputStream* = MemoryOutputStream[string] + BytesOutputStream* = MemoryOutputStream[seq[byte]] + + FileOutputStream* = object of OutputStream + file: File + +const + pageSize = 4096 + +proc prepareOutputImpl[T](s: ptr OutputStream, size: int) = + let s = cast[ptr T](s) + let currentSize = s.output.len + s.output.setLen(s.output.len + pageSize) + s.head = cast[ptr byte](addr s.output[currentSize]) + s.bufferEnd = cast[ptr byte](shift(addr s.output[0], s.output.len)) + +proc finishImpl[T](s: ptr OutputStream) = + let s = cast[ptr T](s) + s.output.setLen(s.output.len - distance(s.head, s.bufferEnd)) + +proc init*(T: type MemoryOutputStream): T = + var vtable {.global.} = (prepareOutputImpl[T], finishImpl[T]) + result.vtable = addr vtable + result.vtable.prepareOutput(addr result, pageSize) + +proc append*(s: var OutputStream, b: byte) = + if s.head == s.bufferEnd: + s.vtable.prepareOutput(addr s, pageSize) + + s.head[] = b + s.head = shift(s.head, 1) + +template append*(s: var OutputStream, c: char) = + s.append byte(c) + +proc flush*(s: var OutputStream) = + s.vtable.finish(addr s) + +proc getOutput*(s: var MemoryOutputStream): auto = + flush s + shallow s.output + return s.output + +proc append*(s: var OutputStream, bytes: openarray[byte]) = + # TODO: this can use copyMem + for b in bytes: + s.append b + +proc append*(s: var OutputStream, chars: openarray[char]) = + # TODO: this can use copyMem + for c in chars: + s.append byte(c) + +template append*(s: var OutputStream, str: string) = + s.append str.toOpenArrayByte(0, str.len - 1) + +# Any stream + +proc appendNumberImpl(s: var OutputStream, number: BiggestInt) = + # TODO: don't allocate + s.append $number + +proc appendNumberImpl(s: var OutputStream, number: BiggestUInt) = + # TODO: don't allocate + s.append $number + +template toBiggestRepr(i: SomeUnsignedInt): BiggestUInt = + BiggestUInt(i) + +template toBiggestRepr(i: SomeSignedInt): BiggestInt = + BiggestInt(i) + +template appendNumber*(s: var OutputStream, i: SomeInteger) = + # TODO: specify radix/base + appendNumberImpl(s, toBiggestRepr(i)) + diff --git a/tests/test_output_stream.nim b/tests/test_output_stream.nim new file mode 100644 index 0000000..32e276f --- /dev/null +++ b/tests/test_output_stream.nim @@ -0,0 +1,21 @@ +import + unittest, + ../faststreams + +suite "output stream": + test "string output": + var s = init StringOutputStream + var altOutput = "" + + for i in 0 .. 1000: + s.appendNumber i + s.append " bottles on the wall" + s.append byte('\n') + + altOutput.add $i + altOutput.add " bottles on the wall" + altOutput.add '\n' + + let streamOutput = s.getOutput + check streamOutput == altOutput +