From 2d636d636dbf1e9331f3cf137ab5ad0727fb750a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 10 Mar 2020 16:21:40 -0600 Subject: [PATCH 01/17] Nimble package init --- protobuf_serialization.nim | 7 +++++++ protobuf_serialization.nimble | 14 ++++++++++++++ tests/config.nims | 1 + 3 files changed, 22 insertions(+) create mode 100644 protobuf_serialization.nim create mode 100644 protobuf_serialization.nimble create mode 100644 tests/config.nims diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim new file mode 100644 index 0000000..4b2a270 --- /dev/null +++ b/protobuf_serialization.nim @@ -0,0 +1,7 @@ +# This is just an example to get you started. A typical library package +# exports the main API in this file. Note that you cannot rename this file +# but you can remove it if you wish. + +proc add*(x, y: int): int = + ## Adds two files together. + return x + y diff --git a/protobuf_serialization.nimble b/protobuf_serialization.nimble new file mode 100644 index 0000000..90b1304 --- /dev/null +++ b/protobuf_serialization.nimble @@ -0,0 +1,14 @@ +# Package + +version = "0.1.0" +author = "Joey Yakimowich-Payne" +description = "Protobuf implementation compatible with the nim-serialization framework." +license = "MIT" +srcDir = "src" +skipDirs = @["tests"] + + + +# Dependencies + +requires "nim >= 1.0.6" diff --git a/tests/config.nims b/tests/config.nims new file mode 100644 index 0000000..3bb69f8 --- /dev/null +++ b/tests/config.nims @@ -0,0 +1 @@ +switch("path", "$projectDir/../src") \ No newline at end of file From 9d60f49e6f689810006c2a4821a7310f0d9fcba4 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 10 Mar 2020 16:24:23 -0600 Subject: [PATCH 02/17] Add CI files --- .appveyor.yml | 37 +++++++++++++++++++++++++++++++++++++ .travis.yml | 27 +++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 .appveyor.yml create mode 100644 .travis.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..54a4e61 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,37 @@ +version: '{build}' + +image: Visual Studio 2015 + +cache: + - NimBinaries + +matrix: + # We always want 32 and 64-bit compilation + fast_finish: false + +platform: + - x86 + - x64 + +# when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X" +clone_depth: 10 + +install: + # use the newest versions documented here: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin + - IF "%PLATFORM%" == "x86" SET PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH% + - IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + + # build nim from our own branch - this to avoid the day-to-day churn and + # regressions of the fast-paced Nim development while maintaining the + # flexibility to apply patches + - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh + - env MAKE="mingw32-make -j2" ARCH_OVERRIDE=%PLATFORM% bash build_nim.sh Nim csources dist/nimble NimBinaries + - SET PATH=%CD%\Nim\bin;%PATH% + +build_script: + - nimble install -y + +test_script: + - nimble test + +deploy: off diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..9ee1b9e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,27 @@ +language: c + +# https://docs.travis-ci.com/user/caching/ +cache: + directories: + - NimBinaries + +git: + # when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X" + depth: 10 + +os: + - linux + - osx + +install: + # build nim from our own branch - this to avoid the day-to-day churn and + # regressions of the fast-paced Nim development while maintaining the + # flexibility to apply patches + - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh + - env MAKE="make -j2" bash build_nim.sh Nim csources dist/nimble NimBinaries + - export PATH=$PWD/Nim/bin:$PATH + +script: + - nimble install -y + - nimble test + From fe9c5fa1f30336ac08aa22fdec34796017704885 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 14 Mar 2020 20:15:03 -0600 Subject: [PATCH 03/17] Add preliminary encode/decode of varints --- protobuf_serialization.nim | 118 ++++++++++++++++++++++++++++++++-- protobuf_serialization.nimble | 2 +- 2 files changed, 113 insertions(+), 7 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index 4b2a270..8cbf33b 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -1,7 +1,113 @@ -# This is just an example to get you started. A typical library package -# exports the main API in this file. Note that you cannot rename this file -# but you can remove it if you wish. +import faststreams -proc add*(x, y: int): int = - ## Adds two files together. - return x + y +const + MaxMessageSize* = 1'u shl 22 + +type + ProtoBuffer* = ref object + fieldNum: int + outstream: OutputStreamVar + + ProtoWireType* = enum + ## Protobuf's field types enum + Varint, Fixed64, Length, StartGroup, EndGroup, Fixed32 + + ProtoField* = object + ## Protobuf's message field representation object + index: int + case kind: ProtoWireType + of Varint: + vint*: uint64 + of Fixed64: + vfloat64*: float64 + of Length: + vbuffer*: OutputStreamVar + of Fixed32: + vfloat32*: float32 + of StartGroup, EndGroup: + discard + + SomeSVarint* = int | int64 | int32 | int16 | int8 | enum + SomeUVarint* = uint | uint64 | uint32 | uint16 | uint8 | byte | bool + SomeVarint* = SomeSVarint | SomeUVarint + +proc newProtoBuffer*(): ProtoBuffer = + ProtoBuffer(outstream: OutputStream.init(), fieldNum: 1) + +# Main interface +proc encode*(): ProtoBuffer = + discard + +proc decode*[T](source: ProtoBuffer): T = + discard + +template wireType(firstByte: byte): ProtoWireType = + (firstByte and 0b111).ProtoWireType + +template fieldNumber(firstByte: byte): uint = + (firstByte shr 3) and 0b1111 + +template protoHeader*(fieldNum: int, wire: ProtoWireType): byte = + ## Get protobuf's field header integer for ``index`` and ``wire``. + ((cast[uint](fieldNum) shl 3) or cast[uint](wire)).byte + +proc encodeVarint(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.inline.} = + let header = protoHeader(fieldNum, Varint) + stream.append header + + when value is enum: + var value = cast[type(ord(value))](value) + elif value is bool: + var value = cast[byte](value) + else: + var value = value + + when type(value) is SomeSVarint: + if value < type(value)(0): + value = not(value shl type(value)(1)) + else: + value = value shl type(value)(1) + + while value > type(value)(0b0111_1111): + stream.append byte((value and 0b0111_1111) or 0b1000_0000) + value = value shr 7 + stream.append byte(value and 0b1111_1111) + +proc encode(protobuf: ProtoBuffer, value: SomeVarint) {.inline.} = + protobuf.outstream.encodeVarint(protobuf.fieldNum, value) + inc protobuf.fieldNum + +proc decode[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[fieldNum: uint, value: T] {.inline.} = + # Only up to 128 bits supported by the spec + assert (bytes.len - 1) <= 16 + + let wireTy = wireType(bytes[offset]) + if wireTy != Varint: + raise newException(Exception, "Not a varint!") + + result.fieldNum = fieldNumber(bytes[offset]) + result.value = cast[ty](0) + var shiftAmount = 0 + var i = offset + 1 + while true: + result.value += T(bytes[i] and 0b0111_1111) shl shiftAmount + shiftAmount += 7 + if (bytes[i] shr 7) == 0: + break + i += 1 + + when ty is SomeSVarint: + if (result.value and T(1)) != T(0): + result.value = cast[T](not(result.value shr T(1))) + else: + result.value = cast[T](result.value shr T(1)) + +proc main() = + let proto = newProtoBuffer() + proto.encode(-1500000) + var input: seq[byte] = proto.outstream.getOutput + echo input + + echo decode(input, int64) + +main() \ No newline at end of file diff --git a/protobuf_serialization.nimble b/protobuf_serialization.nimble index 90b1304..8512041 100644 --- a/protobuf_serialization.nimble +++ b/protobuf_serialization.nimble @@ -11,4 +11,4 @@ skipDirs = @["tests"] # Dependencies -requires "nim >= 1.0.6" +requires "nim >= 1.0.6", "faststreams" From ec0e5a190a4b165bf8b65801fa9e67fd9b15a60c Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 14 Mar 2020 21:00:19 -0600 Subject: [PATCH 04/17] Add tests, fix unsigned int --- protobuf_serialization.nim | 34 +++++++++++++------------- tests/config.nims | 2 +- tests/test_serialization.nim | 46 ++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 tests/test_serialization.nim diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index 8cbf33b..e31be9a 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -41,6 +41,9 @@ proc encode*(): ProtoBuffer = proc decode*[T](source: ProtoBuffer): T = discard +proc output*(proto: ProtoBuffer): seq[byte] {.inline.} = + proto.outstream.getOutput + template wireType(firstByte: byte): ProtoWireType = (firstByte and 0b111).ProtoWireType @@ -73,11 +76,11 @@ proc encodeVarint(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.i value = value shr 7 stream.append byte(value and 0b1111_1111) -proc encode(protobuf: ProtoBuffer, value: SomeVarint) {.inline.} = +proc encode*(protobuf: ProtoBuffer, value: SomeVarint) {.inline.} = protobuf.outstream.encodeVarint(protobuf.fieldNum, value) inc protobuf.fieldNum -proc decode[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[fieldNum: uint, value: T] {.inline.} = +proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[fieldNum: uint, value: T, bytesProcessed: int] {.inline.} = # Only up to 128 bits supported by the spec assert (bytes.len - 1) <= 16 @@ -86,28 +89,25 @@ proc decode[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): t raise newException(Exception, "Not a varint!") result.fieldNum = fieldNumber(bytes[offset]) - result.value = cast[ty](0) + when T is enum: + var value: type(ord(result.value)) + else: + var value: T var shiftAmount = 0 var i = offset + 1 while true: - result.value += T(bytes[i] and 0b0111_1111) shl shiftAmount + value += type(value)(bytes[i] and 0b0111_1111) shl shiftAmount shiftAmount += 7 if (bytes[i] shr 7) == 0: break i += 1 + result.bytesProcessed = i + 1 + when ty is SomeSVarint: - if (result.value and T(1)) != T(0): - result.value = cast[T](not(result.value shr T(1))) + if (value and type(value)(1)) != type(value)(0): + result.value = cast[T](not(value shr type(value)(1))) else: - result.value = cast[T](result.value shr T(1)) - -proc main() = - let proto = newProtoBuffer() - proto.encode(-1500000) - var input: seq[byte] = proto.outstream.getOutput - echo input - - echo decode(input, int64) - -main() \ No newline at end of file + result.value = cast[T](value shr type(value)(1)) + else: + result.value = value \ No newline at end of file diff --git a/tests/config.nims b/tests/config.nims index 3bb69f8..e355636 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -1 +1 @@ -switch("path", "$projectDir/../src") \ No newline at end of file +switch("path", "$projectDir/../") \ No newline at end of file diff --git a/tests/test_serialization.nim b/tests/test_serialization.nim new file mode 100644 index 0000000..1ee0a57 --- /dev/null +++ b/tests/test_serialization.nim @@ -0,0 +1,46 @@ +import unittest + +import protobuf_serialization + +type + MyEnum = enum + ME1, ME2, ME3 + +suite "Test Varint Encoding": + test "Can encode enum": + let proto = newProtoBuffer() + proto.encode(ME3) + proto.encode(ME2) + var output = proto.output + assert output == @[8.byte, 4, 16, 2] + + let decodedME3 = decode(output, MyEnum) + assert decodedME3.value == ME3 + assert decodedME3.fieldNum == 1 + + let decodedME2 = decode(output, MyEnum, offset=decodedME3.bytesProcessed) + assert decodedME2.value == ME2 + assert decodedME2.fieldNum == 2 + + test "Can encode negative number": + let proto = newProtoBuffer() + let num = -153452 + proto.encode(num) + var output = proto.output + assert output == @[8.byte, 215, 221, 18] + + let decoded = decode(output, int) + assert decoded.value == num + assert decoded.fieldNum == 1 + + test "Can encode unsigned number": + let proto = newProtoBuffer() + let num = 123151.uint + proto.encode(num) + var output = proto.output + assert output == @[8.byte, 143, 194, 7] + + let decoded = decode(output, uint) + echo decoded.value + assert decoded.value == num + assert decoded.fieldNum == 1 \ No newline at end of file From c924821d0ac07d9f8e5d79891ae4b711d4d5097b Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 15 Mar 2020 18:39:14 -0600 Subject: [PATCH 05/17] Add string encoding/decoding --- protobuf_serialization.nim | 101 ++++++++++++++++++++++++++++------- tests/test_serialization.nim | 18 +++++-- 2 files changed, 97 insertions(+), 22 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index e31be9a..9a2596b 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -1,3 +1,4 @@ +import macros, strformat import faststreams const @@ -10,7 +11,7 @@ type ProtoWireType* = enum ## Protobuf's field types enum - Varint, Fixed64, Length, StartGroup, EndGroup, Fixed32 + Varint, Fixed64, LengthDelimited, StartGroup, EndGroup, Fixed32 ProtoField* = object ## Protobuf's message field representation object @@ -20,7 +21,7 @@ type vint*: uint64 of Fixed64: vfloat64*: float64 - of Length: + of LengthDelimited: vbuffer*: OutputStreamVar of Fixed32: vfloat32*: float32 @@ -30,6 +31,7 @@ type SomeSVarint* = int | int64 | int32 | int16 | int8 | enum SomeUVarint* = uint | uint64 | uint32 | uint16 | uint8 | byte | bool SomeVarint* = SomeSVarint | SomeUVarint + SomeLengthDelimited* = string | seq[byte] | seq[uint8] | cstring proc newProtoBuffer*(): ProtoBuffer = ProtoBuffer(outstream: OutputStream.init(), fieldNum: 1) @@ -54,10 +56,7 @@ template protoHeader*(fieldNum: int, wire: ProtoWireType): byte = ## Get protobuf's field header integer for ``index`` and ``wire``. ((cast[uint](fieldNum) shl 3) or cast[uint](wire)).byte -proc encodeVarint(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.inline.} = - let header = protoHeader(fieldNum, Varint) - stream.append header - +proc putVarint(stream: OutputStreamVar, value: SomeVarint) {.inline.} = when value is enum: var value = cast[type(ord(value))](value) elif value is bool: @@ -76,25 +75,35 @@ proc encodeVarint(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.i value = value shr 7 stream.append byte(value and 0b1111_1111) +proc encode(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.inline.} = + stream.append protoHeader(fieldNum, Varint) + stream.putVarint(value) + proc encode*(protobuf: ProtoBuffer, value: SomeVarint) {.inline.} = - protobuf.outstream.encodeVarint(protobuf.fieldNum, value) + protobuf.outstream.encode(protobuf.fieldNum, value) inc protobuf.fieldNum -proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[fieldNum: uint, value: T, bytesProcessed: int] {.inline.} = +proc putLengthDelimited(stream: OutputStreamVar, value: SomeLengthDelimited) {.inline.} = + for b in value: + stream.append byte(b) + +proc encode(stream: OutputStreamVar, fieldNum: int, value: SomeLengthDelimited) {.inline.} = + stream.append protoHeader(fieldNum, LengthDelimited) + stream.putVarint(len(value).uint) + stream.putLengthDelimited(value) + +proc encode*(protobuf: ProtoBuffer, value: SomeLengthDelimited) {.inline.} = + protobuf.outstream.encode(protobuf.fieldNum, value) + inc protobuf.fieldNum + +proc getVarint[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[value: T, bytesProcessed: int] {.inline.} = # Only up to 128 bits supported by the spec - assert (bytes.len - 1) <= 16 - - let wireTy = wireType(bytes[offset]) - if wireTy != Varint: - raise newException(Exception, "Not a varint!") - - result.fieldNum = fieldNumber(bytes[offset]) when T is enum: var value: type(ord(result.value)) else: var value: T var shiftAmount = 0 - var i = offset + 1 + var i = offset while true: value += type(value)(bytes[i] and 0b0111_1111) shl shiftAmount shiftAmount += 7 @@ -102,7 +111,7 @@ proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): break i += 1 - result.bytesProcessed = i + 1 + result.bytesProcessed = i when ty is SomeSVarint: if (value and type(value)(1)) != type(value)(0): @@ -110,4 +119,60 @@ proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): else: result.value = cast[T](value shr type(value)(1)) else: - result.value = value \ No newline at end of file + result.value = value + +proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[fieldNum: uint, value: T, bytesProcessed: int] {.inline.} = + # Only up to 128 bits supported by the spec + assert (bytes.len - 1) <= 16 + + let wireTy = wireType(bytes[offset]) + if wireTy != Varint: + raise newException(Exception, fmt"Not a varint at offset {offset}! Received a {wireTy}") + + result.fieldNum = fieldNumber(bytes[offset]) + var offset = offset + 1 + + let varGet = getVarint(bytes, ty, offset) + result.value = varGet.value + result.bytesProcessed = varGet.bytesProcessed + offset + +proc getLengthDelimited*[T: SomeLengthDelimited]( + bytes: var seq[byte], + ty: typedesc[T], offset = 0 +): tuple[value: T, bytesProcessed: int] {.inline.} = + + var offset = offset + let decodedSize = getVarint(bytes, uint, offset = offset) + offset += decodedSize.bytesProcessed + let length = decodedSize.value.int + + when T is string: + result.value = newString(length) + for i in offset ..< (offset + length): + result.value[i - offset] = bytes[i].chr + elif T is cstring: + result.value = cast[cstring](bytes[offset ..< (offset + length)]) + else: + result.value = newSeq(length) + for i in offset ..< (offset + length): + result.value[i - offset] = bytes[i].chr + + result.bytesProcessed += length + +proc decode*[T: SomeLengthDelimited]( + bytes: var seq[byte], + ty: typedesc[T], offset = 0 +): tuple[fieldNum: uint, value: T, bytesProcessed: int] {.inline.} = + var offset = offset + + let wireTy = wireType(bytes[offset]) + if wireTy != LengthDelimited: + raise newException(Exception, fmt"Not a length delimited value at offset {offset}! Received a {wireTy}") + + result.fieldNum = fieldNumber(bytes[offset]) + + offset += 1 + + let lengthDelimited = getLengthDelimited(bytes, ty, offset) + result.bytesProcessed = offset + lengthDelimited.bytesProcessed + result.value = lengthDelimited.value \ No newline at end of file diff --git a/tests/test_serialization.nim b/tests/test_serialization.nim index 1ee0a57..2a11db7 100644 --- a/tests/test_serialization.nim +++ b/tests/test_serialization.nim @@ -7,7 +7,7 @@ type ME1, ME2, ME3 suite "Test Varint Encoding": - test "Can encode enum": + test "Can encode/decode enum": let proto = newProtoBuffer() proto.encode(ME3) proto.encode(ME2) @@ -22,7 +22,7 @@ suite "Test Varint Encoding": assert decodedME2.value == ME2 assert decodedME2.fieldNum == 2 - test "Can encode negative number": + test "Can encode/decode negative number": let proto = newProtoBuffer() let num = -153452 proto.encode(num) @@ -33,7 +33,7 @@ suite "Test Varint Encoding": assert decoded.value == num assert decoded.fieldNum == 1 - test "Can encode unsigned number": + test "Can encode/decode unsigned number": let proto = newProtoBuffer() let num = 123151.uint proto.encode(num) @@ -41,6 +41,16 @@ suite "Test Varint Encoding": assert output == @[8.byte, 143, 194, 7] let decoded = decode(output, uint) - echo decoded.value assert decoded.value == num + assert decoded.fieldNum == 1 + + test "Can encode/decode string": + let proto = newProtoBuffer() + let str = "hey this is a string" + proto.encode(str) + var output = proto.output + assert output == @[10.byte, 20, 104, 101, 121, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 115, 116, 114, 105, 110, 103] + + let decoded = decode(output, string) + assert decoded.value == str assert decoded.fieldNum == 1 \ No newline at end of file From 5f2219bae7b7b3fe5144240606c6d01f4e8c52c5 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 3 Apr 2020 17:44:50 -0600 Subject: [PATCH 06/17] WIP Object serialization/deserialization --- protobuf_serialization.nim | 285 ++++++++++++++++++++++++++--------- tests/test_serialization.nim | 59 ++++++-- 2 files changed, 260 insertions(+), 84 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index 9a2596b..fa2dcd3 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -1,11 +1,20 @@ -import macros, strformat +import macros, strformat, typetraits, options import faststreams +template sint32*() {.pragma.} +template sint64*() {.pragma.} +template sfixed32*() {.pragma.} +template sfixed64*() {.pragma.} +template fixed32*() {.pragma.} +template fixed64*() {.pragma.} +template float*() {.pragma.} +template double*() {.pragma.} + const MaxMessageSize* = 1'u shl 22 type - ProtoBuffer* = ref object + ProtoBuffer* = object fieldNum: int outstream: OutputStreamVar @@ -13,23 +22,16 @@ type ## Protobuf's field types enum Varint, Fixed64, LengthDelimited, StartGroup, EndGroup, Fixed32 - ProtoField* = object + EncodingKind* = enum + ekNormal, ekZigzag + + ProtoField*[T] = object ## Protobuf's message field representation object - index: int - case kind: ProtoWireType - of Varint: - vint*: uint64 - of Fixed64: - vfloat64*: float64 - of LengthDelimited: - vbuffer*: OutputStreamVar - of Fixed32: - vfloat32*: float32 - of StartGroup, EndGroup: - discard + index*: int + value*: T SomeSVarint* = int | int64 | int32 | int16 | int8 | enum - SomeUVarint* = uint | uint64 | uint32 | uint16 | uint8 | byte | bool + SomeUVarint* = uint | uint64 | uint32 | uint16 | uint8 | byte | bool | char SomeVarint* = SomeSVarint | SomeUVarint SomeLengthDelimited* = string | seq[byte] | seq[uint8] | cstring @@ -49,22 +51,32 @@ proc output*(proto: ProtoBuffer): seq[byte] {.inline.} = template wireType(firstByte: byte): ProtoWireType = (firstByte and 0b111).ProtoWireType -template fieldNumber(firstByte: byte): uint = - (firstByte shr 3) and 0b1111 +template fieldNumber(firstByte: byte): int = + ((firstByte shr 3) and 0b1111).int template protoHeader*(fieldNum: int, wire: ProtoWireType): byte = ## Get protobuf's field header integer for ``index`` and ``wire``. ((cast[uint](fieldNum) shl 3) or cast[uint](wire)).byte -proc putVarint(stream: OutputStreamVar, value: SomeVarint) {.inline.} = +template increaseBytesRead(amount = 1) = + mixin isSome + bytesRead += amount + outOffset += amount + outBytesProcessed += amount + if numBytesToRead.isSome(): + if (bytesRead > numBytesToRead.get()).unlikely: + raise newException(Exception, "Number of bytes read exceeded") + +proc put(stream: OutputStreamVar, value: SomeVarint) {.inline.} = when value is enum: var value = cast[type(ord(value))](value) - elif value is bool: + elif value is bool or value is char: var value = cast[byte](value) else: var value = value when type(value) is SomeSVarint: + # Encode using zigzag if value < type(value)(0): value = not(value shl type(value)(1)) else: @@ -77,102 +89,235 @@ proc putVarint(stream: OutputStreamVar, value: SomeVarint) {.inline.} = proc encode(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.inline.} = stream.append protoHeader(fieldNum, Varint) - stream.putVarint(value) + stream.put(value) -proc encode*(protobuf: ProtoBuffer, value: SomeVarint) {.inline.} = +proc encode*(protobuf: var ProtoBuffer, value: SomeVarint) {.inline.} = protobuf.outstream.encode(protobuf.fieldNum, value) inc protobuf.fieldNum -proc putLengthDelimited(stream: OutputStreamVar, value: SomeLengthDelimited) {.inline.} = +proc put(stream: OutputStreamVar, value: SomeLengthDelimited) {.inline.} = for b in value: stream.append byte(b) proc encode(stream: OutputStreamVar, fieldNum: int, value: SomeLengthDelimited) {.inline.} = stream.append protoHeader(fieldNum, LengthDelimited) - stream.putVarint(len(value).uint) - stream.putLengthDelimited(value) + stream.put(len(value).uint) + stream.put(value) -proc encode*(protobuf: ProtoBuffer, value: SomeLengthDelimited) {.inline.} = +proc encode*(protobuf: var ProtoBuffer, value: SomeLengthDelimited) {.inline.} = protobuf.outstream.encode(protobuf.fieldNum, value) inc protobuf.fieldNum -proc getVarint[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[value: T, bytesProcessed: int] {.inline.} = +proc put(stream: OutputStreamVar, value: object) {.inline.} + +proc encode(stream: OutputStreamVar, fieldNum: int, value: object) {.inline.} = + #TODO Encode generic objects + stream.append protoHeader(fieldNum, LengthDelimited) + let objStream = OutputStream.init() + objStream.put(value) + let objOutput = objStream.getOutput() + stream.put(len(objOutput).uint) + stream.put(objOutput) + +proc encode*(protobuf: var ProtoBuffer, value: object) {.inline.} = + protobuf.outstream.encode(protobuf.fieldNum, value) + inc protobuf.fieldNum + +proc put(stream: OutputStreamVar, value: object) {.inline.} = + var fieldNum = 1 + for field, val in value.fieldPairs: + stream.encode(fieldNum, val) + fieldNum += 1 + +proc getVarint[T: SomeVarint]( + bytes: var seq[byte], + ty: typedesc[T], + outOffset: var int, + outBytesProcessed: var int, + numBytesToRead = none(int) +): T {.inline.} = + var bytesRead = 0 # Only up to 128 bits supported by the spec when T is enum: - var value: type(ord(result.value)) + var value: type(ord(result)) else: var value: T var shiftAmount = 0 - var i = offset while true: - value += type(value)(bytes[i] and 0b0111_1111) shl shiftAmount + value += type(value)(bytes[outOffset] and 0b0111_1111) shl shiftAmount shiftAmount += 7 - if (bytes[i] shr 7) == 0: + if (bytes[outOffset] shr 7) == 0: break - i += 1 + increaseBytesRead() - result.bytesProcessed = i + increaseBytesRead() when ty is SomeSVarint: if (value and type(value)(1)) != type(value)(0): - result.value = cast[T](not(value shr type(value)(1))) + result = cast[T](not(value shr type(value)(1))) else: - result.value = cast[T](value shr type(value)(1)) + result = cast[T](value shr type(value)(1)) else: - result.value = value + result = value -proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[fieldNum: uint, value: T, bytesProcessed: int] {.inline.} = +proc decode*[T: SomeVarint]( + bytes: var seq[byte], + ty: typedesc[T], + outOffset: var int, + outBytesProcessed: var int, + numBytesToRead = none(int) +): ProtoField[T] {.inline.} = # Only up to 128 bits supported by the spec assert (bytes.len - 1) <= 16 - let wireTy = wireType(bytes[offset]) + var bytesRead = 0 + + let wireTy = wireType(bytes[outOffset]) if wireTy != Varint: - raise newException(Exception, fmt"Not a varint at offset {offset}! Received a {wireTy}") + raise newException(Exception, fmt"Not a varint at offset {outOffset}! Received a {wireTy}") - result.fieldNum = fieldNumber(bytes[offset]) - var offset = offset + 1 + result.index = fieldNumber(bytes[outOffset]) + increaseBytesRead() + + result.value = getVarint(bytes, ty, outOffset, outBytesProcessed, numBytesToRead) - let varGet = getVarint(bytes, ty, offset) - result.value = varGet.value - result.bytesProcessed = varGet.bytesProcessed + offset proc getLengthDelimited*[T: SomeLengthDelimited]( bytes: var seq[byte], - ty: typedesc[T], offset = 0 -): tuple[value: T, bytesProcessed: int] {.inline.} = - - var offset = offset - let decodedSize = getVarint(bytes, uint, offset = offset) - offset += decodedSize.bytesProcessed - let length = decodedSize.value.int + ty: typedesc[T], outOffset: var int, + outBytesProcessed: var int, + numBytesToRead = none(int) +): T {.inline.} = + var bytesRead = 0 + let decodedSize = getVarint(bytes, uint, outOffset, outBytesProcessed, numBytesToRead) + let length = decodedSize.int when T is string: - result.value = newString(length) - for i in offset ..< (offset + length): - result.value[i - offset] = bytes[i].chr + result = newString(length) + for i in outOffset ..< (outOffset + length): + result[i - outOffset] = bytes[i].chr elif T is cstring: - result.value = cast[cstring](bytes[offset ..< (offset + length)]) + result = cast[cstring](bytes[outOffset ..< (outOffset + length)]) else: - result.value = newSeq(length) - for i in offset ..< (offset + length): - result.value[i - offset] = bytes[i].chr + result.setLen(length) + for i in outOffset ..< (outOffset + length): + result[i - outOffset] = type(result[0])(bytes[i]) - result.bytesProcessed += length + increaseBytesRead(length) proc decode*[T: SomeLengthDelimited]( bytes: var seq[byte], - ty: typedesc[T], offset = 0 -): tuple[fieldNum: uint, value: T, bytesProcessed: int] {.inline.} = - var offset = offset - - let wireTy = wireType(bytes[offset]) + ty: typedesc[T], + outOffset: var int, + outBytesProcessed: var int, + numBytesToRead = none(int) +): ProtoField[T] {.inline.} = + var bytesRead = 0 + let wireTy = wireType(bytes[outOffset]) if wireTy != LengthDelimited: - raise newException(Exception, fmt"Not a length delimited value at offset {offset}! Received a {wireTy}") + raise newException(Exception, fmt"Not a length delimited value at offset {outOffset}! Received a {wireTy}") - result.fieldNum = fieldNumber(bytes[offset]) + result.index = fieldNumber(bytes[outOffset]) + increaseBytesRead() - offset += 1 + result.value = getLengthDelimited(bytes, ty, outOffset, outBytesProcessed, numBytesToRead) - let lengthDelimited = getLengthDelimited(bytes, ty, offset) - result.bytesProcessed = offset + lengthDelimited.bytesProcessed - result.value = lengthDelimited.value \ No newline at end of file +type + Test1 = object + a: uint + + Test3 = object + g {.sfixed32.}: int + h: int + i: Test1 + +macro getField(obj: typed, fieldNum: int, ty: typedesc): untyped = + template fieldTypeCheck(obj, field, fieldNum, ty) = + when type(obj.field) is type(ty): + obj.field + else: + let fnum {.inject.} = fieldNum + raise newException(Exception, fmt"Could not find field at position {fnum}.") + + let typeImpl = obj.getTypeInst.getImpl + let typeFields = obj.getTypeInst.getType + + let objFields = typeFields[2] + expectKind objFields, nnkRecList + + result = newStmtList() + let caseStmt = newNimNode(nnkCaseStmt) + caseStmt.add(fieldNum) + + for i in 0 ..< len(objFields) - 1: + let field = objFields[i] + let ofBranch = newNimNode(nnkOfBranch) + ofBranch.add(newLit(i+1)) + ofBranch.add(getAst(fieldTypeCheck(obj, field, fieldNum, ty))) + caseStmt.add(ofBranch) + + let field = objFields[len(objFields) - 1] + let elseBranch = newNimNode(nnkElse) + elseBranch.add( + nnkStmtList.newTree(getAst(fieldTypeCheck(obj, field, fieldNum, ty))) + ) + caseStmt.add(elseBranch) + + result.add(caseStmt) + +macro setField(obj: typed, fieldNum: int, offset: int, bytesProcessed: int, bytesToRead: Option[int], value: untyped): untyped = + let typeImpl = obj.getTypeInst.getImpl + let typeFields = obj.getTypeInst.getType + + let objFields = typeFields[2] + expectKind objFields, nnkRecList + + result = newStmtList() + + let caseStmt = newNimNode(nnkCaseStmt) + caseStmt.add(fieldNum) + + for i in 0 ..< len(objFields) - 1: + let field = objFields[i] + let ofBranch = newNimNode(nnkOfBranch) + ofBranch.add(newLit(i+1)) + ofBranch.add( + quote do: + `obj`.`field` = decode(`value`, type(`obj`.`field`), `offset`, `bytesProcessed`, `bytesToRead`).value + ) + caseStmt.add(ofBranch) + + let field = objFields[len(objFields) - 1] + let elseBranch = newNimNode(nnkElse) + elseBranch.add( + nnkStmtList.newTree( + quote do: + `obj`.`field` = decode(`value`, type(`obj`.`field`), `offset`, `bytesProcessed`, `bytesToRead`).value + ) + ) + caseStmt.add(elseBranch) + + result.add(caseStmt) + +proc decode*[T: object]( + bytes: var seq[byte], + ty: typedesc[T], + outOffset: var int, + outBytesProcessed: var int, + numBytesToRead = none(int) +): ProtoField[T] {.inline.} = + var bytesRead = 0 + + let wireTy = wireType(bytes[outOffset]) + result.index = fieldNumber(bytes[outOffset]) + + if wireTy == LengthDelimited: + # read LD header + # then read only amount of bytes needed + increaseBytesRead() + + let decodedSize = getVarint(bytes, uint, outOffset, outBytesProcessed, numBytesToRead) + let bytesToRead = some(decodedSize.int) + setField(result.value, result.index, outOffset, outBytesProcessed, bytesToRead, bytes) + else: + setField(result.value, result.index, outOffset, outBytesProcessed, numBytesToRead, bytes) \ No newline at end of file diff --git a/tests/test_serialization.nim b/tests/test_serialization.nim index 2a11db7..37ae148 100644 --- a/tests/test_serialization.nim +++ b/tests/test_serialization.nim @@ -5,52 +5,83 @@ import protobuf_serialization type MyEnum = enum ME1, ME2, ME3 +type + Test1 = object + a: uint + + Test3 = object + g {.sfixed32.}: int + h: int + i: Test1 suite "Test Varint Encoding": test "Can encode/decode enum": - let proto = newProtoBuffer() + var proto = newProtoBuffer() + var bytesProcessed: int proto.encode(ME3) proto.encode(ME2) var output = proto.output assert output == @[8.byte, 4, 16, 2] + var offset = 0 - let decodedME3 = decode(output, MyEnum) + let decodedME3 = decode(output, MyEnum, offset, bytesProcessed) assert decodedME3.value == ME3 - assert decodedME3.fieldNum == 1 + assert decodedME3.index == 1 - let decodedME2 = decode(output, MyEnum, offset=decodedME3.bytesProcessed) + let decodedME2 = decode(output, MyEnum, offset, bytesProcessed) assert decodedME2.value == ME2 - assert decodedME2.fieldNum == 2 + assert decodedME2.index == 2 test "Can encode/decode negative number": - let proto = newProtoBuffer() + var proto = newProtoBuffer() let num = -153452 + var bytesProcessed: int proto.encode(num) var output = proto.output assert output == @[8.byte, 215, 221, 18] - let decoded = decode(output, int) + var offset = 0 + let decoded = decode(output, int, offset, bytesProcessed) assert decoded.value == num - assert decoded.fieldNum == 1 + assert decoded.index == 1 test "Can encode/decode unsigned number": - let proto = newProtoBuffer() + var proto = newProtoBuffer() let num = 123151.uint + var bytesProcessed: int proto.encode(num) var output = proto.output assert output == @[8.byte, 143, 194, 7] + var offset = 0 - let decoded = decode(output, uint) + let decoded = decode(output, uint, offset, bytesProcessed) assert decoded.value == num - assert decoded.fieldNum == 1 + assert decoded.index == 1 test "Can encode/decode string": - let proto = newProtoBuffer() + var proto = newProtoBuffer() let str = "hey this is a string" + var bytesProcessed: int proto.encode(str) var output = proto.output assert output == @[10.byte, 20, 104, 101, 121, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 115, 116, 114, 105, 110, 103] - let decoded = decode(output, string) + var offset = 0 + let decoded = decode(output, string, offset, bytesProcessed) assert decoded.value == str - assert decoded.fieldNum == 1 \ No newline at end of file + assert decoded.index == 1 + + test "Can encode/decode object": + var proto = newProtoBuffer() + + let obj = Test3(g: 300, h: 200, i: Test1(a: 100)) + + proto.encode(obj) + var offset, bytesProcessed: int + + var output = proto.output + let decoded = decode(output, Test3, offset, bytesProcessed) + echo decoded + + echo output + assert false \ No newline at end of file From 2db4d48802baf2976f70db7cbdb1d2d879d2a0a9 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 3 Apr 2020 18:50:00 -0600 Subject: [PATCH 07/17] Add object field decoding. Refactor proc names to be inline with behavior --- protobuf_serialization.nim | 64 +++++++++++++++++++++--------------- tests/test_serialization.nim | 39 +++++++++++++--------- 2 files changed, 61 insertions(+), 42 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index fa2dcd3..b518b0a 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -65,7 +65,7 @@ template increaseBytesRead(amount = 1) = outBytesProcessed += amount if numBytesToRead.isSome(): if (bytesRead > numBytesToRead.get()).unlikely: - raise newException(Exception, "Number of bytes read exceeded") + raise newException(Exception, &"Number of bytes read ({bytesRead}) exceeded bytes requested ({numBytesToRead})") proc put(stream: OutputStreamVar, value: SomeVarint) {.inline.} = when value is enum: @@ -87,30 +87,30 @@ proc put(stream: OutputStreamVar, value: SomeVarint) {.inline.} = value = value shr 7 stream.append byte(value and 0b1111_1111) -proc encode(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.inline.} = +proc encodeField(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.inline.} = stream.append protoHeader(fieldNum, Varint) stream.put(value) -proc encode*(protobuf: var ProtoBuffer, value: SomeVarint) {.inline.} = - protobuf.outstream.encode(protobuf.fieldNum, value) +proc encodeField*(protobuf: var ProtoBuffer, value: SomeVarint) {.inline.} = + protobuf.outstream.encodeField(protobuf.fieldNum, value) inc protobuf.fieldNum proc put(stream: OutputStreamVar, value: SomeLengthDelimited) {.inline.} = for b in value: stream.append byte(b) -proc encode(stream: OutputStreamVar, fieldNum: int, value: SomeLengthDelimited) {.inline.} = +proc encodeField(stream: OutputStreamVar, fieldNum: int, value: SomeLengthDelimited) {.inline.} = stream.append protoHeader(fieldNum, LengthDelimited) stream.put(len(value).uint) stream.put(value) -proc encode*(protobuf: var ProtoBuffer, value: SomeLengthDelimited) {.inline.} = - protobuf.outstream.encode(protobuf.fieldNum, value) +proc encodeField*(protobuf: var ProtoBuffer, value: SomeLengthDelimited) {.inline.} = + protobuf.outstream.encodeField(protobuf.fieldNum, value) inc protobuf.fieldNum proc put(stream: OutputStreamVar, value: object) {.inline.} -proc encode(stream: OutputStreamVar, fieldNum: int, value: object) {.inline.} = +proc encodeField(stream: OutputStreamVar, fieldNum: int, value: object) {.inline.} = #TODO Encode generic objects stream.append protoHeader(fieldNum, LengthDelimited) let objStream = OutputStream.init() @@ -119,14 +119,14 @@ proc encode(stream: OutputStreamVar, fieldNum: int, value: object) {.inline.} = stream.put(len(objOutput).uint) stream.put(objOutput) -proc encode*(protobuf: var ProtoBuffer, value: object) {.inline.} = - protobuf.outstream.encode(protobuf.fieldNum, value) +proc encodeField*(protobuf: var ProtoBuffer, value: object) {.inline.} = + protobuf.outstream.encodeField(protobuf.fieldNum, value) inc protobuf.fieldNum proc put(stream: OutputStreamVar, value: object) {.inline.} = var fieldNum = 1 for field, val in value.fieldPairs: - stream.encode(fieldNum, val) + stream.encodeField(fieldNum, val) fieldNum += 1 proc getVarint[T: SomeVarint]( @@ -160,7 +160,7 @@ proc getVarint[T: SomeVarint]( else: result = value -proc decode*[T: SomeVarint]( +proc decodeField*[T: SomeVarint]( bytes: var seq[byte], ty: typedesc[T], outOffset: var int, @@ -205,7 +205,7 @@ proc getLengthDelimited*[T: SomeLengthDelimited]( increaseBytesRead(length) -proc decode*[T: SomeLengthDelimited]( +proc decodeField*[T: SomeLengthDelimited]( bytes: var seq[byte], ty: typedesc[T], outOffset: var int, @@ -283,7 +283,7 @@ macro setField(obj: typed, fieldNum: int, offset: int, bytesProcessed: int, byte ofBranch.add(newLit(i+1)) ofBranch.add( quote do: - `obj`.`field` = decode(`value`, type(`obj`.`field`), `offset`, `bytesProcessed`, `bytesToRead`).value + `obj`.`field` = decodeField(`value`, type(`obj`.`field`), `offset`, `bytesProcessed`, `bytesToRead`).value ) caseStmt.add(ofBranch) @@ -292,14 +292,13 @@ macro setField(obj: typed, fieldNum: int, offset: int, bytesProcessed: int, byte elseBranch.add( nnkStmtList.newTree( quote do: - `obj`.`field` = decode(`value`, type(`obj`.`field`), `offset`, `bytesProcessed`, `bytesToRead`).value + `obj`.`field` = decodeField(`value`, type(`obj`.`field`), `offset`, `bytesProcessed`, `bytesToRead`).value ) ) caseStmt.add(elseBranch) - result.add(caseStmt) -proc decode*[T: object]( +proc decodeField*[T: object]( bytes: var seq[byte], ty: typedesc[T], outOffset: var int, @@ -309,15 +308,28 @@ proc decode*[T: object]( var bytesRead = 0 let wireTy = wireType(bytes[outOffset]) + assert wireTy == LengthDelimited + result.index = fieldNumber(bytes[outOffset]) - if wireTy == LengthDelimited: - # read LD header - # then read only amount of bytes needed - increaseBytesRead() + # read LD header + # then read only amount of bytes needed + increaseBytesRead() + var index = 1 + let decodedSize = getVarint(bytes, uint, outOffset, outBytesProcessed, numBytesToRead) + let bytesToRead = some(decodedSize.int) + for field, val in result.value.fieldPairs: + setField(result.value, index, outOffset, outBytesProcessed, bytesToRead, bytes) + index += 1 - let decodedSize = getVarint(bytes, uint, outOffset, outBytesProcessed, numBytesToRead) - let bytesToRead = some(decodedSize.int) - setField(result.value, result.index, outOffset, outBytesProcessed, bytesToRead, bytes) - else: - setField(result.value, result.index, outOffset, outBytesProcessed, numBytesToRead, bytes) \ No newline at end of file +proc decode*[T: object]( + bytes: var seq[byte], + ty: typedesc[T], +): T {.inline.} = + var bytesRead = 0 + var offset = 0 + + var fieldNum = 1 + for field, val in result.fieldPairs: + setField(result, fieldNum, offset, bytesRead, none(int), bytes) + fieldNum += 1 \ No newline at end of file diff --git a/tests/test_serialization.nim b/tests/test_serialization.nim index 37ae148..86c97f9 100644 --- a/tests/test_serialization.nim +++ b/tests/test_serialization.nim @@ -18,17 +18,20 @@ suite "Test Varint Encoding": test "Can encode/decode enum": var proto = newProtoBuffer() var bytesProcessed: int - proto.encode(ME3) - proto.encode(ME2) + + proto.encodeField(ME3) + proto.encodeField(ME2) + var output = proto.output assert output == @[8.byte, 4, 16, 2] + var offset = 0 - let decodedME3 = decode(output, MyEnum, offset, bytesProcessed) + let decodedME3 = decodeField(output, MyEnum, offset, bytesProcessed) assert decodedME3.value == ME3 assert decodedME3.index == 1 - let decodedME2 = decode(output, MyEnum, offset, bytesProcessed) + let decodedME2 = decodeField(output, MyEnum, offset, bytesProcessed) assert decodedME2.value == ME2 assert decodedME2.index == 2 @@ -36,12 +39,14 @@ suite "Test Varint Encoding": var proto = newProtoBuffer() let num = -153452 var bytesProcessed: int - proto.encode(num) + + proto.encodeField(num) + var output = proto.output assert output == @[8.byte, 215, 221, 18] var offset = 0 - let decoded = decode(output, int, offset, bytesProcessed) + let decoded = decodeField(output, int, offset, bytesProcessed) assert decoded.value == num assert decoded.index == 1 @@ -49,12 +54,14 @@ suite "Test Varint Encoding": var proto = newProtoBuffer() let num = 123151.uint var bytesProcessed: int - proto.encode(num) + + proto.encodeField(num) + var output = proto.output assert output == @[8.byte, 143, 194, 7] var offset = 0 - let decoded = decode(output, uint, offset, bytesProcessed) + let decoded = decodeField(output, uint, offset, bytesProcessed) assert decoded.value == num assert decoded.index == 1 @@ -62,12 +69,14 @@ suite "Test Varint Encoding": var proto = newProtoBuffer() let str = "hey this is a string" var bytesProcessed: int - proto.encode(str) + + proto.encodeField(str) + var output = proto.output assert output == @[10.byte, 20, 104, 101, 121, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 115, 116, 114, 105, 110, 103] var offset = 0 - let decoded = decode(output, string, offset, bytesProcessed) + let decoded = decodeField(output, string, offset, bytesProcessed) assert decoded.value == str assert decoded.index == 1 @@ -76,12 +85,10 @@ suite "Test Varint Encoding": let obj = Test3(g: 300, h: 200, i: Test1(a: 100)) - proto.encode(obj) + proto.encodeField(obj) var offset, bytesProcessed: int var output = proto.output - let decoded = decode(output, Test3, offset, bytesProcessed) - echo decoded - - echo output - assert false \ No newline at end of file + let decoded = decodeField(output, Test3, offset, bytesProcessed) + assert decoded.value == obj + assert decoded.index == 1 \ No newline at end of file From aae968ac3b91c63d559aacee0f02a7a2cd6b354b Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 3 Apr 2020 19:13:11 -0600 Subject: [PATCH 08/17] Add object encoding/decoding --- protobuf_serialization.nim | 27 ++++++++++++++++----------- tests/test_serialization.nim | 22 ++++++++++++++++------ 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index b518b0a..2b9fbdd 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -38,13 +38,6 @@ type proc newProtoBuffer*(): ProtoBuffer = ProtoBuffer(outstream: OutputStream.init(), fieldNum: 1) -# Main interface -proc encode*(): ProtoBuffer = - discard - -proc decode*[T](source: ProtoBuffer): T = - discard - proc output*(proto: ProtoBuffer): seq[byte] {.inline.} = proto.outstream.getOutput @@ -59,6 +52,8 @@ template protoHeader*(fieldNum: int, wire: ProtoWireType): byte = ((cast[uint](fieldNum) shl 3) or cast[uint](wire)).byte template increaseBytesRead(amount = 1) = + ## Convenience template for increasing + ## all of the counts mixin isSome bytesRead += amount outOffset += amount @@ -111,10 +106,14 @@ proc encodeField*(protobuf: var ProtoBuffer, value: SomeLengthDelimited) {.inlin proc put(stream: OutputStreamVar, value: object) {.inline.} proc encodeField(stream: OutputStreamVar, fieldNum: int, value: object) {.inline.} = - #TODO Encode generic objects stream.append protoHeader(fieldNum, LengthDelimited) + + # This is currently needed in order to get the size + # of the output before adding it to the stream. + # Maybe there is a better way to do this let objStream = OutputStream.init() objStream.put(value) + let objOutput = objStream.getOutput() stream.put(len(objOutput).uint) stream.put(objOutput) @@ -123,11 +122,17 @@ proc encodeField*(protobuf: var ProtoBuffer, value: object) {.inline.} = protobuf.outstream.encodeField(protobuf.fieldNum, value) inc protobuf.fieldNum +proc encode*(protobuf: var ProtoBuffer, value: object) {.inline.} = + var fieldNum = 1 + for field, val in value.fieldPairs: + protobuf.outstream.encodeField(fieldNum, val) + inc fieldNum + proc put(stream: OutputStreamVar, value: object) {.inline.} = var fieldNum = 1 for field, val in value.fieldPairs: stream.encodeField(fieldNum, val) - fieldNum += 1 + inc fieldNum proc getVarint[T: SomeVarint]( bytes: var seq[byte], @@ -320,7 +325,7 @@ proc decodeField*[T: object]( let bytesToRead = some(decodedSize.int) for field, val in result.value.fieldPairs: setField(result.value, index, outOffset, outBytesProcessed, bytesToRead, bytes) - index += 1 + inc index proc decode*[T: object]( bytes: var seq[byte], @@ -332,4 +337,4 @@ proc decode*[T: object]( var fieldNum = 1 for field, val in result.fieldPairs: setField(result, fieldNum, offset, bytesRead, none(int), bytes) - fieldNum += 1 \ No newline at end of file + inc fieldNum \ No newline at end of file diff --git a/tests/test_serialization.nim b/tests/test_serialization.nim index 86c97f9..b6b2051 100644 --- a/tests/test_serialization.nim +++ b/tests/test_serialization.nim @@ -15,7 +15,7 @@ type i: Test1 suite "Test Varint Encoding": - test "Can encode/decode enum": + test "Can encode/decode enum field": var proto = newProtoBuffer() var bytesProcessed: int @@ -35,7 +35,7 @@ suite "Test Varint Encoding": assert decodedME2.value == ME2 assert decodedME2.index == 2 - test "Can encode/decode negative number": + test "Can encode/decode negative number field": var proto = newProtoBuffer() let num = -153452 var bytesProcessed: int @@ -50,7 +50,7 @@ suite "Test Varint Encoding": assert decoded.value == num assert decoded.index == 1 - test "Can encode/decode unsigned number": + test "Can encode/decode unsigned number field": var proto = newProtoBuffer() let num = 123151.uint var bytesProcessed: int @@ -65,7 +65,7 @@ suite "Test Varint Encoding": assert decoded.value == num assert decoded.index == 1 - test "Can encode/decode string": + test "Can encode/decode string field": var proto = newProtoBuffer() let str = "hey this is a string" var bytesProcessed: int @@ -80,7 +80,7 @@ suite "Test Varint Encoding": assert decoded.value == str assert decoded.index == 1 - test "Can encode/decode object": + test "Can encode/decode object field": var proto = newProtoBuffer() let obj = Test3(g: 300, h: 200, i: Test1(a: 100)) @@ -91,4 +91,14 @@ suite "Test Varint Encoding": var output = proto.output let decoded = decodeField(output, Test3, offset, bytesProcessed) assert decoded.value == obj - assert decoded.index == 1 \ No newline at end of file + assert decoded.index == 1 + + test "Can encode/decode object": + var proto = newProtoBuffer() + + let obj = Test3(g: 300, h: 200, i: Test1(a: 100)) + + proto.encode(obj) + var output = proto.output + let decoded = output.decode(Test3) + assert decoded == obj \ No newline at end of file From 307cb7f9f31e86ba44e1b66cb0080e0a9034030d Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 3 Apr 2020 19:23:08 -0600 Subject: [PATCH 09/17] Fix bug with size detection. Add string to test objects --- protobuf_serialization.nim | 2 +- tests/test_serialization.nim | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index 2b9fbdd..2bfd54c 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -173,7 +173,7 @@ proc decodeField*[T: SomeVarint]( numBytesToRead = none(int) ): ProtoField[T] {.inline.} = # Only up to 128 bits supported by the spec - assert (bytes.len - 1) <= 16 + assert sizeof(T) <= 16 var bytesRead = 0 diff --git a/tests/test_serialization.nim b/tests/test_serialization.nim index b6b2051..16278d4 100644 --- a/tests/test_serialization.nim +++ b/tests/test_serialization.nim @@ -8,11 +8,13 @@ type type Test1 = object a: uint + b: string Test3 = object g {.sfixed32.}: int h: int i: Test1 + j: string suite "Test Varint Encoding": test "Can encode/decode enum field": @@ -83,7 +85,7 @@ suite "Test Varint Encoding": test "Can encode/decode object field": var proto = newProtoBuffer() - let obj = Test3(g: 300, h: 200, i: Test1(a: 100)) + let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test"), j: "testing") proto.encodeField(obj) var offset, bytesProcessed: int @@ -96,7 +98,7 @@ suite "Test Varint Encoding": test "Can encode/decode object": var proto = newProtoBuffer() - let obj = Test3(g: 300, h: 200, i: Test1(a: 100)) + let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test"), j: "testing") proto.encode(obj) var output = proto.output From ccb729d219b16ec02ee72146619bc9070b3c6ecf Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 3 Apr 2020 21:16:18 -0600 Subject: [PATCH 10/17] Fix bug with out of order deserialization --- protobuf_serialization.nim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index 2bfd54c..ff3ff9a 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -334,7 +334,6 @@ proc decode*[T: object]( var bytesRead = 0 var offset = 0 - var fieldNum = 1 - for field, val in result.fieldPairs: - setField(result, fieldNum, offset, bytesRead, none(int), bytes) - inc fieldNum \ No newline at end of file + while bytesRead < bytes.len: + let fieldNum = fieldNumber(bytes[offset]) + setField(result, fieldNum, offset, bytesRead, none(int), bytes) \ No newline at end of file From e1905e13cef567d1ddbe3fd407077dced2aa6b42 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 3 Apr 2020 22:18:21 -0600 Subject: [PATCH 11/17] Actually fix out of order decoding --- protobuf_serialization.nim | 32 +++++++++++++++++++++++--------- tests/test_serialization.nim | 13 +++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index ff3ff9a..32489f3 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -90,6 +90,9 @@ proc encodeField*(protobuf: var ProtoBuffer, value: SomeVarint) {.inline.} = protobuf.outstream.encodeField(protobuf.fieldNum, value) inc protobuf.fieldNum +proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: SomeVarint) {.inline.} = + protobuf.outstream.encodeField(fieldNum, value) + proc put(stream: OutputStreamVar, value: SomeLengthDelimited) {.inline.} = for b in value: stream.append byte(b) @@ -103,6 +106,9 @@ proc encodeField*(protobuf: var ProtoBuffer, value: SomeLengthDelimited) {.inlin protobuf.outstream.encodeField(protobuf.fieldNum, value) inc protobuf.fieldNum +proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: SomeLengthDelimited) {.inline.} = + protobuf.outstream.encodeField(fieldNum, value) + proc put(stream: OutputStreamVar, value: object) {.inline.} proc encodeField(stream: OutputStreamVar, fieldNum: int, value: object) {.inline.} = @@ -122,17 +128,24 @@ proc encodeField*(protobuf: var ProtoBuffer, value: object) {.inline.} = protobuf.outstream.encodeField(protobuf.fieldNum, value) inc protobuf.fieldNum +proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: object) {.inline.} = + protobuf.outstream.encodeField(fieldNum, value) + proc encode*(protobuf: var ProtoBuffer, value: object) {.inline.} = var fieldNum = 1 - for field, val in value.fieldPairs: - protobuf.outstream.encodeField(fieldNum, val) + for _, val in value.fieldPairs: + # Only store the value + if default(type(val)) != val: + protobuf.outstream.encodeField(fieldNum, val) inc fieldNum proc put(stream: OutputStreamVar, value: object) {.inline.} = var fieldNum = 1 - for field, val in value.fieldPairs: + for _, val in value.fieldPairs: + # Only store the value + if default(type(val)) != val: stream.encodeField(fieldNum, val) - inc fieldNum + inc fieldNum proc getVarint[T: SomeVarint]( bytes: var seq[byte], @@ -320,12 +333,13 @@ proc decodeField*[T: object]( # read LD header # then read only amount of bytes needed increaseBytesRead() - var index = 1 let decodedSize = getVarint(bytes, uint, outOffset, outBytesProcessed, numBytesToRead) let bytesToRead = some(decodedSize.int) - for field, val in result.value.fieldPairs: - setField(result.value, index, outOffset, outBytesProcessed, bytesToRead, bytes) - inc index + + let oldOffset = outOffset + while outOffset < oldOffset + bytesToRead.get(): + let fieldNum = fieldNumber(bytes[outOffset]) + setField(result.value, fieldNum, outOffset, outBytesProcessed, bytesToRead, bytes) proc decode*[T: object]( bytes: var seq[byte], @@ -334,6 +348,6 @@ proc decode*[T: object]( var bytesRead = 0 var offset = 0 - while bytesRead < bytes.len: + while offset < bytes.len - 1: let fieldNum = fieldNumber(bytes[offset]) setField(result, fieldNum, offset, bytesRead, none(int), bytes) \ No newline at end of file diff --git a/tests/test_serialization.nim b/tests/test_serialization.nim index 16278d4..0247586 100644 --- a/tests/test_serialization.nim +++ b/tests/test_serialization.nim @@ -101,6 +101,19 @@ suite "Test Varint Encoding": let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test"), j: "testing") proto.encode(obj) + var output = proto.output + let decoded = output.decode(Test3) + assert decoded == obj + + test "Can encode/decode out of order object": + var proto = newProtoBuffer() + + let obj = Test3(g: 400, h: 100, i: Test1(a: 100, b: "this is a test"), j: "testing") + proto.encodeField(2, 100) + proto.encodeField(4, "testing") + proto.encodeField(1, 400) + proto.encodeField(3, Test1(a: 100, b: "this is a test")) + var output = proto.output let decoded = output.decode(Test3) assert decoded == obj \ No newline at end of file From d81f0f0cbe4e6146d69e013a6e5d648f7234b734 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 4 Apr 2020 07:38:14 -0600 Subject: [PATCH 12/17] Empty objects should not be encoded --- protobuf_serialization.nim | 8 ++++---- tests/test_serialization.nim | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index 32489f3..bfe06cb 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -112,8 +112,6 @@ proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: SomeLengthDel proc put(stream: OutputStreamVar, value: object) {.inline.} proc encodeField(stream: OutputStreamVar, fieldNum: int, value: object) {.inline.} = - stream.append protoHeader(fieldNum, LengthDelimited) - # This is currently needed in order to get the size # of the output before adding it to the stream. # Maybe there is a better way to do this @@ -121,8 +119,10 @@ proc encodeField(stream: OutputStreamVar, fieldNum: int, value: object) {.inline objStream.put(value) let objOutput = objStream.getOutput() - stream.put(len(objOutput).uint) - stream.put(objOutput) + if objOutput.len > 0: + stream.append protoHeader(fieldNum, LengthDelimited) + stream.put(len(objOutput).uint) + stream.put(objOutput) proc encodeField*(protobuf: var ProtoBuffer, value: object) {.inline.} = protobuf.outstream.encodeField(protobuf.fieldNum, value) diff --git a/tests/test_serialization.nim b/tests/test_serialization.nim index 0247586..87a4fbe 100644 --- a/tests/test_serialization.nim +++ b/tests/test_serialization.nim @@ -116,4 +116,29 @@ suite "Test Varint Encoding": var output = proto.output let decoded = output.decode(Test3) + + assert decoded == obj + + test "Empty object field does not get encoded": + var proto = newProtoBuffer() + + let obj = Test1() + proto.encodeField(1, obj) + + var output = proto.output + assert output.len == 0 + + let decoded = output.decode(Test1) + assert decoded == obj + + test "Empty object does not get encoded": + var proto = newProtoBuffer() + + let obj = Test1() + proto.encode(obj) + + var output = proto.output + assert output.len == 0 + + let decoded = output.decode(Test1) assert decoded == obj \ No newline at end of file From d750cac498e31040e337a27016745e2e08536ada Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 4 Apr 2020 08:03:00 -0600 Subject: [PATCH 13/17] Fix support for char, bool, byte --- protobuf_serialization.nim | 11 ++++-- tests/test_serialization.nim | 74 ++++++++++++++++++++++++++++++++++-- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index bfe06cb..31a8340 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -31,9 +31,10 @@ type value*: T SomeSVarint* = int | int64 | int32 | int16 | int8 | enum - SomeUVarint* = uint | uint64 | uint32 | uint16 | uint8 | byte | bool | char + SomeByte* = byte | bool | char | uint8 + SomeUVarint* = uint | uint64 | uint32 | uint16 | SomeByte SomeVarint* = SomeSVarint | SomeUVarint - SomeLengthDelimited* = string | seq[byte] | seq[uint8] | cstring + SomeLengthDelimited* = string | seq[SomeByte] | cstring proc newProtoBuffer*(): ProtoBuffer = ProtoBuffer(outstream: OutputStream.init(), fieldNum: 1) @@ -156,8 +157,10 @@ proc getVarint[T: SomeVarint]( ): T {.inline.} = var bytesRead = 0 # Only up to 128 bits supported by the spec - when T is enum: + when T is enum or T is char: var value: type(ord(result)) + elif T is bool: + var value: byte else: var value: T var shiftAmount = 0 @@ -176,7 +179,7 @@ proc getVarint[T: SomeVarint]( else: result = cast[T](value shr type(value)(1)) else: - result = value + result = T(value) proc decodeField*[T: SomeVarint]( bytes: var seq[byte], diff --git a/tests/test_serialization.nim b/tests/test_serialization.nim index 87a4fbe..69f3255 100644 --- a/tests/test_serialization.nim +++ b/tests/test_serialization.nim @@ -1,4 +1,5 @@ import unittest +import sequtils import protobuf_serialization @@ -9,12 +10,14 @@ type Test1 = object a: uint b: string + c: char Test3 = object g {.sfixed32.}: int h: int i: Test1 j: string + k: bool suite "Test Varint Encoding": test "Can encode/decode enum field": @@ -52,6 +55,38 @@ suite "Test Varint Encoding": assert decoded.value == num assert decoded.index == 1 + test "Can encode/decode bool field": + var proto = newProtoBuffer() + let boolean = true + var bytesProcessed: int + + proto.encodeField(boolean) + + var output = proto.output + assert output == @[8.byte, 1] + + var offset = 0 + let decoded = decodeField(output, bool, offset, bytesProcessed) + assert bytesProcessed == 2 + assert decoded.value == boolean + assert decoded.index == 1 + + test "Can encode/decode char field": + var proto = newProtoBuffer() + let charVal = 'G' + var bytesProcessed: int + + proto.encodeField(charVal) + + var output = proto.output + assert output == @[8.byte, ord(charVal).byte] + + var offset = 0 + let decoded = decodeField(output, char, offset, bytesProcessed) + assert bytesProcessed == 2 + assert decoded.value == charVal + assert decoded.index == 1 + test "Can encode/decode unsigned number field": var proto = newProtoBuffer() let num = 123151.uint @@ -82,10 +117,40 @@ suite "Test Varint Encoding": assert decoded.value == str assert decoded.index == 1 + test "Can encode/decode char seq field": + var proto = newProtoBuffer() + let charSeq = "hey this is a string".toSeq + var bytesProcessed: int + + proto.encodeField(charSeq) + + var output = proto.output + assert output == @[10.byte, 20, 104, 101, 121, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 115, 116, 114, 105, 110, 103] + + var offset = 0 + let decoded = decodeField(output, seq[char], offset, bytesProcessed) + assert decoded.value == charSeq + assert decoded.index == 1 + + test "Can encode/decode uint8 seq field": + var proto = newProtoBuffer() + let uint8Seq = cast[seq[uint8]]("hey this is a string".toSeq) + var bytesProcessed: int + + proto.encodeField(uint8Seq) + + var output = proto.output + assert output == @[10.byte, 20, 104, 101, 121, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 115, 116, 114, 105, 110, 103] + + var offset = 0 + let decoded = decodeField(output, seq[uint8], offset, bytesProcessed) + assert decoded.value == uint8Seq + assert decoded.index == 1 + test "Can encode/decode object field": var proto = newProtoBuffer() - let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test"), j: "testing") + let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test", c: 'H'), j: "testing", k: true) proto.encodeField(obj) var offset, bytesProcessed: int @@ -98,7 +163,7 @@ suite "Test Varint Encoding": test "Can encode/decode object": var proto = newProtoBuffer() - let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test"), j: "testing") + let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test", c: 'H'), j: "testing", k: true) proto.encode(obj) var output = proto.output @@ -108,11 +173,12 @@ suite "Test Varint Encoding": test "Can encode/decode out of order object": var proto = newProtoBuffer() - let obj = Test3(g: 400, h: 100, i: Test1(a: 100, b: "this is a test"), j: "testing") + let obj = Test3(g: 400, h: 100, i: Test1(a: 100, b: "this is a test", c: 'H'), j: "testing", k: true) proto.encodeField(2, 100) proto.encodeField(4, "testing") proto.encodeField(1, 400) - proto.encodeField(3, Test1(a: 100, b: "this is a test")) + proto.encodeField(3, Test1(a: 100, b: "this is a test", c: 'H')) + proto.encodeField(5, true) var output = proto.output let decoded = output.decode(Test3) From f26b941bd7a6d8ca02cd99ba75da5911494484be Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 5 Apr 2020 14:46:50 -0600 Subject: [PATCH 14/17] Add support for float32 and float64 --- protobuf_serialization.nim | 107 ++++++++++++++++++++++------------- tests/test_serialization.nim | 30 ++++++++++ 2 files changed, 99 insertions(+), 38 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index 31a8340..53590fe 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -35,6 +35,11 @@ type SomeUVarint* = uint | uint64 | uint32 | uint16 | SomeByte SomeVarint* = SomeSVarint | SomeUVarint SomeLengthDelimited* = string | seq[SomeByte] | cstring + SomeFixed64* = float64 + SomeFixed32* = float32 + SomeFixed* = SomeFixed32 | SomeFixed64 + + AnyProtoType* = SomeVarint | SomeLengthDelimited | SomeFixed | object proc newProtoBuffer*(): ProtoBuffer = ProtoBuffer(outstream: OutputStream.init(), fieldNum: 1) @@ -87,12 +92,23 @@ proc encodeField(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.in stream.append protoHeader(fieldNum, Varint) stream.put(value) -proc encodeField*(protobuf: var ProtoBuffer, value: SomeVarint) {.inline.} = - protobuf.outstream.encodeField(protobuf.fieldNum, value) - inc protobuf.fieldNum +proc put(stream: OutputStreamVar, value: SomeFixed) {.inline.} = + when typeof(value) is SomeFixed64: + var value = cast[int64](value) + else: + var value = cast[int32](value) -proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: SomeVarint) {.inline.} = - protobuf.outstream.encodeField(fieldNum, value) + for _ in 0 ..< sizeof(value): + stream.append byte(value and 0b1111_1111) + value = value shr 8 + +proc encodeField(stream: OutputStreamVar, fieldNum: int, value: SomeFixed64) {.inline.} = + stream.append protoHeader(fieldNum, Fixed64) + stream.put(value) + +proc encodeField(stream: OutputStreamVar, fieldNum: int, value: SomeFixed32) {.inline.} = + stream.append protoHeader(fieldNum, Fixed32) + stream.put(value) proc put(stream: OutputStreamVar, value: SomeLengthDelimited) {.inline.} = for b in value: @@ -103,13 +119,6 @@ proc encodeField(stream: OutputStreamVar, fieldNum: int, value: SomeLengthDelimi stream.put(len(value).uint) stream.put(value) -proc encodeField*(protobuf: var ProtoBuffer, value: SomeLengthDelimited) {.inline.} = - protobuf.outstream.encodeField(protobuf.fieldNum, value) - inc protobuf.fieldNum - -proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: SomeLengthDelimited) {.inline.} = - protobuf.outstream.encodeField(fieldNum, value) - proc put(stream: OutputStreamVar, value: object) {.inline.} proc encodeField(stream: OutputStreamVar, fieldNum: int, value: object) {.inline.} = @@ -125,21 +134,6 @@ proc encodeField(stream: OutputStreamVar, fieldNum: int, value: object) {.inline stream.put(len(objOutput).uint) stream.put(objOutput) -proc encodeField*(protobuf: var ProtoBuffer, value: object) {.inline.} = - protobuf.outstream.encodeField(protobuf.fieldNum, value) - inc protobuf.fieldNum - -proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: object) {.inline.} = - protobuf.outstream.encodeField(fieldNum, value) - -proc encode*(protobuf: var ProtoBuffer, value: object) {.inline.} = - var fieldNum = 1 - for _, val in value.fieldPairs: - # Only store the value - if default(type(val)) != val: - protobuf.outstream.encodeField(fieldNum, val) - inc fieldNum - proc put(stream: OutputStreamVar, value: object) {.inline.} = var fieldNum = 1 for _, val in value.fieldPairs: @@ -148,6 +142,37 @@ proc put(stream: OutputStreamVar, value: object) {.inline.} = stream.encodeField(fieldNum, val) inc fieldNum +proc encode*(protobuf: var ProtoBuffer, value: object) {.inline.} = + protobuf.outstream.put(value) + +proc encodeField*(protobuf: var ProtoBuffer, value: AnyProtoType) {.inline.} = + protobuf.outstream.encodeField(protobuf.fieldNum, value) + inc protobuf.fieldNum + +proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: AnyProtoType) {.inline.} = + protobuf.outstream.encodeField(fieldNum, value) + +proc getFixed*[T: SomeFixed]( + bytes: var seq[byte], + ty: typedesc[T], + outOffset: var int, + outBytesProcessed: var int, + numBytesToRead = none(int) +): T {.inline.} = + var bytesRead = 0 + when T is SomeFixed64: + var value: int64 + else: + var value: int32 + var shiftAmount = 0 + + for _ in 0 ..< sizeof(T): + value += type(value)(bytes[outOffset]) shl shiftAmount + shiftAmount += 8 + increaseBytesRead() + + result = cast[T](value) + proc getVarint[T: SomeVarint]( bytes: var seq[byte], ty: typedesc[T], @@ -202,6 +227,23 @@ proc decodeField*[T: SomeVarint]( result.value = getVarint(bytes, ty, outOffset, outBytesProcessed, numBytesToRead) +proc decodeField*[T: SomeFixed]( + bytes: var seq[byte], + ty: typedesc[T], + outOffset: var int, + outBytesProcessed: var int, + numBytesToRead = none(int) +): ProtoField[T] {.inline.} = + var bytesRead = 0 + + let wireTy = wireType(bytes[outOffset]) + if wireTy notin {Fixed32, Fixed64}: + raise newException(Exception, fmt"Not a fixed32 or fixed64 at offset {outOffset}! Received a {wireTy}") + + result.index = fieldNumber(bytes[outOffset]) + increaseBytesRead() + + result.value = getFixed(bytes, ty, outOffset, outBytesProcessed, numBytesToRead) proc getLengthDelimited*[T: SomeLengthDelimited]( bytes: var seq[byte], @@ -243,15 +285,6 @@ proc decodeField*[T: SomeLengthDelimited]( result.value = getLengthDelimited(bytes, ty, outOffset, outBytesProcessed, numBytesToRead) -type - Test1 = object - a: uint - - Test3 = object - g {.sfixed32.}: int - h: int - i: Test1 - macro getField(obj: typed, fieldNum: int, ty: typedesc): untyped = template fieldTypeCheck(obj, field, fieldNum, ty) = when type(obj.field) is type(ty): @@ -260,7 +293,6 @@ macro getField(obj: typed, fieldNum: int, ty: typedesc): untyped = let fnum {.inject.} = fieldNum raise newException(Exception, fmt"Could not find field at position {fnum}.") - let typeImpl = obj.getTypeInst.getImpl let typeFields = obj.getTypeInst.getType let objFields = typeFields[2] @@ -287,7 +319,6 @@ macro getField(obj: typed, fieldNum: int, ty: typedesc): untyped = result.add(caseStmt) macro setField(obj: typed, fieldNum: int, offset: int, bytesProcessed: int, bytesToRead: Option[int], value: untyped): untyped = - let typeImpl = obj.getTypeInst.getImpl let typeFields = obj.getTypeInst.getType let objFields = typeFields[2] diff --git a/tests/test_serialization.nim b/tests/test_serialization.nim index 69f3255..9b81f5b 100644 --- a/tests/test_serialization.nim +++ b/tests/test_serialization.nim @@ -55,6 +55,36 @@ suite "Test Varint Encoding": assert decoded.value == num assert decoded.index == 1 + test "Can encode/decode float32 number field": + var proto = newProtoBuffer() + let num = float32(1234.164423) + var bytesProcessed: int + + proto.encodeField(num) + + var output = proto.output + assert output == @[13.byte, 67, 69, 154, 68] + + var offset = 0 + let decoded = decodeField(output, float32, offset, bytesProcessed) + assert decoded.value == num + assert decoded.index == 1 + + test "Can encode/decode float64 number field": + var proto = newProtoBuffer() + let num = 12343121537452.1644232341'f64 + var bytesProcessed: int + + proto.encodeField(num) + + var output = proto.output + assert output == @[9.byte, 84, 88, 211, 191, 182, 115, 166, 66] + + var offset = 0 + let decoded = decodeField(output, float64, offset, bytesProcessed) + assert decoded.value == num + assert decoded.index == 1 + test "Can encode/decode bool field": var proto = newProtoBuffer() let boolean = true From 5b49b86b236c50b03379df3aff69eb4d891905ae Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 5 Apr 2020 21:36:01 -0600 Subject: [PATCH 15/17] Refactor decodeField and get* --- protobuf_serialization.nim | 117 +++++++++++-------------------------- 1 file changed, 35 insertions(+), 82 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index 53590fe..e4c4293 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -41,6 +41,8 @@ type AnyProtoType* = SomeVarint | SomeLengthDelimited | SomeFixed | object + UnexpectedTypeError* = object of ValueError + proc newProtoBuffer*(): ProtoBuffer = ProtoBuffer(outstream: OutputStream.init(), fieldNum: 1) @@ -111,12 +113,12 @@ proc encodeField(stream: OutputStreamVar, fieldNum: int, value: SomeFixed32) {.i stream.put(value) proc put(stream: OutputStreamVar, value: SomeLengthDelimited) {.inline.} = + stream.put(len(value).uint) for b in value: stream.append byte(b) proc encodeField(stream: OutputStreamVar, fieldNum: int, value: SomeLengthDelimited) {.inline.} = stream.append protoHeader(fieldNum, LengthDelimited) - stream.put(len(value).uint) stream.put(value) proc put(stream: OutputStreamVar, value: object) {.inline.} @@ -131,7 +133,6 @@ proc encodeField(stream: OutputStreamVar, fieldNum: int, value: object) {.inline let objOutput = objStream.getOutput() if objOutput.len > 0: stream.append protoHeader(fieldNum, LengthDelimited) - stream.put(len(objOutput).uint) stream.put(objOutput) proc put(stream: OutputStreamVar, value: object) {.inline.} = @@ -145,14 +146,14 @@ proc put(stream: OutputStreamVar, value: object) {.inline.} = proc encode*(protobuf: var ProtoBuffer, value: object) {.inline.} = protobuf.outstream.put(value) -proc encodeField*(protobuf: var ProtoBuffer, value: AnyProtoType) {.inline.} = - protobuf.outstream.encodeField(protobuf.fieldNum, value) - inc protobuf.fieldNum - proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: AnyProtoType) {.inline.} = protobuf.outstream.encodeField(fieldNum, value) -proc getFixed*[T: SomeFixed]( +proc encodeField*(protobuf: var ProtoBuffer, value: AnyProtoType) {.inline.} = + protobuf.encodeField(protobuf.fieldNum, value) + inc protobuf.fieldNum + +proc get*[T: SomeFixed]( bytes: var seq[byte], ty: typedesc[T], outOffset: var int, @@ -173,7 +174,7 @@ proc getFixed*[T: SomeFixed]( result = cast[T](value) -proc getVarint[T: SomeVarint]( +proc get[T: SomeVarint]( bytes: var seq[byte], ty: typedesc[T], outOffset: var int, @@ -188,6 +189,7 @@ proc getVarint[T: SomeVarint]( var value: byte else: var value: T + var shiftAmount = 0 while true: value += type(value)(bytes[outOffset] and 0b0111_1111) shl shiftAmount @@ -206,53 +208,30 @@ proc getVarint[T: SomeVarint]( else: result = T(value) -proc decodeField*[T: SomeVarint]( - bytes: var seq[byte], - ty: typedesc[T], - outOffset: var int, - outBytesProcessed: var int, - numBytesToRead = none(int) -): ProtoField[T] {.inline.} = - # Only up to 128 bits supported by the spec - assert sizeof(T) <= 16 - - var bytesRead = 0 - - let wireTy = wireType(bytes[outOffset]) +proc checkType[T: SomeVarint](tyByte: byte, ty: typedesc[T], offset: int) {.inline.} = + let wireTy = wireType(tyByte) if wireTy != Varint: - raise newException(Exception, fmt"Not a varint at offset {outOffset}! Received a {wireTy}") + raise newException(UnexpectedTypeError, fmt"Not a varint at offset {offset}! Received a {wireTy}") - result.index = fieldNumber(bytes[outOffset]) - increaseBytesRead() +proc checkType[T: SomeFixed](tyByte: byte, ty: typedesc[T], offset: int) {.inline.} = + let wireTy = wireType(tyByte) + if wireTy notin {Fixed32, Fixed64}: + raise newException(UnexpectedTypeError, fmt"Not a fixed32 or fixed64 at offset {offset}! Received a {wireTy}") - result.value = getVarint(bytes, ty, outOffset, outBytesProcessed, numBytesToRead) +proc checkType[T: SomeLengthDelimited](tyByte: byte, ty: typedesc[T], offset: int) {.inline.} = + let wireTy = wireType(tyByte) + if wireTy != LengthDelimited: + raise newException(UnexpectedTypeError, fmt"Not a length delimited value at offset {offset}! Received a {wireTy}") -proc decodeField*[T: SomeFixed]( +proc get*[T: SomeLengthDelimited]( bytes: var seq[byte], ty: typedesc[T], outOffset: var int, outBytesProcessed: var int, numBytesToRead = none(int) -): ProtoField[T] {.inline.} = - var bytesRead = 0 - - let wireTy = wireType(bytes[outOffset]) - if wireTy notin {Fixed32, Fixed64}: - raise newException(Exception, fmt"Not a fixed32 or fixed64 at offset {outOffset}! Received a {wireTy}") - - result.index = fieldNumber(bytes[outOffset]) - increaseBytesRead() - - result.value = getFixed(bytes, ty, outOffset, outBytesProcessed, numBytesToRead) - -proc getLengthDelimited*[T: SomeLengthDelimited]( - bytes: var seq[byte], - ty: typedesc[T], outOffset: var int, - outBytesProcessed: var int, - numBytesToRead = none(int) ): T {.inline.} = var bytesRead = 0 - let decodedSize = getVarint(bytes, uint, outOffset, outBytesProcessed, numBytesToRead) + let decodedSize = bytes.get(uint, outOffset, outBytesProcessed, numBytesToRead) let length = decodedSize.int when T is string: @@ -268,7 +247,7 @@ proc getLengthDelimited*[T: SomeLengthDelimited]( increaseBytesRead(length) -proc decodeField*[T: SomeLengthDelimited]( +proc decodeField*[T: SomeFixed | SomeVarint | SomeLengthDelimited]( bytes: var seq[byte], ty: typedesc[T], outOffset: var int, @@ -276,47 +255,21 @@ proc decodeField*[T: SomeLengthDelimited]( numBytesToRead = none(int) ): ProtoField[T] {.inline.} = var bytesRead = 0 - let wireTy = wireType(bytes[outOffset]) - if wireTy != LengthDelimited: - raise newException(Exception, fmt"Not a length delimited value at offset {outOffset}! Received a {wireTy}") + + checkType(bytes[outOffset], ty, outOffset) result.index = fieldNumber(bytes[outOffset]) increaseBytesRead() - result.value = getLengthDelimited(bytes, ty, outOffset, outBytesProcessed, numBytesToRead) + result.value = bytes.get(ty, outOffset, outBytesProcessed, numBytesToRead) -macro getField(obj: typed, fieldNum: int, ty: typedesc): untyped = - template fieldTypeCheck(obj, field, fieldNum, ty) = - when type(obj.field) is type(ty): - obj.field - else: - let fnum {.inject.} = fieldNum - raise newException(Exception, fmt"Could not find field at position {fnum}.") - - let typeFields = obj.getTypeInst.getType - - let objFields = typeFields[2] - expectKind objFields, nnkRecList - - result = newStmtList() - let caseStmt = newNimNode(nnkCaseStmt) - caseStmt.add(fieldNum) - - for i in 0 ..< len(objFields) - 1: - let field = objFields[i] - let ofBranch = newNimNode(nnkOfBranch) - ofBranch.add(newLit(i+1)) - ofBranch.add(getAst(fieldTypeCheck(obj, field, fieldNum, ty))) - caseStmt.add(ofBranch) - - let field = objFields[len(objFields) - 1] - let elseBranch = newNimNode(nnkElse) - elseBranch.add( - nnkStmtList.newTree(getAst(fieldTypeCheck(obj, field, fieldNum, ty))) - ) - caseStmt.add(elseBranch) - - result.add(caseStmt) +proc decodeField*[T: object]( + bytes: var seq[byte], + ty: typedesc[T], + outOffset: var int, + outBytesProcessed: var int, + numBytesToRead = none(int) +): ProtoField[T] {.inline.} macro setField(obj: typed, fieldNum: int, offset: int, bytesProcessed: int, bytesToRead: Option[int], value: untyped): untyped = let typeFields = obj.getTypeInst.getType @@ -367,7 +320,7 @@ proc decodeField*[T: object]( # read LD header # then read only amount of bytes needed increaseBytesRead() - let decodedSize = getVarint(bytes, uint, outOffset, outBytesProcessed, numBytesToRead) + let decodedSize = bytes.get(uint, outOffset, outBytesProcessed, numBytesToRead) let bytesToRead = some(decodedSize.int) let oldOffset = outOffset From 86c8143567254adf964d8f44929d885c79325d83 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 5 Apr 2020 21:39:21 -0600 Subject: [PATCH 16/17] Add checkType for object --- protobuf_serialization.nim | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index e4c4293..c9d3b32 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -223,6 +223,11 @@ proc checkType[T: SomeLengthDelimited](tyByte: byte, ty: typedesc[T], offset: in if wireTy != LengthDelimited: raise newException(UnexpectedTypeError, fmt"Not a length delimited value at offset {offset}! Received a {wireTy}") +proc checkType[T: object](tyByte: byte, ty: typedesc[T], offset: int) {.inline.} = + let wireTy = wireType(tyByte) + if wireTy != LengthDelimited: + raise newException(UnexpectedTypeError, fmt"Not an object value at offset {offset}! Received a {wireTy}") + proc get*[T: SomeLengthDelimited]( bytes: var seq[byte], ty: typedesc[T], @@ -312,8 +317,7 @@ proc decodeField*[T: object]( ): ProtoField[T] {.inline.} = var bytesRead = 0 - let wireTy = wireType(bytes[outOffset]) - assert wireTy == LengthDelimited + checkType(bytes[outOffset], ty, outOffset) result.index = fieldNumber(bytes[outOffset]) From fbdce1712d9f0e6283c420f26447608503a19532 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 5 Apr 2020 22:27:40 -0600 Subject: [PATCH 17/17] Add support for encoding/decoding custom types --- protobuf_serialization.nim | 32 ++++++++++++++++++++++++ tests/test_serialization.nim | 48 +++++++++++++++++++++++++++++++++--- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index c9d3b32..ba3110a 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -70,6 +70,10 @@ template increaseBytesRead(amount = 1) = if (bytesRead > numBytesToRead.get()).unlikely: raise newException(Exception, &"Number of bytes read ({bytesRead}) exceeded bytes requested ({numBytesToRead})") +proc encodeField*[T: not AnyProtoType](protobuf: var ProtoBuffer, value: T) {.inline.} +proc encodeField*[T: not AnyProtoType](protobuf: var ProtoBuffer, fieldNum: int, value: T) {.inline.} +proc encodeField[T: not AnyProtoType](stream: OutputStreamVar, fieldNum: int, value: T) {.inline.} + proc put(stream: OutputStreamVar, value: SomeVarint) {.inline.} = when value is enum: var value = cast[type(ord(value))](value) @@ -153,6 +157,16 @@ proc encodeField*(protobuf: var ProtoBuffer, value: AnyProtoType) {.inline.} = protobuf.encodeField(protobuf.fieldNum, value) inc protobuf.fieldNum +proc encodeField[T: not AnyProtoType](stream: OutputStreamVar, fieldNum: int, value: T) {.inline.} = + stream.encodeField(fieldNum, value.toBytes) + +proc encodeField*[T: not AnyProtoType](protobuf: var ProtoBuffer, fieldNum: int, value: T) {.inline.} = + protobuf.outstream.encodeField(fieldNum, value.toBytes) + +proc encodeField*[T: not AnyProtoType](protobuf: var ProtoBuffer, value: T) {.inline.} = + protobuf.encodeField(protobuf.fieldNum, value.toBytes) + inc protobuf.fieldNum + proc get*[T: SomeFixed]( bytes: var seq[byte], ty: typedesc[T], @@ -276,6 +290,24 @@ proc decodeField*[T: object]( numBytesToRead = none(int) ): ProtoField[T] {.inline.} +proc decodeField*[T: not AnyProtoType]( + bytes: var seq[byte], + ty: typedesc[T], + outOffset: var int, + outBytesProcessed: var int, + numBytesToRead = none(int) +): ProtoField[T] {.inline.} = + + var bytesRead = 0 + + checkType(bytes[outOffset], seq[byte], outOffset) + + result.index = fieldNumber(bytes[outOffset]) + increaseBytesRead() + + var value = bytes.get(seq[byte], outOffset, outBytesProcessed, numBytesToRead) + result.value = value.to(T) + macro setField(obj: typed, fieldNum: int, offset: int, bytesProcessed: int, bytesToRead: Option[int], value: untyped): untyped = let typeFields = obj.getTypeInst.getType diff --git a/tests/test_serialization.nim b/tests/test_serialization.nim index 9b81f5b..e3aa176 100644 --- a/tests/test_serialization.nim +++ b/tests/test_serialization.nim @@ -6,7 +6,7 @@ import protobuf_serialization type MyEnum = enum ME1, ME2, ME3 -type + Test1 = object a: uint b: string @@ -18,6 +18,30 @@ type i: Test1 j: string k: bool + l: MyInt + + MyInt = distinct int + +proc to*(bytes: var seq[byte], ty: typedesc[MyInt]): MyInt = + + var value: int + + var shiftAmount = 0 + + for i in 0 ..< len(bytes): + value += int(bytes[i]) shl shiftAmount + shiftAmount += 8 + + result = MyInt(value) + +proc toBytes*(value: MyInt): seq[byte] = + var value = value.int + + while value > 0: + result.add byte(value and 0b1111_1111) + value = value shr 8 + +proc `==`(a, b: MyInt): bool {.borrow.} suite "Test Varint Encoding": test "Can encode/decode enum field": @@ -55,6 +79,21 @@ suite "Test Varint Encoding": assert decoded.value == num assert decoded.index == 1 + test "Can encode/decode distinct number field": + var proto = newProtoBuffer() + let num = 114151.MyInt + var bytesProcessed: int + + proto.encodeField(num) + + var output = proto.output + assert output == @[10.byte, 3, 231, 189, 1] + + var offset = 0 + let decoded = decodeField(output, MyInt, offset, bytesProcessed) + assert decoded.value.int == num.int + assert decoded.index == 1 + test "Can encode/decode float32 number field": var proto = newProtoBuffer() let num = float32(1234.164423) @@ -180,7 +219,7 @@ suite "Test Varint Encoding": test "Can encode/decode object field": var proto = newProtoBuffer() - let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test", c: 'H'), j: "testing", k: true) + let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test", c: 'H'), j: "testing", k: true, l: 124521.MyInt) proto.encodeField(obj) var offset, bytesProcessed: int @@ -193,7 +232,7 @@ suite "Test Varint Encoding": test "Can encode/decode object": var proto = newProtoBuffer() - let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test", c: 'H'), j: "testing", k: true) + let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test", c: 'H'), j: "testing", k: true, l: 124521.MyInt) proto.encode(obj) var output = proto.output @@ -203,7 +242,8 @@ suite "Test Varint Encoding": test "Can encode/decode out of order object": var proto = newProtoBuffer() - let obj = Test3(g: 400, h: 100, i: Test1(a: 100, b: "this is a test", c: 'H'), j: "testing", k: true) + let obj = Test3(g: 400, h: 100, i: Test1(a: 100, b: "this is a test", c: 'H'), j: "testing", k: true, l: 14514.MyInt) + proto.encodeField(6, 14514.MyInt) proto.encodeField(2, 100) proto.encodeField(4, "testing") proto.encodeField(1, 400)