diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index e31be9a..9a2596b 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -1,3 +1,4 @@ +import macros, strformat import faststreams const @@ -10,7 +11,7 @@ type ProtoWireType* = enum ## Protobuf's field types enum - Varint, Fixed64, Length, StartGroup, EndGroup, Fixed32 + Varint, Fixed64, LengthDelimited, StartGroup, EndGroup, Fixed32 ProtoField* = object ## Protobuf's message field representation object @@ -20,7 +21,7 @@ type vint*: uint64 of Fixed64: vfloat64*: float64 - of Length: + of LengthDelimited: vbuffer*: OutputStreamVar of Fixed32: vfloat32*: float32 @@ -30,6 +31,7 @@ type SomeSVarint* = int | int64 | int32 | int16 | int8 | enum SomeUVarint* = uint | uint64 | uint32 | uint16 | uint8 | byte | bool SomeVarint* = SomeSVarint | SomeUVarint + SomeLengthDelimited* = string | seq[byte] | seq[uint8] | cstring proc newProtoBuffer*(): ProtoBuffer = ProtoBuffer(outstream: OutputStream.init(), fieldNum: 1) @@ -54,10 +56,7 @@ template protoHeader*(fieldNum: int, wire: ProtoWireType): byte = ## Get protobuf's field header integer for ``index`` and ``wire``. ((cast[uint](fieldNum) shl 3) or cast[uint](wire)).byte -proc encodeVarint(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.inline.} = - let header = protoHeader(fieldNum, Varint) - stream.append header - +proc putVarint(stream: OutputStreamVar, value: SomeVarint) {.inline.} = when value is enum: var value = cast[type(ord(value))](value) elif value is bool: @@ -76,25 +75,35 @@ proc encodeVarint(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.i value = value shr 7 stream.append byte(value and 0b1111_1111) +proc encode(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.inline.} = + stream.append protoHeader(fieldNum, Varint) + stream.putVarint(value) + proc encode*(protobuf: ProtoBuffer, value: SomeVarint) {.inline.} = - protobuf.outstream.encodeVarint(protobuf.fieldNum, value) + protobuf.outstream.encode(protobuf.fieldNum, value) inc protobuf.fieldNum -proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[fieldNum: uint, value: T, bytesProcessed: int] {.inline.} = +proc putLengthDelimited(stream: OutputStreamVar, value: SomeLengthDelimited) {.inline.} = + for b in value: + stream.append byte(b) + +proc encode(stream: OutputStreamVar, fieldNum: int, value: SomeLengthDelimited) {.inline.} = + stream.append protoHeader(fieldNum, LengthDelimited) + stream.putVarint(len(value).uint) + stream.putLengthDelimited(value) + +proc encode*(protobuf: ProtoBuffer, value: SomeLengthDelimited) {.inline.} = + protobuf.outstream.encode(protobuf.fieldNum, value) + inc protobuf.fieldNum + +proc getVarint[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[value: T, bytesProcessed: int] {.inline.} = # Only up to 128 bits supported by the spec - assert (bytes.len - 1) <= 16 - - let wireTy = wireType(bytes[offset]) - if wireTy != Varint: - raise newException(Exception, "Not a varint!") - - result.fieldNum = fieldNumber(bytes[offset]) when T is enum: var value: type(ord(result.value)) else: var value: T var shiftAmount = 0 - var i = offset + 1 + var i = offset while true: value += type(value)(bytes[i] and 0b0111_1111) shl shiftAmount shiftAmount += 7 @@ -102,7 +111,7 @@ proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): break i += 1 - result.bytesProcessed = i + 1 + result.bytesProcessed = i when ty is SomeSVarint: if (value and type(value)(1)) != type(value)(0): @@ -110,4 +119,60 @@ proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): else: result.value = cast[T](value shr type(value)(1)) else: - result.value = value \ No newline at end of file + result.value = value + +proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[fieldNum: uint, value: T, bytesProcessed: int] {.inline.} = + # Only up to 128 bits supported by the spec + assert (bytes.len - 1) <= 16 + + let wireTy = wireType(bytes[offset]) + if wireTy != Varint: + raise newException(Exception, fmt"Not a varint at offset {offset}! Received a {wireTy}") + + result.fieldNum = fieldNumber(bytes[offset]) + var offset = offset + 1 + + let varGet = getVarint(bytes, ty, offset) + result.value = varGet.value + result.bytesProcessed = varGet.bytesProcessed + offset + +proc getLengthDelimited*[T: SomeLengthDelimited]( + bytes: var seq[byte], + ty: typedesc[T], offset = 0 +): tuple[value: T, bytesProcessed: int] {.inline.} = + + var offset = offset + let decodedSize = getVarint(bytes, uint, offset = offset) + offset += decodedSize.bytesProcessed + let length = decodedSize.value.int + + when T is string: + result.value = newString(length) + for i in offset ..< (offset + length): + result.value[i - offset] = bytes[i].chr + elif T is cstring: + result.value = cast[cstring](bytes[offset ..< (offset + length)]) + else: + result.value = newSeq(length) + for i in offset ..< (offset + length): + result.value[i - offset] = bytes[i].chr + + result.bytesProcessed += length + +proc decode*[T: SomeLengthDelimited]( + bytes: var seq[byte], + ty: typedesc[T], offset = 0 +): tuple[fieldNum: uint, value: T, bytesProcessed: int] {.inline.} = + var offset = offset + + let wireTy = wireType(bytes[offset]) + if wireTy != LengthDelimited: + raise newException(Exception, fmt"Not a length delimited value at offset {offset}! Received a {wireTy}") + + result.fieldNum = fieldNumber(bytes[offset]) + + offset += 1 + + let lengthDelimited = getLengthDelimited(bytes, ty, offset) + result.bytesProcessed = offset + lengthDelimited.bytesProcessed + result.value = lengthDelimited.value \ No newline at end of file diff --git a/tests/test_serialization.nim b/tests/test_serialization.nim index 1ee0a57..2a11db7 100644 --- a/tests/test_serialization.nim +++ b/tests/test_serialization.nim @@ -7,7 +7,7 @@ type ME1, ME2, ME3 suite "Test Varint Encoding": - test "Can encode enum": + test "Can encode/decode enum": let proto = newProtoBuffer() proto.encode(ME3) proto.encode(ME2) @@ -22,7 +22,7 @@ suite "Test Varint Encoding": assert decodedME2.value == ME2 assert decodedME2.fieldNum == 2 - test "Can encode negative number": + test "Can encode/decode negative number": let proto = newProtoBuffer() let num = -153452 proto.encode(num) @@ -33,7 +33,7 @@ suite "Test Varint Encoding": assert decoded.value == num assert decoded.fieldNum == 1 - test "Can encode unsigned number": + test "Can encode/decode unsigned number": let proto = newProtoBuffer() let num = 123151.uint proto.encode(num) @@ -41,6 +41,16 @@ suite "Test Varint Encoding": assert output == @[8.byte, 143, 194, 7] let decoded = decode(output, uint) - echo decoded.value assert decoded.value == num + assert decoded.fieldNum == 1 + + test "Can encode/decode string": + let proto = newProtoBuffer() + let str = "hey this is a string" + proto.encode(str) + var output = proto.output + assert output == @[10.byte, 20, 104, 101, 121, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 115, 116, 114, 105, 110, 103] + + let decoded = decode(output, string) + assert decoded.value == str assert decoded.fieldNum == 1 \ No newline at end of file