jyapayne/nim-protobuf-serialization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Actually fix out of order decoding

Browse source
This commit is contained in:
Joey Yakimowich-Payne 2020-04-03 22:18:21 -06:00
commit e1905e13ce
2 changed files with 36 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

32
protobuf_serialization.nim
View file

@ -90,6 +90,9 @@ proc encodeField*(protobuf: var ProtoBuffer, value: SomeVarint) {.inline.} =
protobuf.outstream.encodeField(protobuf.fieldNum, value) protobuf.outstream.encodeField(protobuf.fieldNum, value)
inc protobuf.fieldNum inc protobuf.fieldNum
proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: SomeVarint) {.inline.} =
protobuf.outstream.encodeField(fieldNum, value)
proc put(stream: OutputStreamVar, value: SomeLengthDelimited) {.inline.} = proc put(stream: OutputStreamVar, value: SomeLengthDelimited) {.inline.} =
for b in value: for b in value:
stream.append byte(b) stream.append byte(b)
@ -103,6 +106,9 @@ proc encodeField*(protobuf: var ProtoBuffer, value: SomeLengthDelimited) {.inlin
protobuf.outstream.encodeField(protobuf.fieldNum, value) protobuf.outstream.encodeField(protobuf.fieldNum, value)
inc protobuf.fieldNum inc protobuf.fieldNum
proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: SomeLengthDelimited) {.inline.} =
protobuf.outstream.encodeField(fieldNum, value)
proc put(stream: OutputStreamVar, value: object) {.inline.} proc put(stream: OutputStreamVar, value: object) {.inline.}
proc encodeField(stream: OutputStreamVar, fieldNum: int, value: object) {.inline.} = proc encodeField(stream: OutputStreamVar, fieldNum: int, value: object) {.inline.} =
@ -122,17 +128,24 @@ proc encodeField*(protobuf: var ProtoBuffer, value: object) {.inline.} =
protobuf.outstream.encodeField(protobuf.fieldNum, value) protobuf.outstream.encodeField(protobuf.fieldNum, value)
inc protobuf.fieldNum inc protobuf.fieldNum
proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: object) {.inline.} =
protobuf.outstream.encodeField(fieldNum, value)
proc encode*(protobuf: var ProtoBuffer, value: object) {.inline.} = proc encode*(protobuf: var ProtoBuffer, value: object) {.inline.} =
var fieldNum = 1 var fieldNum = 1
for field, val in value.fieldPairs: for _, val in value.fieldPairs:
protobuf.outstream.encodeField(fieldNum, val) # Only store the value
if default(type(val)) != val:
protobuf.outstream.encodeField(fieldNum, val)
inc fieldNum inc fieldNum
proc put(stream: OutputStreamVar, value: object) {.inline.} = proc put(stream: OutputStreamVar, value: object) {.inline.} =
var fieldNum = 1 var fieldNum = 1
for field, val in value.fieldPairs: for _, val in value.fieldPairs:
# Only store the value
if default(type(val)) != val:
stream.encodeField(fieldNum, val) stream.encodeField(fieldNum, val)
inc fieldNum inc fieldNum
proc getVarint[T: SomeVarint]( proc getVarint[T: SomeVarint](
bytes: var seq[byte], bytes: var seq[byte],
@ -320,12 +333,13 @@ proc decodeField*[T: object](
# read LD header # read LD header
# then read only amount of bytes needed # then read only amount of bytes needed
increaseBytesRead() increaseBytesRead()
var index = 1
let decodedSize = getVarint(bytes, uint, outOffset, outBytesProcessed, numBytesToRead) let decodedSize = getVarint(bytes, uint, outOffset, outBytesProcessed, numBytesToRead)
let bytesToRead = some(decodedSize.int) let bytesToRead = some(decodedSize.int)
for field, val in result.value.fieldPairs:
setField(result.value, index, outOffset, outBytesProcessed, bytesToRead, bytes) let oldOffset = outOffset
inc index while outOffset < oldOffset + bytesToRead.get():
let fieldNum = fieldNumber(bytes[outOffset])
setField(result.value, fieldNum, outOffset, outBytesProcessed, bytesToRead, bytes)
proc decode*[T: object]( proc decode*[T: object](
bytes: var seq[byte], bytes: var seq[byte],
@ -334,6 +348,6 @@ proc decode*[T: object](
var bytesRead = 0 var bytesRead = 0
var offset = 0 var offset = 0
while bytesRead < bytes.len: while offset < bytes.len - 1:
let fieldNum = fieldNumber(bytes[offset]) let fieldNum = fieldNumber(bytes[offset])
setField(result, fieldNum, offset, bytesRead, none(int), bytes) setField(result, fieldNum, offset, bytesRead, none(int), bytes)

13
tests/test_serialization.nim
View file

@ -101,6 +101,19 @@ suite "Test Varint Encoding":
let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test"), j: "testing") let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test"), j: "testing")
proto.encode(obj) proto.encode(obj)
var output = proto.output
let decoded = output.decode(Test3)
assert decoded == obj
test "Can encode/decode out of order object":
var proto = newProtoBuffer()
let obj = Test3(g: 400, h: 100, i: Test1(a: 100, b: "this is a test"), j: "testing")
proto.encodeField(2, 100)
proto.encodeField(4, "testing")
proto.encodeField(1, 400)
proto.encodeField(3, Test1(a: 100, b: "this is a test"))
var output = proto.output var output = proto.output
let decoded = output.decode(Test3) let decoded = output.decode(Test3)
assert decoded == obj assert decoded == obj