Add string encoding/decoding

protobuf_serialization.nim

@@ -1,3 +1,4 @@
+import macros, strformat
 import faststreams
 
 const
@@ -10,7 +11,7 @@ type
 
   ProtoWireType* = enum
     ## Protobuf's field types enum
-    Varint, Fixed64, Length, StartGroup, EndGroup, Fixed32
+    Varint, Fixed64, LengthDelimited, StartGroup, EndGroup, Fixed32
 
   ProtoField* = object
     ## Protobuf's message field representation object
@@ -20,7 +21,7 @@ type
       vint*: uint64
     of Fixed64:
       vfloat64*: float64
-    of Length:
+    of LengthDelimited:
       vbuffer*: OutputStreamVar
     of Fixed32:
       vfloat32*: float32
@@ -30,6 +31,7 @@ type
   SomeSVarint* = int | int64 | int32 | int16 | int8 | enum
   SomeUVarint* = uint | uint64 | uint32 | uint16 | uint8 | byte | bool
   SomeVarint* = SomeSVarint | SomeUVarint
+  SomeLengthDelimited* = string | seq[byte] | seq[uint8] | cstring
 
 proc newProtoBuffer*(): ProtoBuffer =
   ProtoBuffer(outstream: OutputStream.init(), fieldNum: 1)
@@ -54,10 +56,7 @@ template protoHeader*(fieldNum: int, wire: ProtoWireType): byte =
   ## Get protobuf's field header integer for ``index`` and ``wire``.
   ((cast[uint](fieldNum) shl 3) or cast[uint](wire)).byte
 
-proc encodeVarint(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.inline.} =
+proc putVarint(stream: OutputStreamVar, value: SomeVarint) {.inline.} =
-  let header = protoHeader(fieldNum, Varint)
-  stream.append header
-
   when value is enum:
     var value = cast[type(ord(value))](value)
   elif value is bool:
@@ -76,25 +75,35 @@ proc encodeVarint(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.i
     value = value shr 7
   stream.append byte(value and 0b1111_1111)
 
+proc encode(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.inline.} =
+  stream.append protoHeader(fieldNum, Varint)
+  stream.putVarint(value)
+
 proc encode*(protobuf: ProtoBuffer, value: SomeVarint) {.inline.} =
-  protobuf.outstream.encodeVarint(protobuf.fieldNum, value)
+  protobuf.outstream.encode(protobuf.fieldNum, value)
   inc protobuf.fieldNum
 
-proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[fieldNum: uint, value: T, bytesProcessed: int] {.inline.} =
+proc putLengthDelimited(stream: OutputStreamVar, value: SomeLengthDelimited) {.inline.} =
+  for b in value:
+    stream.append byte(b)
+
+proc encode(stream: OutputStreamVar, fieldNum: int, value: SomeLengthDelimited) {.inline.} =
+  stream.append protoHeader(fieldNum, LengthDelimited)
+  stream.putVarint(len(value).uint)
+  stream.putLengthDelimited(value)
+
+proc encode*(protobuf: ProtoBuffer, value: SomeLengthDelimited) {.inline.} =
+  protobuf.outstream.encode(protobuf.fieldNum, value)
+  inc protobuf.fieldNum
+
+proc getVarint[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[value: T, bytesProcessed: int] {.inline.} =
   # Only up to 128 bits supported by the spec
-  assert (bytes.len - 1) <= 16
-
-  let wireTy = wireType(bytes[offset])
-  if wireTy != Varint:
-    raise newException(Exception, "Not a varint!")
-
-  result.fieldNum = fieldNumber(bytes[offset])
   when T is enum:
     var value: type(ord(result.value))
   else:
     var value: T
   var shiftAmount = 0
-  var i = offset + 1
+  var i = offset
   while true:
     value += type(value)(bytes[i] and 0b0111_1111) shl shiftAmount
     shiftAmount += 7
@@ -102,7 +111,7 @@ proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0):
       break
     i += 1
 
-  result.bytesProcessed = i + 1
+  result.bytesProcessed = i
 
   when ty is SomeSVarint:
     if (value and type(value)(1)) != type(value)(0):
@@ -110,4 +119,60 @@ proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0):
     else:
       result.value = cast[T](value shr type(value)(1))
   else:
-    result.value = value
+    result.value = value
+
+proc decode*[T: SomeVarint](bytes: var seq[byte], ty: typedesc[T], offset = 0): tuple[fieldNum: uint, value: T, bytesProcessed: int] {.inline.} =
+  # Only up to 128 bits supported by the spec
+  assert (bytes.len - 1) <= 16
+
+  let wireTy = wireType(bytes[offset])
+  if wireTy != Varint:
+    raise newException(Exception, fmt"Not a varint at offset {offset}! Received a {wireTy}")
+
+  result.fieldNum = fieldNumber(bytes[offset])
+  var offset = offset + 1
+
+  let varGet = getVarint(bytes, ty, offset)
+  result.value = varGet.value
+  result.bytesProcessed = varGet.bytesProcessed + offset
+
+proc getLengthDelimited*[T: SomeLengthDelimited](
+  bytes: var seq[byte],
+  ty: typedesc[T], offset = 0
+): tuple[value: T, bytesProcessed: int] {.inline.} =
+
+  var offset = offset
+  let decodedSize = getVarint(bytes, uint, offset = offset)
+  offset += decodedSize.bytesProcessed
+  let length = decodedSize.value.int
+
+  when T is string:
+    result.value = newString(length)
+    for i in offset ..< (offset + length):
+      result.value[i - offset] = bytes[i].chr
+  elif T is cstring:
+    result.value = cast[cstring](bytes[offset ..< (offset + length)])
+  else:
+    result.value = newSeq(length)
+    for i in offset ..< (offset + length):
+      result.value[i - offset] = bytes[i].chr
+
+  result.bytesProcessed += length
+
+proc decode*[T: SomeLengthDelimited](
+  bytes: var seq[byte],
+  ty: typedesc[T], offset = 0
+): tuple[fieldNum: uint, value: T, bytesProcessed: int] {.inline.} =
+  var offset = offset
+
+  let wireTy = wireType(bytes[offset])
+  if wireTy != LengthDelimited:
+    raise newException(Exception, fmt"Not a length delimited value at offset {offset}! Received a {wireTy}")
+
+  result.fieldNum = fieldNumber(bytes[offset])
+
+  offset += 1
+
+  let lengthDelimited = getLengthDelimited(bytes, ty, offset)
+  result.bytesProcessed = offset + lengthDelimited.bytesProcessed
+  result.value = lengthDelimited.value

tests/test_serialization.nim

@@ -7,7 +7,7 @@ type
     ME1, ME2, ME3
 
 suite "Test Varint Encoding":
-  test "Can encode enum":
+  test "Can encode/decode enum":
     let proto = newProtoBuffer()
     proto.encode(ME3)
     proto.encode(ME2)
@@ -22,7 +22,7 @@ suite "Test Varint Encoding":
     assert decodedME2.value == ME2
     assert decodedME2.fieldNum == 2
 
-  test "Can encode negative number":
+  test "Can encode/decode negative number":
     let proto = newProtoBuffer()
     let num = -153452
     proto.encode(num)
@@ -33,7 +33,7 @@ suite "Test Varint Encoding":
     assert decoded.value == num
     assert decoded.fieldNum == 1
 
-  test "Can encode unsigned number":
+  test "Can encode/decode unsigned number":
     let proto = newProtoBuffer()
     let num = 123151.uint
     proto.encode(num)
@@ -41,6 +41,16 @@ suite "Test Varint Encoding":
     assert output == @[8.byte, 143, 194, 7]
 
     let decoded = decode(output, uint)
-    echo decoded.value
     assert decoded.value == num
+    assert decoded.fieldNum == 1
+
+  test "Can encode/decode string":
+    let proto = newProtoBuffer()
+    let str = "hey this is a string"
+    proto.encode(str)
+    var output = proto.output
+    assert output == @[10.byte, 20, 104, 101, 121, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 115, 116, 114, 105, 110, 103]
+
+    let decoded = decode(output, string)
+    assert decoded.value == str
     assert decoded.fieldNum == 1