Initial work on pipelines

tests/all_tests.nim

@@ -1,4 +1,5 @@
 import
   test_input_stream,
-  test_output_stream
+  test_output_stream,
+  test_pipelines
 

tests/base64.nim

@@ -0,0 +1,116 @@
+import
+  ../faststreams
+
+template cbBase(a, b): untyped = [
+  'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+  'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+  'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+  'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+  '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', a, b]
+
+const
+  cb64 = cbBase('+', '/')
+  invalidChar = 255
+  paddingByte = byte('=')
+
+template encodeSize(size: int): int = (size * 4 div 3) + 6
+
+proc base64encode*(i: InputStream, o: OutputStream) =
+  var
+    n: uint32
+    b: uint32
+
+  template inputByte(exp: untyped) =
+    b = uint32(i.read)
+    n = exp
+
+  template outputChar(x: typed) =
+    o.append cb64[x and 63]
+
+  while i.readable(3):
+    inputByte(b shl 16)
+    inputByte(n or b shl 8)
+    inputByte(n or b shl 0)
+    outputChar(n shr 18)
+    outputChar(n shr 12)
+    outputChar(n shr 6)
+    outputChar(n shr 0)
+
+  if i.readable:
+    inputByte(b shl 16)
+    if i.readable:
+      inputByte(n or b shl 8)
+      outputChar(n shr 18)
+      outputChar(n shr 12)
+      outputChar(n shr 6)
+      o.append paddingByte
+    else:
+      outputChar(n shr 18)
+      outputChar(n shr 12)
+      o.append paddingByte
+      o.append paddingByte
+
+proc initDecodeTable*(): array[256, char] =
+  # computes a decode table at compile time
+  for i in 0 ..< 256:
+    let ch = char(i)
+    var code = invalidChar
+    if ch >= 'A' and ch <= 'Z': code = i - 0x00000041
+    if ch >= 'a' and ch <= 'z': code = i - 0x00000047
+    if ch >= '0' and ch <= '9': code = i + 0x00000004
+    if ch == '+' or ch == '-': code = 0x0000003E
+    if ch == '/' or ch == '_': code = 0x0000003F
+    result[i] = char(code)
+
+const
+  decodeTable = initDecodeTable()
+
+proc base64decode*(i: InputStream, o: OutputStream) =
+  proc decodeSize(size: int): int =
+    return (size * 3 div 4) + 6
+
+  proc raiseInvalidChar(c: byte, pos: int) {.noReturn.} =
+    raise newException(ValueError,
+      "Invalid base64 format character `" & char(c) & "` at location " & $pos & ".")
+
+  template inputChar(x: untyped) =
+    let c = i.read()
+    let x = int decodeTable[c]
+    if x == invalidChar:
+      raiseInvalidChar(c, i.pos - 1)
+
+  template outputChar(x: untyped) =
+    o.append char(x and 255)
+
+  let inputLen = i.len
+  if inputLen != lengthUnknown:
+    o.prepareRunway decodeSize(inputLen)
+
+  # hot loop: read 4 characters at at time
+  while i.readable(8):
+    inputChar(a)
+    inputChar(b)
+    inputChar(c)
+    inputChar(d)
+    outputChar(a shl 2 or b shr 4)
+    outputChar(b shl 4 or c shr 2)
+    outputChar(c shl 6 or d shr 0)
+
+  if i.readable(4):
+    inputChar(a)
+    inputChar(b)
+    outputChar(a shl 2 or b shr 4)
+
+    if i.peek == paddingByte:
+      let next = i.peekAt(1)
+      if next != paddingByte:
+        raiseInvalidChar(next, i.pos + 1)
+    else:
+      inputChar(c)
+      outputChar(b shl 4 or c shr 2)
+      if i.peek != paddingByte:
+        inputChar(d)
+        outputChar(c shl 6 or d shr 0)
+  elif i.readable:
+    raise newException(ValueError, "The input stream has insufficient nymber of bytes for base64 decoding")
+

tests/test_input_stream.nim

@@ -8,5 +8,5 @@ suite "input stream":
     var stream = memoryInput(input)
 
     check:
-      (stream.readBytes(4) == "1234".toOpenArrayByte(0, 3))
+      (stream.read(4) == "1234".toOpenArrayByte(0, 3))
 

tests/test_pipelines.nim

@@ -0,0 +1,54 @@
+import
+  std/[unittest, strutils, base64],
+  ../faststreams/pipelines,
+  ./base64 as fsBase64
+
+include system/timers
+
+type
+  TestTimes = object
+    fsPipeline: Nanos
+    fsAsyncPipeline: Nanos
+    stdFunctionCalls: Nanos
+
+proc upcaseAllCharacters(i: InputStream, o: OutputStream) =
+  while i.readable:
+    o.append toUpperAscii(char i.read())
+
+template timeit(timerVar: var Nanos, code: untyped) =
+  let t0 = getTicks()
+  code
+  timerVar = int(getTicks() - t0) div 1000000
+
+suite "pipelines":
+  var loremIpsum = """
+    Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod
+    tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
+    veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex
+    ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate
+    velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat
+    cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id
+    est laborum.
+
+  """.repeat(100)
+
+  test "upper-case/base64 pipeline":
+    var
+      times: TestTimes
+      stdRes: string
+      fsRes: string
+
+    timeIt times.fsPipeline:
+      var memOut = memoryOutput()
+      executePipeline(memoryInput(loremIpsum),
+                      upcaseAllCharacters,
+                      base64encode,
+                      base64decode,
+                      memOut)
+      fsRes = memOut.getOutput(string)
+
+    timeIt times.stdFunctionCalls:
+      stdRes = base64.decode(base64.encode(toUpperAscii(loremIpsum)))
+
+    check fsRes == stdRes
+