Implement the async pipelines and fsMultiSync

tests/base64.nim

@@ -13,7 +13,12 @@ const
   invalidChar = 255
   paddingByte = byte('=')
 
-proc base64encode*(i: InputStream, o: OutputStream) =
+template encodeSize(size: int): int = (size * 4 div 3) + 6
+
+import
+  ../faststreams/buffers
+
+proc base64encode*(i: InputStream, o: OutputStream) {.fsMultiSync.} =
   var
     n: uint32
     b: uint32
@@ -25,6 +30,10 @@ proc base64encode*(i: InputStream, o: OutputStream) =
   template outputChar(x: typed) =
     o.write cb64[x and 63]
 
+  let inputLen = i.len
+  if inputLen.isSome:
+    o.ensureRunway encodeSize(inputLen.get)
+
   while i.readable(3):
     inputByte(b shl 16)
     inputByte(n or b shl 8)
@@ -48,6 +57,8 @@ proc base64encode*(i: InputStream, o: OutputStream) =
       o.write paddingByte
       o.write paddingByte
 
+  close o
+
 proc initDecodeTable*(): array[256, char] =
   # computes a decode table at compile time
   for i in 0 ..< 256:
@@ -63,7 +74,7 @@ proc initDecodeTable*(): array[256, char] =
 const
   decodeTable = initDecodeTable()
 
-proc base64decode*(i: InputStream, o: OutputStream) =
+proc base64decode*(i: InputStream, o: OutputStream) {.fsMultiSync.} =
   proc decodeSize(size: int): int =
     return (size * 3 div 4) + 6
 
@@ -112,3 +123,5 @@ proc base64decode*(i: InputStream, o: OutputStream) =
   elif i.readable:
     raise newException(ValueError, "The input stream has insufficient nymber of bytes for base64 decoding")
 
+  close o
+

tests/test_inputs.nim

@@ -74,6 +74,11 @@ procSuite "input stream":
       finally:
         close input
 
+  # TODO: fileInput with offset
+  #        - in the middle of the
+  #        - right at the end of the file
+  #        - past the end of the file
+
   asciiTableFileTest "count lines":
     check countLines(input) == 34
 

tests/test_outputs.nim

@@ -81,10 +81,7 @@ suite "output stream":
 
   template checkOutputsMatch(showResults = false,
                              skipUnbufferedFile = false) =
-    flush fileStream
     close fileStream
-
-    flush unbufferedFileStream
     close unbufferedFileStream
 
     check fileExists(fileOutputPath) and
@@ -122,15 +119,15 @@ suite "output stream":
 
     let outputsMatch =
       nimSeq == makeOpenArray(cast[ptr byte](buffer),
-                              streamWritingToExistingBuffer.pos) and
-      nimSeq == memStreamRes and
-      nimSeq == readFileRes and
-      nimSeq == fileInputRes and
-      nimSeq == memFileInputRes and
-      nimSeq == fileInputWithSmallPagesRes
-
+                              streamWritingToExistingBuffer.pos)
     check outputsMatch
 
+    check nimSeq == memStreamRes
+    check nimSeq == readFileRes
+    check nimSeq == fileInputRes
+    check nimSeq == memFileInputRes
+    check nimSeq == fileInputWithSmallPagesRes
+
     when not skipUnbufferedFile:
       let unbufferedFileRes = readFile(unbufferedFileOutputPath).string.bytes
       check nimSeq == unbufferedFileRes
@@ -236,5 +233,6 @@ suite "output stream":
       finalize dw.cursor
 
     # The final outputs are the same
-    check nimSeq == memStream.getOutput
+    let resultsAreEqual = nimSeq == memStream.getOutput
+    check resultsAreEqual
 

tests/test_pipelines.nim

@@ -1,8 +1,13 @@
 {.used.}
 
 import
-  std/[unittest, strutils, base64],
-  ../faststreams/pipelines,
+  # Std lib:
+  std/[strutils, random, base64, terminal],
+  # Other packages:
+  testutils/unittests,
+  # FastStreams modules:
+  ../faststreams/[pipelines, multisync],
+  # Testing modules:
   ./base64 as fsBase64
 
 include system/timers
@@ -13,17 +18,29 @@ type
     fsAsyncPipeline: Nanos
     stdFunctionCalls: Nanos
 
-proc upcaseAllCharacters(i: InputStream, o: OutputStream) =
+proc upcaseAllCharacters(i: InputStream, o: OutputStream) {.fsMultiSync.} =
+  let inputLen = i.len
+  if inputLen.isSome:
+    o.ensureRunway inputLen.get
+
   while i.readable:
-    o.write toUpperAscii(char i.read())
+    o.write toUpperAscii(i.read.char)
+
+  echo "closing upcase"
+  close o
+
+proc printTimes(t: TestTimes) =
+  styledEcho "  cpu time [FS Sync  ]: ", styleBright, $t.fsPipeline, "ms"
+  styledEcho "  cpu time [FS Async ]: ", styleBright, $t.fsAsyncPipeline, "ms"
+  styledEcho "  cpu time [Std Lib  ]: ", styleBright, $t.stdFunctionCalls, "ms"
 
 template timeit(timerVar: var Nanos, code: untyped) =
   let t0 = getTicks()
   code
   timerVar = int(getTicks() - t0) div 1000000
 
-suite "pipelines":
-  var loremIpsum = """
+procSuite "pipelines":
+  let loremIpsum = """
     Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod
     tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
     veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex
@@ -32,25 +49,83 @@ suite "pipelines":
     cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id
     est laborum.
 
-  """.repeat(100)
+  """
 
-  test "upper-case/base64 pipeline":
+  #[
+  test "upper-case/base64 pipeline benchmark":
     var
       times: TestTimes
       stdRes: string
       fsRes: string
+      fsAsyncRes: string
+
+    let inputText = loremIpsum.repeat(5000)
+
+    when debugHelpers:
+      echo "Input len: ", inputText.len
+      echo "Base 64 len: ", base64.encode(inputText).len
 
     timeIt times.fsPipeline:
-      var memOut = memoryOutput()
-      executePipeline(unsafeMemoryInput(loremIpsum),
-                      upcaseAllCharacters,
-                      base64encode,
-                      base64decode,
-                      memOut)
-      fsRes = memOut.getOutput(string)
+      fsRes = executePipeline(unsafeMemoryInput(inputText),
+                              upcaseAllCharacters,
+                              base64encode,
+                              base64decode,
+                              getOutput string)
+
+    timeIt times.fsAsyncPipeline:
+      fsAsyncRes = waitFor executePipeline(Async unsafeMemoryInput(inputText),
+                                           upcaseAllCharacters,
+                                           base64encode,
+                                           base64decode,
+                                           getOutput string)
 
     timeIt times.stdFunctionCalls:
-      stdRes = base64.decode(base64.encode(toUpperAscii(loremIpsum)))
+      stdRes = base64.decode(base64.encode(toUpperAscii(inputText)))
 
+    check fsAsyncRes == stdRes
     check fsRes == stdRes
 
+    printTimes times
+  ]#
+
+  asyncTest "upper-case/base64 async pipeline":
+    let pipe = asyncPipe()
+    let inputText = repeat(loremIpsum, 8)
+
+    when debugHelpers:
+      echo "Input len: ", inputText.len
+
+    proc pipeFeeder(s: AsyncOutputStream) {.gcsafe, async.} =
+      randomize 1234
+      var pos = 0
+
+      while pos != inputText.len:
+        let bytesToWrite = rand(15)
+
+        if bytesToWrite == 0:
+          s.write inputText[pos]
+          inc pos
+        else:
+          let endPos = min(pos + bytesToWrite, inputText.len)
+          s.writeAndWait inputText[pos ..< endPos]
+          pos = endPos
+
+        let sleep = rand(50) - 45
+        if sleep > 0:
+          echo "written ", pos
+          await sleepAsync(sleep.milliseconds)
+
+      close s
+
+    asyncCheck pipeFeeder(pipe.initWriter)
+
+    let f = executePipeline(pipe.initReader,
+                            upcaseAllCharacters,
+                            base64encode,
+                            base64decode,
+                            getOutput string)
+
+    let fsAsyncres = await f
+
+    check fsAsyncRes == toUpperAscii(inputText)
+