CI: test with multiple Nim versions (#23)

* CI: test with multiple Nim versions

* fix readIntoExImpl() when premature EOF is triggered on some Windows versions
* also fix some unrelated pointer math

.github/workflows/ci.yml

@@ -1,14 +1,16 @@
-name: nim-faststreams CI
+name: CI
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
 
 jobs:
   build:
     strategy:
       fail-fast: false
-      max-parallel: 20
       matrix:
-        branch: [master]
-        test_lang: [c, cpp]
         target:
           - os: linux
             cpu: amd64
@@ -18,27 +20,33 @@ jobs:
             cpu: amd64
           - os: windows
             cpu: amd64
-          - os: windows
+          #- os: windows
-            cpu: i386
+            #cpu: i386
+        branch: [version-1-2, version-1-4, version-1-6, devel]
         include:
           - target:
               os: linux
             builder: ubuntu-18.04
+            shell: bash
           - target:
               os: macos
             builder: macos-10.15
+            shell: bash
           - target:
               os: windows
             builder: windows-2019
+            shell: msys2 {0}
 
-    name: '${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ matrix.test_lang }} (${{ matrix.branch }})'
+    defaults:
+      run:
+        shell: ${{ matrix.shell }}
+
+    name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})'
     runs-on: ${{ matrix.builder }}
+    continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }}
     steps:
-      - name: Checkout nim-faststreams
+      - name: Checkout
         uses: actions/checkout@v2
-        with:
-          path: nim-faststreams
-          submodules: false
 
       - name: Install build dependencies (Linux i386)
         if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
@@ -60,98 +68,93 @@ jobs:
           chmod 755 external/bin/gcc external/bin/g++
           echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
 
-      - name: Restore MinGW-W64 (Windows) from cache
+      - name: MSYS2 (Windows i386)
-        if: runner.os == 'Windows'
+        if: runner.os == 'Windows' && matrix.target.cpu == 'i386'
-        id: windows-mingw-cache
+        uses: msys2/setup-msys2@v2
-        uses: actions/cache@v2
         with:
-          path: external/mingw-${{ matrix.target.cpu }}
+          path-type: inherit
-          key: 'mingw-${{ matrix.target.cpu }}'
+          msystem: MINGW32
+          install: >-
+            base-devel
+            git
+            mingw-w64-i686-toolchain
+
+      - name: MSYS2 (Windows amd64)
+        if: runner.os == 'Windows' && matrix.target.cpu == 'amd64'
+        uses: msys2/setup-msys2@v2
+        with:
+          path-type: inherit
+          install: >-
+            base-devel
+            git
+            mingw-w64-x86_64-toolchain
 
       - name: Restore Nim DLLs dependencies (Windows) from cache
         if: runner.os == 'Windows'
         id: windows-dlls-cache
         uses: actions/cache@v2
         with:
-          path: external/dlls-${{ matrix.target.cpu }}
+          path: external/dlls
-          key: 'dlls-${{ matrix.target.cpu }}'
+          key: 'dlls'
 
-      - name: Install MinGW64 dependency (Windows)
+      - name: Install DLL dependencies (Windows)
-        if: >
-          steps.windows-mingw-cache.outputs.cache-hit != 'true' &&
-          runner.os == 'Windows'
-        shell: bash
-        run: |
-          mkdir -p external
-          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
-            MINGW_URL="https://sourceforge.net/projects/mingw-w64/files/Toolchains targetting Win64/Personal Builds/mingw-builds/8.1.0/threads-posix/seh/x86_64-8.1.0-release-posix-seh-rt_v6-rev0.7z"
-            ARCH=64
-          else
-            MINGW_URL="https://sourceforge.net/projects/mingw-w64/files/Toolchains targetting Win32/Personal Builds/mingw-builds/8.1.0/threads-posix/dwarf/i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z"
-            ARCH=32
-          fi
-          curl -L "$MINGW_URL" -o "external/mingw-${{ matrix.target.cpu }}.7z"
-          7z x -y "external/mingw-${{ matrix.target.cpu }}.7z" -oexternal/
-          mv external/mingw$ARCH external/mingw-${{ matrix.target.cpu }}
-
-      - name: Install DLLs dependencies (Windows)
         if: >
           steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
           runner.os == 'Windows'
-        shell: bash
         run: |
-          mkdir -p external
+          mkdir external
           curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
-          7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }}
+          7z x external/windeps.zip -oexternal/dlls
 
       - name: Path to cached dependencies (Windows)
         if: >
           runner.os == 'Windows'
-        shell: bash
         run: |
-          echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH
+          echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH
-          echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH
 
-      - name: Get latest nimbus-build-system commit hash
+      - name: Derive environment variables
-        id: versions
-        shell: bash
         run: |
-          getHash() {
-            git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1
-          }
-          nbsHash=$(getHash status-im/nimbus-build-system)
-          echo "::set-output name=nimbus_build_system::$nbsHash"
-
-      - name: Restore prebuilt Nim from cache
-        id: nim-cache
-        uses: actions/cache@v2
-        with:
-          path: NimBinaries
-          key: 'NimBinaries-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.nimbus_build_system }}'
-
-      - name: Build Nim and associated tools
-        shell: bash
-        run: |
-          curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
           if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
             PLATFORM=x64
           else
             PLATFORM=x86
           fi
-          if [[ '${{ matrix.target.os }}' == 'windows' ]]; then
+          echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
+
+          ncpu=
+          MAKE_CMD="make"
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
             MAKE_CMD="mingw32-make"
-          else
+            ;;
-            MAKE_CMD="make"
+          esac
-          fi
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
-          env MAKE="$MAKE_CMD -j2" ARCH_OVERRIDE=$PLATFORM CC=gcc bash build_nim.sh nim csources dist/nimble NimBinaries
+          echo "ncpu=$ncpu" >> $GITHUB_ENV
+          echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV
 
-      - name: Setup environment
+      - name: Build Nim and Nimble
-        shell: bash
-        run: echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
-
-      - name: Run nim-faststreams tests
-        shell: bash
-        working-directory: nim-faststreams
         run: |
+          curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
+          env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ matrix.branch }} \
+            QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \
+            bash build_nim.sh nim csources dist/nimble NimBinaries
+          echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
+
+      - name: Run tests
+        run: |
+          if [[ "${{ matrix.target.os }}" == "windows" ]]; then
+            # https://github.com/status-im/nimbus-eth2/issues/3121
+            export NIMFLAGS="-d:nimRawSetjmp"
+          fi
+          nim --version
+          nimble --version
           nimble install -y --depsOnly
-          env TEST_LANG="${{ matrix.test_lang }}" nimble test
+          env TEST_LANG="c" nimble test
+          env TEST_LANG="cpp" nimble test

faststreams.nimble

@@ -14,20 +14,18 @@ requires "nim >= 1.2.0",
          "unittest2"
 
 ### Helper functions
-proc test(env, path: string) =
+proc test(args, path: string) =
   # Compilation language is controlled by TEST_LANG
-  var lang = "c"
+  let lang = getEnv("TEST_LANG", "c")
-  if existsEnv"TEST_LANG":
-    lang = getEnv"TEST_LANG"
 
-  let common_args = "-r -f --hints:off --skipParentCfg --styleCheck:usages --styleCheck:error"
+  let common_args = "-r -f " & getEnv("NIMFLAGS") &  " --hints:off --skipParentCfg --styleCheck:usages --styleCheck:error"
 
-  exec "nim " & lang & " " & env &
+  exec "nim " & lang & " " & args &
     " -d:asyncBackend=none " & common_args & " " & path
-  exec "nim " & lang & " " & env &
+  exec "nim " & lang & " " & args &
     " -d:asyncBackend=chronos " & common_args & " " & path
   # TODO std backend is broken / untested
-  # exec "nim " & lang & " " & env &
+  # exec "nim " & lang & " " & args &
   #  " -d:asyncBackend=asyncdispatch " & common_args & " " & path
 
 task test, "Run all tests":

faststreams/inputs.nim

@@ -796,6 +796,9 @@ template readIntoExImpl(s: InputStream,
     var adjustedDst = offset(dst, totalBytesDrained)
 
     while true:
+      if s.vtable.isNil():
+        break
+
       let newBytesRead = awaiter s.vtable.readOp(s, adjustedDst, bytesDeficit)
 
       s.spanEndPos += newBytesRead
@@ -808,7 +811,7 @@ template readIntoExImpl(s: InputStream,
       if bytesDeficit == 0:
         break
 
-      adjustedDst = offset(dst, newBytesRead)
+      adjustedDst = offset(adjustedDst, newBytesRead)
 
   dstLen - bytesDeficit
 

tests/test_inputs.nim

@@ -58,26 +58,27 @@ suite "input stream":
     var input = memFileInput("files" / "empty_file")
 
   template asciiTableFileTest(name: string, body: untyped) =
-    test name & " of ascii table with regular pageSize":
+    suite name:
-      var input {.inject.} = fileInput(asciiTableFile)
+      test name & " of ascii table with regular pageSize":
-      try:
+        var input {.inject.} = fileInput(asciiTableFile)
-        body
+        try:
-      finally:
+          body
-        close input
+        finally:
+          close input
 
-    test name & " of ascii table with pageSize = 10":
+      test name & " of ascii table with pageSize = 10":
-      var input {.inject.} = fileInput(asciiTableFile, pageSize = 10)
+        var input {.inject.} = fileInput(asciiTableFile, pageSize = 10)
-      try:
+        try:
-        body
+          body
-      finally:
+        finally:
-        close input
+          close input
 
-    test name & " of ascii table with pageSize = 1":
+      test name & " of ascii table with pageSize = 1":
-      var input {.inject.} = fileInput(asciiTableFile, pageSize = 1)
+        var input {.inject.} = fileInput(asciiTableFile, pageSize = 1)
-      try:
+        try:
-        body
+          body
-      finally:
+        finally:
-        close input
+          close input
 
   # TODO: fileInput with offset
   #        - in the middle of the
@@ -124,41 +125,42 @@ suite "input stream":
 
     check fileContents == asciiTableContents
 
-  test "missing file input":
+  suite "misc":
-    const fileName = "there-is-no-such-faststreams-file"
+    test "missing file input":
+      const fileName = "there-is-no-such-faststreams-file"
 
-    check not fileExists(fileName)
+      check not fileExists(fileName)
-    expect CatchableError: discard fileInput(fileName)
+      expect CatchableError: discard fileInput(fileName)
 
-    check not fileExists(fileName)
+      check not fileExists(fileName)
-    expect CatchableError: discard memFileInput(fileName)
+      expect CatchableError: discard memFileInput(fileName)
 
-    check not fileExists(fileName)
+      check not fileExists(fileName)
 
-  test "non-blocking reads":
+    test "non-blocking reads":
-    let s = fileInput(asciiTableFile, pageSize = 100)
+      let s = fileInput(asciiTableFile, pageSize = 100)
-    if s.readable(20):
+      if s.readable(20):
-      s.withReadableRange(20, r):
+        s.withReadableRange(20, r):
-        check r.len.get == 20
+          check r.len.get == 20
-        check r.totalUnconsumedBytes == 20
+          check r.totalUnconsumedBytes == 20
-        check r.readAll.len == 20
+          check r.readAll.len == 20
 
-    check s.readable
+      check s.readable
 
-    if s.readable(200):
+      if s.readable(200):
-      s.withReadableRange(200, r):
+        s.withReadableRange(200, r):
-        check r.len.get == 200
+          check r.len.get == 200
-        check r.totalUnconsumedBytes == 200
+          check r.totalUnconsumedBytes == 200
-        check r.readAll.len == 200
+          check r.readAll.len == 200
 
-    check s.readable
+      check s.readable
 
-  test "simple":
+    test "simple":
-    var input = repeat("1234 5678 90AB CDEF\n", 1000)
+      var input = repeat("1234 5678 90AB CDEF\n", 1000)
-    var stream = unsafeMemoryInput(input)
+      var stream = unsafeMemoryInput(input)
 
-    check:
+      check:
-      (stream.read(4) == "1234".toOpenArrayByte(0, 3))
+        (stream.read(4) == "1234".toOpenArrayByte(0, 3))
 
   template posTest(name: string, setup: untyped) =
     test name:

tests/test_outputs.nim

@@ -1,7 +1,7 @@
 {.used.}
 
 import
-  os, unittest, random, strformat,
+  os, unittest2, random, strformat,
   stew/ranges/ptr_arith,
   ../faststreams, ../faststreams/textio
 
@@ -20,12 +20,10 @@ proc repeat(b: byte, count: int): seq[byte] =
   result = newSeq[byte](count)
   for i in 0 ..< count: result[i] = b
 
-const line = "123456789123456789123456789123456789\n\n\n\n\n"
-
 proc randomBytes(n: int): seq[byte] =
   result.newSeq n
   for i in 0 ..< n:
-    result[i] = byte(rand(line))
+    result[i] = byte(rand(int('1')..int('9')))
 
 proc readAllAndClose(s: InputStream): seq[byte] =
   while s.readable:
@@ -55,6 +53,8 @@ suite "output stream":
       streamWritingToExistingBuffer = unsafeMemoryOutput(buffer, bufferSize)
 
   teardown:
+    fileStream.close()
+    unbufferedFileStream.close()
     removeFile fileOutputPath
     removeFile unbufferedFileOutputPath
     dealloc buffer