remove .github/workflows/ci_ssl.yml; instead run via trunner_thirdparty (#16221)

.github/workflows/ci_ssl.yml

@@ -1,93 +0,0 @@
-name: Nim SSL CI
-on:
-  pull_request:
-    # Run only on changes on related files
-    paths:
-      - 'lib/pure/httpclient.nim'
-      - 'lib/pure/net.nim'
-      - 'lib/pure/ssl_certs.nim'
-      - 'lib/wrappers/openssl.nim'
-      - 'tests/stdlib/thttpclient_ssl*'
-      - 'tests/untestable/thttpclient_ssl*'
-
-jobs:
-  build:
-    if: |
-      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-18.04, macos-10.15, windows-2019]
-        cpu: [amd64]
-    name: '${{ matrix.os }} (${{ matrix.cpu }})'
-    runs-on: ${{ matrix.os }}
-    steps:
-      - name: 'Checkout'
-        uses: actions/checkout@v2
-
-      - name: 'Checkout csources'
-        uses: actions/checkout@v2
-        with:
-          repository: nim-lang/csources
-          path: csources
-
-      - name: 'Install dependencies (Linux amd64)'
-        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
-        run: |
-          sudo apt-fast update -qq
-          DEBIAN_FRONTEND='noninteractive' \
-            sudo apt-fast install --no-install-recommends -y libssl1.1
-      - name: 'Install dependencies (macOS)'
-        if: runner.os == 'macOS'
-        run: brew install make
-      - name: 'Install dependencies (Windows)'
-        if: runner.os == 'Windows'
-        shell: bash
-        run: |
-          mkdir dist
-          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
-          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
-          7z x dist/mingw64.7z -odist
-          7z x dist/dlls.zip -obin
-          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
-
-      - name: 'Add build binaries to PATH'
-        shell: bash
-        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
-
-      - name: 'Build 1-stage compiler from csources'
-        shell: bash
-        run: |
-          ncpu=
-          case '${{ runner.os }}' in
-          'Linux')
-            ncpu=$(nproc)
-            ;;
-          'macOS')
-            ncpu=$(sysctl -n hw.ncpu)
-            ;;
-          'Windows')
-            ncpu=$NUMBER_OF_PROCESSORS
-            ;;
-          esac
-          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
-
-          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
-
-      - name: 'Build koch'
-        shell: bash
-        run: nim c koch
-
-      - name: 'Build the real compiler'
-        shell: bash
-        run: ./koch boot
-
-      - name: 'Run SSL nimDisableCertificateValidation integration tests'
-        shell: bash
-        run: nim c -d:nimDisableCertificateValidation -d:ssl -r -p:. tests/untestable/thttpclient_ssl_disabled.nim
-
-      - name: 'Run SSL certificate check integration tests'
-        # Not supported on Windows due to old openssl version
-        if: runner.os != 'Windows'
-        shell: bash
-        run: nim c -d:ssl -p:. --threads:on -r tests/untestable/thttpclient_ssl.nim

.gitlab-ci.yml

@@ -54,7 +54,7 @@ test-windows:
   script:
     - call ci\deps.bat
     - nim c testament\tester
-    - testament\tester.exe --pedantic all
+    - testament\tester.exe all
   tags:
     - windows
     - fast

koch.nim

@@ -548,7 +548,7 @@ proc runCI(cmd: string) =
     #[
     BUG: with initOptParser, `--batch:'' all` interprets `all` as the argument of --batch
     ]#
-    execFold("Run tester", "nim c -r -d:nimCoroutines testament/testament --pedantic --batch:$1 all -d:nimCoroutines" % ["NIM_TESTAMENT_BATCH".getEnv("_")])
+    execFold("Run tester", "nim c -r -d:nimCoroutines --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 testament/testament --batch:$1 all -d:nimCoroutines" % ["NIM_TESTAMENT_BATCH".getEnv("_")])
 
     block CT_FFI:
       when defined(posix): # windows can be handled in future PR's

testament/categories.nim

@@ -34,7 +34,7 @@ const
     "nimble-packages-2",
     "niminaction",
     "threads",
-    "untestable",
+    "untestable", # see trunner_special
     "testdata",
     "nimcache",
     "coroutines",
@@ -711,8 +711,9 @@ proc processCategory(r: var TResults, cat: Category,
     of "ic":
       icTests(r, testsDir, cat, options)
     of "untestable":
-      # We can't test it because it depends on a third party.
+      # These require special treatment e.g. because they depend on a third party
-      discard # TODO: Move untestable tests to someplace else, i.e. nimble repo.
+      # dependency; see `trunner_special` which runs some of those.
+      discard
     else:
       handled = false
   if not handled:

testament/lib/stdtest/testutils.nim

@@ -1,5 +1,6 @@
 import std/private/miscdollars
 import std/strutils
+from std/os import getEnv
 
 template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) =
   ## API to deal with flaky or failing tests. This avoids disabling entire tests
@@ -34,3 +35,10 @@ proc greedyOrderedSubsetLines*(lhs, rhs: string): bool =
     if currentPos < 0:
       return false
   return true
+
+template enableRemoteNetworking*: bool =
+  ## Allows contolling whether to run some test at a statement-level granularity.
+  ## Using environment variables simplifies propagating this all the way across
+  ## process calls, e.g. `testament all` calls itself, which in turns invokes
+  ## a `nim` invocation (possibly via additional intermediate processes).
+  getEnv("NIM_TESTAMENT_REMOTE_NETWORKING") == "1"

testament/testament.nim

@@ -52,6 +52,9 @@ Options:
 
 On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API
 provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN.
+
+Experimental: using environment variable `NIM_TESTAMENT_REMOTE_NETWORKING=1` enables
+tests with remote networking (as in CI).
 """ % resultsFile
 
 proc isNimRepoTests(): bool =
@@ -680,7 +683,7 @@ proc main() =
     case p.key.normalize
     of "print", "verbose": optPrintResults = true
     of "failing": optFailing = true
-    of "pedantic": discard "now always enabled"
+    of "pedantic": discard # deadcode
     of "targets":
       targetsStr = p.val
       gTargets = parseTargets(targetsStr)

tests/config.nims

@@ -17,3 +17,7 @@ switch("define", "nimLegacyTypeMismatch")
 
 hint("Processing", off)
   # dots can cause annoyances; instead, a single test can test `hintProcessing`
+
+# uncomment to enable all flaky tests disabled by this flag
+# (works through process calls, e.g. tests that invoke nim).
+# switch("define", "nimTestsEnableFlaky")

tests/misc/trunner.nim

@@ -7,7 +7,7 @@ discard """
 ## A few others could be added to here to simplify code.
 ## Note: this test is a bit slow but tests a lot of things; please don't disable.
 
-import std/[strformat,os,osproc,unittest]
+import std/[strformat,os,osproc,unittest,compilesettings]
 from std/sequtils import toSeq,mapIt
 from std/algorithm import sorted
 import stdtest/[specialpaths, unittest_light]
@@ -25,10 +25,7 @@ const
     # useful when you want to turn only some hints on, and some common ones off.
     # pending https://github.com/timotheecour/Nim/issues/453, simplify to: `--hints:off`
   nim = getCurrentCompilerExe()
-  mode =
+  mode = querySetting(backend)
-    when defined(c): "c"
-    elif defined(cpp): "cpp"
-    else: static: doAssert false
   nimcache = buildDir / "nimcacheTrunner"
     # instead of `querySetting(nimcacheDir)`, avoids stomping on other parallel tests
 

tests/misc/trunner_special.nim

@@ -0,0 +1,31 @@
+discard """
+  targets: "c cpp"
+  joinable: false
+"""
+
+#[
+Runs tests that require special treatment, e.g. because they rely on 3rd party code
+or require external networking.
+
+xxx test all tests/untestable/* here, possibly with adjustments to make running times reasonable
+]#
+
+import std/[strformat,os,unittest,compilesettings]
+import stdtest/specialpaths
+
+const
+  nim = getCurrentCompilerExe()
+  mode = querySetting(backend)
+
+proc runCmd(cmd: string) =
+  let ret = execShellCmd(cmd)
+  check ret == 0 # allows more than 1 failure
+
+proc main =
+  let options = fmt"-b:{mode} --hints:off"
+  block: # SSL nimDisableCertificateValidation integration tests
+    runCmd fmt"{nim} r {options} -d:nimDisableCertificateValidation -d:ssl {testsDir}/untestable/thttpclient_ssl_disabled.nim"
+  block: # SSL certificate check integration tests
+    runCmd fmt"{nim} r {options} -d:ssl --threads:on {testsDir}/untestable/thttpclient_ssl_remotenetwork.nim"
+
+main()

tests/untestable/thttpclient_ssl.nim

@@ -1,207 +0,0 @@
-#
-#
-#            Nim - SSL integration tests
-#        (c) Copyright 2017 Nim contributors
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-## Warning: this test performs external networking.
-## Test with:
-## ./bin/nim c -d:ssl -p:. --threads:on -r tests/untestable/thttpclient_ssl.nim
-##
-## See https://github.com/FedericoCeratto/ssl-comparison/blob/master/README.md
-## for a comparison with other clients.
-
-import
-  httpclient,
-  net,
-  strutils,
-  threadpool,
-  unittest
-
-
-type
-  # bad and dubious tests should not pass SSL validation
-  # "_broken" mark the test as skipped. Some tests have different
-  # behavior depending on OS and SSL version!
-  # TODO: chase and fix the broken tests
-  Category = enum
-    good, bad, dubious, good_broken, bad_broken, dubious_broken
-  CertTest = tuple[url:string, category:Category, desc: string]
-
-const certificate_tests: array[0..55, CertTest] = [
-  ("https://wrong.host.badssl.com/", bad, "wrong.host"),
-  ("https://captive-portal.badssl.com/", bad, "captive-portal"),
-  ("https://expired.badssl.com/", bad, "expired"),
-  ("https://google.com/", good, "good"),
-  ("https://self-signed.badssl.com/", bad, "self-signed"),
-  ("https://untrusted-root.badssl.com/", bad, "untrusted-root"),
-  ("https://revoked.badssl.com/", bad_broken, "revoked"),
-  ("https://pinning-test.badssl.com/", bad_broken, "pinning-test"),
-  ("https://no-common-name.badssl.com/", dubious_broken, "no-common-name"),
-  ("https://no-subject.badssl.com/", dubious_broken, "no-subject"),
-  ("https://incomplete-chain.badssl.com/", dubious_broken, "incomplete-chain"),
-  ("https://sha1-intermediate.badssl.com/", bad, "sha1-intermediate"),
-  ("https://sha256.badssl.com/", good, "sha256"),
-  ("https://sha384.badssl.com/", good, "sha384"),
-  ("https://sha512.badssl.com/", good, "sha512"),
-  ("https://1000-sans.badssl.com/", good, "1000-sans"),
-  ("https://10000-sans.badssl.com/", good_broken, "10000-sans"),
-  ("https://ecc256.badssl.com/", good, "ecc256"),
-  ("https://ecc384.badssl.com/", good, "ecc384"),
-  ("https://rsa2048.badssl.com/", good, "rsa2048"),
-  ("https://rsa8192.badssl.com/", dubious_broken, "rsa8192"),
-  ("http://http.badssl.com/", good, "regular http"),
-  ("https://http.badssl.com/", bad_broken, "http on https URL"),  # FIXME
-  ("https://cbc.badssl.com/", dubious, "cbc"),
-  ("https://rc4-md5.badssl.com/", bad, "rc4-md5"),
-  ("https://rc4.badssl.com/", bad, "rc4"),
-  ("https://3des.badssl.com/", bad, "3des"),
-  ("https://null.badssl.com/", bad, "null"),
-  ("https://mozilla-old.badssl.com/", bad_broken, "mozilla-old"),
-  ("https://mozilla-intermediate.badssl.com/", dubious_broken, "mozilla-intermediate"),
-  ("https://mozilla-modern.badssl.com/", good, "mozilla-modern"),
-  ("https://dh480.badssl.com/", bad, "dh480"),
-  ("https://dh512.badssl.com/", bad, "dh512"),
-  ("https://dh1024.badssl.com/", dubious_broken, "dh1024"),
-  ("https://dh2048.badssl.com/", good, "dh2048"),
-  ("https://dh-small-subgroup.badssl.com/", bad_broken, "dh-small-subgroup"),
-  ("https://dh-composite.badssl.com/", bad_broken, "dh-composite"),
-  ("https://static-rsa.badssl.com/", dubious, "static-rsa"),
-  ("https://tls-v1-0.badssl.com:1010/", dubious, "tls-v1-0"),
-  ("https://tls-v1-1.badssl.com:1011/", dubious, "tls-v1-1"),
-  ("https://invalid-expected-sct.badssl.com/", bad, "invalid-expected-sct"),
-  ("https://hsts.badssl.com/", good, "hsts"),
-  ("https://upgrade.badssl.com/", good, "upgrade"),
-  ("https://preloaded-hsts.badssl.com/", good, "preloaded-hsts"),
-  ("https://subdomain.preloaded-hsts.badssl.com/", bad, "subdomain.preloaded-hsts"),
-  ("https://https-everywhere.badssl.com/", good, "https-everywhere"),
-  ("https://long-extended-subdomain-name-containing-many-letters-and-dashes.badssl.com/", good,
-    "long-extended-subdomain-name-containing-many-letters-and-dashes"),
-  ("https://longextendedsubdomainnamewithoutdashesinordertotestwordwrapping.badssl.com/", good,
-    "longextendedsubdomainnamewithoutdashesinordertotestwordwrapping"),
-  ("https://superfish.badssl.com/", bad, "(Lenovo) Superfish"),
-  ("https://edellroot.badssl.com/", bad, "(Dell) eDellRoot"),
-  ("https://dsdtestprovider.badssl.com/", bad, "(Dell) DSD Test Provider"),
-  ("https://preact-cli.badssl.com/", bad, "preact-cli"),
-  ("https://webpack-dev-server.badssl.com/", bad, "webpack-dev-server"),
-  ("https://mitm-software.badssl.com/", bad, "mitm-software"),
-  ("https://sha1-2016.badssl.com/", dubious, "sha1-2016"),
-  ("https://sha1-2017.badssl.com/", bad, "sha1-2017"),
-]
-
-
-template evaluate(exception_msg: string, category: Category, desc: string) =
-  # Evaluate test outcome. Testes flagged as _broken are evaluated and skipped
-  let raised = (exception_msg.len > 0)
-  let should_not_raise = category in {good, dubious_broken, bad_broken}
-  if should_not_raise xor raised:
-    # we are seeing a known behavior
-    if category in {good_broken, dubious_broken, bad_broken}:
-      skip()
-    if raised:
-      # check exception_msg == "No SSL certificate found." or
-      doAssert exception_msg == "No SSL certificate found." or
-        exception_msg == "SSL Certificate check failed." or
-        exception_msg.contains("certificate verify failed") or
-        exception_msg.contains("key too small") or
-        exception_msg.contains("alert handshake failure") or
-        exception_msg.contains("bad dh p length") or
-        # TODO: This one should only triggers for 10000-sans
-        exception_msg.contains("excessive message size"), exception_msg
-
-  else:
-    # this is unexpected
-    if raised:
-      echo "         $# ($#) raised: $#" % [desc, $category, exception_msg]
-    else:
-      echo "         $# ($#) did not raise" % [desc, $category]
-    if category in {good, dubious, bad}:
-      fail()
-
-
-suite "SSL certificate check - httpclient":
-
-  for i, ct in certificate_tests:
-
-    test ct.desc:
-      var ctx = newContext(verifyMode=CVerifyPeer)
-      var client = newHttpClient(sslContext=ctx)
-      let exception_msg =
-        try:
-          let a = $client.getContent(ct.url)
-          ""
-        except:
-          getCurrentExceptionMsg()
-
-      evaluate(exception_msg, ct.category, ct.desc)
-
-
-
-# threaded tests
-
-
-type
-  TTOutcome = ref object
-    desc, exception_msg: string
-    category: Category
-
-proc run_t_test(ct: CertTest): TTOutcome {.thread.} =
-  ## Run test in a {.thread.} - return by ref
-  result = TTOutcome(desc:ct.desc, exception_msg:"", category: ct.category)
-  try:
-    var ctx = newContext(verifyMode=CVerifyPeer)
-    var client = newHttpClient(sslContext=ctx)
-    let a = $client.getContent(ct.url)
-  except:
-    result.exception_msg = getCurrentExceptionMsg()
-
-
-suite "SSL certificate check - httpclient - threaded":
-
-  # Spawn threads before the "test" blocks
-  var outcomes = newSeq[FlowVar[TTOutcome]](certificate_tests.len)
-  for i, ct in certificate_tests:
-    let t = spawn run_t_test(ct)
-    outcomes[i] = t
-
-  # create "test" blocks and handle thread outputs
-  for t in outcomes:
-    let outcome = ^t  # wait for a thread to terminate
-
-    test outcome.desc:
-
-      evaluate(outcome.exception_msg, outcome.category, outcome.desc)
-
-
-# net tests
-
-
-type NetSocketTest = tuple[hostname: string, port: Port, category:Category, desc: string]
-const net_tests:array[0..3, NetSocketTest] = [
-  ("imap.gmail.com", 993.Port, good, "IMAP"),
-  ("wrong.host.badssl.com", 443.Port, bad, "wrong.host"),
-  ("captive-portal.badssl.com", 443.Port, bad, "captive-portal"),
-  ("expired.badssl.com", 443.Port, bad, "expired"),
-]
-# TODO: ("null.badssl.com", 443.Port, bad_broken, "null"),
-
-
-suite "SSL certificate check - sockets":
-
-  for ct in net_tests:
-
-    test ct.desc:
-
-      var sock = newSocket()
-      var ctx = newContext()
-      ctx.wrapSocket(sock)
-      let exception_msg =
-        try:
-          sock.connect(ct.hostname, ct.port)
-          ""
-        except:
-          getCurrentExceptionMsg()
-
-      evaluate(exception_msg, ct.category, ct.desc)

tests/untestable/thttpclient_ssl_disabled.nim

@@ -5,36 +5,32 @@
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 #
-## Warning: this test performs external networking.
 ## Compile and run with:
-## ./bin/nim c -d:nimDisableCertificateValidation -d:ssl -r -p:. tests/untestable/thttpclient_ssl_disabled.nim
+## nim r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimDisableCertificateValidation -d:ssl -p:. tests/untestable/thttpclient_ssl_disabled.nim
 
-import httpclient,
+from stdtest/testutils import enableRemoteNetworking
-  net,
+when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(openbsd)):
-  unittest,
+  import httpclient, net, unittest
-  os
 
-from strutils import contains
+  const expired = "https://expired.badssl.com/"
 
-const expired = "https://expired.badssl.com/"
+  doAssert defined(nimDisableCertificateValidation)
 
-doAssert defined(nimDisableCertificateValidation)
+  suite "SSL certificate check - disabled":
 
-suite "SSL certificate check - disabled":
+    test "httpclient in insecure mode":
+      var ctx = newContext(verifyMode = CVerifyPeer)
+      var client = newHttpClient(sslContext = ctx)
+      let a = $client.getContent(expired)
 
-  test "httpclient in insecure mode":
+    test "httpclient in insecure mode":
-    var ctx = newContext(verifyMode = CVerifyPeer)
+      var ctx = newContext(verifyMode = CVerifyPeerUseEnvVars)
-    var client = newHttpClient(sslContext = ctx)
+      var client = newHttpClient(sslContext = ctx)
-    let a = $client.getContent(expired)
+      let a = $client.getContent(expired)
 
-  test "httpclient in insecure mode":
+    test "net socket in insecure mode":
-    var ctx = newContext(verifyMode = CVerifyPeerUseEnvVars)
+      var sock = newSocket()
-    var client = newHttpClient(sslContext = ctx)
+      var ctx = newContext(verifyMode = CVerifyPeerUseEnvVars)
-    let a = $client.getContent(expired)
+      ctx.wrapSocket(sock)
-
+      sock.connect("expired.badssl.com", 443.Port)
-  test "net socket in insecure mode":
+      sock.close
-    var sock = newSocket()
-    var ctx = newContext(verifyMode = CVerifyPeerUseEnvVars)
-    ctx.wrapSocket(sock)
-    sock.connect("expired.badssl.com", 443.Port)
-    sock.close

tests/untestable/thttpclient_ssl_remotenetwork.nim

@@ -0,0 +1,209 @@
+#
+#
+#            Nim - SSL integration tests
+#        (c) Copyright 2017 Nim contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+## Test with:
+## nim r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:ssl -p:. --threads:on tests/untestable/thttpclient_ssl_remotenetwork.nim
+##
+## See https://github.com/FedericoCeratto/ssl-comparison/blob/master/README.md
+## for a comparison with other clients.
+
+from stdtest/testutils import enableRemoteNetworking
+when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(windows) and not defined(openbsd) and not defined(i386)):
+  # Not supported on Windows due to old openssl version
+  import
+    httpclient,
+    net,
+    strutils,
+    threadpool,
+    unittest
+
+
+  type
+    # bad and dubious tests should not pass SSL validation
+    # "_broken" mark the test as skipped. Some tests have different
+    # behavior depending on OS and SSL version!
+    # TODO: chase and fix the broken tests
+    Category = enum
+      good, bad, dubious, good_broken, bad_broken, dubious_broken
+    CertTest = tuple[url:string, category:Category, desc: string]
+
+  const certificate_tests: array[0..55, CertTest] = [
+    ("https://wrong.host.badssl.com/", bad, "wrong.host"),
+    ("https://captive-portal.badssl.com/", bad, "captive-portal"),
+    ("https://expired.badssl.com/", bad, "expired"),
+    ("https://google.com/", good, "good"),
+    ("https://self-signed.badssl.com/", bad, "self-signed"),
+    ("https://untrusted-root.badssl.com/", bad, "untrusted-root"),
+    ("https://revoked.badssl.com/", bad_broken, "revoked"),
+    ("https://pinning-test.badssl.com/", bad_broken, "pinning-test"),
+    ("https://no-common-name.badssl.com/", dubious_broken, "no-common-name"),
+    ("https://no-subject.badssl.com/", dubious_broken, "no-subject"),
+    ("https://incomplete-chain.badssl.com/", dubious_broken, "incomplete-chain"),
+    ("https://sha1-intermediate.badssl.com/", bad, "sha1-intermediate"),
+    ("https://sha256.badssl.com/", good, "sha256"),
+    ("https://sha384.badssl.com/", good, "sha384"),
+    ("https://sha512.badssl.com/", good, "sha512"),
+    ("https://1000-sans.badssl.com/", good, "1000-sans"),
+    ("https://10000-sans.badssl.com/", good_broken, "10000-sans"),
+    ("https://ecc256.badssl.com/", good, "ecc256"),
+    ("https://ecc384.badssl.com/", good, "ecc384"),
+    ("https://rsa2048.badssl.com/", good, "rsa2048"),
+    ("https://rsa8192.badssl.com/", dubious_broken, "rsa8192"),
+    ("http://http.badssl.com/", good, "regular http"),
+    ("https://http.badssl.com/", bad_broken, "http on https URL"),  # FIXME
+    ("https://cbc.badssl.com/", dubious, "cbc"),
+    ("https://rc4-md5.badssl.com/", bad, "rc4-md5"),
+    ("https://rc4.badssl.com/", bad, "rc4"),
+    ("https://3des.badssl.com/", bad, "3des"),
+    ("https://null.badssl.com/", bad, "null"),
+    ("https://mozilla-old.badssl.com/", bad_broken, "mozilla-old"),
+    ("https://mozilla-intermediate.badssl.com/", dubious_broken, "mozilla-intermediate"),
+    ("https://mozilla-modern.badssl.com/", good, "mozilla-modern"),
+    ("https://dh480.badssl.com/", bad, "dh480"),
+    ("https://dh512.badssl.com/", bad, "dh512"),
+    ("https://dh1024.badssl.com/", dubious_broken, "dh1024"),
+    ("https://dh2048.badssl.com/", good, "dh2048"),
+    ("https://dh-small-subgroup.badssl.com/", bad_broken, "dh-small-subgroup"),
+    ("https://dh-composite.badssl.com/", bad_broken, "dh-composite"),
+    ("https://static-rsa.badssl.com/", dubious, "static-rsa"),
+    ("https://tls-v1-0.badssl.com:1010/", dubious, "tls-v1-0"),
+    ("https://tls-v1-1.badssl.com:1011/", dubious, "tls-v1-1"),
+    ("https://invalid-expected-sct.badssl.com/", bad, "invalid-expected-sct"),
+    ("https://hsts.badssl.com/", good, "hsts"),
+    ("https://upgrade.badssl.com/", good, "upgrade"),
+    ("https://preloaded-hsts.badssl.com/", good, "preloaded-hsts"),
+    ("https://subdomain.preloaded-hsts.badssl.com/", bad, "subdomain.preloaded-hsts"),
+    ("https://https-everywhere.badssl.com/", good, "https-everywhere"),
+    ("https://long-extended-subdomain-name-containing-many-letters-and-dashes.badssl.com/", good,
+      "long-extended-subdomain-name-containing-many-letters-and-dashes"),
+    ("https://longextendedsubdomainnamewithoutdashesinordertotestwordwrapping.badssl.com/", good,
+      "longextendedsubdomainnamewithoutdashesinordertotestwordwrapping"),
+    ("https://superfish.badssl.com/", bad, "(Lenovo) Superfish"),
+    ("https://edellroot.badssl.com/", bad, "(Dell) eDellRoot"),
+    ("https://dsdtestprovider.badssl.com/", bad, "(Dell) DSD Test Provider"),
+    ("https://preact-cli.badssl.com/", bad, "preact-cli"),
+    ("https://webpack-dev-server.badssl.com/", bad, "webpack-dev-server"),
+    ("https://mitm-software.badssl.com/", bad, "mitm-software"),
+    ("https://sha1-2016.badssl.com/", dubious, "sha1-2016"),
+    ("https://sha1-2017.badssl.com/", bad, "sha1-2017"),
+  ]
+
+
+  template evaluate(exception_msg: string, category: Category, desc: string) =
+    # Evaluate test outcome. Testes flagged as _broken are evaluated and skipped
+    let raised = (exception_msg.len > 0)
+    let should_not_raise = category in {good, dubious_broken, bad_broken}
+    if should_not_raise xor raised:
+      # we are seeing a known behavior
+      if category in {good_broken, dubious_broken, bad_broken}:
+        skip()
+      if raised:
+        # check exception_msg == "No SSL certificate found." or
+        doAssert exception_msg == "No SSL certificate found." or
+          exception_msg == "SSL Certificate check failed." or
+          exception_msg.contains("certificate verify failed") or
+          exception_msg.contains("key too small") or
+          exception_msg.contains("alert handshake failure") or
+          exception_msg.contains("bad dh p length") or
+          # TODO: This one should only triggers for 10000-sans
+          exception_msg.contains("excessive message size"), exception_msg
+
+    else:
+      # this is unexpected
+      if raised:
+        echo "         $# ($#) raised: $#" % [desc, $category, exception_msg]
+      else:
+        echo "         $# ($#) did not raise" % [desc, $category]
+      if category in {good, dubious, bad}:
+        fail()
+
+
+  suite "SSL certificate check - httpclient":
+
+    for i, ct in certificate_tests:
+
+      test ct.desc:
+        var ctx = newContext(verifyMode=CVerifyPeer)
+        var client = newHttpClient(sslContext=ctx)
+        let exception_msg =
+          try:
+            let a = $client.getContent(ct.url)
+            ""
+          except:
+            getCurrentExceptionMsg()
+
+        evaluate(exception_msg, ct.category, ct.desc)
+
+
+
+  # threaded tests
+
+
+  type
+    TTOutcome = ref object
+      desc, exception_msg: string
+      category: Category
+
+  proc run_t_test(ct: CertTest): TTOutcome {.thread.} =
+    ## Run test in a {.thread.} - return by ref
+    result = TTOutcome(desc:ct.desc, exception_msg:"", category: ct.category)
+    try:
+      var ctx = newContext(verifyMode=CVerifyPeer)
+      var client = newHttpClient(sslContext=ctx)
+      let a = $client.getContent(ct.url)
+    except:
+      result.exception_msg = getCurrentExceptionMsg()
+
+
+  suite "SSL certificate check - httpclient - threaded":
+
+    # Spawn threads before the "test" blocks
+    var outcomes = newSeq[FlowVar[TTOutcome]](certificate_tests.len)
+    for i, ct in certificate_tests:
+      let t = spawn run_t_test(ct)
+      outcomes[i] = t
+
+    # create "test" blocks and handle thread outputs
+    for t in outcomes:
+      let outcome = ^t  # wait for a thread to terminate
+
+      test outcome.desc:
+
+        evaluate(outcome.exception_msg, outcome.category, outcome.desc)
+
+
+  # net tests
+
+
+  type NetSocketTest = tuple[hostname: string, port: Port, category:Category, desc: string]
+  const net_tests:array[0..3, NetSocketTest] = [
+    ("imap.gmail.com", 993.Port, good, "IMAP"),
+    ("wrong.host.badssl.com", 443.Port, bad, "wrong.host"),
+    ("captive-portal.badssl.com", 443.Port, bad, "captive-portal"),
+    ("expired.badssl.com", 443.Port, bad, "expired"),
+  ]
+  # TODO: ("null.badssl.com", 443.Port, bad_broken, "null"),
+
+
+  suite "SSL certificate check - sockets":
+
+    for ct in net_tests:
+
+      test ct.desc:
+
+        var sock = newSocket()
+        var ctx = newContext()
+        ctx.wrapSocket(sock)
+        let exception_msg =
+          try:
+            sock.connect(ct.hostname, ct.port)
+            ""
+          except:
+            getCurrentExceptionMsg()
+
+        evaluate(exception_msg, ct.category, ct.desc)