Fix issue #10726 - HTTP response without Content-Length is not accessible (#11904)

* Add patch by @xenogenesi

* Async test for HTTP/1.1 without Content-Length

* Apply suggestions from code review

Co-Authored-By: Dominik Picheta <dominikpicheta@googlemail.com>

lib/pure/httpclient.nim

@@ -340,7 +340,10 @@ proc parseBody(s: Socket, headers: HttpHeaders, httpVersion: string, timeout: in
 
       # -REGION- Connection: Close
       # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5
-      if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0":
+      let implicitConnectionClose =
+        httpVersion == "1.0" or
+        httpVersion == "1.1" # This doesn't match the HTTP spec, but it fixes issues for non-conforming servers.
+      if headers.getOrDefault"Connection" == "close" or implicitConnectionClose:
         var buf = ""
         while true:
           buf = newString(4000)
@@ -811,7 +814,10 @@ proc parseBody(client: HttpClient | AsyncHttpClient,
 
       # -REGION- Connection: Close
       # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5
-      if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0":
+      let implicitConnectionClose =
+        httpVersion == "1.0" or
+        httpVersion == "1.1" # This doesn't match the HTTP spec, but it fixes issues for non-conforming servers.
+      if headers.getOrDefault"Connection" == "close" or implicitConnectionClose:
         while true:
           let recvLen = await client.recvFull(4000, client.timeout, true)
           if recvLen != 4000:

tests/stdlib/thttpclient.nim

@@ -13,6 +13,31 @@ import nativesockets, os, httpclient, asyncdispatch
 
 const manualTests = false
 
+proc makeIPv6HttpServer(hostname: string, port: Port,
+    message: string): AsyncFD =
+  let fd = newNativeSocket(AF_INET6)
+  setSockOptInt(fd, SOL_SOCKET, SO_REUSEADDR, 1)
+  var aiList = getAddrInfo(hostname, port, AF_INET6)
+  if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32:
+    freeAddrInfo(aiList)
+    raiseOSError(osLastError())
+  freeAddrInfo(aiList)
+  if listen(fd) != 0:
+    raiseOSError(osLastError())
+  setBlocking(fd, false)
+
+  var serverFd = fd.AsyncFD
+  register(serverFd)
+  result = serverFd
+
+  proc onAccept(fut: Future[AsyncFD]) {.gcsafe.} =
+    if not fut.failed:
+      let clientFd = fut.read()
+      clientFd.send(message).callback = proc() =
+        clientFd.closeSocket()
+      serverFd.accept().callback = onAccept
+  serverFd.accept().callback = onAccept
+
 proc asyncTest() {.async.} =
   var client = newAsyncHttpClient()
   var resp = await client.request("http://example.com/")
@@ -46,7 +71,7 @@ proc asyncTest() {.async.} =
     data["output"] = "soap12"
     data["uploaded_file"] = ("test.html", "text/html",
       "<html><head></head><body><p>test</p></body></html>")
-    resp = await client.post("http://validator.w3.org/check", multipart=data)
+    resp = await client.post("http://validator.w3.org/check", multipart = data)
     doAssert(resp.code.is2xx)
 
   # onProgressChanged
@@ -58,6 +83,16 @@ proc asyncTest() {.async.} =
     await client.downloadFile("http://speedtest-ams2.digitalocean.com/100mb.test",
                               "100mb.test")
 
+  # HTTP/1.1 without Content-Length - issue #10726
+  var serverFd = makeIPv6HttpServer("::1", Port(18473),
+     "HTTP/1.1 200 \c\L" &
+     "\c\L" &
+     "Here comes reply")
+  resp = await client.request("http://[::1]:18473/")
+  body = await resp.body
+  doAssert(body == "Here comes reply")
+  serverFd.closeSocket()
+
   client.close()
 
   # Proxy test
@@ -96,7 +131,7 @@ proc syncTest() =
     data["output"] = "soap12"
     data["uploaded_file"] = ("test.html", "text/html",
       "<html><head></head><body><p>test</p></body></html>")
-    resp = client.post("http://validator.w3.org/check", multipart=data)
+    resp = client.post("http://validator.w3.org/check", multipart = data)
     doAssert(resp.code.is2xx)
 
   # onProgressChanged
@@ -122,40 +157,17 @@ proc syncTest() =
     except:
       doAssert false, "TimeoutError should have been raised."
 
-proc makeIPv6HttpServer(hostname: string, port: Port): AsyncFD =
-  let fd = newNativeSocket(AF_INET6)
-  setSockOptInt(fd, SOL_SOCKET, SO_REUSEADDR, 1)
-  var aiList = getAddrInfo(hostname, port, AF_INET6)
-  if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32:
-    freeAddrInfo(aiList)
-    raiseOSError(osLastError())
-  freeAddrInfo(aiList)
-  if listen(fd) != 0:
-    raiseOSError(osLastError())
-  setBlocking(fd, false)
-
-  var serverFd = fd.AsyncFD
-  register(serverFd)
-  result = serverFd
-
-  proc onAccept(fut: Future[AsyncFD]) {.gcsafe.} =
-    if not fut.failed:
-      let clientFd = fut.read()
-      clientFd.send("HTTP/1.1 200 OK\r\LContent-Length: 0\r\LConnection: Closed\r\L\r\L").callback = proc() =
-        clientFd.closeSocket()
-      serverFd.accept().callback = onAccept
-  serverFd.accept().callback = onAccept
-
 proc ipv6Test() =
   var client = newAsyncHttpClient()
-  let serverFd = makeIPv6HttpServer("::1", Port(18473))
+  let serverFd = makeIPv6HttpServer("::1", Port(18473),
+    "HTTP/1.1 200 OK\r\LContent-Length: 0\r\LConnection: Closed\r\L\r\L")
   var resp = waitFor client.request("http://[::1]:18473/")
   doAssert(resp.status == "200 OK")
   serverFd.closeSocket()
   client.close()
 
+ipv6Test()
 syncTest()
 waitFor(asyncTest())
-ipv6Test()
 
 echo "OK"