Merge pull request #1867 from def-/httpclient-multipart

Add multipart parameter to httpclient's post and postContent

lib/pure/httpclient.nim

@@ -32,21 +32,12 @@
 ## the server.
 ##
 ## .. code-block:: Nim
-##   var headers: string = "Content-Type: multipart/form-data; boundary=xyz\c\L"
+##   var data = newMultipartData()
-##   var body: string = "--xyz\c\L"
+##   data["output"] = "soap12"
-##   # soap 1.2 output
+##   data["uploaded_file"] = ("test.html", "text/html",
-##   body.add("Content-Disposition: form-data; name=\"output\"\c\L")
+##     "<html><head></head><body><p>test</p></body></html>")
-##   body.add("\c\Lsoap12\c\L")
 ##
-##   # html
+##   echo postContent("http://validator.w3.org/check", multipart=data)
-##   body.add("--xyz\c\L")
-##   body.add("Content-Disposition: form-data; name=\"uploaded_file\";" &
-##            " filename=\"test.html\"\c\L")
-##   body.add("Content-Type: text/html\c\L")
-##   body.add("\c\L<html><head></head><body><p>test</p></body></html>\c\L")
-##   body.add("--xyz--")
-##
-##   echo(postContent("http://validator.w3.org/check", headers, body))
 ##
 ## Asynchronous HTTP requests
 ## ==========================
@@ -88,7 +79,7 @@
 ## constructor should be used for this purpose. However,
 ## currently only basic authentication is supported.
 
-import net, strutils, uri, parseutils, strtabs, base64, os
+import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes, math
 import asyncnet, asyncdispatch
 import rawsockets
 
@@ -103,6 +94,10 @@ type
     url*: Uri
     auth*: string
 
+  MultipartEntries* = openarray[tuple[name, content: string]]
+  MultipartData* = ref object
+    content: seq[string]
+
   ProtocolError* = object of IOError   ## exception that is raised when server
                                        ## does not conform to the implemented
                                        ## protocol
@@ -282,6 +277,109 @@ proc newProxy*(url: string, auth = ""): Proxy =
   ## Constructs a new ``TProxy`` object.
   result = Proxy(url: parseUri(url), auth: auth)
 
+proc newMultipartData*: MultipartData =
+  ## Constructs a new ``MultipartData`` object.
+  MultipartData(content: @[])
+
+proc add*(p: var MultipartData, name, content: string, filename: string = nil,
+          contentType: string = nil) =
+  ## Add a value to the multipart data. Raises a `ValueError` exception if
+  ## `name`, `filename` or `contentType` contain newline characters.
+
+  if {'\c','\L'} in name:
+    raise newException(ValueError, "name contains a newline character")
+  if filename != nil and {'\c','\L'} in filename:
+    raise newException(ValueError, "filename contains a newline character")
+  if contentType != nil and {'\c','\L'} in contentType:
+    raise newException(ValueError, "contentType contains a newline character")
+
+  var str = "Content-Disposition: form-data; name=\"" & name & "\""
+  if filename != nil:
+    str.add("; filename=\"" & filename & "\"")
+  str.add("\c\L")
+  if contentType != nil:
+    str.add("Content-Type: " & contentType & "\c\L")
+  str.add("\c\L" & content & "\c\L")
+
+  p.content.add(str)
+
+proc add*(p: var MultipartData, xs: MultipartEntries): MultipartData
+         {.discardable.} =
+  ## Add a list of multipart entries to the multipart data `p`. All values are
+  ## added without a filename and without a content type.
+  ##
+  ## .. code-block:: Nim
+  ##   data.add({"action": "login", "format": "json"})
+  for name, content in xs.items:
+    p.add(name, content)
+  result = p
+
+proc newMultipartData*(xs: MultipartEntries): MultipartData =
+  ## Create a new multipart data object and fill it with the entries `xs`
+  ## directly.
+  ##
+  ## .. code-block:: Nim
+  ##   var data = newMultipartData({"action": "login", "format": "json"})
+  result = MultipartData(content: @[])
+  result.add(xs)
+
+proc addFiles*(p: var MultipartData, xs: openarray[tuple[name, file: string]]):
+              MultipartData {.discardable.} =
+  ## Add files to a multipart data object. The file will be opened from your
+  ## disk, read and sent with the automatically determined MIME type. Raises an
+  ## `IOError` if the file cannot be opened or reading fails. To manually
+  ## specify file content, filename and MIME type, use `[]=` instead.
+  ##
+  ## .. code-block:: Nim
+  ##   data.addFiles({"uploaded_file": "public/test.html"})
+  var m = newMimetypes()
+  for name, file in xs.items:
+    var contentType: string
+    let (dir, fName, ext) = splitFile(file)
+    if ext.len > 0:
+      contentType = m.getMimetype(ext[1..ext.high], nil)
+    p.add(name, readFile(file), fName & ext, contentType)
+  result = p
+
+proc `[]=`*(p: var MultipartData, name, content: string) =
+  ## Add a multipart entry to the multipart data `p`. The value is added
+  ## without a filename and without a content type.
+  ##
+  ## .. code-block:: Nim
+  ##   data["username"] = "NimUser"
+  p.add(name, content)
+
+proc `[]=`*(p: var MultipartData, name: string,
+            file: tuple[name, contentType, content: string]) =
+  ## Add a file to the multipart data `p`, specifying filename, contentType and
+  ## content manually.
+  ##
+  ## .. code-block:: Nim
+  ##   data["uploaded_file"] = ("test.html", "text/html",
+  ##     "<html><head></head><body><p>test</p></body></html>")
+  p.add(name, file.content, file.name, file.contentType)
+
+proc format(p: MultipartData): tuple[header, body: string] =
+  if p == nil or p.content == nil or p.content.len == 0:
+    return ("", "")
+
+  # Create boundary that is not in the data to be formatted
+  var bound: string
+  while true:
+    bound = $random(int.high)
+    var found = false
+    for s in p.content:
+      if bound in s:
+        found = true
+    if not found:
+      break
+
+  result.header = "Content-Type: multipart/form-data; boundary=" & bound & "\c\L"
+  result.body = ""
+  for s in p.content:
+    result.body.add("--" & bound & "\c\L" & s)
+  result.body.add("--" & bound & "--\c\L")
+
 proc request*(url: string, httpMethod = httpGET, extraHeaders = "",
               body = "",
               sslContext: SSLContext = defaultSSLContext,
@@ -389,15 +487,30 @@ proc post*(url: string, extraHeaders = "", body = "",
            maxRedirects = 5,
            sslContext: SSLContext = defaultSSLContext,
            timeout = -1, userAgent = defUserAgent,
-           proxy: Proxy = nil): Response =
+           proxy: Proxy = nil,
+           multipart: MultipartData = nil): Response =
   ## | POSTs ``body`` to the ``url`` and returns a ``Response`` object.
   ## | This proc adds the necessary Content-Length header.
   ## | This proc also handles redirection.
   ## | Extra headers can be specified and must be separated by ``\c\L``.
   ## | An optional timeout can be specified in miliseconds, if reading from the
   ## server takes longer than specified an ETimeout exception will be raised.
-  var xh = extraHeaders & "Content-Length: " & $len(body) & "\c\L"
+  ## | The optional ``multipart`` parameter can be used to create
-  result = request(url, httpPOST, xh, body, sslContext, timeout, userAgent,
+  ## ``multipart/form-data`` POSTs comfortably.
+  let (mpHeaders, mpBody) = format(multipart)
+
+  template withNewLine(x): expr =
+    if x.len > 0 and not x.endsWith("\c\L"):
+      x & "\c\L"
+    else:
+      x
+
+  var xb = mpBody.withNewLine() & body.withNewLine()
+
+  var xh = extraHeaders.withNewLine() & mpHeaders.withNewLine() &
+    withNewLine("Content-Length: " & $len(xb))
+
+  result = request(url, httpPOST, xh, xb, sslContext, timeout, userAgent,
                    proxy)
   var lastUrl = ""
   for i in 1..maxRedirects:
@@ -412,14 +525,17 @@ proc postContent*(url: string, extraHeaders = "", body = "",
                   maxRedirects = 5,
                   sslContext: SSLContext = defaultSSLContext,
                   timeout = -1, userAgent = defUserAgent,
-                  proxy: Proxy = nil): string =
+                  proxy: Proxy = nil,
+                  multipart: MultipartData = nil): string =
   ## | POSTs ``body`` to ``url`` and returns the response's body as a string
   ## | Raises exceptions for the status codes ``4xx`` and ``5xx``
   ## | Extra headers can be specified and must be separated by ``\c\L``.
   ## | An optional timeout can be specified in miliseconds, if reading from the
   ## server takes longer than specified an ETimeout exception will be raised.
+  ## | The optional ``multipart`` parameter can be used to create
+  ## ``multipart/form-data`` POSTs comfortably.
   var r = post(url, extraHeaders, body, maxRedirects, sslContext, timeout,
-               userAgent, proxy)
+               userAgent, proxy, multipart)
   if r.status[0] in {'4','5'}:
     raise newException(HttpRequestError, r.status)
   else:
@@ -710,18 +826,9 @@ when isMainModule:
     #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com&
     #  charset=%28detect+automatically%29&doctype=Inline&group=0")
 
-    var headers: string = "Content-Type: multipart/form-data; boundary=xyz\c\L"
+    var data = newMultipartData()
-    var body: string = "--xyz\c\L"
+    data["output"] = "soap12"
-    # soap 1.2 output
+    data["uploaded_file"] = ("test.html", "text/html",
-    body.add("Content-Disposition: form-data; name=\"output\"\c\L")
+      "<html><head></head><body><p>test</p></body></html>")
-    body.add("\c\Lsoap12\c\L")
 
-    # html
+    echo postContent("http://validator.w3.org/check", multipart=data)
-    body.add("--xyz\c\L")
-    body.add("Content-Disposition: form-data; name=\"uploaded_file\";" &
-             " filename=\"test.html\"\c\L")
-    body.add("Content-Type: text/html\c\L")
-    body.add("\c\L<html><head></head><body><p>test</p></body></html>\c\L")
-    body.add("--xyz--")
-
-    echo(postContent("http://validator.w3.org/check", headers, body))