sockets.recv optimizations; stdlib now supports taint mode

doc/manual.txt

@@ -3402,3 +3402,25 @@ The interaction between threads and exceptions is simple: A *handled* exception
 in one thread cannot affect any other thread. However, an *unhandled* 
 exception in one thread terminates the whole *process*!
 
+Taint mode
+==========
+
+The Nimrod compiler and standard library support a `taint mode`:idx:. 
+Input strings are declared with the `TaintedString`:idx: string type declared
+in the ``system`` module.
+
+If the taint mode is turned on (via the ``--taintMode:on`` command line 
+option) it is a distinct string type which helps to detect input
+validation errors:
+
+.. code-block:: nimrod
+  echo "your name: "
+  var name: TaintedString = stdin.readline
+  # it is safe here to output the name without any input validation, so
+  # we simply convert `name` to string to make the compiler happy: 
+  echo "hi, ", name.string
+
+If the taint mode is turned off, ``TaintedString`` is simply an alias for
+``string``.
+
+

lib/impure/rdstdin.nim

@@ -14,7 +14,7 @@
 ## wanted functionality.
 
 when defined(Windows):
-  proc ReadLineFromStdin*(prompt: string): string = 
+  proc ReadLineFromStdin*(prompt: string): TaintedString = 
     ## Reads a line from stdin.
     stdout.write(prompt)
     result = readLine(stdin)
@@ -22,11 +22,11 @@ when defined(Windows):
 else:
   import readline, history
     
-  proc ReadLineFromStdin*(prompt: string): string = 
+  proc ReadLineFromStdin*(prompt: string): TaintedString = 
     var buffer = readline.readLine(prompt)
     if isNil(buffer): quit(0)
-    result = $buffer
+    result = TaintedString($buffer)
-    if result.len > 0:
+    if result.string.len > 0:
       add_history(buffer)
     readline.free(buffer)
 

lib/impure/ssl.nim

@@ -50,11 +50,11 @@ proc connect*(sock: var TSecureSocket, address: string,
   
   result = SSL_get_verify_result(sock.ssl)
 
-proc recvLine*(sock: TSecureSocket, line: var string): bool =
+proc recvLine*(sock: TSecureSocket, line: var TaintedString): bool =
   ## Acts in a similar fashion to the `recvLine` in the sockets module.
   ## Returns false when no data is available to be read.
   ## `Line` must be initialized and not nil!
-  setLen(line, 0)
+  setLen(line.string, 0)
   while True:
     var c: array[0..0, char]
     var n = BIO_read(sock.bio, c, c.len)
@@ -66,7 +66,7 @@ proc recvLine*(sock: TSecureSocket, line: var string): bool =
       elif n <= 0:
         return False
     elif c[0] == '\L': return True
-    add(line, c)
+    add(line.string, c)
 
 
 proc send*(sock: TSecureSocket, data: string) =

lib/impure/web.nim

@@ -43,19 +43,19 @@ proc URLretrieveStream*(url: string): PStream =
   if easy_perform(hCurl) != E_OK: return nil
   easy_cleanup(hCurl)
   
-proc URLretrieveString*(url: string): string = 
+proc URLretrieveString*(url: string): TaintedString = 
   ## retrieves the given `url` and returns the contents. Returns nil if an
   ## error occurs.
   var stream = newStringStream()
   var hCurl = easy_init()
-  if hCurl == nil: return nil
+  if hCurl == nil: return
-  if easy_setopt(hCurl, OPT_URL, url) != E_OK: return nil
+  if easy_setopt(hCurl, OPT_URL, url) != E_OK: return
   if easy_setopt(hCurl, OPT_WRITEFUNCTION, 
-                      curlwrapperWrite) != E_OK: return nil
+                      curlwrapperWrite) != E_OK: return
-  if easy_setopt(hCurl, OPT_WRITEDATA, stream) != E_OK: return nil
+  if easy_setopt(hCurl, OPT_WRITEDATA, stream) != E_OK: return
-  if easy_perform(hCurl) != E_OK: return nil
+  if easy_perform(hCurl) != E_OK: return
   easy_cleanup(hCurl)
-  result = stream.data
+  result = stream.data.TaintedString
 
 when isMainModule:
   echo URLretrieveString("http://nimrod.ethexor.com/")

lib/pure/cgi.nim

@@ -111,19 +111,19 @@ proc getEncodedData(allowedMethods: set[TRequestMethod]): string =
   of "POST":
     if methodPost notin allowedMethods:
       cgiError("'REQUEST_METHOD' 'POST' is not supported")
-    var L = parseInt(getenv("CONTENT_LENGTH"))
+    var L = parseInt(getenv("CONTENT_LENGTH").string)
     result = newString(L)
     if readBuffer(stdin, addr(result[0]), L) != L:
       cgiError("cannot read from stdin")
   of "GET":
     if methodGet notin allowedMethods:
       cgiError("'REQUEST_METHOD' 'GET' is not supported")
-    result = getenv("QUERY_STRING")
+    result = getenv("QUERY_STRING").string
   else:
     if methodNone notin allowedMethods:
       cgiError("'REQUEST_METHOD' must be 'POST' or 'GET'")
 
-iterator decodeData*(data: string): tuple[key, value: string] =
+iterator decodeData*(data: string): tuple[key, value: TaintedString] =
   ## Reads and decodes CGI data and yields the (name, value) pairs the
   ## data consists of.
   var i = 0
@@ -160,13 +160,13 @@ iterator decodeData*(data: string): tuple[key, value: string] =
       of '&', '\0': break
       else: add(value, data[i])
       inc(i)
-    yield (name, value)
+    yield (name.TaintedString, value.TaintedString)
     if data[i] == '&': inc(i)
     elif data[i] == '\0': break
     else: cgiError("'&' expected")
 
 iterator decodeData*(allowedMethods: set[TRequestMethod] =
-       {methodNone, methodPost, methodGet}): tuple[key, value: string] =
+       {methodNone, methodPost, methodGet}): tuple[key, value: TaintedString] =
   ## Reads and decodes CGI data and yields the (name, value) pairs the
   ## data consists of. If the client does not use a method listed in the
   ## `allowedMethods` set, an `ECgi` exception is raised.
@@ -181,7 +181,7 @@ proc readData*(allowedMethods: set[TRequestMethod] =
   ## `allowedMethods` set, an `ECgi` exception is raised.
   result = newStringTable()
   for name, value in decodeData(allowedMethods):
-    result[name] = value
+    result[name.string] = value.string
 
 proc validateData*(data: PStringTable, validKeys: openarray[string]) =
   ## validates data; raises `ECgi` if this fails. This checks that each variable

lib/pure/httpclient.nim

@@ -75,7 +75,7 @@ proc fileError(msg: string) =
 proc charAt(d: var string, i: var int, s: TSocket): char {.inline.} = 
   result = d[i]
   while result == '\0':
-    d = s.recv()
+    d = string(s.recv())
     i = 0
     result = d[i]
 
@@ -98,7 +98,7 @@ proc parseChunks(d: var string, start: int, s: TSocket): string =
         digitFound = true
         chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('A') + 10)
       of '\0': 
-        d = s.recv()
+        d = string(s.recv())
         i = -1
       else: break
       inc(i)
@@ -123,7 +123,7 @@ proc parseChunks(d: var string, start: int, s: TSocket): string =
         inc(L, bytesRead)
         dec(missing, bytesRead)
       # next chunk:
-      d = s.recv()
+      d = string(s.recv())
       i = 0
     # skip trailing CR-LF:
     while charAt(d, i, s) in {'\C', '\L'}: inc(i)
@@ -139,7 +139,7 @@ proc parseBody(d: var string, start: int, s: TSocket,
     var contentLengthHeader = headers["Content-Length"]
     if contentLengthHeader != "":
       var length = contentLengthHeader.parseint()
-      while result.len() < length: result.add(s.recv())
+      while result.len() < length: result.add(s.recv.string)
     else:
       # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO
       
@@ -147,12 +147,12 @@ proc parseBody(d: var string, start: int, s: TSocket,
       # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5
       if headers["Connection"] == "close":
         while True:
-          var moreData = recv(s)
+          var moreData = recv(s).string
           if moreData.len == 0: break
           result.add(moreData)
 
 proc parseResponse(s: TSocket): TResponse =
-  var d = s.recv()
+  var d = s.recv.string
   var i = 0
 
   # Parse the version

lib/pure/httpserver.nim

@@ -113,11 +113,11 @@ proc executeCgi(client: TSocket, path, query: string, meth: TRequestMethod) =
     env["REQUEST_METHOD"] = "GET"
     env["QUERY_STRING"] = query
   of reqPost:
-    var buf = ""
+    var buf = TaintedString""
     var dataAvail = false
     while dataAvail:
       dataAvail = recvLine(client, buf)
-      var L = toLower(buf)
+      var L = toLower(buf.string)
       if L.startsWith("content-length:"):
         var i = len("content-length:")
         while L[i] in Whitespace: inc(i)
@@ -146,7 +146,7 @@ proc executeCgi(client: TSocket, path, query: string, meth: TRequestMethod) =
   var outp = process.outputStream
   while running(process) or not outp.atEnd(outp):
     var line = outp.readLine()
-    send(client, line)
+    send(client, line.string)
     send(client, wwwNL)
 
 # --------------- Server Setup -----------------------------------------------
@@ -154,10 +154,10 @@ proc executeCgi(client: TSocket, path, query: string, meth: TRequestMethod) =
 proc acceptRequest(client: TSocket) =
   var cgi = false
   var query = ""
-  var buf = ""
+  var buf = TaintedString""
   discard recvLine(client, buf)
   var path = ""
-  var data = buf.split()
+  var data = buf.string.split()
   var meth = reqGet
 
   var q = find(data[1], '?')
@@ -231,7 +231,7 @@ proc next*(s: var TServer) =
   ## proceed to the first/next request.
   s.client = accept(s.socket)
   headers(s.client, "")
-  var data = recv(s.client)
+  var data = recv(s.client).string
   #discard recvLine(s.client, data)
   
   var i = skipWhitespace(data)

lib/pure/irc.nim

@@ -205,15 +205,15 @@ proc poll*(irc: var TIRC, ev: var TIRCEvent,
   ## This function should be called often as it also handles pinging
   ## the server.
   if not irc.connected: ev.typ = EvDisconnected
-  var line = ""
+  var line = TaintedString""
   var socks = @[irc.sock]
   var ret = socks.select(timeout)
   if socks.len() == 0 and ret == 1:
     if irc.sock.recvLine(line):
-      if line == "":
+      if line.string.len == 0:
         ev.typ = EvDisconnected
       else:
-        ev = parseMessage(line)
+        ev = parseMessage(line.string)
         if ev.cmd == MPing:
           irc.send("PONG " & ev.params[0])
         if ev.cmd == MPong:

lib/pure/osproc.nim

@@ -41,7 +41,7 @@ type
 
 proc execProcess*(command: string,
                   options: set[TProcessOption] = {poStdErrToStdOut,
-                                                  poUseShell}): string {.
+                                                  poUseShell}): TaintedString {.
                                                   rtl, extern: "nosp$1".}
   ## A convenience procedure that executes ``command`` with ``startProcess``
   ## and returns its output as a string.
@@ -203,13 +203,13 @@ proc select*(readfds: var seq[PProcess], timeout = 500): int
 when not defined(useNimRtl):
   proc execProcess(command: string,
                    options: set[TProcessOption] = {poStdErrToStdOut,
-                                                   poUseShell}): string =
+                                                   poUseShell}): TaintedString =
     var p = startProcessAux(command, options=options)
     var outp = outputStream(p)
-    result = ""
+    result = TaintedString""
     while running(p) or not outp.atEnd(outp):
-      result.add(outp.readLine())
+      result.string.add(outp.readLine().string)
-      result.add("\n")
+      result.string.add("\n")
     outp.close(outp)
     close(p)
 

lib/pure/redis.nim

@@ -49,7 +49,7 @@ proc raiseNoOK(status: string) =
     raise newException(EInvalidReply, "Expected \"OK\" got \"$1\"" % status)
 
 proc parseStatus(r: TRedis): TRedisStatus =
-  var line = r.socket.recv()
+  var line = r.socket.recv.string
   
   if line[0] == '-':
     raise newException(ERedis, strip(line))
@@ -59,7 +59,7 @@ proc parseStatus(r: TRedis): TRedisStatus =
   return line.substr(1, line.len-3) # Strip '+' and \c\L.
   
 proc parseInteger(r: TRedis): TRedisInteger =
-  var line = r.socket.recv()
+  var line = r.socket.recv.string
 
   if line[0] == '-':
     raise newException(ERedis, strip(line))
@@ -70,14 +70,14 @@ proc parseInteger(r: TRedis): TRedisInteger =
   if parseBiggestInt(line, result, 1) == 0:
     raise newException(EInvalidReply, "Unable to parse integer.") 
 
-proc recv(sock: TSocket, size: int): string =
+proc recv(sock: TSocket, size: int): TaintedString =
-  result = newString(size)
+  result = newString(size).TaintedString
   if sock.recv(cstring(result), size) != size:
     raise newException(EInvalidReply, "recv failed")
 
 proc parseBulk(r: TRedis, allowMBNil = False): TRedisString =
   var line = ""
-  if not r.socket.recvLine(line):
+  if not r.socket.recvLine(line.TaintedString):
     raise newException(EInvalidReply, "recvLine failed")
   
   # Error.
@@ -97,17 +97,17 @@ proc parseBulk(r: TRedis, allowMBNil = False): TRedisString =
     return RedisNil
 
   var s = r.socket.recv(numBytes+2)
-  result = strip(s)
+  result = strip(s.string)
 
 proc parseMultiBulk(r: TRedis): TRedisList =
-  var line = ""
+  var line = TaintedString""
   if not r.socket.recvLine(line):
     raise newException(EInvalidReply, "recvLine failed")
     
-  if line[0] != '*':
+  if line.string[0] != '*':
-    raiseInvalidReply('*', line[0])
+    raiseInvalidReply('*', line.string[0])
   
-  var numElems = parseInt(line.substr(1))
+  var numElems = parseInt(line.string.substr(1))
   if numElems == -1: return nil
   result = @[]
   for i in 1..numElems:
@@ -839,7 +839,7 @@ proc shutdown*(r: TRedis) =
   ## Synchronously save the dataset to disk and then shut down the server
   r.sendCommand("SHUTDOWN")
   var s = r.socket.recv()
-  if s != "": raise newException(ERedis, s)
+  if s.string.len != 0: raise newException(ERedis, s.string)
 
 proc slaveof*(r: TRedis, host: string, port: string) =
   ## Make the server a slave of another instance, or promote it as master

lib/pure/smtp.nim

@@ -59,8 +59,8 @@ proc debugSend(smtp: TSMTP, cmd: string) =
     when not defined(noSSL):
       smtp.sslSock.send(cmd)
 
-proc debugRecv(smtp: TSMTP): string =
+proc debugRecv(smtp: TSMTP): TaintedString =
-  var line = ""
+  var line = TaintedString""
   var ret = False
   if not smtp.ssl:
     ret = smtp.sock.recvLine(line)
@@ -69,11 +69,11 @@ proc debugRecv(smtp: TSMTP): string =
       ret = smtp.sslSock.recvLine(line)
   if ret:
     if smtp.debug:
-      echo("S:" & line)
+      echo("S:" & line.string)
     return line
   else:
     OSError()
-    return ""
+    return TaintedString""
 
 proc quitExcpt(smtp: TSMTP, msg: string) =
   smtp.debugSend("QUIT")
@@ -81,8 +81,8 @@ proc quitExcpt(smtp: TSMTP, msg: string) =
 
 proc checkReply(smtp: TSMTP, reply: string) =
   var line = smtp.debugRecv()
-  if not line.startswith(reply):
+  if not line.string.startswith(reply):
-    quitExcpt(smtp, "Expected " & reply & " reply, got: " & line)
+    quitExcpt(smtp, "Expected " & reply & " reply, got: " & line.string)
 
 proc connect*(address: string, port = 25, 
               ssl = false, debug = false): TSMTP =

lib/pure/sockets.nim

@@ -553,17 +553,28 @@ proc recv*(socket: TSocket): TaintedString =
   ## If socket is not a connectionless socket and socket is not connected
   ## ``""`` will be returned.
   const bufSize = 1000
-  var buf = newString(bufSize)
+  result = newStringOfCap(bufSize).TaintedString
-  result = TaintedString""
+  var pos = 0
   while true:
-    var bytesRead = recv(socket, cstring(buf), bufSize-1)
+    var bytesRead = recv(socket, addr(string(result)[pos]), bufSize-1)
-    # Error
     if bytesRead == -1: OSError()
-    
+    setLen(result.string, pos + bytesRead)
-    buf[bytesRead] = '\0' # might not be necessary
-    setLen(buf, bytesRead)
-    add(result.string, buf)
     if bytesRead != bufSize-1: break
+    # increase capacity:
+    setLen(result.string, result.string.len + bufSize)
+    inc(pos, bytesRead)
+  when false:
+    var buf = newString(bufSize)
+    result = TaintedString""
+    while true:
+      var bytesRead = recv(socket, cstring(buf), bufSize-1)
+      # Error
+      if bytesRead == -1: OSError()
+      
+      buf[bytesRead] = '\0' # might not be necessary
+      setLen(buf, bytesRead)
+      add(result.string, buf)
+      if bytesRead != bufSize-1: break
 
 proc recvAsync*(socket: TSocket, s: var TaintedString): bool =
   ## receives all the data from a non-blocking socket. If socket is non-blocking 
@@ -572,11 +583,12 @@ proc recvAsync*(socket: TSocket, s: var TaintedString): bool =
   ## If socket is not a connectionless socket and socket is not connected
   ## ``s`` will be set to ``""``.
   const bufSize = 1000
-  var buf = newString(bufSize)
+  # ensure bufSize capacity:
-  s = ""
+  setLen(s.string, bufSize)
+  setLen(s.string, 0)
+  var pos = 0
   while true:
-    var bytesRead = recv(socket, cstring(buf), bufSize-1)
+    var bytesRead = recv(socket, addr(string(s)[pos]), bufSize-1)
-    # Error
     if bytesRead == -1:
       when defined(windows):
         # TODO: Test on Windows
@@ -588,11 +600,12 @@ proc recvAsync*(socket: TSocket, s: var TaintedString): bool =
         if errno == EAGAIN or errno == EWOULDBLOCK:
           return False
         else: OSError()
-    
+
-    buf[bytesRead] = '\0' # might not be necessary
+    setLen(s.string, pos + bytesRead)
-    setLen(buf, bytesRead)
-    add(s, buf)
     if bytesRead != bufSize-1: break
+    # increase capacity:
+    setLen(s.string, s.string.len + bufSize)
+    inc(pos, bytesRead)
   result = True
   
 proc skip*(socket: TSocket) =

todo.txt

@@ -1,7 +1,6 @@
 Version 0.8.14
 ==============
 
-- taint mode
 - 'let x = y'; const ptr/ref
 - fix actors.nim
 - make threadvar efficient again on linux after testing

web/news.txt

@@ -61,6 +61,10 @@ Compiler Additions
   and Objective C somewhat easier.
 - Added a ``--nimcache:PATH`` configuration option for control over the output
   directory for generated code.
+- The compiler and standard library now support a *taint mode*. Input strings
+  are declared with the ``TaintedString`` string type. If the taint
+  mode is turned on it is a distinct string type which helps to detect input
+  validation errors.
 
 
 Library Additions