Merge branch 'devel' into sighashes

compiler/ccgexprs.nim

@@ -1147,8 +1147,8 @@ proc handleConstExpr(p: BProc, n: PNode, d: var TLoc): bool =
     result = false
 
 proc genObjConstr(p: BProc, e: PNode, d: var TLoc) =
-  if handleConstExpr(p, e, d): return
   #echo rendertree e, " ", e.isDeepConstExpr
+  if handleConstExpr(p, e, d): return
   var tmp: TLoc
   var t = e.typ.skipTypes(abstractInst)
   getTemp(p, t, tmp)
@@ -2148,6 +2148,11 @@ proc genNamedConstExpr(p: BProc, n: PNode): Rope =
 proc genConstSimpleList(p: BProc, n: PNode): Rope =
   var length = sonsLen(n)
   result = rope("{")
+  let t = n.typ.skipTypes(abstractInst)
+  if n.kind == nkObjConstr and not isObjLackingTypeField(t) and
+      not p.module.compileToCpp:
+    addf(result, "{$1}", [genTypeInfo(p.module, t)])
+    if n.len > 1: add(result, ",")
   for i in countup(ord(n.kind == nkObjConstr), length - 2):
     addf(result, "$1,$n", [genNamedConstExpr(p, n.sons[i])])
   if length > ord(n.kind == nkObjConstr):

compiler/ccgmerge.nim

@@ -27,8 +27,8 @@ const
     cfsFieldInfo: "NIM_merge_FIELD_INFO",
     cfsTypeInfo: "NIM_merge_TYPE_INFO",
     cfsProcHeaders: "NIM_merge_PROC_HEADERS",
-    cfsData: "NIM_merge_DATA",
     cfsVars: "NIM_merge_VARS",
+    cfsData: "NIM_merge_DATA",
     cfsProcs: "NIM_merge_PROCS",
     cfsInitProc: "NIM_merge_INIT_PROC",
     cfsTypeInit1: "NIM_merge_TYPE_INIT1",

compiler/ccgtypes.nim

@@ -857,7 +857,8 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope) =
   if flags != 0:
     addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [name, rope(flags)])
   if isDefined("nimTypeNames"):
-    addf(m.s[cfsTypeInit3], "$1.name = $2;$n", [name, makeCstring typeToString origType])
+    addf(m.s[cfsTypeInit3], "$1.name = $2;$n",
+        [name, makeCstring typeToString(origType, preferName)])
   discard cgsym(m, "TNimType")
   addf(m.s[cfsVars], "TNimType $1; /* $2 */$n",
        [name, rope(typeToString(typ))])

compiler/cgen.nim

@@ -17,6 +17,7 @@ import
   lowerings, semparallel, tables
 
 from modulegraphs import ModuleGraph
+from dynlib import libCandidates
 
 import strutils except `%` # collides with ropes.`%`
 

compiler/cgendata.nim

@@ -28,8 +28,8 @@ type
     cfsFieldInfo,             # section for field information
     cfsTypeInfo,              # section for type information
     cfsProcHeaders,           # section for C procs prototypes
-    cfsData,                  # section for C constant data
     cfsVars,                  # section for C variable declarations
+    cfsData,                  # section for C constant data
     cfsProcs,                 # section for C procs that are not inline
     cfsInitProc,              # section for the C init proc
     cfsTypeInit1,             # section 1 for declarations of type information

compiler/commands.nim

@@ -226,6 +226,8 @@ proc testCompileOptionArg*(switch, arg: string, info: TLineInfo): bool =
     of "staticlib": result = contains(gGlobalOptions, optGenStaticLib) and
                       not contains(gGlobalOptions, optGenGuiApp)
     else: localError(info, errGuiConsoleOrLibExpectedButXFound, arg)
+  of "dynliboverride":
+    result = isDynlibOverride(arg)
   else: invalidCmdLineOption(passCmd1, switch, info)
 
 proc testCompileOption*(switch: string, info: TLineInfo): bool =

compiler/docgen.nim

@@ -13,8 +13,10 @@
 
 import
   ast, strutils, strtabs, options, msgs, os, ropes, idents,
-  wordrecg, syntaxes, renderer, lexer, rstast, rst, rstgen, times, highlite,
+  wordrecg, syntaxes, renderer, lexer, packages/docutils/rstast,
-  importer, sempass2, json, xmltree, cgi, typesrenderer, astalgo
+  packages/docutils/rst, packages/docutils/rstgen, times,
+  packages/docutils/highlite, importer, sempass2, json, xmltree, cgi,
+  typesrenderer, astalgo
 
 type
   TSections = array[TSymKind, Rope]

compiler/extccomp.nim

@@ -34,7 +34,7 @@ type
   TInfoCCProps* = set[TInfoCCProp]
   TInfoCC* = tuple[
     name: string,        # the short name of the compiler
-    objExt: string,      # the compiler's object file extenstion
+    objExt: string,      # the compiler's object file extension
     optSpeed: string,    # the options for optimization for speed
     optSize: string,     # the options for optimization for size
     compilerExe: string, # the compiler's executable

compiler/lambdalifting.nim

@@ -139,10 +139,10 @@ proc createStateField(iter: PSym): PSym =
   result = newSym(skField, getIdent(":state"), iter, iter.info)
   result.typ = createStateType(iter)
 
-proc createEnvObj(owner: PSym): PType =
+proc createEnvObj(owner: PSym; info: TLineInfo): PType =
   # YYY meh, just add the state field for every closure for now, it's too
   # hard to figure out if it comes from a closure iterator:
-  result = createObj(owner, owner.info)
+  result = createObj(owner, info)
   rawAddField(result, createStateField(owner))
 
 proc getIterResult(iter: PSym): PSym =
@@ -296,18 +296,19 @@ This is why need to store the 'ownerToType' table and use it
 during .closure'fication.
 """
 
-proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym): PType =
+proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
+                        info: TLineInfo): PType =
   result = c.ownerToType.getOrDefault(owner.id)
   if result.isNil:
     result = newType(tyRef, owner)
-    let obj = createEnvObj(owner)
+    let obj = createEnvObj(owner, info)
     rawAddSon(result, obj)
     c.ownerToType[owner.id] = result
 
-proc createUpField(c: var DetectionPass; dest, dep: PSym) =
+proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
-  let refObj = c.getEnvTypeForOwner(dest) # getHiddenParam(dest).typ
+  let refObj = c.getEnvTypeForOwner(dest, info) # getHiddenParam(dest).typ
   let obj = refObj.lastSon
-  let fieldType = c.getEnvTypeForOwner(dep) #getHiddenParam(dep).typ
+  let fieldType = c.getEnvTypeForOwner(dep, info) #getHiddenParam(dep).typ
   if refObj == fieldType:
     localError(dep.info, "internal error: invalid up reference computed")
 
@@ -347,10 +348,10 @@ Consider:
 
 """
 
-proc addClosureParam(c: var DetectionPass; fn: PSym) =
+proc addClosureParam(c: var DetectionPass; fn: PSym; info: TLineInfo) =
   var cp = getEnvParam(fn)
   let owner = if fn.kind == skIterator: fn else: fn.skipGenericOwner
-  let t = c.getEnvTypeForOwner(owner)
+  let t = c.getEnvTypeForOwner(owner, info)
   if cp == nil:
     cp = newSym(skParam, getIdent(paramName), fn, fn.info)
     incl(cp.flags, sfFromGeneric)
@@ -367,7 +368,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
     if s.kind in {skProc, skMethod, skConverter, skIterator} and s.typ != nil and s.typ.callConv == ccClosure:
       # this handles the case that the inner proc was declared as
       # .closure but does not actually capture anything:
-      addClosureParam(c, s)
+      addClosureParam(c, s, n.info)
       c.somethingToDo = true
 
     let innerProc = isInnerProc(s)
@@ -379,7 +380,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
     if ow == owner:
       if owner.isIterator:
         c.somethingToDo = true
-        addClosureParam(c, owner)
+        addClosureParam(c, owner, n.info)
         if interestingIterVar(s):
           if not c.capturedVars.containsOrIncl(s.id):
             let obj = getHiddenParam(owner).typ.lastSon
@@ -403,11 +404,11 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       # mark 'owner' as taking a closure:
       c.somethingToDo = true
       markAsClosure(owner, n)
-      addClosureParam(c, owner)
+      addClosureParam(c, owner, n.info)
       #echo "capturing ", n.info
       # variable 's' is actually captured:
       if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id):
-        let obj = c.getEnvTypeForOwner(ow).lastSon
+        let obj = c.getEnvTypeForOwner(ow, n.info).lastSon
         #getHiddenParam(owner).typ.lastSon
         addField(obj, s)
       # create required upFields:
@@ -428,8 +429,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
           let up = w.skipGenericOwner
           #echo "up for ", w.name.s, " up ", up.name.s
           markAsClosure(w, n)
-          addClosureParam(c, w) # , ow
+          addClosureParam(c, w, n.info) # , ow
-          createUpField(c, w, up)
+          createUpField(c, w, up, n.info)
           w = up
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit,
      nkTemplateDef, nkTypeSection:
@@ -793,7 +794,7 @@ proc liftLambdas*(fn: PSym, body: PNode; tooEarly: var bool): PNode =
     var d = initDetectionPass(fn)
     detectCapturedVars(body, fn, d)
     if not d.somethingToDo and fn.isIterator:
-      addClosureParam(d, fn)
+      addClosureParam(d, fn, body.info)
       d.somethingToDo = true
     if d.somethingToDo:
       var c = initLiftingPass(fn)

compiler/lowerings.nim

@@ -114,6 +114,10 @@ proc createObj*(owner: PSym, info: TLineInfo): PType =
   rawAddSon(result, nil)
   incl result.flags, tfFinal
   result.n = newNodeI(nkRecList, info)
+  let s = newSym(skType, getIdent("Env_" & info.toFilename & "_" & $info.line),
+                 owner, info)
+  s.typ = result
+  result.sym = s
 
 proc rawAddField*(obj: PType; field: PSym) =
   assert field.kind == skField

compiler/nim.cfg

@@ -4,8 +4,6 @@ hint[XDeclaredButNotUsed]:off
 path:"llvm"
 path:"$projectPath/.."
 
-path:"$lib/packages/docutils"
-
 define:booting
 #import:"$projectpath/testability"
 

compiler/options.nim

@@ -227,8 +227,16 @@ proc setDefaultLibpath*() =
       libpath = parentNimLibPath
 
 proc canonicalizePath*(path: string): string =
-  when not FileSystemCaseSensitive: result = path.expandFilename.toLowerAscii
+  # on Windows, 'expandFilename' calls getFullPathName which doesn't do
-  else: result = path.expandFilename
+  # case corrections, so we have to use this convoluted way of retrieving
+  # the true filename (see tests/modules and Nimble uses 'import Uri' instead
+  # of 'import uri'):
+  when defined(windows):
+    result = path.expandFilename
+    for x in walkFiles(result):
+      return x
+  else:
+    result = path.expandFilename
 
 proc shortenDir*(dir: string): string =
   ## returns the interesting part of a dir
@@ -373,17 +381,6 @@ proc findModule*(modulename, currentModule: string): string =
     result = findFile(m)
   patchModule()
 
-proc libCandidates*(s: string, dest: var seq[string]) =
-  var le = strutils.find(s, '(')
-  var ri = strutils.find(s, ')', le+1)
-  if le >= 0 and ri > le:
-    var prefix = substr(s, 0, le - 1)
-    var suffix = substr(s, ri + 1)
-    for middle in split(substr(s, le + 1, ri - 1), '|'):
-      libCandidates(prefix & middle & suffix, dest)
-  else:
-    add(dest, s)
-
 proc canonDynlibName(s: string): string =
   let start = if s.startsWith("lib"): 3 else: 0
   let ende = strutils.find(s, {'(', ')', '.'})

compiler/pragmas.nim

@@ -490,6 +490,7 @@ proc pragmaLine(c: PContext, n: PNode) =
       elif y.kind != nkIntLit:
         localError(n.info, errIntLiteralExpected)
       else:
+        # XXX this produces weird paths which are not properly resolved:
         n.info.fileIndex = msgs.fileInfoIdx(x.strVal)
         n.info.line = int16(y.intVal)
     else:

compiler/semexprs.nim

@@ -1276,7 +1276,8 @@ proc propertyWriteAccess(c: PContext, n, nOrig, a: PNode): PNode =
   # this is ugly. XXX Semantic checking should use the ``nfSem`` flag for
   # nodes?
   let aOrig = nOrig[0]
-  result = newNode(nkCall, n.info, sons = @[setterId, a[0], semExpr(c, n[1])])
+  result = newNode(nkCall, n.info, sons = @[setterId, a[0],
+                                            semExprWithType(c, n[1])])
   result.flags.incl nfDotSetter
   let orig = newNode(nkCall, n.info, sons = @[setterId, aOrig[0], nOrig[1]])
   result = semOverloadedCallAnalyseEffects(c, result, orig, {})

lib/pure/asyncdispatch.nim

@@ -893,9 +893,11 @@ when defined(windows) or defined(nimdoc):
               deallocShared(cast[pointer](pcd))
               raiseOSError(osLastError())
             else:
-              # we ref pcd.ovl one more time, because it will be unrefed in
+              # we incref `pcd.ovl` and `protect` callback one more time,
-              # poll()
+              # because it will be unrefed and disposed in `poll()` after
+              # callback finishes.
               GC_ref(pcd.ovl)
+              pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb))
     )
     # We need to protect our callback environment value, so GC will not free it
     # accidentally.
@@ -956,15 +958,6 @@ when defined(windows) or defined(nimdoc):
   initAll()
 else:
   import selectors
-  when defined(windows):
-    import winlean
-    const
-      EINTR = WSAEINPROGRESS
-      EINPROGRESS = WSAEINPROGRESS
-      EWOULDBLOCK = WSAEWOULDBLOCK
-      EAGAIN = EINPROGRESS
-      MSG_NOSIGNAL = 0
-  else:
   from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK,
                     MSG_NOSIGNAL
 

lib/pure/dynlib.nim

@@ -11,20 +11,22 @@
 ## libraries. On POSIX this uses the ``dlsym`` mechanism, on
 ## Windows ``LoadLibrary``.
 
+import strutils
+
 type
   LibHandle* = pointer ## a handle to a dynamically loaded library
 
 {.deprecated: [TLibHandle: LibHandle].}
 
-proc loadLib*(path: string, global_symbols=false): LibHandle
+proc loadLib*(path: string, global_symbols=false): LibHandle {.gcsafe.}
   ## loads a library from `path`. Returns nil if the library could not
   ## be loaded.
 
-proc loadLib*(): LibHandle
+proc loadLib*(): LibHandle {.gcsafe.}
   ## gets the handle from the current executable. Returns nil if the
   ## library could not be loaded.
 
-proc unloadLib*(lib: LibHandle)
+proc unloadLib*(lib: LibHandle) {.gcsafe.}
   ## unloads the library `lib`
 
 proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} =
@@ -34,7 +36,7 @@ proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} =
   e.msg = "could not find symbol: " & $name
   raise e
 
-proc symAddr*(lib: LibHandle, name: cstring): pointer
+proc symAddr*(lib: LibHandle, name: cstring): pointer {.gcsafe.}
   ## retrieves the address of a procedure/variable from `lib`. Returns nil
   ## if the symbol could not be found.
 
@@ -44,6 +46,28 @@ proc checkedSymAddr*(lib: LibHandle, name: cstring): pointer =
   result = symAddr(lib, name)
   if result == nil: raiseInvalidLibrary(name)
 
+proc libCandidates*(s: string, dest: var seq[string]) =
+  ## given a library name pattern `s` write possible library names to `dest`.
+  var le = strutils.find(s, '(')
+  var ri = strutils.find(s, ')', le+1)
+  if le >= 0 and ri > le:
+    var prefix = substr(s, 0, le - 1)
+    var suffix = substr(s, ri + 1)
+    for middle in split(substr(s, le + 1, ri - 1), '|'):
+      libCandidates(prefix & middle & suffix, dest)
+  else:
+    add(dest, s)
+
+proc loadLibPattern*(pattern: string, global_symbols=false): LibHandle =
+  ## loads a library with name matching `pattern`, similar to what `dlimport`
+  ## pragma does. Returns nil if the library could not be loaded.
+  ## Warning: this proc uses the GC and so cannot be used to load the GC.
+  var candidates = newSeq[string]()
+  libCandidates(pattern, candidates)
+  for c in candidates:
+    result = loadLib(c, global_symbols)
+    if not result.isNil: break
+
 when defined(posix):
   #
   # =========================================================================

lib/pure/net.nim

@@ -90,8 +90,8 @@ when defineSsl:
 
     SslContext* = ref object
       context*: SslCtx
-      extraInternalIndex: int
       referencedData: HashSet[int]
+      extraInternal: SslContextExtraInternal
 
     SslAcceptResult* = enum
       AcceptNoClient = 0, AcceptNoHandshake, AcceptSuccess
@@ -103,6 +103,10 @@ when defineSsl:
 
     SslServerGetPskFunc* = proc(identity: string): string
 
+    SslContextExtraInternal = ref object of RootRef
+      serverGetPskFunc: SslServerGetPskFunc
+      clientGetPskFunc: SslClientGetPskFunc
+
   {.deprecated: [ESSL: SSLError, TSSLCVerifyMode: SSLCVerifyMode,
     TSSLProtVersion: SSLProtVersion, PSSLContext: SSLContext,
     TSSLAcceptResult: SSLAcceptResult].}
@@ -240,11 +244,6 @@ when defineSsl:
   ErrLoadBioStrings()
   OpenSSL_add_all_algorithms()
 
-  type
-    SslContextExtraInternal = ref object of RootRef
-      serverGetPskFunc: SslServerGetPskFunc
-      clientGetPskFunc: SslClientGetPskFunc
-
   proc raiseSSLError*(s = "") =
     ## Raises a new SSL error.
     if s != "":
@@ -257,12 +256,6 @@ when defineSsl:
     var errStr = ErrErrorString(err, nil)
     raise newException(SSLError, $errStr)
 
-  proc getExtraDataIndex*(ctx: SSLContext): int =
-    ## Retrieves unique index for storing extra data in SSLContext.
-    result = SSL_CTX_get_ex_new_index(0, nil, nil, nil, nil).int
-    if result < 0:
-      raiseSSLError()
-
   proc getExtraData*(ctx: SSLContext, index: int): RootRef =
     ## Retrieves arbitrary data stored inside SSLContext.
     if index notin ctx.referencedData:
@@ -347,15 +340,11 @@ when defineSsl:
     discard newCTX.SSLCTXSetMode(SSL_MODE_AUTO_RETRY)
     newCTX.loadCertificates(certFile, keyFile)
 
-    result = SSLContext(context: newCTX, extraInternalIndex: 0,
+    result = SSLContext(context: newCTX, referencedData: initSet[int](),
-        referencedData: initSet[int]())
+      extraInternal: new(SslContextExtraInternal))
-    result.extraInternalIndex = getExtraDataIndex(result)
-
-    let extraInternal = new(SslContextExtraInternal)
-    result.setExtraData(result.extraInternalIndex, extraInternal)
 
   proc getExtraInternal(ctx: SSLContext): SslContextExtraInternal =
-    return SslContextExtraInternal(ctx.getExtraData(ctx.extraInternalIndex))
+    return ctx.extraInternal
 
   proc destroyContext*(ctx: SSLContext) =
     ## Free memory referenced by SSLContext.
@@ -379,7 +368,7 @@ when defineSsl:
 
   proc pskClientCallback(ssl: SslPtr; hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr cuchar;
     max_psk_len: cuint): cuint {.cdecl.} =
-    let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX, extraInternalIndex: 0)
+    let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX)
     let hintString = if hint == nil: nil else: $hint
     let (identityString, pskString) = (ctx.clientGetPskFunc)(hintString)
     if psk.len.cuint > max_psk_len:
@@ -398,8 +387,6 @@ when defineSsl:
     ##
     ## Only used in PSK ciphersuites.
     ctx.getExtraInternal().clientGetPskFunc = fun
-    assert ctx.extraInternalIndex == 0,
-          "The pskClientCallback assumes the extraInternalIndex is 0"
     ctx.context.SSL_CTX_set_psk_client_callback(
         if fun == nil: nil else: pskClientCallback)
 
@@ -407,7 +394,7 @@ when defineSsl:
     return ctx.getExtraInternal().serverGetPskFunc
 
   proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr cuchar; max_psk_len: cint): cuint {.cdecl.} =
-    let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX, extraInternalIndex: 0)
+    let ctx = SSLContext(context: ssl.SSL_get_SSL_CTX)
     let pskString = (ctx.serverGetPskFunc)($identity)
     if psk.len.cint > max_psk_len:
       return 0

lib/pure/smtp.nim

@@ -256,24 +256,48 @@ proc close*(smtp: AsyncSmtp) {.async.} =
   smtp.sock.close()
 
 when not defined(testing) and isMainModule:
-  #var msg = createMessage("Test subject!",
+  # To test with a real SMTP service, create a smtp.ini file, e.g.:
-  #     "Hello, my name is dom96.\n What\'s yours?", @["dominik@localhost"])
+  # username = ""
-  #echo(msg)
+  # password = ""
+  # smtphost = "smtp.gmail.com"
+  # port = 465
+  # use_tls = true
+  # sender = ""
+  # recipient = ""
 
-  #var smtpConn = connect("localhost", Port 25, false, true)
+  import parsecfg
-  #smtpConn.sendmail("root@localhost", @["dominik@localhost"], $msg)
 
-  #echo(decode("a17sm3701420wbe.12"))
+  proc `[]`(c: Config, key: string): string = c.getSectionValue("", key)
-  proc main() {.async.} =
+
-    var client = newAsyncSmtp("smtp.gmail.com", Port(465), true)
+  let
+    conf = loadConfig("smtp.ini")
+    msg = createMessage("Hello from Nim's SMTP!",
+      "Hello!\n Is this awesome or what?", @[conf["recipient"]])
+
+  assert conf["smtphost"] != ""
+
+  proc async_test() {.async.} =
+    let client = newAsyncSmtp(
+      conf["smtphost"],
+      conf["port"].parseInt.Port,
+      conf["use_tls"].parseBool
+    )
     await client.connect()
-    await client.auth("johndoe", "foo")
+    await client.auth(conf["username"], conf["password"])
-    var msg = createMessage("Hello from Nim's SMTP!",
+    await client.sendMail(conf["sender"], @[conf["recipient"]], $msg)
-                            "Hello!!!!.\n Is this awesome or what?",
-                            @["blah@gmail.com"])
-    echo(msg)
-    await client.sendMail("blah@gmail.com", @["blah@gmail.com"], $msg)
-
     await client.close()
+    echo "async email sent"
 
-  waitFor main()
+  proc sync_test() =
+    var smtpConn = connect(
+      conf["smtphost"],
+      conf["port"].parseInt.Port,
+      conf["use_tls"].parseBool,
+      true, # debug
+    )
+    smtpConn.auth(conf["username"], conf["password"])
+    smtpConn.sendmail(conf["sender"], @[conf["recipient"]], $msg)
+    echo "sync email sent"
+
+  waitFor async_test()
+  sync_test()

lib/pure/terminal.nim

@@ -13,6 +13,8 @@
 ## Windows API.
 ## Changing the style is permanent even after program termination! Use the
 ## code ``system.addQuitProc(resetAttributes)`` to restore the defaults.
+## Similarly, if you hide the cursor, make sure to unhide it with
+## ``showCursor`` before quitting.
 
 import macros
 
@@ -29,6 +31,8 @@ when defined(windows):
     BACKGROUND_GREEN = 32
     BACKGROUND_RED = 64
     BACKGROUND_INTENSITY = 128
+    FOREGROUND_RGB = FOREGROUND_RED or FOREGROUND_GREEN or FOREGROUND_BLUE
+    BACKGROUND_RGB = BACKGROUND_RED or BACKGROUND_GREEN or BACKGROUND_BLUE
 
   type
     SHORT = int16
@@ -49,6 +53,10 @@ when defined(windows):
       srWindow: SMALL_RECT
       dwMaximumWindowSize: COORD
 
+    CONSOLE_CURSOR_INFO = object
+      dwSize: DWORD
+      bVisible: WINBOOL
+
   proc duplicateHandle(hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE,
                        hTargetProcessHandle: HANDLE, lpTargetHandle: ptr HANDLE,
                        dwDesiredAccess: DWORD, bInheritHandle: WINBOOL,
@@ -60,6 +68,14 @@ when defined(windows):
     lpConsoleScreenBufferInfo: ptr CONSOLE_SCREEN_BUFFER_INFO): WINBOOL{.stdcall,
     dynlib: "kernel32", importc: "GetConsoleScreenBufferInfo".}
 
+  proc getConsoleCursorInfo(hConsoleOutput: HANDLE,
+      lpConsoleCursorInfo: ptr CONSOLE_CURSOR_INFO): WINBOOL{.
+      stdcall, dynlib: "kernel32", importc: "GetConsoleCursorInfo".}
+
+  proc setConsoleCursorInfo(hConsoleOutput: HANDLE,
+      lpConsoleCursorInfo: ptr CONSOLE_CURSOR_INFO): WINBOOL{.
+      stdcall, dynlib: "kernel32", importc: "SetConsoleCursorInfo".}
+
   proc terminalWidthIoctl*(handles: openArray[Handle]): int =
     var csbi: CONSOLE_SCREEN_BUFFER_INFO
     for h in handles:
@@ -179,6 +195,30 @@ else:
       return w
     return 80                               #Finally default to venerable value
 
+when defined(windows):
+  proc setCursorVisibility(f: File, visible: bool) =
+    var ccsi: CONSOLE_CURSOR_INFO
+    let h = conHandle(f)
+    if getConsoleCursorInfo(h, addr(ccsi)) == 0:
+      raiseOSError(osLastError())
+    ccsi.bVisible = if visible: 1 else: 0
+    if setConsoleCursorInfo(h, addr(ccsi)) == 0:
+      raiseOSError(osLastError())
+
+proc hideCursor*(f: File) =
+  ## Hides the cursor.
+  when defined(windows):
+    setCursorVisibility(f, false)
+  else:
+    f.write("\e[?25l")
+
+proc showCursor*(f: File) =
+  ## Shows the cursor.
+  when defined(windows):
+    setCursorVisibility(f, true)
+  else:
+    f.write("\e[?25h")
+
 proc setCursorPos*(f: File, x, y: int) =
   ## Sets the terminal's cursor to the (x,y) position.
   ## (0,0) is the upper left of the screen.
@@ -369,12 +409,13 @@ proc setStyle*(f: File, style: set[Style]) =
   ## Sets the terminal style.
   when defined(windows):
     let h = conHandle(f)
+    var old = getAttributes(h) and (FOREGROUND_RGB or BACKGROUND_RGB)
     var a = 0'i16
     if styleBright in style: a = a or int16(FOREGROUND_INTENSITY)
     if styleBlink in style: a = a or int16(BACKGROUND_INTENSITY)
     if styleReverse in style: a = a or 0x4000'i16 # COMMON_LVB_REVERSE_VIDEO
     if styleUnderscore in style: a = a or 0x8000'i16 # COMMON_LVB_UNDERSCORE
-    discard setConsoleTextAttribute(h, a)
+    discard setConsoleTextAttribute(h, old or a)
   else:
     for s in items(style):
       f.write("\e[" & $ord(s) & 'm')
@@ -423,7 +464,7 @@ proc setForegroundColor*(f: File, fg: ForegroundColor, bright=false) =
   ## Sets the terminal's foreground color.
   when defined(windows):
     let h = conHandle(f)
-    var old = getAttributes(h) and not 0x0007
+    var old = getAttributes(h) and not FOREGROUND_RGB
     if bright:
       old = old or FOREGROUND_INTENSITY
     const lookup: array[ForegroundColor, int] = [
@@ -445,7 +486,7 @@ proc setBackgroundColor*(f: File, bg: BackgroundColor, bright=false) =
   ## Sets the terminal's background color.
   when defined(windows):
     let h = conHandle(f)
-    var old = getAttributes(h) and not 0x0070
+    var old = getAttributes(h) and not BACKGROUND_RGB
     if bright:
       old = old or BACKGROUND_INTENSITY
     const lookup: array[BackgroundColor, int] = [
@@ -558,6 +599,8 @@ proc getch*(): char =
     discard fd.tcsetattr(TCSADRAIN, addr oldMode)
 
 # Wrappers assuming output to stdout:
+template hideCursor*() = hideCursor(stdout)
+template showCursor*() = showCursor(stdout)
 template setCursorPos*(x, y: int) = setCursorPos(stdout, x, y)
 template setCursorXPos*(x: int)   = setCursorXPos(stdout, x)
 when defined(windows):

lib/upcoming/asyncdispatch.nim

@@ -827,7 +827,7 @@ when defined(windows) or defined(nimdoc):
                                        cast[pointer](p.ovl))
   {.pop.}
 
-  template registerWaitableEvent(mask) =
+  proc registerWaitableEvent(fd: AsyncFD, cb: Callback; mask: Dword) =
     let p = getGlobalDispatcher()
     var flags = (WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE).Dword
     var hEvent = wsaCreateEvent()
@@ -878,9 +878,11 @@ when defined(windows) or defined(nimdoc):
               deallocShared(cast[pointer](pcd))
               raiseOSError(osLastError())
             else:
-              # we ref pcd.ovl one more time, because it will be unrefed in
+              # we incref `pcd.ovl` and `protect` callback one more time,
-              # poll()
+              # because it will be unrefed and disposed in `poll()` after
+              # callback finishes.
               GC_ref(pcd.ovl)
+              pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb))
     )
     # We need to protect our callback environment value, so GC will not free it
     # accidentally.
@@ -919,7 +921,7 @@ when defined(windows) or defined(nimdoc):
     ## Be sure your callback ``cb`` returns ``true``, if you want to remove
     ## watch of `read` notifications, and ``false``, if you want to continue
     ## receiving notifies.
-    registerWaitableEvent(FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE)
+    registerWaitableEvent(fd, cb, FD_READ or FD_ACCEPT or FD_OOB or FD_CLOSE)
 
   proc addWrite*(fd: AsyncFD, cb: Callback) =
     ## Start watching the file descriptor for write availability and then call
@@ -936,7 +938,7 @@ when defined(windows) or defined(nimdoc):
     ## Be sure your callback ``cb`` returns ``true``, if you want to remove
     ## watch of `write` notifications, and ``false``, if you want to continue
     ## receiving notifies.
-    registerWaitableEvent(FD_WRITE or FD_CONNECT or FD_CLOSE)
+    registerWaitableEvent(fd, cb, FD_WRITE or FD_CONNECT or FD_CLOSE)
 
   template registerWaitableHandle(p, hEvent, flags, pcd, timeout, handleCallback) =
     let handleFD = AsyncFD(hEvent)
@@ -944,7 +946,8 @@ when defined(windows) or defined(nimdoc):
     pcd.handleFd = handleFD
     var ol = PCustomOverlapped()
     GC_ref(ol)
-    ol.data = CompletionData(fd: handleFD, cb: handleCallback)
+    ol.data.fd = handleFD
+    ol.data.cb = handleCallback
     # We need to protect our callback environment value, so GC will not free it
     # accidentally.
     ol.data.cell = system.protect(rawEnv(ol.data.cb))
@@ -986,6 +989,12 @@ when defined(windows) or defined(nimdoc):
         discard closeHandle(hEvent)
         deallocShared(cast[pointer](pcd))
         p.handles.excl(fd)
+      else:
+        # if callback returned `false`, then it wants to be called again, so
+        # we need to ref and protect `pcd.ovl` again, because it will be
+        # unrefed and disposed in `poll()`.
+        GC_ref(pcd.ovl)
+        pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb))
 
     registerWaitableHandle(p, hEvent, flags, pcd, timeout, timercb)
 
@@ -1068,6 +1077,12 @@ when defined(windows) or defined(nimdoc):
         # called in callback.
         if ev.hWaiter != 0: unregister(ev)
         deallocShared(cast[pointer](pcd))
+      else:
+        # if callback returned `false`, then it wants to be called again, so
+        # we need to ref and protect `pcd.ovl` again, because it will be
+        # unrefed and disposed in `poll()`.
+        GC_ref(pcd.ovl)
+        pcd.ovl.data.cell = system.protect(rawEnv(pcd.ovl.data.cb))
 
     registerWaitableHandle(p, hEvent, flags, pcd, INFINITE, eventcb)
     ev.hWaiter = pcd.waitFd
@@ -1075,15 +1090,6 @@ when defined(windows) or defined(nimdoc):
   initAll()
 else:
   import ioselectors
-  when defined(windows):
-    import winlean
-    const
-      EINTR = WSAEINPROGRESS
-      EINPROGRESS = WSAEINPROGRESS
-      EWOULDBLOCK = WSAEWOULDBLOCK
-      EAGAIN = EINPROGRESS
-      MSG_NOSIGNAL = 0
-  else:
   from posix import EINTR, EAGAIN, EINPROGRESS, EWOULDBLOCK, MSG_PEEK,
                     MSG_NOSIGNAL
 

lib/wrappers/openssl.nim

@@ -37,6 +37,8 @@ else:
       DLLUtilName = "libcrypto.so" & versions
   from posix import SocketHandle
 
+import dynlib
+
 type
   SslStruct {.final, pure.} = object
   SslPtr* = ptr SslStruct
@@ -185,16 +187,74 @@ const
   BIO_C_DO_STATE_MACHINE = 101
   BIO_C_GET_SSL = 110
 
-proc SSL_library_init*(): cInt{.cdecl, dynlib: DLLSSLName, importc, discardable.}
-proc SSL_load_error_strings*(){.cdecl, dynlib: DLLSSLName, importc.}
-proc ERR_load_BIO_strings*(){.cdecl, dynlib: DLLUtilName, importc.}
-
-proc SSLv23_client_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.}
-proc SSLv23_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.}
-proc SSLv2_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.}
-proc SSLv3_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.}
 proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.}
 
+when compileOption("dynlibOverride", "ssl"):
+  proc SSL_library_init*(): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.}
+  proc SSL_load_error_strings*() {.cdecl, dynlib: DLLSSLName, importc.}
+  proc SSLv23_client_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.}
+
+  proc SSLv23_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.}
+  proc SSLv2_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.}
+  proc SSLv3_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.}
+
+  template OpenSSL_add_all_algorithms*() = discard
+else:
+  # Here we're trying to stay compatible with openssl 1.0.* and 1.1.*. Some
+  # symbols are loaded dynamically and we don't use them if not found.
+  proc thisModule(): LibHandle {.inline.} =
+    var thisMod {.global.}: LibHandle
+    if thisMod.isNil: thisMod = loadLib()
+    result = thisMod
+
+  proc sslModule(): LibHandle {.inline.} =
+    var sslMod {.global.}: LibHandle
+    if sslMod.isNil: sslMod = loadLibPattern(DLLSSLName)
+    result = sslMod
+
+  proc sslSym(name: string): pointer =
+    var dl = thisModule()
+    if not dl.isNil:
+      result = symAddr(dl, name)
+    if result.isNil:
+      dl = sslModule()
+      if not dl.isNil:
+        result = symAddr(dl, name)
+
+  proc SSL_library_init*(): cint {.discardable.} =
+    let theProc = cast[proc(): cint {.cdecl.}](sslSym("SSL_library_init"))
+    if not theProc.isNil: result = theProc()
+
+  proc SSL_load_error_strings*() =
+    let theProc = cast[proc() {.cdecl.}](sslSym("SSL_load_error_strings"))
+    if not theProc.isNil: theProc()
+
+  proc SSLv23_client_method*(): PSSL_METHOD =
+    let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_client_method"))
+    if not theProc.isNil: result = theProc()
+    else: result = TLSv1_method()
+
+  proc SSLv23_method*(): PSSL_METHOD =
+    let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_method"))
+    if not theProc.isNil: result = theProc()
+    else: result = TLSv1_method()
+
+  proc SSLv2_method*(): PSSL_METHOD =
+    let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv2_method"))
+    if not theProc.isNil: result = theProc()
+    else: result = TLSv1_method()
+
+  proc SSLv3_method*(): PSSL_METHOD =
+    let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv3_method"))
+    if not theProc.isNil: result = theProc()
+    else: result = TLSv1_method()
+
+  proc OpenSSL_add_all_algorithms*() =
+    let theProc = cast[proc() {.cdecl.}](sslSym("OPENSSL_add_all_algorithms_conf"))
+    if not theProc.isNil: theProc()
+
+proc ERR_load_BIO_strings*(){.cdecl, dynlib: DLLUtilName, importc.}
+
 proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.}
 proc SSL_free*(ssl: SslPtr){.cdecl, dynlib: DLLSSLName, importc.}
 proc SSL_get_SSL_CTX*(ssl: SslPtr): SslCtx {.cdecl, dynlib: DLLSSLName, importc.}
@@ -261,11 +321,6 @@ proc ERR_error_string*(e: cInt, buf: cstring): cstring{.cdecl,
 proc ERR_get_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.}
 proc ERR_peek_last_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.}
 
-when defined(android):
-    template OpenSSL_add_all_algorithms*() = discard
-else:
-    proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSSL_add_all_algorithms_conf".}
-
 proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.}
 
 when not useWinVersion and not defined(macosx) and not defined(android):

tests/ccgbugs/tmissingderef2.nim

@@ -0,0 +1,25 @@
+discard """
+  output: "c"
+"""
+
+# bug #5079
+
+import tables, strutils
+
+type Test = ref object
+  s: string
+
+proc `test=`(t: Test, s: string) =
+  t.s = s
+
+var t = Test()
+
+#t.test = spaces(2) # -- works
+
+var a = newTable[string, string]()
+a["b"] = "c"
+
+#t.s = a["b"] # -- works
+#t.test a["b"] # -- works
+t.test = a["b"] # -- prints "out of memory" and quits
+echo t.s

tests/ccgbugs/tobjconstr_regression.nim

@@ -0,0 +1,14 @@
+discard """
+  output: "@[(username: user, role: admin, description: desc, email_addr: email), (username: user, role: admin, description: desc, email_addr: email)]"
+"""
+
+type
+  User = object of RootObj
+    username, role, description, email_addr: string
+
+# bug 5055
+let us4 = @[
+  User(username:"user", role:"admin", description:"desc", email_addr:"email"),
+  User(username:"user", role:"admin", description:"desc", email_addr:"email"),
+]
+echo us4

tests/cpp/tasync_cpp.nim

@@ -8,4 +8,7 @@ discard """
 import jester
 import asyncdispatch, asyncnet
 
+# bug #5081
+#import nre
+
 echo "hello"

tests/modules/UpperCased.nim

@@ -0,0 +1,6 @@
+
+# bug #5076
+
+var str*: string
+
+UpperCased.str = "hello"

tests/modules/tuppercased.nim

@@ -0,0 +1,8 @@
+discard """
+  output: "hello"
+"""
+
+import UpperCased
+
+# stress normalization rules:
+echo Upper_Cased.str

web/news/e029_version_0_16_0.rst

@@ -39,6 +39,10 @@ Library Additions
   ``deques`` provides a superset of ``queues`` API with clear naming.
   ``queues`` module is now deprecated and will be removed in the future.
 
+- Added ``hideCursor`` and ``showCursor`` to the ``terminal``
+  `(doc) <http://nim-lang.org/docs/terminal.html>`_ module.
+
+
 Tool Additions
 --------------