Merge remote-tracking branch 'upstream/devel' into better-error-message

changelog.md

@@ -90,6 +90,8 @@ proc enumToString*(enums: openArray[enum]): string =
 - There is a new stdlib module `std/diff` to compute the famous "diff"
   of two texts by line.
 
+- Added `os.relativePath`.
+
 ### Library changes
 
 - The string output of `macros.lispRepr` proc has been tweaked

compiler/cmdlinehelper.nim

@@ -48,6 +48,15 @@ proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: Confi
   if self.suggestMode:
     conf.command = "nimsuggest"
 
+  # These defines/options should not be enabled while processing nimscript
+  # bug #4446, #9420, #8991, #9589, #9153
+  undefSymbol(conf.symbols, "profiler")
+  undefSymbol(conf.symbols, "memProfiler")
+  undefSymbol(conf.symbols, "nodejs")
+
+  # bug #9120
+  conf.globalOptions.excl(optTaintMode)
+
   proc runNimScriptIfExists(path: AbsoluteFile)=
     if fileExists(path):
       runNimScript(cache, path, freshDefines = false, conf)
@@ -79,6 +88,9 @@ proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: Confi
         # 'nimsuggest foo.nims' means to just auto-complete the NimScript file
         discard
 
+  # Reload configuration from .cfg file
+  loadConfigs(DefaultConfig, cache, conf)
+
   # now process command line arguments again, because some options in the
   # command line can overwite the config file's settings
   extccomp.initVars(conf)

compiler/extccomp.nim

@@ -780,7 +780,7 @@ proc linkViaResponseFile(conf: ConfigRef; cmd: string) =
   let linkerArgs = conf.projectName & "_" & "linkerArgs.txt"
   let args = cmd.substr(i)
   # GCC's response files don't support backslashes. Junk.
-  if conf.cCompiler == ccGcc:
+  if conf.cCompiler == ccGcc or conf.cCompiler == ccCLang:
     writeFile(linkerArgs, args.replace('\\', '/'))
   else:
     writeFile(linkerArgs, args)

compiler/pathutils.nim

@@ -9,9 +9,8 @@
 
 ## Path handling utilities for Nim. Strictly typed code in order
 ## to avoid the never ending time sink in getting path handling right.
-## Might be a candidate for the stdlib later.
 
-import os, strutils
+import os, strutils, pathnorm
 
 type
   AbsoluteFile* = distinct string
@@ -45,130 +44,9 @@ proc cmpPaths*(x, y: AbsoluteDir): int {.borrow.}
 
 proc createDir*(x: AbsoluteDir) {.borrow.}
 
-type
-  PathIter = object
-    i, prev: int
-    notFirst: bool
-
-proc hasNext(it: PathIter; x: string): bool =
-  it.i < x.len
-
-proc next(it: var PathIter; x: string): (int, int) =
-  it.prev = it.i
-  if not it.notFirst and x[it.i] in {DirSep, AltSep}:
-    # absolute path:
-    inc it.i
-  else:
-    while it.i < x.len and x[it.i] notin {DirSep, AltSep}: inc it.i
-  if it.i > it.prev:
-    result = (it.prev, it.i-1)
-  elif hasNext(it, x):
-    result = next(it, x)
-
-  # skip all separators:
-  while it.i < x.len and x[it.i] in {DirSep, AltSep}: inc it.i
-  it.notFirst = true
-
-iterator dirs(x: string): (int, int) =
-  var it: PathIter
-  while hasNext(it, x): yield next(it, x)
-
-proc isDot(x: string; bounds: (int, int)): bool =
-  bounds[1] == bounds[0] and x[bounds[0]] == '.'
-
-proc isDotDot(x: string; bounds: (int, int)): bool =
-  bounds[1] == bounds[0] + 1 and x[bounds[0]] == '.' and x[bounds[0]+1] == '.'
-
-proc isSlash(x: string; bounds: (int, int)): bool =
-  bounds[1] == bounds[0] and x[bounds[0]] in {DirSep, AltSep}
-
-const canonDirSep = when isMainModule: '/' else: DirSep
-
-proc canon(x: string; result: var string; state: var int) =
-  # state: 0th bit set if isAbsolute path. Other bits count
-  # the number of path components.
-  for b in dirs(x):
-    if (state shr 1 == 0) and isSlash(x, b):
-      result.add canonDirSep
-      state = state or 1
-    elif result.len > (state and 1) and isDotDot(x, b):
-      var d = result.len
-      # f/..
-      while (d-1) > (state and 1) and result[d-1] notin {DirSep, AltSep}:
-        dec d
-      if d > 0: setLen(result, d-1)
-    elif isDot(x, b):
-      discard "discard the dot"
-    elif b[1] >= b[0]:
-      if result.len > 0 and result[^1] notin {DirSep, AltSep}:
-        result.add canonDirSep
-      result.add substr(x, b[0], b[1])
-    inc state, 2
-
-proc canon(x: string): string =
-  # - Turn multiple slashes into single slashes.
-  # - Resolve '/foo/../bar' to '/bar'.
-  # - Remove './' from the path.
-  result = newStringOfCap(x.len)
-  var state = 0
-  canon(x, result, state)
-
-when FileSystemCaseSensitive:
-  template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b)
-else:
-  template `!=?`(a, b: char): bool = a != b
-
-proc relativeTo(full, base: string; sep = canonDirSep): string =
-  if full.len == 0: return ""
-  var f, b: PathIter
-  var ff = (0, -1)
-  var bb = (0, -1) # (int, int)
-  result = newStringOfCap(full.len)
-  # skip the common prefix:
-  while f.hasNext(full) and b.hasNext(base):
-    ff = next(f, full)
-    bb = next(b, base)
-    let diff = ff[1] - ff[0]
-    if diff != bb[1] - bb[0]: break
-    var same = true
-    for i in 0..diff:
-      if full[i + ff[0]] !=? base[i + bb[0]]:
-        same = false
-        break
-    if not same: break
-    ff = (0, -1)
-    bb = (0, -1)
-  #  for i in 0..diff:
-  #    result.add base[i + bb[0]]
-
-  # /foo/bar/xxx/ -- base
-  # /foo/bar/baz  -- full path
-  #   ../baz
-  # every directory that is in 'base', needs to add '..'
-  while true:
-    if bb[1] >= bb[0]:
-      if result.len > 0 and result[^1] != sep:
-        result.add sep
-      result.add ".."
-    if not b.hasNext(base): break
-    bb = b.next(base)
-
-  # add the rest of 'full':
-  while true:
-    if ff[1] >= ff[0]:
-      if result.len > 0 and result[^1] != sep:
-        result.add sep
-      for i in 0..ff[1] - ff[0]:
-        result.add full[i + ff[0]]
-    if not f.hasNext(full): break
-    ff = f.next(full)
-
 when true:
-  proc eqImpl(x, y: string): bool =
+  proc eqImpl(x, y: string): bool {.inline.} =
-    when FileSystemCaseSensitive:
+    result = cmpPaths(x, y) == 0
-      result = cmpIgnoreCase(canon x, canon y) == 0
-    else:
-      result = canon(x) == canon(y)
 
   proc `==`*(x, y: AbsoluteFile): bool = eqImpl(x.string, y.string)
   proc `==`*(x, y: AbsoluteDir): bool = eqImpl(x.string, y.string)
@@ -180,20 +58,20 @@ when true:
     assert(not isAbsolute(f.string))
     result = AbsoluteFile newStringOfCap(base.string.len + f.string.len)
     var state = 0
-    canon(base.string, result.string, state)
+    addNormalizePath(base.string, result.string, state)
-    canon(f.string, result.string, state)
+    addNormalizePath(f.string, result.string, state)
 
   proc `/`*(base: AbsoluteDir; f: RelativeDir): AbsoluteDir =
     #assert isAbsolute(base.string)
     assert(not isAbsolute(f.string))
     result = AbsoluteDir newStringOfCap(base.string.len + f.string.len)
     var state = 0
-    canon(base.string, result.string, state)
+    addNormalizePath(base.string, result.string, state)
-    canon(f.string, result.string, state)
+    addNormalizePath(f.string, result.string, state)
 
   proc relativeTo*(fullPath: AbsoluteFile, baseFilename: AbsoluteDir;
-                   sep = canonDirSep): RelativeFile =
+                   sep = DirSep): RelativeFile =
-    RelativeFile(relativeTo(fullPath.string, baseFilename.string, sep))
+    RelativeFile(relativePath(fullPath.string, baseFilename.string, sep))
 
   proc toAbsolute*(file: string; base: AbsoluteDir): AbsoluteFile =
     if isAbsolute(file): result = AbsoluteFile(file)
@@ -208,37 +86,8 @@ when true:
   proc writeFile*(x: AbsoluteFile; content: string) {.borrow.}
 
 when isMainModule:
-  doAssert canon"/foo/../bar" == "/bar"
-  doAssert canon"foo/../bar" == "bar"
-
-  doAssert canon"/f/../bar///" == "/bar"
-  doAssert canon"f/..////bar" == "bar"
-
-  doAssert canon"../bar" == "../bar"
-  doAssert canon"/../bar" == "/../bar"
-
-  doAssert canon("foo/../../bar/") == "../bar"
-  doAssert canon("./bla/blob/") == "bla/blob"
-  doAssert canon(".hiddenFile") == ".hiddenFile"
-  doAssert canon("./bla/../../blob/./zoo.nim") == "../blob/zoo.nim"
-
-  doAssert canon("C:/file/to/this/long") == "C:/file/to/this/long"
-  doAssert canon("") == ""
-  doAssert canon("foobar") == "foobar"
-  doAssert canon("f/////////") == "f"
-
-  doAssert relativeTo("/foo/bar//baz.nim", "/foo") == "bar/baz.nim"
-
-  doAssert relativeTo("/Users/me/bar/z.nim", "/Users/other/bad") == "../../me/bar/z.nim"
-
-  doAssert relativeTo("/Users/me/bar/z.nim", "/Users/other") == "../me/bar/z.nim"
-  doAssert relativeTo("/Users///me/bar//z.nim", "//Users/") == "me/bar/z.nim"
-  doAssert relativeTo("/Users/me/bar/z.nim", "/Users/me") == "bar/z.nim"
-  doAssert relativeTo("", "/users/moo") == ""
-  doAssert relativeTo("foo", "") == "foo"
-
   doAssert AbsoluteDir"/Users/me///" / RelativeFile"z.nim" == AbsoluteFile"/Users/me/z.nim"
-  doAssert relativeTo("/foo/bar.nim", "/foo/") == "bar.nim"
+  doAssert relativePath("/foo/bar.nim", "/foo/", '/') == "bar.nim"
 
 when isMainModule and defined(windows):
   let nasty = string(AbsoluteDir(r"C:\Users\rumpf\projects\nim\tests\nimble\nimbleDir\linkedPkgs\pkgB-#head\../../simplePkgs/pkgB-#head/") / RelativeFile"pkgA/module.nim")

compiler/semstmts.nim

@@ -807,11 +807,13 @@ proc semCase(c: PContext, n: PNode; flags: TExprFlags): PNode =
       typ = commonType(typ, x.sons[1])
       closeScope(c)
     of nkElse:
-      chckCovered = false
       checkSonsLen(x, 1, c.config)
       x.sons[0] = semExprBranchScope(c, x.sons[0])
       typ = commonType(typ, x.sons[0])
       hasElse = true
+      if chckCovered and covered == toCover(c, n.sons[0].typ):
+        localError(c.config, x.info, "invalid else, all cases are already covered")
+      chckCovered = false
     else:
       illFormedAst(x, c.config)
   if chckCovered:

compiler/semtypes.nim

@@ -90,7 +90,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
         if sonsLen(v) == 2:
           strVal = v.sons[1] # second tuple part is the string value
           if skipTypes(strVal.typ, abstractInst).kind in {tyString, tyCString}:
-            if not isOrdinalType(v.sons[0].typ):
+            if not isOrdinalType(v.sons[0].typ, allowEnumWithHoles=true):
               localError(c.config, v.sons[0].info, errOrdinalTypeExpected & "; given: " & typeToString(v.sons[0].typ, preferDesc))
             x = getOrdValue(v.sons[0]) # first tuple part is the ordinal
           else:
@@ -101,7 +101,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
         strVal = v
         x = counter
       else:
-        if not isOrdinalType(v.typ):
+        if not isOrdinalType(v.typ, allowEnumWithHoles=true):
           localError(c.config, v.info, errOrdinalTypeExpected & "; given: " & typeToString(v.typ, preferDesc))
         x = getOrdValue(v)
       if i != 1:

compiler/transf.nim

@@ -368,6 +368,11 @@ proc transformYield(c: PTransf, n: PNode): PTransNode =
   else:
     # we need to introduce new local variables:
     add(result, introduceNewLocalVars(c, c.transCon.forLoopBody.PNode))
+  if result.len > 0:
+    var changeNode = PNode(result[0])
+    changeNode.info = c.transCon.forStmt.info
+    for i, child in changeNode:
+      child.info = changeNode.info
 
 proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PTransNode =
   result = transformSons(c, n)

lib/deprecated/pure/sockets.nim

@@ -219,7 +219,6 @@ when defined(Posix):
     of AF_UNIX:        result = posix.AF_UNIX
     of AF_INET:        result = posix.AF_INET
     of AF_INET6:       result = posix.AF_INET6
-    else: discard
 
   proc toInt(typ: SockType): cint =
     case typ
@@ -227,7 +226,6 @@ when defined(Posix):
     of SOCK_DGRAM:     result = posix.SOCK_DGRAM
     of SOCK_SEQPACKET: result = posix.SOCK_SEQPACKET
     of SOCK_RAW:       result = posix.SOCK_RAW
-    else: discard
 
   proc toInt(p: Protocol): cint =
     case p
@@ -237,7 +235,6 @@ when defined(Posix):
     of IPPROTO_IPV6:   result = posix.IPPROTO_IPV6
     of IPPROTO_RAW:    result = posix.IPPROTO_RAW
     of IPPROTO_ICMP:   result = posix.IPPROTO_ICMP
-    else: discard
 
 else:
   proc toInt(domain: Domain): cint =
@@ -853,7 +850,6 @@ proc connect*(socket: Socket, address: string, port = Port(0),
       of AF_UNIX: s.sin_family = posix.AF_UNIX
       of AF_INET: s.sin_family = posix.AF_INET
       of AF_INET6: s.sin_family = posix.AF_INET6
-      else: nil
     if connect(socket.fd, cast[ptr TSockAddr](addr(s)), sizeof(s).cint) < 0'i32:
       OSError()
 

lib/pure/asyncnet.nim

@@ -622,6 +622,61 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {.
     raiseOSError(osLastError())
   freeAddrInfo(aiList)
 
+when defined(posix):
+
+  proc connectUnix*(socket: AsyncSocket, path: string): Future[void] =
+    ## Binds Unix socket to `path`.
+    ## This only works on Unix-style systems: Mac OS X, BSD and Linux
+    when not defined(nimdoc):
+      let retFuture = newFuture[void]("connectUnix")
+      result = retFuture
+  
+      proc cb(fd: AsyncFD): bool =
+        let ret = SocketHandle(fd).getSockOptInt(cint(SOL_SOCKET), cint(SO_ERROR))
+        if ret == 0:
+          retFuture.complete()
+          return true
+        elif ret == EINTR:
+          return false
+        else:
+          retFuture.fail(newException(OSError, osErrorMsg(OSErrorCode(ret))))
+          return true
+  
+      var socketAddr = makeUnixAddr(path)
+      let ret = socket.fd.connect(cast[ptr SockAddr](addr socketAddr),
+                       (sizeof(socketAddr.sun_family) + path.len).Socklen)
+      if ret == 0:
+        # Request to connect completed immediately.
+        retFuture.complete()
+      else:
+        let lastError = osLastError()
+        if lastError.int32 == EINTR or lastError.int32 == EINPROGRESS:
+          addWrite(AsyncFD(socket.fd), cb)
+        else:
+          retFuture.fail(newException(OSError, osErrorMsg(lastError)))
+
+  proc bindUnix*(socket: AsyncSocket, path: string)  {.
+    tags: [ReadIOEffect].} =
+    ## Binds Unix socket to `path`.
+    ## This only works on Unix-style systems: Mac OS X, BSD and Linux
+    when not defined(nimdoc):
+      var socketAddr = makeUnixAddr(path)
+      if socket.fd.bindAddr(cast[ptr SockAddr](addr socketAddr),
+                            (sizeof(socketAddr.sun_family) + path.len).Socklen) != 0'i32:
+        raiseOSError(osLastError())
+
+elif defined(nimdoc):
+
+  proc connectUnix*(socket: AsyncSocket, path: string): Future[void] =
+    ## Binds Unix socket to `path`.
+    ## This only works on Unix-style systems: Mac OS X, BSD and Linux
+    discard
+  
+  proc bindUnix*(socket: AsyncSocket, path: string) =
+    ## Binds Unix socket to `path`.
+    ## This only works on Unix-style systems: Mac OS X, BSD and Linux
+    discard
+
 proc close*(socket: AsyncSocket) =
   ## Closes the socket.
   defer:

lib/pure/includes/osseps.nim

@@ -0,0 +1,130 @@
+# Include file that implements 'DirSep' and friends. Do not import this when
+# you also import ``os.nim``!
+
+const
+  doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS)
+
+when defined(Nimdoc): # only for proper documentation:
+  const
+    CurDir* = '.'
+      ## The constant string used by the operating system to refer to the
+      ## current directory.
+      ##
+      ## For example: '.' for POSIX or ':' for the classic Macintosh.
+
+    ParDir* = ".."
+      ## The constant string used by the operating system to refer to the
+      ## parent directory.
+      ##
+      ## For example: ".." for POSIX or "::" for the classic Macintosh.
+
+    DirSep* = '/'
+      ## The character used by the operating system to separate pathname
+      ## components, for example, '/' for POSIX or ':' for the classic
+      ## Macintosh.
+
+    AltSep* = '/'
+      ## An alternative character used by the operating system to separate
+      ## pathname components, or the same as `DirSep` if only one separator
+      ## character exists. This is set to '/' on Windows systems
+      ## where `DirSep` is a backslash.
+
+    PathSep* = ':'
+      ## The character conventionally used by the operating system to separate
+      ## search patch components (as in PATH), such as ':' for POSIX
+      ## or ';' for Windows.
+
+    FileSystemCaseSensitive* = true
+      ## true if the file system is case sensitive, false otherwise. Used by
+      ## `cmpPaths` to compare filenames properly.
+
+    ExeExt* = ""
+      ## The file extension of native executables. For example:
+      ## "" for POSIX, "exe" on Windows.
+
+    ScriptExt* = ""
+      ## The file extension of a script file. For example: "" for POSIX,
+      ## "bat" on Windows.
+
+    DynlibFormat* = "lib$1.so"
+      ## The format string to turn a filename into a `DLL`:idx: file (also
+      ## called `shared object`:idx: on some operating systems).
+
+elif defined(macos):
+  const
+    CurDir* = ':'
+    ParDir* = "::"
+    DirSep* = ':'
+    AltSep* = Dirsep
+    PathSep* = ','
+    FileSystemCaseSensitive* = false
+    ExeExt* = ""
+    ScriptExt* = ""
+    DynlibFormat* = "$1.dylib"
+
+  #  MacOS paths
+  #  ===========
+  #  MacOS directory separator is a colon ":" which is the only character not
+  #  allowed in filenames.
+  #
+  #  A path containing no colon or which begins with a colon is a partial
+  #  path.
+  #  E.g. ":kalle:petter" ":kalle" "kalle"
+  #
+  #  All other paths are full (absolute) paths. E.g. "HD:kalle:" "HD:"
+  #  When generating paths, one is safe if one ensures that all partial paths
+  #  begin with a colon, and all full paths end with a colon.
+  #  In full paths the first name (e g HD above) is the name of a mounted
+  #  volume.
+  #  These names are not unique, because, for instance, two diskettes with the
+  #  same names could be inserted. This means that paths on MacOS are not
+  #  waterproof. In case of equal names the first volume found will do.
+  #  Two colons "::" are the relative path to the parent. Three is to the
+  #  grandparent etc.
+elif doslikeFileSystem:
+  const
+    CurDir* = '.'
+    ParDir* = ".."
+    DirSep* = '\\' # seperator within paths
+    AltSep* = '/'
+    PathSep* = ';' # seperator between paths
+    FileSystemCaseSensitive* = false
+    ExeExt* = "exe"
+    ScriptExt* = "bat"
+    DynlibFormat* = "$1.dll"
+elif defined(PalmOS) or defined(MorphOS):
+  const
+    DirSep* = '/'
+    AltSep* = Dirsep
+    PathSep* = ';'
+    ParDir* = ".."
+    FileSystemCaseSensitive* = false
+    ExeExt* = ""
+    ScriptExt* = ""
+    DynlibFormat* = "$1.prc"
+elif defined(RISCOS):
+  const
+    DirSep* = '.'
+    AltSep* = '.'
+    ParDir* = ".." # is this correct?
+    PathSep* = ','
+    FileSystemCaseSensitive* = true
+    ExeExt* = ""
+    ScriptExt* = ""
+    DynlibFormat* = "lib$1.so"
+else: # UNIX-like operating system
+  const
+    CurDir* = '.'
+    ParDir* = ".."
+    DirSep* = '/'
+    AltSep* = DirSep
+    PathSep* = ':'
+    FileSystemCaseSensitive* = when defined(macosx): false else: true
+    ExeExt* = ""
+    ScriptExt* = ""
+    DynlibFormat* = when defined(macosx): "lib$1.dylib" else: "lib$1.so"
+
+const
+  ExtSep* = '.'
+    ## The character which separates the base filename from the extension;
+    ## for example, the '.' in ``os.nim``.

lib/pure/nativesockets.nim

@@ -105,6 +105,7 @@ else:
     osInvalidSocket* = posix.INVALID_SOCKET
     nativeAfInet = posix.AF_INET
     nativeAfInet6 = posix.AF_INET6
+    nativeAfUnix = posix.AF_UNIX
 
 proc `==`*(a, b: Port): bool {.borrow.}
   ## ``==`` for ports.
@@ -482,8 +483,18 @@ proc getAddrString*(sockAddr: ptr SockAddr): string =
         raiseOSError(osLastError())
     setLen(result, len(cstring(result)))
   else:
+    when defined(posix) and not defined(nimdoc):
+      if sockAddr.sa_family.cint == nativeAfUnix:
+        return "unix"
     raise newException(IOError, "Unknown socket family in getAddrString")
 
+when defined(posix) and not defined(nimdoc):
+  proc makeUnixAddr*(path: string): Sockaddr_un =
+    result.sun_family = AF_UNIX.uint16
+    if path.len >= Sockaddr_un_path_length:
+      raise newException(ValueError, "socket path too long")
+    copyMem(addr result.sun_path, path.cstring, path.len + 1)
+
 proc getSockName*(socket: SocketHandle): Port =
   ## returns the socket's associated port number.
   var name: Sockaddr_in

lib/pure/net.nim

@@ -947,13 +947,6 @@ proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {
   var valuei = cint(if value: 1 else: 0)
   setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei)
 
-when defined(posix) and not defined(nimdoc):
-  proc makeUnixAddr(path: string): Sockaddr_un =
-    result.sun_family = AF_UNIX.uint16
-    if path.len >= Sockaddr_un_path_length:
-      raise newException(ValueError, "socket path too long")
-    copyMem(addr result.sun_path, path.cstring, path.len + 1)
-
 when defined(posix) or defined(nimdoc):
   proc connectUnix*(socket: Socket, path: string) =
     ## Connects to Unix socket on `path`.

lib/pure/os.nim

@@ -17,7 +17,7 @@
 include "system/inclrtl"
 
 import
-  strutils
+  strutils, pathnorm
 
 when defined(nimscript):
   discard
@@ -51,133 +51,7 @@ type
 
   OSErrorCode* = distinct int32 ## Specifies an OS Error Code.
 
-const
+include "includes/osseps"
-  doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS)
-
-when defined(Nimdoc): # only for proper documentation:
-  const
-    CurDir* = '.'
-      ## The constant string used by the operating system to refer to the
-      ## current directory.
-      ##
-      ## For example: '.' for POSIX or ':' for the classic Macintosh.
-
-    ParDir* = ".."
-      ## The constant string used by the operating system to refer to the
-      ## parent directory.
-      ##
-      ## For example: ".." for POSIX or "::" for the classic Macintosh.
-
-    DirSep* = '/'
-      ## The character used by the operating system to separate pathname
-      ## components, for example, '/' for POSIX or ':' for the classic
-      ## Macintosh.
-
-    AltSep* = '/'
-      ## An alternative character used by the operating system to separate
-      ## pathname components, or the same as `DirSep` if only one separator
-      ## character exists. This is set to '/' on Windows systems
-      ## where `DirSep` is a backslash.
-
-    PathSep* = ':'
-      ## The character conventionally used by the operating system to separate
-      ## search patch components (as in PATH), such as ':' for POSIX
-      ## or ';' for Windows.
-
-    FileSystemCaseSensitive* = true
-      ## true if the file system is case sensitive, false otherwise. Used by
-      ## `cmpPaths` to compare filenames properly.
-
-    ExeExt* = ""
-      ## The file extension of native executables. For example:
-      ## "" for POSIX, "exe" on Windows.
-
-    ScriptExt* = ""
-      ## The file extension of a script file. For example: "" for POSIX,
-      ## "bat" on Windows.
-
-    DynlibFormat* = "lib$1.so"
-      ## The format string to turn a filename into a `DLL`:idx: file (also
-      ## called `shared object`:idx: on some operating systems).
-
-elif defined(macos):
-  const
-    CurDir* = ':'
-    ParDir* = "::"
-    DirSep* = ':'
-    AltSep* = Dirsep
-    PathSep* = ','
-    FileSystemCaseSensitive* = false
-    ExeExt* = ""
-    ScriptExt* = ""
-    DynlibFormat* = "$1.dylib"
-
-  #  MacOS paths
-  #  ===========
-  #  MacOS directory separator is a colon ":" which is the only character not
-  #  allowed in filenames.
-  #
-  #  A path containing no colon or which begins with a colon is a partial
-  #  path.
-  #  E.g. ":kalle:petter" ":kalle" "kalle"
-  #
-  #  All other paths are full (absolute) paths. E.g. "HD:kalle:" "HD:"
-  #  When generating paths, one is safe if one ensures that all partial paths
-  #  begin with a colon, and all full paths end with a colon.
-  #  In full paths the first name (e g HD above) is the name of a mounted
-  #  volume.
-  #  These names are not unique, because, for instance, two diskettes with the
-  #  same names could be inserted. This means that paths on MacOS are not
-  #  waterproof. In case of equal names the first volume found will do.
-  #  Two colons "::" are the relative path to the parent. Three is to the
-  #  grandparent etc.
-elif doslikeFileSystem:
-  const
-    CurDir* = '.'
-    ParDir* = ".."
-    DirSep* = '\\' # seperator within paths
-    AltSep* = '/'
-    PathSep* = ';' # seperator between paths
-    FileSystemCaseSensitive* = false
-    ExeExt* = "exe"
-    ScriptExt* = "bat"
-    DynlibFormat* = "$1.dll"
-elif defined(PalmOS) or defined(MorphOS):
-  const
-    DirSep* = '/'
-    AltSep* = Dirsep
-    PathSep* = ';'
-    ParDir* = ".."
-    FileSystemCaseSensitive* = false
-    ExeExt* = ""
-    ScriptExt* = ""
-    DynlibFormat* = "$1.prc"
-elif defined(RISCOS):
-  const
-    DirSep* = '.'
-    AltSep* = '.'
-    ParDir* = ".." # is this correct?
-    PathSep* = ','
-    FileSystemCaseSensitive* = true
-    ExeExt* = ""
-    ScriptExt* = ""
-    DynlibFormat* = "lib$1.so"
-else: # UNIX-like operating system
-  const
-    CurDir* = '.'
-    ParDir* = ".."
-    DirSep* = '/'
-    AltSep* = DirSep
-    PathSep* = ':'
-    FileSystemCaseSensitive* = when defined(macosx): false else: true
-    ExeExt* = ""
-    ScriptExt* = ""
-    DynlibFormat* = when defined(macosx): "lib$1.dylib" else: "lib$1.so"
-
-const
-  ExtSep* = '.'
-    ## The character which separates the base filename from the extension;
-    ## for example, the '.' in ``os.nim``.
 
 proc normalizePathEnd(path: var string, trailingSep = false) =
   ## ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on
@@ -228,6 +102,14 @@ proc joinPath*(head, tail: string): string {.
   ##   assert joinPath("", "lib") == "lib"
   ##   assert joinPath("", "/lib") == "/lib"
   ##   assert joinPath("usr/", "/lib") == "usr/lib"
+  result = newStringOfCap(head.len + tail.len)
+  var state = 0
+  addNormalizePath(head, result, state, DirSep)
+  if tail.len == 0:
+    result.add DirSep
+  else:
+    addNormalizePath(tail, result, state, DirSep)
+  when false:
     if len(head) == 0:
       result = tail
     elif head[len(head)-1] in {DirSep, AltSep}:
@@ -246,9 +128,12 @@ proc joinPath*(parts: varargs[string]): string {.noSideEffect,
   ## The same as `joinPath(head, tail)`, but works with any number of
   ## directory parts. You need to pass at least one element or the proc
   ## will assert in debug builds and crash on release builds.
-  result = parts[0]
+  var estimatedLen = 0
-  for i in 1..high(parts):
+  for p in parts: estimatedLen += p.len
-    result = joinPath(result, parts[i])
+  result = newStringOfCap(estimatedLen)
+  var state = 0
+  for i in 0..high(parts):
+    addNormalizePath(parts[i], result, state, DirSep)
 
 proc `/`*(head, tail: string): string {.noSideEffect.} =
   ## The same as ``joinPath(head, tail)``
@@ -287,6 +172,71 @@ proc splitPath*(path: string): tuple[head, tail: string] {.
     result.head = ""
     result.tail = path
 
+when FileSystemCaseSensitive:
+  template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b)
+else:
+  template `!=?`(a, b: char): bool = a != b
+
+proc relativePath*(path, base: string; sep = DirSep): string {.
+  noSideEffect, rtl, extern: "nos$1", raises: [].} =
+  ## Converts `path` to a path relative to `base`.
+  ## The `sep` is used for the path normalizations, this can be useful to
+  ## ensure the relative path only contains '/' so that it can be used for
+  ## URL constructions.
+  runnableExamples:
+    doAssert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim"
+    doAssert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim"
+    doAssert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim"
+    doAssert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim"
+    doAssert relativePath("", "/users/moo", '/') == ""
+
+
+  # Todo: If on Windows, path and base do not agree on the drive letter,
+  # return `path` as is.
+  if path.len == 0: return ""
+  var f, b: PathIter
+  var ff = (0, -1)
+  var bb = (0, -1) # (int, int)
+  result = newStringOfCap(path.len)
+  # skip the common prefix:
+  while f.hasNext(path) and b.hasNext(base):
+    ff = next(f, path)
+    bb = next(b, base)
+    let diff = ff[1] - ff[0]
+    if diff != bb[1] - bb[0]: break
+    var same = true
+    for i in 0..diff:
+      if path[i + ff[0]] !=? base[i + bb[0]]:
+        same = false
+        break
+    if not same: break
+    ff = (0, -1)
+    bb = (0, -1)
+  #  for i in 0..diff:
+  #    result.add base[i + bb[0]]
+
+  # /foo/bar/xxx/ -- base
+  # /foo/bar/baz  -- path path
+  #   ../baz
+  # every directory that is in 'base', needs to add '..'
+  while true:
+    if bb[1] >= bb[0]:
+      if result.len > 0 and result[^1] != sep:
+        result.add sep
+      result.add ".."
+    if not b.hasNext(base): break
+    bb = b.next(base)
+
+  # add the rest of 'path':
+  while true:
+    if ff[1] >= ff[0]:
+      if result.len > 0 and result[^1] != sep:
+        result.add sep
+      for i in 0..ff[1] - ff[0]:
+        result.add path[i + ff[0]]
+    if not f.hasNext(path): break
+    ff = f.next(path)
+
 proc parentDirPos(path: string): int =
   var q = 1
   if len(path) >= 1 and path[len(path)-1] in {DirSep, AltSep}: q = 2
@@ -478,14 +428,17 @@ proc cmpPaths*(pathA, pathB: string): int {.
       doAssert cmpPaths("foo", "Foo") == 0
     elif defined(posix):
       doAssert cmpPaths("foo", "Foo") > 0
+
+  let a = normalizePath(pathA)
+  let b = normalizePath(pathB)
   if FileSystemCaseSensitive:
-    result = cmp(pathA, pathB)
+    result = cmp(a, b)
   else:
     when defined(nimscript):
-      result = cmpic(pathA, pathB)
+      result = cmpic(a, b)
     elif defined(nimdoc): discard
     else:
-      result = cmpIgnoreCase(pathA, pathB)
+      result = cmpIgnoreCase(a, b)
 
 proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1".} =
   ## Checks whether a given `path` is absolute.
@@ -523,12 +476,10 @@ proc unixToNativePath*(path: string, drive=""): string {.
   ## which drive label to use during absolute path conversion.
   ## `drive` defaults to the drive of the current working directory, and is
   ## ignored on systems that do not have a concept of "drives".
-
   when defined(unix):
     result = path
   else:
-    if path.len == 0:
+    if path.len == 0: return ""
-        return ""
 
     var start: int
     if path[0] == '/':

lib/pure/pathnorm.nim

@@ -0,0 +1,95 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2018 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## OS-Path normalization. Used by ``os.nim`` but also
+## generally useful for dealing with paths. Note that this module
+## does not provide a stable API.
+
+# Yes, this uses import here, not include so that
+# we don't end up exporting these symbols from pathnorm and os:
+import "includes/osseps"
+
+type
+  PathIter* = object
+    i, prev: int
+    notFirst: bool
+
+proc hasNext*(it: PathIter; x: string): bool =
+  it.i < x.len
+
+proc next*(it: var PathIter; x: string): (int, int) =
+  it.prev = it.i
+  if not it.notFirst and x[it.i] in {DirSep, AltSep}:
+    # absolute path:
+    inc it.i
+  else:
+    while it.i < x.len and x[it.i] notin {DirSep, AltSep}: inc it.i
+  if it.i > it.prev:
+    result = (it.prev, it.i-1)
+  elif hasNext(it, x):
+    result = next(it, x)
+
+  # skip all separators:
+  while it.i < x.len and x[it.i] in {DirSep, AltSep}: inc it.i
+  it.notFirst = true
+
+iterator dirs(x: string): (int, int) =
+  var it: PathIter
+  while hasNext(it, x): yield next(it, x)
+
+proc isDot(x: string; bounds: (int, int)): bool =
+  bounds[1] == bounds[0] and x[bounds[0]] == '.'
+
+proc isDotDot(x: string; bounds: (int, int)): bool =
+  bounds[1] == bounds[0] + 1 and x[bounds[0]] == '.' and x[bounds[0]+1] == '.'
+
+proc isSlash(x: string; bounds: (int, int)): bool =
+  bounds[1] == bounds[0] and x[bounds[0]] in {DirSep, AltSep}
+
+proc addNormalizePath*(x: string; result: var string; state: var int; dirSep = DirSep) =
+  ## Low level proc. Undocumented.
+
+  # state: 0th bit set if isAbsolute path. Other bits count
+  # the number of path components.
+  var it: PathIter
+  it.notFirst = (state shr 1) > 0
+  if it.notFirst:
+    while it.i < x.len and x[it.i] in {DirSep, AltSep}: inc it.i
+  while hasNext(it, x):
+    let b = next(it, x)
+    if (state shr 1 == 0) and isSlash(x, b):
+      result.add dirSep
+      state = state or 1
+    elif result.len > (state and 1) and isDotDot(x, b):
+      var d = result.len
+      # f/..
+      while (d-1) > (state and 1) and result[d-1] notin {DirSep, AltSep}:
+        dec d
+      if d > 0: setLen(result, d-1)
+    elif isDot(x, b):
+      discard "discard the dot"
+    elif b[1] >= b[0]:
+      if result.len > 0 and result[^1] notin {DirSep, AltSep}:
+        result.add dirSep
+      result.add substr(x, b[0], b[1])
+    inc state, 2
+
+proc normalizePath*(path: string; dirSep = DirSep): string =
+  ## Example:
+  ##
+  ## .. code-block:: nim
+  ##   assert normalizePath("./foo//bar/../baz") == "foo/baz"
+  ##
+  ##
+  ## - Turns multiple slashes into single slashes.
+  ## - Resolves '/foo/../bar' to '/bar'.
+  ## - Removes './' from the path.
+  result = newStringOfCap(path.len)
+  var state = 0
+  addNormalizePath(path, result, state, dirSep)

lib/pure/unittest.nim

@@ -239,7 +239,6 @@ method testEnded*(formatter: ConsoleOutputFormatter, testResult: TestResult) =
                     of OK: fgGreen
                     of FAILED: fgRed
                     of SKIPPED: fgYellow
-                    else: fgWhite
         styledEcho styleBright, color, prefix, "[", $testResult.status, "] ", resetStyle, testResult.testName
       else:
         rawPrint()

testament/tester.nim

@@ -292,8 +292,7 @@ proc generatedFile(test: TTest, target: TTarget): string =
   let (_, name, _) = test.name.splitFile
   let ext = targetToExt[target]
   result = nimcacheDir(test.name, test.options, target) /
-    (if target == targetJS: "" else: "compiler_") &
+    ((if target == targetJS: "" else: "compiler_") & name.changeFileExt(ext))
-    name.changeFileExt(ext)
 
 proc needsCodegenCheck(spec: TSpec): bool =
   result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0

tests/stdlib/tospaths.nim

@@ -3,7 +3,7 @@ discard """
 """
 # test the ospaths module
 
-import os
+import os, pathnorm
 
 doAssert unixToNativePath("") == ""
 doAssert unixToNativePath(".") == $CurDir
@@ -61,3 +61,39 @@ block lastPathPartTest:
   when doslikeFileSystem:
     doAssert lastPathPart(r"foo\bar.txt") == "bar.txt"
     doAssert lastPathPart(r"foo\") == "foo"
+
+template canon(x): untyped = normalizePath(x, '/')
+doAssert canon"/foo/../bar" == "/bar"
+doAssert canon"foo/../bar" == "bar"
+
+doAssert canon"/f/../bar///" == "/bar"
+doAssert canon"f/..////bar" == "bar"
+
+doAssert canon"../bar" == "../bar"
+doAssert canon"/../bar" == "/../bar"
+
+doAssert canon("foo/../../bar/") == "../bar"
+doAssert canon("./bla/blob/") == "bla/blob"
+doAssert canon(".hiddenFile") == ".hiddenFile"
+doAssert canon("./bla/../../blob/./zoo.nim") == "../blob/zoo.nim"
+
+doAssert canon("C:/file/to/this/long") == "C:/file/to/this/long"
+doAssert canon("") == ""
+doAssert canon("foobar") == "foobar"
+doAssert canon("f/////////") == "f"
+
+doAssert relativePath("/foo/bar//baz.nim", "/foo", '/') == "bar/baz.nim"
+doAssert normalizePath("./foo//bar/../baz", '/') == "foo/baz"
+
+doAssert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim"
+
+doAssert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim"
+doAssert relativePath("/Users///me/bar//z.nim", "//Users/", '/') == "me/bar/z.nim"
+doAssert relativePath("/Users/me/bar/z.nim", "/Users/me", '/') == "bar/z.nim"
+doAssert relativePath("", "/users/moo", '/') == ""
+doAssert relativePath("foo", "", '/') == "foo"
+
+doAssert joinPath("usr", "") == unixToNativePath"usr/"
+doAssert joinPath("", "lib") == "lib"
+doAssert joinPath("", "/lib") == unixToNativePath"/lib"
+doAssert joinPath("usr/", "/lib") == unixToNativePath"usr/lib"

tools/vccexe/vccexe.nim

@@ -2,11 +2,11 @@ import strutils, strtabs, os, osproc, vcvarsall, vccenv
 
 type
   VccVersion* = enum ## VCC compiler backend versions
-    vccUndefined = (0, ""), ## VCC version undefined, resolves to the latest recognizable VCC version
+    vccUndefined = 0,   ## VCC version undefined, resolves to the latest recognizable VCC version
-    vcc90  =  vs90, ## Visual Studio 2008 (Version 9.0)
+    vcc90  = vs90  ## Visual Studio 2008 (Version 9.0)
-    vcc100 = vs100, ## Visual Studio 2010 (Version 10.0)
+    vcc100 = vs100 ## Visual Studio 2010 (Version 10.0)
-    vcc110 = vs110, ## Visual Studio 2012 (Version 11.0)
+    vcc110 = vs110 ## Visual Studio 2012 (Version 11.0)
-    vcc120 = vs120, ## Visual Studio 2013 (Version 12.0)
+    vcc120 = vs120 ## Visual Studio 2013 (Version 12.0)
     vcc140 = vs140 ## Visual Studio 2015 (Version 14.0)
 
 proc discoverVccVcVarsAllPath*(version: VccVersion = vccUndefined): string =
@@ -101,12 +101,12 @@ command was specified
 when isMainModule:
   var vccversionArg: seq[string] = @[]
   var printPathArg: bool = false
-  var vcvarsallArg: string = nil
+  var vcvarsallArg: string
-  var commandArg: string = nil
+  var commandArg: string
   var noCommandArg: bool = false
   var platformArg: VccArch
   var sdkTypeArg: VccPlatformType
-  var sdkVersionArg: string = nil
+  var sdkVersionArg: string
   var verboseArg: bool = false
 
   var clArgs: seq[TaintedString] = @[]

tools/vccexe/vcvarsall.nim

@@ -34,7 +34,7 @@ type
     vccplatUWP = "uwp", ## Universal Windows Platform (UWP) Application
     vccplatOneCore = "onecore" # Undocumented platform type in the Windows SDK, probably XBox One SDK platform type.
 
-proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type: VccPlatformType = vccplatEmpty, sdk_version: string = nil, verbose: bool = false): StringTableRef =
+proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type: VccPlatformType = vccplatEmpty, sdk_version: string = "", verbose: bool = false): StringTableRef =
   ## Returns a string table containing the proper process environment to successfully execute VCC compile commands for the specified SDK version, CPU architecture and platform type.
   ##
   ## path
@@ -50,7 +50,7 @@ proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type
 
   var vccvarsallpath = path
   # Assume that default executable is in current directory or in PATH
-  if path == nil or path.len < 1:
+  if path == "":
     vccvarsallpath = vcvarsallDefaultPath
   
   var args: seq[string] = @[]