Merge branch 'master' into native-pkg-support

changelog.markdown

@@ -3,6 +3,46 @@
 
 # Nimble changelog
 
+## 0.8.0 - 01/01/2017
+
+This is a large release containing multiple new features and many bug fixes.
+
+* Implemented a completely new output system.
+  * Supports different message types and priorities. Each is differently
+    encoded using a color and a brightness.
+  * The amount of messages shown can be changed by using the new ``--verbose``
+    and ``--debug`` flags, by default only high priority messages are shown.
+  * Duplicate warnings are filtered out to prevent too much noise.
+* Package namespaces are now validated. You will see a warning whenever an
+  incorrectly namespaced package is read by Nimble, this can occur either
+  during installation or when the installed package database is being loaded.
+  The namespacing rules are described in Nimble's
+  [readme](https://github.com/nim-lang/nimble#libraries).
+  **Consider these warnings to be unstable, if you see something that you
+  think is incorrect please report it**.
+* Special version dependencies are now installed into a directory with that
+  special version in its name. For example, ``compiler@#head`` will be installed
+  into ``~/.nimble/pkgs/compiler-#head``. This reduces the amount of redundant
+  installs. See [#88](https://github.com/nim-lang/nimble/issues/88) for
+  more information.
+* Nimble now supports package aliases in the packages.json files.
+* Fixed regression that caused transitive dependencies to not be installed.
+* Fixed problem with ``install`` command when a ``src`` directory is present
+  in the current directory.
+* Improved quoting of process execution arguments.
+* Many improvements to custom ``--nimbleDir`` handling. All commands should now
+  support it correctly.
+* Running ``nimble -v`` will no longer read the Nimble config before displaying
+  the version.
+* Refresh command now supports a package list name as argument.
+* Fixes issues with symlinks not being removed correctly.
+* Changed the way the ``dump`` command locates the .nimble file.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.7.10...v0.8.0
+Full list of issues which have been closed: https://github.com/nim-lang/nimble/issues?utf8=%E2%9C%93&q=is%3Aissue+closed%3A%222016-10-09+..+2017-01-01%22+
+
 ## 0.7.10 - 09/10/2016
 
 This release includes multiple bug fixes.

readme.markdown

@@ -464,7 +464,7 @@ To make testing even more convenient, you may wish to define a ``test`` task
 in your ``.nimble`` file. Like so:
 
 ```nim
-task "test", "Runs the test suite":
+task test, "Runs the test suite":
   exec "nim c -r tests/tester"
 ```
 

src/nimble.nim

@@ -45,45 +45,6 @@ proc refresh(options: Options) =
     else:
       ""
 
-  proc downloadList(list: PackageList, options: Options) =
-    display("Downloading", list.name & " package list", priority = HighPriority)
-
-    var lastError = ""
-    for i in 0 .. <list.urls.len:
-      let url = list.urls[i]
-      display("Trying", url)
-      let tempPath = options.getNimbleDir() / "packages_temp.json"
-
-      # Grab the proxy
-      let proxy = getProxy(options)
-      if not proxy.isNil:
-        var maskedUrl = proxy.url
-        if maskedUrl.password.len > 0: maskedUrl.password = "***"
-        display("Connecting", "to proxy at " & $maskedUrl,
-                priority = LowPriority)
-
-      try:
-        downloadFile(url, tempPath, proxy = getProxy(options))
-      except:
-        let message = "Could not download: " & getCurrentExceptionMsg()
-        display("Warning:", message, Warning)
-        lastError = message
-        continue
-
-      if not validatePackagesList(tempPath):
-        lastError = "Downloaded packages.json file is invalid"
-        display("Warning:", lastError & ", discarding.", Warning)
-        continue
-
-      copyFile(tempPath,
-          options.getNimbleDir() / "packages_$1.json" % list.name.toLowerAscii())
-      display("Success", "Package list downloaded.", Success, HighPriority)
-      lastError = ""
-      break
-
-    if lastError.len != 0:
-      raise newException(NimbleError, "Refresh failed\n" & lastError)
-
   if parameter.len > 0:
     if parameter.isUrl:
       let cmdLine = PackageList(name: "commandline", urls: @[parameter])
@@ -275,17 +236,27 @@ proc processDeps(pkginfo: PackageInfo, options: Options): seq[string] =
         let msg = "Unsatisfied dependency: " & dep.name & " (" & $dep.ver & ")"
         raise newException(NimbleError, msg)
     else:
-      let depDesc = "$1@$2" % [dep.name, $dep.ver]
+      let resolvedDep = dep.resolveAlias(options)
-      display("Checking", "for $1" % depDesc, priority = MediumPriority)
+      display("Checking", "for $1" % $resolvedDep, priority = MediumPriority)
       var pkg: PackageInfo
-      if not findPkg(pkglist, dep, pkg):
+      var found = findPkg(pkgList, resolvedDep, pkg)
-        display("Installing", depDesc, priority = HighPriority)
+      # Check if the original name exists.
-        let (paths, installedPkg) = install(@[(dep.name, dep.ver)], options)
+      if not found and resolvedDep.name != dep.name:
+        display("Checking", "for $1" % $dep, priority = MediumPriority)
+        found = findPkg(pkgList, dep, pkg)
+        if found:
+          display("Warning:", "Installed package $1 should be renamed to $2" %
+                  [dep.name, resolvedDep.name], Warning, HighPriority)
+
+      if not found:
+        display("Installing", $resolvedDep, priority = HighPriority)
+        let toInstall = @[(resolvedDep.name, resolvedDep.ver)]
+        let (paths, installedPkg) = install(toInstall, options)
         result.add(paths)
 
         pkg = installedPkg # For addRevDep
       else:
-        display("Info:", "Dependency on $1 already satisfied" % depDesc,
+        display("Info:", "Dependency on $1 already satisfied" % $dep,
                 priority = HighPriority)
         result.add(pkg.mypath.splitFile.dir)
         # Process the dependencies of this dependency.
@@ -496,6 +467,10 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options,
       when defined(unix):
         display("Creating", "symlink: $1 -> $2" %
                 [pkgDestDir / bin, binDir / cleanBin], priority = MediumPriority)
+        if existsFile(binDir / cleanBin):
+          display("Warning:", "Symlink already exists in $1. Replacing." % binDir,
+                  Warning, HighPriority)
+          removeFile(binDir / cleanBin)
         createSymlink(pkgDestDir / bin, binDir / cleanBin)
         binariesInstalled.incl(cleanBin)
       elif defined(windows):
@@ -619,15 +594,6 @@ proc install(packages: seq[PkgTuple],
   if packages == @[]:
     result = installFromDir(getCurrentDir(), newVRAny(), options, "")
   else:
-    # If packages.json is not present ask the user if they want to download it.
-    if needsRefresh(options):
-      if doPrompt and
-          options.prompt("No local packages.json found, download it from " &
-              "internet?"):
-        refresh(options)
-      else:
-        raise newException(NimbleError, "Please run nimble refresh.")
-
     # Install each package.
     for pv in packages:
       let (meth, url) = getDownloadInfo(pv, options, doPrompt)

src/nimblepkg/common.nim

@@ -50,4 +50,4 @@ when not defined(nimscript):
     raise exc
 
 const
-  nimbleVersion* = "0.7.11"
+  nimbleVersion* = "0.8.0"

src/nimblepkg/packageinfo.nim

@@ -1,7 +1,13 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
+
+# Stdlib imports
+import system except TResult
 import parsecfg, json, streams, strutils, parseutils, os, sets, tables
-import version, tools, common, options, cli
+import httpclient
+
+# Local imports
+import version, tools, common, options, cli, config
 
 type
   Package* = object ## Definition of package from packages.json.
@@ -16,6 +22,7 @@ type
     version*: string
     dvcsTag*: string
     web*: string # Info url for humans.
+    alias*: string ## A name of another package, that this package aliases.
 
   MetaData* = object
     url*: string
@@ -103,16 +110,20 @@ proc fromJson(obj: JSonNode): Package =
   ##
   ## Aborts execution if the JSON node doesn't contain the required fields.
   result.name = obj.requiredField("name")
-  result.version = obj.optionalField("version")
+  if obj.hasKey("alias"):
-  result.url = obj.requiredField("url")
+    result.alias = obj.requiredField("alias")
-  result.downloadMethod = obj.requiredField("method")
+  else:
-  result.dvcsTag = obj.optionalField("dvcs-tag")
+    result.alias = ""
-  result.license = obj.requiredField("license")
+    result.version = obj.optionalField("version")
-  result.tags = @[]
+    result.url = obj.requiredField("url")
-  for t in obj["tags"]:
+    result.downloadMethod = obj.requiredField("method")
-    result.tags.add(t.str)
+    result.dvcsTag = obj.optionalField("dvcs-tag")
-  result.description = obj.requiredField("description")
+    result.license = obj.requiredField("license")
-  result.web = obj.optionalField("web")
+    result.tags = @[]
+    for t in obj["tags"]:
+      result.tags.add(t.str)
+    result.description = obj.requiredField("description")
+    result.web = obj.optionalField("web")
 
 proc readMetaData*(path: string): MetaData =
   ## Reads the metadata present in ``~/.nimble/pkgs/pkg-0.1/nimblemeta.json``
@@ -127,21 +138,107 @@ proc readMetaData*(path: string): MetaData =
   let jsonmeta = parseJson(cont)
   result.url = jsonmeta["url"].str
 
-proc getPackage*(pkg: string, options: Options,
+proc needsRefresh*(options: Options): bool =
-    resPkg: var Package): bool =
+  ## Determines whether a ``nimble refresh`` is needed.
+  ##
+  ## In the future this will check a stored time stamp to determine how long
+  ## ago the package list was refreshed.
+  result = true
+  for name, list in options.config.packageLists:
+    if fileExists(options.getNimbleDir() / "packages_" & name & ".json"):
+      result = false
+
+proc validatePackagesList(path: string): bool =
+  ## Determines whether package list at ``path`` is valid.
+  try:
+    let pkgList = parseFile(path)
+    if pkgList.kind == JArray:
+      if pkgList.len == 0:
+        display("Warning:", path & " contains no packages.", Warning,
+                HighPriority)
+      return true
+  except ValueError, JsonParsingError:
+    return false
+
+proc downloadList*(list: PackageList, options: Options) =
+  ## Downloads the specified package list and saves it in $nimbleDir.
+  display("Downloading", list.name & " package list", priority = HighPriority)
+
+  var lastError = ""
+  for i in 0 .. <list.urls.len:
+    let url = list.urls[i]
+    display("Trying", url)
+    let tempPath = options.getNimbleDir() / "packages_temp.json"
+
+    # Grab the proxy
+    let proxy = getProxy(options)
+    if not proxy.isNil:
+      var maskedUrl = proxy.url
+      if maskedUrl.password.len > 0: maskedUrl.password = "***"
+      display("Connecting", "to proxy at " & $maskedUrl,
+              priority = LowPriority)
+
+    try:
+      downloadFile(url, tempPath, proxy = getProxy(options))
+    except:
+      let message = "Could not download: " & getCurrentExceptionMsg()
+      display("Warning:", message, Warning)
+      lastError = message
+      continue
+
+    if not validatePackagesList(tempPath):
+      lastError = "Downloaded packages.json file is invalid"
+      display("Warning:", lastError & ", discarding.", Warning)
+      continue
+
+    copyFile(tempPath,
+        options.getNimbleDir() / "packages_$1.json" % list.name.toLowerAscii())
+    display("Success", "Package list downloaded.", Success, HighPriority)
+    lastError = ""
+    break
+
+  if lastError.len != 0:
+    raise newException(NimbleError, "Refresh failed\n" & lastError)
+
+proc readPackageList(name: string, options: Options): JsonNode =
+  # If packages.json is not present ask the user if they want to download it.
+  if needsRefresh(options):
+    if options.prompt("No local packages.json found, download it from " &
+            "internet?"):
+      for name, list in options.config.packageLists:
+        downloadList(list, options)
+    else:
+      raise newException(NimbleError, "Please run nimble refresh.")
+  return parseFile(options.getNimbleDir() / "packages_" &
+                   name.toLowerAscii() & ".json")
+
+proc getPackage*(pkg: string, options: Options, resPkg: var Package): bool
+proc resolveAlias(pkg: Package, options: Options): Package =
+  result = pkg
+  # Resolve alias.
+  if pkg.alias.len > 0:
+    display("Warning:", "The $1 package has been renamed to $2" %
+            [pkg.name, pkg.alias], Warning, HighPriority)
+    if not getPackage(pkg.alias, options, result):
+      raise newException(NimbleError, "Alias for package not found: " &
+                         pkg.alias)
+
+proc getPackage*(pkg: string, options: Options, resPkg: var Package): bool =
   ## Searches any packages.json files defined in ``options.config.packageLists``
   ## Saves the found package into ``resPkg``.
   ##
   ## Pass in ``pkg`` the name of the package you are searching for. As
   ## convenience the proc returns a boolean specifying if the ``resPkg`` was
   ## successfully filled with good data.
+  ##
+  ## Aliases are handled and resolved.
   for name, list in options.config.packageLists:
     display("Reading", "$1 package list" % name, priority = LowPriority)
-    let packages = parseFile(options.getNimbleDir() /
+    let packages = readPackageList(name, options)
-        "packages_" & name.toLowerAscii() & ".json")
     for p in packages:
       if normalize(p["name"].str) == normalize(pkg):
         resPkg = p.fromJson()
+        resPkg = resolveAlias(resPkg, options)
         return true
 
 proc getPackageList*(options: Options): seq[Package] =
@@ -149,8 +246,7 @@ proc getPackageList*(options: Options): seq[Package] =
   result = @[]
   var namesAdded = initSet[string]()
   for name, list in options.config.packageLists:
-    let packages = parseFile(options.getNimbleDir() /
+    let packages = readPackageList(name, options)
-        "packages_" & name.toLowerAscii() & ".json")
     for p in packages:
       let pkg: Package = p.fromJson()
       if pkg.name notin namesAdded:
@@ -208,6 +304,17 @@ proc withinRange*(pkgInfo: PackageInfo, verRange: VersionRange): bool =
   return withinRange(newVersion(pkgInfo.version), verRange) or
          withinRange(newVersion(pkgInfo.specialVersion), verRange)
 
+proc resolveAlias*(dep: PkgTuple, options: Options): PkgTuple =
+  ## Looks up the specified ``dep.name`` in the packages.json files to resolve
+  ## a potential alias into the package's real name.
+  result = dep
+  var pkg: Package
+  # TODO: This needs better caching.
+  if getPackage(dep.name, options, pkg):
+    # The resulting ``pkg`` will contain the resolved name or the original if
+    # no alias is present.
+    result.name = pkg.name
+
 proc findPkg*(pkglist: seq[tuple[pkgInfo: PackageInfo, meta: MetaData]],
              dep: PkgTuple,
              r: var PackageInfo): bool =
@@ -255,12 +362,15 @@ proc getOutputDir*(pkgInfo: PackageInfo, bin: string): string =
 
 proc echoPackage*(pkg: Package) =
   echo(pkg.name & ":")
-  echo("  url:         " & pkg.url & " (" & pkg.downloadMethod & ")")
+  if pkg.alias.len > 0:
-  echo("  tags:        " & pkg.tags.join(", "))
+    echo("  Alias for ", pkg.alias)
-  echo("  description: " & pkg.description)
+  else:
-  echo("  license:     " & pkg.license)
+    echo("  url:         " & pkg.url & " (" & pkg.downloadMethod & ")")
-  if pkg.web.len > 0:
+    echo("  tags:        " & pkg.tags.join(", "))
-    echo("  website:     " & pkg.web)
+    echo("  description: " & pkg.description)
+    echo("  license:     " & pkg.license)
+    if pkg.web.len > 0:
+      echo("  website:     " & pkg.web)
 
 proc getDownloadDirName*(pkg: Package, verRange: VersionRange): string =
   result = pkg.name
@@ -269,28 +379,6 @@ proc getDownloadDirName*(pkg: Package, verRange: VersionRange): string =
     result.add "_"
     result.add verSimple
 
-proc needsRefresh*(options: Options): bool =
-  ## Determines whether a ``nimble refresh`` is needed.
-  ##
-  ## In the future this will check a stored time stamp to determine how long
-  ## ago the package list was refreshed.
-  result = true
-  for name, list in options.config.packageLists:
-    if fileExists(options.getNimbleDir() / "packages_" & name & ".json"):
-      result = false
-
-proc validatePackagesList*(path: string): bool =
-  ## Determines whether package list at ``path`` is valid.
-  try:
-    let pkgList = parseFile(path)
-    if pkgList.kind == JArray:
-      if pkgList.len == 0:
-        display("Warning:", path & " contains no packages.", Warning,
-                HighPriority)
-      return true
-  except ValueError, JsonParsingError:
-    return false
-
 proc checkInstallFile(pkgInfo: PackageInfo,
                       origDir, file: string): bool =
   ## Checks whether ``file`` should be installed.

src/nimblepkg/publish.nim

@@ -26,6 +26,8 @@ proc getGithubAuth(): Auth =
   display("Info:", "Please create a new personal access token on Github in" &
          " order to allow Nimble to fork the packages repository.",
          priority = HighPriority)
+  display("Hint:", "Make sure to give the access token access to public repos" &
+          " (public_repo scope)!", Warning, HighPriority)
   sleep(5000)
   display("Info:", "Your default browser should open with the following URL: " &
           "https://github.com/settings/tokens/new", priority = HighPriority)
@@ -54,8 +56,12 @@ proc forkExists(a: Auth): bool =
     result = false
 
 proc createFork(a: Auth) =
-  discard postContent("https://api.github.com/repos/nim-lang/packages/forks",
+  try:
-      extraHeaders=createHeaders(a))
+    discard postContent("https://api.github.com/repos/nim-lang/packages/forks",
+        extraHeaders=createHeaders(a))
+  except HttpRequestError:
+    raise newException(NimbleError, "Unable to create fork. Access token" &
+                       " might not have enough permissions.")
 
 proc createPullRequest(a: Auth, packageName, branch: string) =
   display("Info", "Creating PR", priority = HighPriority)

src/nimblepkg/version.nim

@@ -291,6 +291,9 @@ proc findLatest*(verRange: VersionRange,
     if ver > result.ver:
       result = (ver, tag)
 
+proc `$`*(dep: PkgTuple): string =
+  return dep.name & "@" & $dep.ver
+
 when isMainModule:
   doAssert(newVersion("1.0") < newVersion("1.4"))
   doAssert(newVersion("1.0.1") > newVersion("1.0"))

tests/tester.nim

@@ -172,7 +172,7 @@ test "can use nimscript's setCommand":
     let (output, exitCode) = execNimble("--verbose", "cTest")
     let lines = output.strip.splitLines()
     check exitCode == QuitSuccess
-    check "Compilation finished".normalize in lines[^1].normalize
+    check "Execution finished".normalize in lines[^1].normalize
 
 test "can use nimscript's setCommand with flags":
   cd "nimscript":