Merge branch 'master' into native-pkg-support

nimble.nimble

@@ -1,4 +1,7 @@
-when fileExists("src/nimblepkg/common.nim"):
+import ospaths
+template thisModuleFile: string = instantiationInfo(fullPaths = true).filename
+
+when fileExists(thisModuleFile.parentDir / "src/nimblepkg/common.nim"):
   # In the git repository the Nimble sources are in a ``src`` directory.
   import src/nimblepkg/common
 else:

readme.markdown

@@ -10,7 +10,6 @@ Interested in learning **how to create a package**? Skip directly to that sectio
 
 - [Requirements](#requirements)
 - [Installation](#installation)
-- [Nimble's folder structure and packages](#nimbles-folder-structure-and-packages)
 - [Nimble usage](#nimble-usage)
   - [nimble refresh](#nimble-refresh)
   - [nimble install](#nimble-install)
@@ -43,6 +42,8 @@ Interested in learning **how to create a package**? Skip directly to that sectio
   - [[Deps]/[Dependencies]](#depsdependencies)
     - [Optional](#optional)
 - [Troubleshooting](#troubleshooting)
+- [Nimble's folder structure and packages](#nimbles-folder-structure-and-packages)
+- [Repository information](#repository-information)
 - [Contribution](#contribution)
 - [About](#about)
 
@@ -103,19 +104,6 @@ This will install Nimble to the default Nimble packages location:
 ``~/.nimble/pkgs``. The binary will be installed to ``~/.nimble/bin``, so you
 will need to add this directory to your PATH.
 
-## Nimble's folder structure and packages
-
-Nimble stores everything that has been installed in ``~/.nimble`` on Unix systems
-and in your ``$home/.nimble`` on Windows. Libraries are stored in
-``$nimbleDir/pkgs``, and binaries are stored in ``$nimbleDir/bin``. Most Nimble
-packages will provide ``.nim`` files and some documentation. The Nim
-compiler is aware of Nimble and will automatically find the modules so you can
-``import modulename`` and have that working without additional setup.
-
-However, some Nimble packages can provide additional tools or commands. If you
-don't add their location (``$nimbleDir/bin``) to your ``$PATH`` they will not
-work properly and you won't be able to run them.
-
 ## Nimble usage
 
 Once you have Nimble installed on your system you can run the ``nimble`` command
@@ -517,9 +505,9 @@ structure may be enforced in the future.
 
 All files and folders in the directory of where the .nimble file resides will be
 copied as-is, you can however skip some directories or files by setting
-the ``SkipDirs``, ``SkipFiles`` or ``SkipExt`` options in your .nimble file.
+the ``skipDirs``, ``skipFiles`` or ``skipExt`` options in your .nimble file.
 Directories and files can also be specified on a *whitelist* basis, if you
-specify either of ``InstallDirs``, ``InstallFiles`` or ``InstallExt`` then
+specify either of ``installDirs``, ``installFiles`` or ``installExt`` then
 Nimble will **only** install the files specified.
 
 ### Binary packages
@@ -540,7 +528,7 @@ created instead.
 Other files will be copied in the same way as they are for library packages.
 
 Binary packages should not install .nim files so include
-``SkipExt = "nim"`` in your .nimble file, unless you intend for your package to
+``skipExt = @["nim"]`` in your .nimble file, unless you intend for your package to
 be a binary/library combo which is fine.
 
 Dependencies are automatically installed before building.
@@ -672,25 +660,25 @@ Nimble includes a ``publish`` command which does this for you automatically.
 
 #### Optional
 
-* ``SkipDirs`` - A list of directory names which should be skipped during
+* ``skipDirs`` - A list of directory names which should be skipped during
   installation, separated by commas.
-* ``SkipFiles`` - A list of file names which should be skipped during
+* ``skipFiles`` - A list of file names which should be skipped during
   installation, separated by commas.
-* ``SkipExt`` - A list of file extensions which should be skipped during
+* ``skipExt`` - A list of file extensions which should be skipped during
   installation, the extensions should be specified without a leading ``.`` and
   should be separated by commas.
-* ``InstallDirs`` - A list of directories which should exclusively be installed,
+* ``installDirs`` - A list of directories which should exclusively be installed,
   if this option is specified nothing else will be installed except the dirs
-  listed here, the files listed in ``InstallFiles``, the files which share the
+  listed here, the files listed in ``installFiles``, the files which share the
-  extensions listed in ``InstallExt``, the .nimble file and the binary
+  extensions listed in ``installExt``, the .nimble file and the binary
   (if ``bin`` is specified). Separated by commas.
-* ``InstallFiles`` - A list of files which should be exclusively installed,
+* ``installFiles`` - A list of files which should be exclusively installed,
-  this complements ``InstallDirs`` and ``InstallExt``. Only the files listed
+  this complements ``installDirs`` and ``installExt``. Only the files listed
-  here, directories listed in ``InstallDirs``, files which share the extension
+  here, directories listed in ``installDirs``, files which share the extension
-  listed in ``InstallExt``, the .nimble file and the binary (if ``bin`` is
+  listed in ``installExt``, the .nimble file and the binary (if ``bin`` is
   specified) will be installed. Separated by commas.
-* ``InstallExt`` - A list of file extensions which should be exclusively
+* ``installExt`` - A list of file extensions which should be exclusively
-  installed, this complements ``InstallDirs`` and ``InstallFiles``.
+  installed, this complements ``installDirs`` and ``installFiles``.
   Separated by commas.
 * ``srcDir`` - Specifies the directory which contains the .nim source files.
   **Default**: The directory in which the .nimble file resides; i.e. root dir of
@@ -716,6 +704,19 @@ Nimble includes a ``publish`` command which does this for you automatically.
   **Example**: ``nim >= 0.10.0, jester``; with this value your package will
   depend on ``nim`` version 0.10.0 or greater and on any version of ``jester``.
 
+## Nimble's folder structure and packages
+
+Nimble stores everything that has been installed in ``~/.nimble`` on Unix systems
+and in your ``$home/.nimble`` on Windows. Libraries are stored in
+``$nimbleDir/pkgs``, and binaries are stored in ``$nimbleDir/bin``. Most Nimble
+packages will provide ``.nim`` files and some documentation. The Nim
+compiler is aware of Nimble and will automatically find the modules so you can
+``import modulename`` and have that working without additional setup.
+
+However, some Nimble packages can provide additional tools or commands. If you
+don't add their location (``$nimbleDir/bin``) to your ``$PATH`` they will not
+work properly and you won't be able to run them.
+
 ## Troubleshooting
 
 * ```SSL support is not available. Cannot connect over SSL. [HttpRequestError]```
@@ -725,6 +726,28 @@ flag to the file ```src/nimble.nim.cfg```.
 After that, you can run ```src/nimble install``` and overwrite the existing
 installation.
 
+## Repository information
+
+This repository has two main branches: ``master`` and ``stable``.
+
+The ``master`` branch is...
+
+* default
+* bleeding edge
+* tested to compile with the latest Nim version
+
+The ``stable`` branch is...
+
+* installed by ``koch tools``/``koch nimble``
+* relatively stable
+* should compile with Nim HEAD as well as the latest Nim version
+
+Note: The travis build only tests whether Nimble works with the latest Nim
+version.
+
+A new Nim release (via ``koch xz``) will always bundle the latest tagged
+Nimble release.
+
 ## Contribution
 
 If you would like to help, feel free to fork and make any additions you see fit

src/nimble.nim

@@ -163,7 +163,7 @@ proc copyFilesRec(origDir, currentDir, dest: string,
         if options.prompt("Missing file " & src & ". Continue?"):
           continue
         else:
-          quit(QuitSuccess)
+          raise NimbleQuit(msg: "")
       createDir(dest / file.splitFile.dir)
       result.incl copyFileD(src, dest / file)
 
@@ -174,7 +174,7 @@ proc copyFilesRec(origDir, currentDir, dest: string,
         if options.prompt("Missing directory " & src & ". Continue?"):
           continue
         else:
-          quit(QuitSuccess)
+          raise NimbleQuit(msg: "")
       result.incl copyDirD(origDir / dir, dest / dir)
 
     result.incl copyWithExt(origDir, currentDir, dest, pkgInfo)
@@ -210,7 +210,7 @@ proc addRevDep(options: Options, dep: tuple[name, version: string],
     options.nimbleData["reverseDeps"][dep.name] = newJObject()
   if not options.nimbleData["reverseDeps"][dep.name].hasKey(dep.version):
     options.nimbleData["reverseDeps"][dep.name][dep.version] = newJArray()
-  let revDep = %{ "name": %pkg.name, "version": %pkg.version}
+  let revDep = %{ "name": %pkg.name, "version": %pkg.specialVersion}
   let thisDep = options.nimbleData["reverseDeps"][dep.name][dep.version]
   if revDep notin thisDep:
     thisDep.add revDep
@@ -225,7 +225,7 @@ proc removeRevDep(options: Options, pkg: PackageInfo) =
         var newVal = newJArray()
         for revDep in val:
           if not (revDep["name"].str == pkg.name and
-                  revDep["version"].str == pkg.version):
+                  revDep["version"].str == pkg.specialVersion):
             newVal.add revDep
         thisDep[ver] = newVal
 
@@ -263,7 +263,7 @@ proc processDeps(pkginfo: PackageInfo, options: Options): seq[string] =
   result = @[]
   assert(not pkginfo.isMinimal, "processDeps needs pkginfo.requires")
   display("Verifying",
-          "dependencies for $1 v$2" % [pkginfo.name, pkginfo.version],
+          "dependencies for $1@$2" % [pkginfo.name, pkginfo.specialVersion],
           priority = HighPriority)
 
   let pkglist = getInstalledPkgs(options.getPkgsDir(), options)
@@ -275,7 +275,7 @@ proc processDeps(pkginfo: PackageInfo, options: Options): seq[string] =
         let msg = "Unsatisfied dependency: " & dep.name & " (" & $dep.ver & ")"
         raise newException(NimbleError, msg)
     else:
-      let depDesc = "$1 ($2)" % [dep.name, $dep.ver]
+      let depDesc = "$1@$2" % [dep.name, $dep.ver]
       display("Checking", "for $1" % depDesc, priority = MediumPriority)
       var pkg: PackageInfo
       if not findPkg(pkglist, dep, pkg):
@@ -290,18 +290,19 @@ proc processDeps(pkginfo: PackageInfo, options: Options): seq[string] =
         result.add(pkg.mypath.splitFile.dir)
         # Process the dependencies of this dependency.
         result.add(processDeps(pkg, options))
-      reverseDeps.add((pkg.name, pkg.version))
+      reverseDeps.add((pkg.name, pkg.specialVersion))
 
   # Check if two packages of the same name (but different version) are listed
   # in the path.
   var pkgsInPath: StringTableRef = newStringTable(modeCaseSensitive)
   for p in result:
-    let (name, version) = getNameVersion(p)
+    let pkgInfo = getPkgInfo(p, options)
-    if pkgsInPath.hasKey(name) and pkgsInPath[name] != version:
+    if pkgsInPath.hasKey(pkgInfo.name) and
+       pkgsInPath[pkgInfo.name] != pkgInfo.version:
       raise newException(NimbleError,
         "Cannot satisfy the dependency on $1 $2 and $1 $3" %
-          [name, version, pkgsInPath[name]])
+          [pkgInfo.name, pkgInfo.version, pkgsInPath[pkgInfo.name]])
-    pkgsInPath[name] = version
+    pkgsInPath[pkgInfo.name] = pkgInfo.version
 
   # We add the reverse deps to the JSON file here because we don't want
   # them added if the above errorenous condition occurs
@@ -337,7 +338,15 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: seq[string], forRelease: bool) =
       raise newException(BuildFailed, "Build failed for package: " &
                          pkgInfo.name)
 
-proc saveNimbleMeta(pkgDestDir, url, vcsRevision: string, filesInstalled: HashSet[string]) =
+proc saveNimbleMeta(pkgDestDir, url, vcsRevision: string,
+                    filesInstalled, bins: HashSet[string]) =
+  ## Saves the specified data into a ``nimblemeta.json`` file inside
+  ## ``pkgDestDir``.
+  ##
+  ## filesInstalled - A list of absolute paths to files which have been
+  ##                  installed.
+  ## bins - A list of binary filenames which have been installed for this
+  ##        package.
   var nimblemeta = %{"url": %url}
   if not vcsRevision.isNil:
     nimblemeta["vcsRevision"] = %vcsRevision
@@ -345,6 +354,10 @@ proc saveNimbleMeta(pkgDestDir, url, vcsRevision: string, filesInstalled: HashSe
   nimblemeta["files"] = files
   for file in filesInstalled:
     files.add(%changeRoot(pkgDestDir, "", file))
+  let binaries = newJArray()
+  nimblemeta["binaries"] = binaries
+  for bin in bins:
+    binaries.add(%bin)
   writeFile(pkgDestDir / "nimblemeta.json", $nimblemeta)
 
 proc removePkgDir(dir: string, options: Options) =
@@ -365,6 +378,15 @@ proc removePkgDir(dir: string, options: Options) =
     else:
       display("Warning:", ("Cannot completely remove $1. Files not installed " &
               "by nimble are present.") % dir, Warning, HighPriority)
+
+    # Remove binaries.
+    if nimblemeta.hasKey("binaries"):
+      for binary in nimblemeta["binaries"]:
+        removeFile(options.getBinDir() / binary.str)
+    else:
+      display("Warning:", ("Cannot completely remove $1. Binary symlinks may " &
+                          "have been left over in $2.") %
+                          [dir, options.getBinDir()])
   except OSError, JsonParsingError:
     display("Warning", "Unable to read nimblemeta.json: " &
             getCurrentExceptionMsg(), Warning, HighPriority)
@@ -390,7 +412,7 @@ proc vcsRevisionInDir(dir: string): string =
     except:
       discard
 
-proc installFromDir(dir: string, latest: bool, options: Options,
+proc installFromDir(dir: string, requestedVer: VersionRange, options: Options,
                     url: string): tuple[paths: seq[string], pkg: PackageInfo] =
   ## Returns where package has been installed to, together with paths
   ## to the packages this package depends on.
@@ -400,24 +422,29 @@ proc installFromDir(dir: string, latest: bool, options: Options,
   let realDir = pkgInfo.getRealDir()
   let binDir = options.getBinDir()
   let pkgsDir = options.getPkgsDir()
-  var deps_options = options
+  var depsOptions = options
-  deps_options.depsOnly = false
+  depsOptions.depsOnly = false
+
+  # Overwrite the version if the requested version is "#head" or similar.
+  if requestedVer.kind == verSpecial:
+    pkgInfo.specialVersion = $requestedVer.spe
 
   # Dependencies need to be processed before the creation of the pkg dir.
-  result.paths = processDeps(pkginfo, deps_options)
+  result.paths = processDeps(pkgInfo, depsOptions)
 
   if options.depsOnly:
     result.pkg = pkgInfo
     return result
 
-  display("Installing", "$1 v$2" % [pkginfo.name, pkginfo.version],
+  display("Installing", "$1@$2" % [pkginfo.name, pkginfo.specialVersion],
           priority = HighPriority)
 
   # Build before removing an existing package (if one exists). This way
   # if the build fails then the old package will still be installed.
   if pkgInfo.bin.len > 0: buildFromDir(pkgInfo, result.paths, true)
 
-  let versionStr = (if latest: "" else: '-' & pkgInfo.version)
+  let versionStr = '-' & pkgInfo.specialVersion
+
   let pkgDestDir = pkgsDir / (pkgInfo.name & versionStr)
   if existsDir(pkgDestDir) and existsFile(pkgDestDir / "nimblemeta.json"):
     if not options.prompt(pkgInfo.name & versionStr &
@@ -430,38 +457,47 @@ proc installFromDir(dir: string, latest: bool, options: Options,
       when defined(windows):
         removeFile(binDir / bin.changeFileExt("cmd"))
         removeFile(binDir / bin.changeFileExt(""))
-        # TODO: Remove this later.
-        # Remove .bat file too from previous installs.
-        removeFile(binDir / bin.changeFileExt("bat"))
       else:
         removeFile(binDir / bin)
 
-  ## Will contain a list of files which have been installed.
-  var filesInstalled: HashSet[string]
 
   createDir(pkgDestDir)
+  # Copy this package's files based on the preferences specified in PkgInfo.
+  var filesInstalled = initSet[string]()
+  for file in getInstallFiles(realDir, pkgInfo, options):
+    createDir(changeRoot(realDir, pkgDestDir, file.splitFile.dir))
+    let dest = changeRoot(realDir, pkgDestDir, file)
+    filesInstalled.incl copyFileD(file, dest)
+
+  # Copy the .nimble file.
+  let dest = changeRoot(pkgInfo.mypath.splitFile.dir, pkgDestDir,
+                        pkgInfo.mypath)
+  filesInstalled.incl copyFileD(pkgInfo.mypath, dest)
+
+  var binariesInstalled = initSet[string]()
   if pkgInfo.bin.len > 0:
+    # Make sure ~/.nimble/bin directory is created.
     createDir(binDir)
-    # Copy all binaries and files that are not skipped
-    filesInstalled = copyFilesRec(realDir, realDir, pkgDestDir, options,
-                                  pkgInfo)
     # Set file permissions to +x for all binaries built,
     # and symlink them on *nix OS' to $nimbleDir/bin/
     for bin in pkgInfo.bin:
       if not existsFile(pkgDestDir / bin):
-        filesInstalled.incl copyFileD(pkgInfo.getOutputDir(bin),
+        display("Warning:", ("Binary '$1' was already installed from source" &
-            pkgDestDir / bin)
+                            " directory. Will be overwritten.") % bin, Warning,
+                HighPriority)
+
+      # Copy the binary file.
+      filesInstalled.incl copyFileD(pkgInfo.getOutputDir(bin),
+                                    pkgDestDir / bin)
 
       let currentPerms = getFilePermissions(pkgDestDir / bin)
       setFilePermissions(pkgDestDir / bin, currentPerms + {fpUserExec})
       let cleanBin = bin.extractFilename
       when defined(unix):
-        # TODO: Verify that we are removing an old bin of this package, not
-        # some other package's binary!
-        if existsFile(binDir / cleanBin): removeFile(binDir / cleanBin)
         display("Creating", "symlink: $1 -> $2" %
                 [pkgDestDir / bin, binDir / cleanBin], priority = MediumPriority)
         createSymlink(pkgDestDir / bin, binDir / cleanBin)
+        binariesInstalled.incl(cleanBin)
       elif defined(windows):
         # There is a bug on XP, described here:
         # http://stackoverflow.com/questions/2182568/batch-script-is-not-executed-if-chcp-was-called
@@ -473,6 +509,7 @@ proc installFromDir(dir: string, latest: bool, options: Options,
             "Can't detect OS version: GetVersionExA call failed")
         let fixChcp = osver.dwMajorVersion <= 5
 
+        # Create cmd.exe/powershell stub.
         let dest = binDir / cleanBin.changeFileExt("cmd")
         display("Creating", "stub: $1 -> $2" % [pkgDestDir / bin, dest],
                 priority = MediumPriority)
@@ -483,21 +520,21 @@ proc installFromDir(dir: string, latest: bool, options: Options,
           else: contents.add "chcp 65001 > nul\n@"
         contents.add "\"" & pkgDestDir / bin & "\" %*\n"
         writeFile(dest, contents)
+        binariesInstalled.incl(dest.extractFilename)
         # For bash on Windows (Cygwin/Git bash).
         let bashDest = dest.changeFileExt("")
         display("Creating", "Cygwin stub: $1 -> $2" %
                 [pkgDestDir / bin, bashDest], priority = MediumPriority)
         writeFile(bashDest, "\"" & pkgDestDir / bin & "\" \"$@\"\n")
+        binariesInstalled.incl(bashDest.extractFilename)
       else:
         {.error: "Sorry, your platform is not supported.".}
-  else:
-    filesInstalled = copyFilesRec(realDir, realDir, pkgDestDir, options,
-                                  pkgInfo)
 
   let vcsRevision = vcsRevisionInDir(realDir)
 
   # Save a nimblemeta.json file.
-  saveNimbleMeta(pkgDestDir, url, vcsRevision, filesInstalled)
+  saveNimbleMeta(pkgDestDir, url, vcsRevision, filesInstalled,
+                 binariesInstalled)
 
   # Save the nimble data (which might now contain reverse deps added in
   # processDeps).
@@ -527,7 +564,7 @@ proc getNimbleTempDir(): string =
 
 proc downloadPkg(url: string, verRange: VersionRange,
                  downMethod: DownloadMethod,
-                 options: Options): (string, VersionRange) =
+                 options: Options): (string, Version) =
   ## Downloads the repository as specified by ``url`` and ``verRange`` using
   ## the download method specified.
   ##
@@ -580,7 +617,7 @@ proc install(packages: seq[PkgTuple],
              options: Options,
              doPrompt = true): tuple[paths: seq[string], pkg: PackageInfo] =
   if packages == @[]:
-    result = installFromDir(getCurrentDir(), false, options, "")
+    result = installFromDir(getCurrentDir(), newVRAny(), options, "")
   else:
     # If packages.json is not present ask the user if they want to download it.
     if needsRefresh(options):
@@ -597,11 +634,11 @@ proc install(packages: seq[PkgTuple],
       let (downloadDir, downloadVersion) =
           downloadPkg(url, pv.ver, meth, options)
       try:
-        result = installFromDir(downloadDir, false, options, url)
+        result = installFromDir(downloadDir, pv.ver, options, url)
       except BuildFailed:
         # The package failed to build.
         # Check if we tried building a tagged version of the package.
-        let headVer = parseVersionRange("#" & getHeadName(meth))
+        let headVer = getHeadName(meth)
         if pv.ver.kind != verSpecial and downloadVersion != headVer:
           # If we tried building a tagged version of the package then
           # ask the user whether they want to try building #head.
@@ -610,8 +647,8 @@ proc install(packages: seq[PkgTuple],
                   " like to try installing '$1@#head' (latest unstable)?") %
                   [pv.name, $downloadVersion])
           if promptResult:
-
+            let toInstall = @[(pv.name, headVer.toVersionRange())]
-            result = install(@[(pv.name, headVer)], options, doPrompt)
+            result = install(toInstall, options, doPrompt)
           else:
             raise newException(BuildFailed,
               "Aborting installation due to build failure")
@@ -624,10 +661,10 @@ proc build(options: Options) =
   let paths = processDeps(pkginfo, options)
   buildFromDir(pkgInfo, paths, false)
 
-proc compile(options: Options) =
+proc execBackend(options: Options) =
   let bin = options.action.file
   if bin == "":
-    raise newException(NimbleError, "You need to specify a file to compile.")
+    raise newException(NimbleError, "You need to specify a file.")
 
   if not fileExists(bin):
     raise newException(NimbleError, "Specified file does not exist.")
@@ -647,11 +684,15 @@ proc compile(options: Options) =
     else:
       pkgInfo.backend
 
-  display("Compiling", "$1 (from package $2) using $3 backend" %
+  if options.action.typ == actionCompile:
-          [bin, pkgInfo.name, backend], priority = HighPriority)
+    display("Compiling", "$1 (from package $2) using $3 backend" %
-  doCmd("\"" & getNimBin() & "\" $# --noBabelPath $# \"$#\"" %
+            [bin, pkgInfo.name, backend], priority = HighPriority)
+  else:
+    display("Generating", ("documentation for $1 (from package $2) using $3 " &
+            "backend") % [bin, pkgInfo.name, backend], priority = HighPriority)
+  doCmd("\"" & getNimBin() & "\" $# --noNimblePath $# \"$#\"" %
         [backend, args, bin])
-  display("Success:", "Compilation finished", Success, HighPriority)
+  display("Success:", "Execution finished", Success, HighPriority)
 
 proc search(options: Options) =
   ## Searches for matches in ``options.action.search``.
@@ -702,7 +743,7 @@ proc listInstalled(options: Options) =
   for x in pkgs.items():
     let
       pName = x.pkginfo.name
-      pVer = x.pkginfo.version
+      pVer = x.pkginfo.specialVersion
     if not h.hasKey(pName): h[pName] = @[]
     var s = h[pName]
     add(s, pVer)
@@ -744,15 +785,15 @@ proc listPaths(options: Options) =
       if hasSpec:
         var pkgInfo = getPkgInfo(path, options)
         var v: VersionAndPath
-        v.version = newVersion(pkgInfo.version)
+        v.version = newVersion(pkgInfo.specialVersion)
-        v.path = options.getPkgsDir / (pkgInfo.name & '-' & pkgInfo.version)
+        v.path = options.getPkgsDir / (pkgInfo.name & '-' & pkgInfo.specialVersion)
         installed.add(v)
       else:
         display("Warning:", "No .nimble file found for " & path, Warning,
                 MediumPriority)
 
     if installed.len > 0:
-      sort(installed, system.cmp[VersionAndPath], Descending)
+      sort(installed, cmp[VersionAndPath], Descending)
       # The output for this command is used by tools so we do not use display().
       echo installed[0].path
     else:
@@ -770,10 +811,30 @@ proc join(x: seq[PkgTuple]; y: string): string =
     result.add y
     result.add x[i][0] & " " & $x[i][1]
 
+proc getPackageByPattern(pattern: string, options: Options): PackageInfo =
+  ## Search for a package file using multiple strategies.
+  if pattern == "":
+    # Not specified - using current directory
+    result = getPkgInfo(os.getCurrentDir(), options)
+  elif pattern.splitFile.ext == ".nimble" and pattern.existsFile:
+    # project file specified
+    result = getPkgInfoFromFile(pattern, options)
+  elif pattern.existsDir:
+    # project directory specified
+    result = getPkgInfo(pattern, options)
+  else:
+    # Last resort - attempt to read as package identifier
+    let packages = getInstalledPkgsMin(options.getPkgsDir(), options)
+    let identTuple = parseRequires(pattern)
+    var skeletonInfo: PackageInfo
+    if not findPkg(packages, identTuple, skeletonInfo):
+      raise newException(NimbleError,
+          "Specified package not found"
+      )
+    result = getPkgInfoFromFile(skeletonInfo.myPath, options)
+
 proc dump(options: Options) =
-  let proj = addFileExt(options.action.projName, "nimble")
+  let p = getPackageByPattern(options.action.projName, options)
-  let p = if fileExists(proj): readPackageInfo(proj, options)
-          else: getPkgInfo(os.getCurrentDir(), options)
   echo "name: ", p.name.escape
   echo "version: ", p.version.escape
   echo "author: ", p.author.escape
@@ -882,7 +943,7 @@ proc uninstall(options: Options) =
     for pkg in pkgList:
       # Check whether any packages depend on the ones the user is trying to
       # uninstall.
-      let thisPkgsDep = options.nimbleData["reverseDeps"]{pkg.name}{pkg.version}
+      let thisPkgsDep = options.nimbleData["reverseDeps"]{pkg.name}{pkg.specialVersion}
       if not thisPkgsDep.isNil:
         var reason = ""
         if thisPkgsDep.len == 1:
@@ -896,7 +957,7 @@ proc uninstall(options: Options) =
               reason.add ", "
           reason.add " depend on it"
         errors.add("Cannot uninstall $1 ($2) because $3" % [pkgTup.name,
-                   pkg.version, reason])
+                   pkg.specialVersion, reason])
       else:
         pkgsToDelete.add pkg
 
@@ -907,7 +968,7 @@ proc uninstall(options: Options) =
   for i in 0 .. <pkgsToDelete.len:
     if i != 0: pkgNames.add ", "
     let pkg = pkgsToDelete[i]
-    pkgNames.add("$1 ($2)" % [pkg.name, pkg.version])
+    pkgNames.add("$1 ($2)" % [pkg.name, pkg.specialVersion])
 
   # Let's confirm that the user wants these packages removed.
   let msg = ("The following packages will be removed:\n  $1\n" &
@@ -920,10 +981,10 @@ proc uninstall(options: Options) =
 
     # removeRevDep needs the package dependency info, so we can't just pass
     # a minimal pkg info.
-    let pkgFull = readPackageInfo(pkg.mypath, options, false)
+    let pkgFull = getPkgInfo(pkg.mypath.splitFile.dir, options) # TODO: Simplify
     removeRevDep(options, pkgFull)
-    removePkgDir(options.getPkgsDir / (pkg.name & '-' & pkg.version), options)
+    removePkgDir(options.getPkgsDir / (pkg.name & '-' & pkg.specialVersion), options)
-    display("Removed", "$1 ($2)" % [pkg.name, $pkg.version], Success,
+    display("Removed", "$1 ($2)" % [pkg.name, $pkg.specialVersion], Success,
             HighPriority)
 
 proc listTasks(options: Options) =
@@ -938,7 +999,7 @@ proc execHook(options: Options, before: bool): bool =
     nimbleFile = findNimbleFile(getCurrentDir(), true)
   except NimbleError: return true
   # PackageInfos are cached so we can read them as many times as we want.
-  let pkgInfo = readPackageInfo(nimbleFile, options)
+  let pkgInfo = getPkgInfo(nimbleFile.splitFile.dir, options) # TODO: Simplify
   let actionName =
     if options.action.typ == actionCustom: options.action.command
     else: ($options.action.typ)[6 .. ^1]
@@ -982,8 +1043,8 @@ proc doAction(options: Options) =
     listPaths(options)
   of actionBuild:
     build(options)
-  of actionCompile:
+  of actionCompile, actionDoc:
-    compile(options)
+    execBackend(options)
   of actionInit:
     init(options)
   of actionPublish:

src/nimblepkg/common.nim

@@ -14,14 +14,17 @@ when not defined(nimscript):
     BuildFailed* = object of NimbleError
 
     PackageInfo* = object
-      mypath*: string ## The path of this .nimble file
+      myPath*: string ## The path of this .nimble file
       isNimScript*: bool ## Determines if this pkg info was read from a nims file
       isMinimal*: bool
       isInstalled*: bool ## Determines if the pkg this info belongs to is installed
       postHooks*: HashSet[string] ## Useful to know so that Nimble doesn't execHook unnecessarily
       preHooks*: HashSet[string]
       name*: string
+      ## The version specified in the .nimble file.Assuming info is non-minimal,
+      ## it will always be a non-special version such as '0.1.4'
       version*: string
+      specialVersion*: string ## Either `myVersion` or a special version such as #head.
       author*: string
       description*: string
       license*: string

src/nimblepkg/download.nim

@@ -91,8 +91,10 @@ proc getTagsListRemote*(url: string, meth: DownloadMethod): seq[string] =
       raise newException(OSError, "Unable to query remote tags for " & url &
           ". Git returned: " & output)
     for i in output.splitLines():
-      if i == "": continue
+      let refStart = i.find("refs/tags/")
-      let start = i.find("refs/tags/")+"refs/tags/".len
+      # git outputs warnings, empty lines, etc
+      if refStart == -1: continue
+      let start = refStart+"refs/tags/".len
       let tag = i[start .. i.len-1]
       if not tag.endswith("^{}"): result.add(tag)
 
@@ -116,12 +118,12 @@ proc getDownloadMethod*(meth: string): DownloadMethod =
   else:
     raise newException(NimbleError, "Invalid download method: " & meth)
 
-proc getHeadName*(meth: DownloadMethod): string =
+proc getHeadName*(meth: DownloadMethod): Version =
   ## Returns the name of the download method specific head. i.e. for git
   ## it's ``head`` for hg it's ``tip``.
   case meth
-  of DownloadMethod.git: "head"
+  of DownloadMethod.git: newVersion("#head")
-  of DownloadMethod.hg: "tip"
+  of DownloadMethod.hg: newVersion("#tip")
 
 proc checkUrlType*(url: string): DownloadMethod =
   ## Determines the download method based on the URL.
@@ -136,7 +138,7 @@ proc isURL*(name: string): bool =
   name.startsWith(peg" @'://' ")
 
 proc doDownload*(url: string, downloadDir: string, verRange: VersionRange,
-                 downMethod: DownloadMethod, options: Options): VersionRange =
+                 downMethod: DownloadMethod, options: Options): Version =
   ## Downloads the repository specified by ``url`` using the specified download
   ## method.
   ##
@@ -152,7 +154,7 @@ proc doDownload*(url: string, downloadDir: string, verRange: VersionRange,
     # https://github.com/nim-lang/nimble/issues/22
     meth
     if $latest.ver != "":
-      result = parseVersionRange($latest.ver)
+      result = latest.ver
     else:
       # Result should already be set to #head here.
       assert(not result.isNil)
@@ -170,20 +172,20 @@ proc doDownload*(url: string, downloadDir: string, verRange: VersionRange,
   removeDir(downloadDir)
   if verRange.kind == verSpecial:
     # We want a specific commit/branch/tag here.
-    if verRange.spe == newSpecial(getHeadName(downMethod)):
+    if verRange.spe == getHeadName(downMethod):
       doClone(downMethod, url, downloadDir) # Grab HEAD.
     else:
       # Grab the full repo.
       doClone(downMethod, url, downloadDir, tip = false)
       # Then perform a checkout operation to get the specified branch/commit.
       doCheckout(downMethod, downloadDir, $verRange.spe)
-    result = verRange
+    result = verRange.spe
   else:
     case downMethod
     of DownloadMethod.git:
       # For Git we have to query the repo remotely for its tags. This is
       # necessary as cloning with a --depth of 1 removes all tag info.
-      result = parseVersionRange("#head")
+      result = getHeadName(downMethod)
       let versions = getTagsListRemote(url, downMethod).getVersionList()
       if versions.len > 0:
         getLatestByTag:
@@ -197,7 +199,7 @@ proc doDownload*(url: string, downloadDir: string, verRange: VersionRange,
       verifyClone()
     of DownloadMethod.hg:
       doClone(downMethod, url, downloadDir)
-      result = parseVersionRange("#tip")
+      result = getHeadName(downMethod)
       let versions = getTagsList(downloadDir, downMethod).getVersionList()
 
       if versions.len > 0:

src/nimblepkg/options.nim

@@ -23,7 +23,7 @@ type
     actionNil, actionRefresh, actionInit, actionDump, actionPublish,
     actionInstall, actionSearch,
     actionList, actionBuild, actionPath, actionUninstall, actionCompile,
-    actionCustom, actionTasks
+    actionDoc, actionCustom, actionTasks
 
   Action* = object
     case typ*: ActionType
@@ -36,7 +36,7 @@ type
       search*: seq[string] # Search string.
     of actionInit, actionDump:
       projName*: string
-    of actionCompile:
+    of actionCompile, actionDoc:
       file*: string
       backend*: string
       compileOptions*: seq[string]
@@ -61,6 +61,8 @@ Commands:
   build                           Builds a package.
   c, cc, js    [opts, ...] f.nim  Builds a file inside a package. Passes options
                                   to the Nim compiler.
+  doc, doc2    [opts, ...] f.nim  Builds documentation for a file inside a
+                                  package. Passes options to the Nim compiler.
   refresh      [url]              Refreshes the package list. A package list URL
                                   can be optionally specified.
   search       pkg/tag            Searches for a specified package. Search is
@@ -74,7 +76,10 @@ Commands:
   path         pkgname ...        Shows absolute path to the installed packages
                                   specified.
   dump         [pkgname]          Outputs Nimble package information for
-                                  external tools.
+                                  external tools. The argument can be a
+                                  .nimble file, a project directory or
+                                  the name of an installed package.
+
 
 Options:
   -h, --help                      Print this help message.
@@ -111,6 +116,8 @@ proc parseActionType*(action: string): ActionType =
     result = actionBuild
   of "c", "compile", "js", "cpp", "cc":
     result = actionCompile
+  of "doc", "doc2":
+    result = actionDoc
   of "init":
     result = actionInit
   of "dump":
@@ -137,7 +144,7 @@ proc initAction*(options: var Options, key: string) =
   case options.action.typ
   of actionInstall, actionPath:
     options.action.packages = @[]
-  of actionCompile:
+  of actionCompile, actionDoc:
     options.action.compileOptions = @[]
     options.action.file = ""
     if keyNorm == "c" or keyNorm == "compile": options.action.backend = ""
@@ -219,7 +226,7 @@ proc parseArgument*(key: string, result: var Options) =
       raise newException(NimbleError,
           "Can only initialize one package at a time.")
     result.action.projName = key
-  of actionCompile:
+  of actionCompile, actionDoc:
     result.action.file = key
   of actionList, actionBuild, actionPublish:
     writeHelp()
@@ -258,7 +265,7 @@ proc parseFlag*(flag, val: string, result: var Options) =
       wasFlagHandled = false
   else:
     case result.action.typ
-    of actionCompile:
+    of actionCompile, actionDoc:
       if val == "":
         result.action.compileOptions.add("--" & flag)
       else:

src/nimblepkg/packageinfo.nim

@@ -4,7 +4,7 @@ import parsecfg, json, streams, strutils, parseutils, os, sets, tables
 import version, tools, common, options, cli
 
 type
-  Package* = object
+  Package* = object ## Definition of package from packages.json.
     # Required fields in a package.
     name*: string
     url*: string # Download location.
@@ -21,7 +21,8 @@ type
     url*: string
 
 proc initPackageInfo*(path: string): PackageInfo =
-  result.mypath = path
+  result.myPath = path
+  result.specialVersion = ""
   result.preHooks.init()
   result.postHooks.init()
   # reasonable default:
@@ -58,7 +59,6 @@ proc getNameVersion*(pkgpath: string): tuple[name, version: string] =
   ##
   ## Also works for file paths like:
   ##   ``/home/user/.nimble/pkgs/package-0.1/package.nimble``
-
   if pkgPath.splitFile.ext == ".nimble" or pkgPath.splitFile.ext == ".babel":
     return getNameVersion(pkgPath.splitPath.head)
 
@@ -196,11 +196,19 @@ proc getInstalledPkgsMin*(libsDir: string, options: Options):
         var pkg = initPackageInfo(nimbleFile)
         pkg.name = name
         pkg.version = version
+        pkg.specialVersion = version
         pkg.isMinimal = true
         pkg.isInstalled = true
         result.add((pkg, meta))
 
-proc findPkg*(pkglist: seq[tuple[pkginfo: PackageInfo, meta: MetaData]],
+proc withinRange*(pkgInfo: PackageInfo, verRange: VersionRange): bool =
+  ## Determines whether the specified package's version is within the
+  ## specified range. The check works with ordinary versions as well as
+  ## special ones.
+  return withinRange(newVersion(pkgInfo.version), verRange) or
+         withinRange(newVersion(pkgInfo.specialVersion), verRange)
+
+proc findPkg*(pkglist: seq[tuple[pkgInfo: PackageInfo, meta: MetaData]],
              dep: PkgTuple,
              r: var PackageInfo): bool =
   ## Searches ``pkglist`` for a package of which version is within the range
@@ -212,26 +220,27 @@ proc findPkg*(pkglist: seq[tuple[pkginfo: PackageInfo, meta: MetaData]],
   for pkg in pkglist:
     if cmpIgnoreStyle(pkg.pkginfo.name, dep.name) != 0 and
        cmpIgnoreStyle(pkg.meta.url, dep.name) != 0: continue
-    if withinRange(newVersion(pkg.pkginfo.version), dep.ver):
+    if withinRange(pkg.pkgInfo, dep.ver):
-      if not result or newVersion(r.version) < newVersion(pkg.pkginfo.version):
+      let isNewer = (not r.version.isNil) and
+                    newVersion(r.version) < newVersion(pkg.pkginfo.version)
+      if not result or isNewer:
         r = pkg.pkginfo
         result = true
 
-proc findAllPkgs*(pkglist: seq[tuple[pkginfo: PackageInfo, meta: MetaData]],
+proc findAllPkgs*(pkglist: seq[tuple[pkgInfo: PackageInfo, meta: MetaData]],
                   dep: PkgTuple): seq[PackageInfo] =
   ## Searches ``pkglist`` for packages of which version is within the range
   ## of ``dep.ver``. This is similar to ``findPkg`` but returns multiple
   ## packages if multiple are found.
   result = @[]
   for pkg in pkglist:
-    if cmpIgnoreStyle(pkg.pkginfo.name, dep.name) != 0 and
+    if cmpIgnoreStyle(pkg.pkgInfo.name, dep.name) != 0 and
        cmpIgnoreStyle(pkg.meta.url, dep.name) != 0: continue
-    if withinRange(newVersion(pkg.pkginfo.version), dep.ver):
+    if withinRange(pkg.pkgInfo, dep.ver):
       result.add pkg.pkginfo
 
 proc getRealDir*(pkgInfo: PackageInfo): string =
-  ## Returns the ``pkgInfo.srcDir`` or the .mypath directory if package does
+  ## Returns the directory containing the package source files.
-  ## not specify the src dir.
   if pkgInfo.srcDir != "" and not pkgInfo.isInstalled:
     result = pkgInfo.mypath.splitFile.dir / pkgInfo.srcDir
   else:
@@ -282,6 +291,104 @@ proc validatePackagesList*(path: string): bool =
   except ValueError, JsonParsingError:
     return false
 
+proc checkInstallFile(pkgInfo: PackageInfo,
+                      origDir, file: string): bool =
+  ## Checks whether ``file`` should be installed.
+  ## ``True`` means file should be skipped.
+
+  for ignoreFile in pkgInfo.skipFiles:
+    if ignoreFile.endswith("nimble"):
+      raise newException(NimbleError, ignoreFile & " must be installed.")
+    if samePaths(file, origDir / ignoreFile):
+      result = true
+      break
+
+  for ignoreExt in pkgInfo.skipExt:
+    if file.splitFile.ext == ('.' & ignoreExt):
+      result = true
+      break
+
+  if file.splitFile().name[0] == '.': result = true
+
+proc checkInstallDir(pkgInfo: PackageInfo,
+                     origDir, dir: string): bool =
+  ## Determines whether ``dir`` should be installed.
+  ## ``True`` means dir should be skipped.
+  for ignoreDir in pkgInfo.skipDirs:
+    if samePaths(dir, origDir / ignoreDir):
+      result = true
+      break
+
+  let thisDir = splitPath(dir).tail
+  assert thisDir != ""
+  if thisDir[0] == '.': result = true
+  if thisDir == "nimcache": result = true
+
+proc findWithExt(dir: string, pkgInfo: PackageInfo): seq[string] =
+  ## Returns the filenames of the files that should be copied.
+  result = @[]
+  for kind, path in walkDir(dir):
+    if kind == pcDir:
+      result.add findWithExt(path, pkgInfo)
+    else:
+      if path.splitFile.ext[1 .. ^1] in pkgInfo.installExt:
+        result.add path
+
+proc getFilesInDir(dir: string): seq[string] =
+  ## Returns a list of paths to files inside the specified directory and any
+  ## subdirectories that are in it.
+  result = @[]
+  for kind, path in walkDir(dir):
+    if kind == pcDir:
+      result.add getFilesInDir(path)
+    else:
+      result.add path
+
+proc getInstallFiles*(realDir: string, pkgInfo: PackageInfo,
+                      options: Options): seq[string] =
+  ## Returns a list of files within the ``realDir`` that should be installed.
+  result = @[]
+  let whitelistMode =
+          pkgInfo.installDirs.len != 0 or
+          pkgInfo.installFiles.len != 0 or
+          pkgInfo.installExt.len != 0
+  if whitelistMode:
+    for file in pkgInfo.installFiles:
+      let src = realDir / file
+      if not src.existsFile():
+        if options.prompt("Missing file " & src & ". Continue?"):
+          continue
+        else:
+          raise NimbleQuit(msg: "")
+      result.add src
+
+    for dir in pkgInfo.installDirs:
+      # TODO: Allow skipping files inside dirs?
+      let src = realDir / dir
+      if not src.existsDir():
+        if options.prompt("Missing directory " & src & ". Continue?"):
+          continue
+        else:
+          raise NimbleQuit(msg: "")
+
+      result.add getFilesInDir(src)
+
+    result.add findWithExt(realDir, pkgInfo)
+  else:
+    for kind, file in walkDir(realDir):
+      if kind == pcDir:
+        let skip = pkgInfo.checkInstallDir(realDir, file)
+
+        if skip: continue
+
+        result.add getInstallFiles(file, pkgInfo, options)
+      else:
+        let skip = pkgInfo.checkInstallFile(realDir, file)
+
+        if skip: continue
+
+        result.add file
+
 when isMainModule:
   doAssert getNameVersion("/home/user/.nimble/libs/packagea-0.1") ==
       ("packagea", "0.1")
@@ -289,14 +396,8 @@ when isMainModule:
       ("package-a", "0.1")
   doAssert getNameVersion("/home/user/.nimble/libs/package-a-0.1/package.nimble") ==
       ("package-a", "0.1")
-
+  doAssert getNameVersion("/home/user/.nimble/libs/package-#head") ==
-  validatePackageName("foo_bar")
+      ("package", "#head")
-  validatePackageName("f_oo_b_a_r")
-  try:
-    validatePackageName("foo__bar")
-    assert false
-  except NimbleError:
-    assert true
 
   doAssert toValidPackageName("foo__bar") == "foo_bar"
   doAssert toValidPackageName("jhbasdh!£$@%#^_&*_()qwe") == "jhbasdh_qwe"

src/nimblepkg/packageparser.nim

@@ -14,13 +14,18 @@ type
 
   ValidationError* = object of NimbleError
     warnInstalled*: bool # Determines whether to show a warning for installed pkgs
+    warnAll*: bool
 
-proc newValidationError(msg: string, warnInstalled: bool): ref ValidationError =
+proc newValidationError(msg: string, warnInstalled: bool,
+                        hint: string, warnAll: bool): ref ValidationError =
   result = newException(ValidationError, msg)
   result.warnInstalled = warnInstalled
+  result.warnAll = warnAll
+  result.hint = hint
 
-proc raiseNewValidationError(msg: string, warnInstalled: bool) =
+proc raiseNewValidationError(msg: string, warnInstalled: bool,
-  raise newValidationError(msg, warnInstalled)
+                             hint: string = "", warnAll = false) =
+  raise newValidationError(msg, warnInstalled, hint, warnAll)
 
 proc validatePackageName*(name: string) =
   ## Raises an error if specified package name contains invalid characters.
@@ -49,6 +54,10 @@ proc validatePackageName*(name: string) =
     else:
       prevWasUnderscore = false
 
+  if name.endsWith("pkg"):
+    raiseNewValidationError("\"$1\" is an invalid package name: cannot end" &
+                            " with \"pkg\"" % name, false)
+
 proc validateVersion*(ver: string) =
   for c in ver:
     if c notin ({'.'} + Digits):
@@ -56,7 +65,57 @@ proc validateVersion*(ver: string) =
           "Version may only consist of numbers and the '.' character " &
           "but found '" & c & "'.", false)
 
-proc validatePackageInfo(pkgInfo: PackageInfo, path: string) =
+proc validatePackageStructure(pkgInfo: PackageInfo, options: Options) =
+  ## This ensures that a package's source code does not leak into
+  ## another package's namespace.
+  ## https://github.com/nim-lang/nimble/issues/144
+  let realDir = pkgInfo.getRealDir()
+  for path in getInstallFiles(realDir, pkgInfo, options):
+    # Remove the root to leave only the package subdirectories.
+    # ~/package-0.1/package/utils.nim -> package/utils.nim.
+    var trailPath = changeRoot(realDir, "", path)
+    if trailPath.startsWith(DirSep): trailPath = trailPath[1 .. ^1]
+    let (dir, file, ext) = trailPath.splitFile
+    # We're only interested in nim files, because only they can pollute our
+    # namespace.
+    if ext != (ExtSep & "nim"):
+      continue
+
+    if dir.len == 0:
+      if file != pkgInfo.name:
+        let msg = ("File inside package '$1' is outside of permitted " &
+                   "namespace, should be " &
+                   "named '$2' but was named '$3' instead. This will be an error" &
+                   " in the future.") %
+                   [pkgInfo.name, pkgInfo.name & ext, file & ext]
+        let hint = ("Rename this file to '$1', move it into a '$2' " &
+                "subdirectory, or prevent its installation by adding " &
+                "`skipFiles = @[\"$3\"]` to the .nimble file. See " &
+                "https://github.com/nim-lang/nimble#libraries for more info.") %
+                [pkgInfo.name & ext, pkgInfo.name & DirSep, file & ext]
+        raiseNewValidationError(msg, true, hint, true)
+    else:
+      assert(not pkgInfo.isMinimal)
+      let correctDir =
+        if pkgInfo.name in pkgInfo.bin:
+          pkgInfo.name & "pkg"
+        else:
+          pkgInfo.name
+
+      if not (dir.startsWith(correctDir & DirSep) or dir == correctDir):
+        let msg = ("File '$1' inside package '$2' is outside of the" &
+                " permitted namespace" &
+                ", should be inside a directory named '$3' but is in a" &
+                " directory named '$4' instead. This will be an error in the " &
+                "future.") %
+                [file & ext, pkgInfo.name, correctDir, dir]
+        let hint = ("Rename the directory to '$1' or prevent its " &
+                "installation by adding `skipDirs = @[\"$2\"]` to the " &
+                ".nimble file.") % [correctDir, dir]
+        raiseNewValidationError(msg, true, hint, true)
+
+proc validatePackageInfo(pkgInfo: PackageInfo, options: Options) =
+  let path = pkgInfo.myPath
   if pkgInfo.name == "":
     raiseNewValidationError("Incorrect .nimble file: " & path &
         " does not contain a name field.", false)
@@ -83,7 +142,8 @@ proc validatePackageInfo(pkgInfo: PackageInfo, path: string) =
       raiseNewValidationError("'" & pkgInfo.backend &
           "' is an invalid backend.", false)
 
-  validateVersion(pkgInfo.version)
+  validatePackageStructure(pkginfo, options)
+
 
 proc nimScriptHint*(pkgInfo: PackageInfo) =
   if not pkgInfo.isNimScript:
@@ -172,7 +232,7 @@ proc readPackageInfoFromNimble(path: string; result: var PackageInfo) =
   else:
     raise newException(ValueError, "Cannot open package info: " & path)
 
-proc readPackageInfo*(nf: NimbleFile, options: Options,
+proc readPackageInfo(nf: NimbleFile, options: Options,
     onlyMinimalInfo=false): PackageInfo =
   ## Reads package info from the specified Nimble file.
   ##
@@ -182,17 +242,20 @@ proc readPackageInfo*(nf: NimbleFile, options: Options,
   ## If both fail then returns an error.
   ##
   ## When ``onlyMinimalInfo`` is true, only the `name` and `version` fields are
-  ## populated. The isNimScript field can also be relied on.
+  ## populated. The ``isNimScript`` field can also be relied on.
   ##
   ## This version uses a cache stored in ``options``, so calling it multiple
   ## times on the same ``nf`` shouldn't require re-evaluation of the Nimble
   ## file.
 
+  assert fileExists(nf)
+
   # Check the cache.
   if options.pkgInfoCache.hasKey(nf):
     return options.pkgInfoCache[nf]
 
   result = initPackageInfo(nf)
+  let minimalInfo = getNameVersion(nf)
 
   validatePackageName(nf.splitFile.name)
 
@@ -208,9 +271,8 @@ proc readPackageInfo*(nf: NimbleFile, options: Options,
 
   if not success:
     if onlyMinimalInfo:
-      let tmp = getNameVersion(nf)
+      result.name = minimalInfo.name
-      result.name = tmp.name
+      result.version = minimalInfo.version
-      result.version = tmp.version
       result.isNimScript = true
       result.isMinimal = true
     else:
@@ -225,14 +287,40 @@ proc readPackageInfo*(nf: NimbleFile, options: Options,
                   "    " & getCurrentExceptionMsg() & "."
         raise newException(NimbleError, msg)
 
-  validatePackageInfo(result, nf)
+  # By default specialVersion is the same as version.
+  result.specialVersion = result.version
+
+  # The package directory name may include a "special" version
+  # (example #head). If so, it is given higher priority and therefore
+  # overwrites the .nimble file's version.
+  let version = parseVersionRange(minimalInfo.version)
+  if version.kind == verSpecial:
+    result.specialVersion = minimalInfo.version
+
   if not result.isMinimal:
     options.pkgInfoCache[nf] = result
 
+  # Validate the rest of the package info last.
+  validateVersion(result.version)
+  validatePackageInfo(result, options)
+
+proc getPkgInfoFromFile*(file: NimbleFile, options: Options): PackageInfo =
+  ## Reads the specified .nimble file and returns its data as a PackageInfo
+  ## object. Any validation errors are handled and displayed as warnings.
+  try:
+    result = readPackageInfo(file, options)
+  except ValidationError:
+    let exc = (ref ValidationError)(getCurrentException())
+    if exc.warnAll:
+      display("Warning:", exc.msg, Warning, HighPriority)
+      display("Hint:", exc.hint, Warning, HighPriority)
+    else:
+      raise
+
 proc getPkgInfo*(dir: string, options: Options): PackageInfo =
   ## Find the .nimble file in ``dir`` and parses it, returning a PackageInfo.
   let nimbleFile = findNimbleFile(dir, true)
-  result = readPackageInfo(nimbleFile, options)
+  getPkgInfoFromFile(nimbleFile, options)
 
 proc getInstalledPkgs*(libsDir: string, options: Options):
         seq[tuple[pkginfo: PackageInfo, meta: MetaData]] =
@@ -240,7 +328,7 @@ proc getInstalledPkgs*(libsDir: string, options: Options):
   ##
   ## ``libsDir`` is in most cases: ~/.nimble/pkgs/
   const
-    readErrorMsg = "Installed package $1 v$2 is outdated or corrupt."
+    readErrorMsg = "Installed package '$1@$2' is outdated or corrupt."
     validationErrorMsg = readErrorMsg & "\nPackage did not pass validation: $3"
     hintMsg = "The corrupted package will need to be removed manually. To fix" &
               " this error message, remove $1."
@@ -257,16 +345,17 @@ proc getInstalledPkgs*(libsDir: string, options: Options):
       let nimbleFile = findNimbleFile(path, false)
       if nimbleFile != "":
         let meta = readMetaData(path)
+        var pkg: PackageInfo
         try:
-          var pkg = readPackageInfo(nimbleFile, options, onlyMinimalInfo=false)
+          pkg = readPackageInfo(nimbleFile, options, onlyMinimalInfo=false)
-          pkg.isInstalled = true
-          result.add((pkg, meta))
         except ValidationError:
           let exc = (ref ValidationError)(getCurrentException())
           exc.msg = createErrorMsg(validationErrorMsg, path, exc.msg)
           exc.hint = hintMsg % path
-          if exc.warnInstalled:
+          if exc.warnInstalled or exc.warnAll:
             display("Warning:", exc.msg, Warning, HighPriority)
+            # Don't show hints here because they are only useful for package
+            # owners.
           else:
             raise exc
         except:
@@ -276,5 +365,19 @@ proc getInstalledPkgs*(libsDir: string, options: Options):
           exc.hint = hintMsg % path
           raise exc
 
+        pkg.isInstalled = true
+        result.add((pkg, meta))
+
 proc isNimScript*(nf: string, options: Options): bool =
   result = readPackageInfo(nf, options).isNimScript
+
+when isMainModule:
+  validatePackageName("foo_bar")
+  validatePackageName("f_oo_b_a_r")
+  try:
+    validatePackageName("foo__bar")
+    assert false
+  except NimbleError:
+    assert true
+
+  echo("Everything passed!")

src/nimblepkg/version.nim

@@ -5,7 +5,6 @@
 import strutils, tables, hashes, parseutils
 type
   Version* = distinct string
-  Special* = distinct string
 
   VersionRangeEnum* = enum
     verLater, # > V
@@ -23,7 +22,7 @@ type
     of verLater, verEarlier, verEqLater, verEqEarlier, verEq:
       ver*: Version
     of verSpecial:
-      spe*: Special
+      spe*: Version
     of verIntersect:
       verILeft, verIRight: VersionRange
     of verAny:
@@ -36,18 +35,25 @@ type
   NimbleError* = object of Exception
     hint*: string
 
-proc newVersion*(ver: string): Version = return Version(ver)
+proc newVersion*(ver: string): Version =
-proc newSpecial*(spe: string): Special = return Special(spe)
+  doAssert(ver[0] in {'#', '\0'} + Digits)
+  return Version(ver)
 
 proc `$`*(ver: Version): string {.borrow.}
 
 proc hash*(ver: Version): Hash {.borrow.}
 
-proc `$`*(ver: Special): string {.borrow.}
+proc isNil*(ver: Version): bool {.borrow.}
 
-proc hash*(ver: Special): Hash {.borrow.}
+proc isSpecial*(ver: Version): bool =
+  return ($ver)[0] == '#'
 
 proc `<`*(ver: Version, ver2: Version): bool =
+  # Handling for special versions such as "#head" or "#branch".
+  if ver.isSpecial or ver2.isSpecial:
+    return false
+
+  # Handling for normal versions such as "0.1.0" or "1.0".
   var sVer = string(ver).split('.')
   var sVer2 = string(ver2).split('.')
   for i in 0..max(sVer.len, sVer2.len)-1:
@@ -65,6 +71,9 @@ proc `<`*(ver: Version, ver2: Version): bool =
       return false
 
 proc `==`*(ver: Version, ver2: Version): bool =
+  if ver.isSpecial or ver2.isSpecial:
+    return ($ver).toLowerAscii() == ($ver2).toLowerAscii()
+
   var sVer = string(ver).split('.')
   var sVer2 = string(ver2).split('.')
   for i in 0..max(sVer.len, sVer2.len)-1:
@@ -79,9 +88,6 @@ proc `==`*(ver: Version, ver2: Version): bool =
     else:
       return false
 
-proc `==`*(spe: Special, spe2: Special): bool =
-  return ($spe).toLowerAscii() == ($spe2).toLowerAscii()
-
 proc `<=`*(ver: Version, ver2: Version): bool =
   return (ver == ver2) or (ver < ver2)
 
@@ -97,39 +103,36 @@ proc `==`*(range1: VersionRange, range2: VersionRange): bool =
   of verAny: true
 
 proc withinRange*(ver: Version, ran: VersionRange): bool =
-  case ran.kind
+  if ver.isSpecial:
-  of verLater:
+    case ran.kind
-    return ver > ran.ver
+    of verLater, verEarlier, verEqLater, verEqEarlier, verEq, verIntersect:
-  of verEarlier:
+      return false
-    return ver < ran.ver
+    of verSpecial:
-  of verEqLater:
+      return ver == ran.spe
-    return ver >= ran.ver
+    of verAny:
-  of verEqEarlier:
+      return true
-    return ver <= ran.ver
+  else:
-  of verEq:
+    case ran.kind
-    return ver == ran.ver
+    of verLater:
-  of verSpecial:
+      return ver > ran.ver
-    return false
+    of verEarlier:
-  of verIntersect:
+      return ver < ran.ver
-    return withinRange(ver, ran.verILeft) and withinRange(ver, ran.verIRight)
+    of verEqLater:
-  of verAny:
+      return ver >= ran.ver
-    return true
+    of verEqEarlier:
-
+      return ver <= ran.ver
-proc withinRange*(spe: Special, ran: VersionRange): bool =
+    of verEq:
-  case ran.kind
+      return ver == ran.ver
-  of verLater, verEarlier, verEqLater, verEqEarlier, verEq, verIntersect:
+    of verSpecial:
-    return false
+      return false
-  of verSpecial:
+    of verIntersect:
-    return spe == ran.spe
+      return withinRange(ver, ran.verILeft) and withinRange(ver, ran.verIRight)
-  of verAny:
+    of verAny:
-    return true
+      return true
 
 proc contains*(ran: VersionRange, ver: Version): bool =
   return withinRange(ver, ran)
 
-proc contains*(ran: VersionRange, spe: Special): bool =
-  return withinRange(spe, ran)
-
 proc makeRange*(version: string, op: string): VersionRange =
   new(result)
   if version == "":
@@ -153,9 +156,13 @@ proc makeRange*(version: string, op: string): VersionRange =
 proc parseVersionRange*(s: string): VersionRange =
   # >= 1.5 & <= 1.8
   new(result)
+  if s.len == 0:
+    result.kind = verAny
+    return
+
   if s[0] == '#':
     result.kind = verSpecial
-    result.spe = s[1 .. s.len-1].Special
+    result.spe = s.Version
     return
 
   var i = 0
@@ -200,6 +207,15 @@ proc parseVersionRange*(s: string): VersionRange =
           "Unexpected char in version range: " & s[i])
     inc(i)
 
+proc toVersionRange*(ver: Version): VersionRange =
+  ## Converts a version to either a verEq or verSpecial VersionRange.
+  new(result)
+  if ver.isSpecial:
+    result.kind = verSpecial
+    result.spe = ver
+  else:
+    result.kind = verEq
+    result.ver = ver
 
 proc parseRequires*(req: string): PkgTuple =
   try:
@@ -231,7 +247,7 @@ proc `$`*(verRange: VersionRange): string =
   of verEq:
     result = ""
   of verSpecial:
-    return "#" & $verRange.spe
+    return $verRange.spe
   of verIntersect:
     return $verRange.verILeft & " & " & $verRange.verIRight
   of verAny:
@@ -312,13 +328,26 @@ when isMainModule:
   #doAssert newVersion("0.1-rc1") < newVersion("0.1")
 
   # Special tests
-  doAssert newSpecial("ab26sgdt362") != newSpecial("ab26saggdt362")
+  doAssert newVersion("#ab26sgdt362") != newVersion("#qwersaggdt362")
-  doAssert newSpecial("ab26saggdt362") == newSpecial("ab26saggdt362")
+  doAssert newVersion("#ab26saggdt362") == newVersion("#ab26saggdt362")
-  doAssert newSpecial("head") == newSpecial("HEAD")
+  doAssert newVersion("#head") == newVersion("#HEAD")
-  doAssert newSpecial("head") == newSpecial("head")
+  doAssert newVersion("#head") == newVersion("#head")
 
   var sp = parseVersionRange("#ab26sgdt362")
-  doAssert newSpecial("ab26sgdt362") in sp
+  doAssert newVersion("#ab26sgdt362") in sp
-  doAssert newSpecial("ab26saggdt362") notin sp
+  doAssert newVersion("#ab26saggdt362") notin sp
+
+  doAssert newVersion("#head") in parseVersionRange("#head")
+
+  # TODO: It may be worth changing this in the future, although we can't be
+  # certain that #head is in fact newer than v0.1.0.
+  doAssert(not(newVersion("#head") > newVersion("0.1.0")))
+
+  # An empty version range should give verAny
+  doAssert parseVersionRange("").kind == verAny
+
+  # toVersionRange tests
+  doAssert toVersionRange(newVersion("#head")).kind == verSpecial
+  doAssert toVersionRange(newVersion("0.2.0")).kind == verEq
 
   echo("Everything works!")

tests/issue289/issue289.nim

@@ -0,0 +1 @@
+echo 42

tests/issue289/issue289.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Package reproducing issues depending on #head and concrete version of the same package."
+license       = "MIT"
+
+bin = @["issue289"]
+
+# Dependencies
+
+requires "nim >= 0.15.0", "https://github.com/nimble-test/packagea.git 0.6.0"
+requires "https://github.com/nimble-test/packagea.git#head"
+

tests/packageStructure/a/a.nim

@@ -0,0 +1 @@
+

tests/packageStructure/a/a.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package A"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/b/b.nim

@@ -0,0 +1 @@
+

tests/packageStructure/b/b.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package B"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/b/b/foobar.nim

@@ -0,0 +1 @@
+

tests/packageStructure/c/c.nim

@@ -0,0 +1 @@
+

tests/packageStructure/c/c.nimble

@@ -0,0 +1,13 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package C"
+license       = "MIT"
+
+bin = @["c"]
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/c/cpkg/foobar.nim

@@ -0,0 +1 @@
+

tests/packageStructure/x/x.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Incorrectly structured package X."
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/y/y.nimble

@@ -0,0 +1,13 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Incorrectly structured package Y"
+license       = "MIT"
+
+bin = @["y"]
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/y/yWrong/foobar.nim

@@ -0,0 +1 @@
+

tests/packageStructure/z/z.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Incorrect package structure Z."
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/testdump/testdump.nimble

@@ -0,0 +1,4 @@
+description = "Test package for dump command"
+version = "0.1.0"
+author = "nigredo-tori"
+license = "BSD"

tests/tester.nim

@@ -2,6 +2,10 @@
 # BSD License. Look at license.txt for more info.
 import osproc, streams, unittest, strutils, os, sequtils, future
 
+# TODO: Each test should start off with a clean slate. Currently installed
+# packages are shared between each test which causes a multitude of issues
+# and is really fragile.
+
 var rootDir = getCurrentDir().parentDir()
 var nimblePath = rootDir / "src" / addFileExt("nimble", ExeExt)
 var installDir = rootDir / "tests" / "nimbleDir"
@@ -37,10 +41,55 @@ proc inLines(lines: seq[string], line: string): bool =
   for i in lines:
     if line.normalize in i.normalize: return true
 
+test "can build with #head and versioned package (#289)":
+  cd "issue289":
+    check execNimble(["install", "-y"]).exitCode == QuitSuccess
+
+  check execNimble(["uninstall", "issue289", "-y"]).exitCode == QuitSuccess
+  check execNimble(["uninstall", "packagea", "-y"]).exitCode == QuitSuccess
+
+test "can validate package structure (#144)":
+  # Test that no warnings are produced for correctly structured packages.
+  for package in ["a", "b", "c"]:
+    cd "packageStructure/" & package:
+      let (output, exitCode) = execNimble(["install", "-y"])
+      check exitCode == QuitSuccess
+      let lines = output.strip.splitLines()
+      check(not inLines(lines, "warning"))
+
+  # Test that warnings are produced for the incorrectly structured packages.
+  for package in ["x", "y", "z"]:
+    cd "packageStructure/" & package:
+      let (output, exitCode) = execNimble(["install", "-y"])
+      check exitCode == QuitSuccess
+      let lines = output.strip.splitLines()
+      case package
+      of "x":
+        check inLines(lines, "File 'foobar.nim' inside package 'x' is outside" &
+                             " of the permitted namespace, should be inside a" &
+                             " directory named 'x' but is in a directory named" &
+                             " 'incorrect' instead.")
+      of "y":
+        check inLines(lines, "File 'foobar.nim' inside package 'y' is outside" &
+                             " of the permitted namespace, should be inside a" &
+                             " directory named 'ypkg' but is in a directory" &
+                             " named 'yWrong' instead.")
+      of "z":
+        check inLines(lines, "File inside package 'z' is outside of permitted" &
+                             " namespace, should be named 'z.nim' but was" &
+                             " named 'incorrect.nim' instead.")
+      else:
+        assert false
+
 test "issue 129 (installing commit hash)":
   let arguments = @["install", "-y",
                    "https://github.com/nimble-test/packagea.git@#1f9cb289c89"]
   check execNimble(arguments).exitCode == QuitSuccess
+  # Verify that it was installed correctly.
+  check dirExists(installDir / "pkgs" / "PackageA-#1f9cb289c89")
+  # Remove it so that it doesn't interfere with the uninstall tests.
+  check execNimble("uninstall", "-y", "packagea@#1f9cb289c89").exitCode ==
+        QuitSuccess
 
 test "issue 113 (uninstallation problems)":
   cd "issue113/c":
@@ -247,6 +296,34 @@ test "can uninstall":
 
   check execNimble("uninstall", "-y", "PackageA@0.2", "issue27b").exitCode ==
       QuitSuccess
-  check(not dirExists(getHomeDir() / ".nimble" / "pkgs" / "PackageA-0.2.0"))
+  check(not dirExists(installDir / "pkgs" / "PackageA-0.2.0"))
 
   check execNimble("uninstall", "-y", "nimscript").exitCode == QuitSuccess
+
+test "can dump for current project":
+  cd "testdump":
+    let (outp, exitCode) = execNimble("dump")
+    check: exitCode == 0
+    check: outp.processOutput.inLines("desc: \"Test package for dump command\"")
+
+test "can dump for project directory":
+  let (outp, exitCode) = execNimble("dump", "testdump")
+  check: exitCode == 0
+  check: outp.processOutput.inLines("desc: \"Test package for dump command\"")
+
+test "can dump for project file":
+  let (outp, exitCode) = execNimble("dump", "testdump" / "testdump.nimble")
+  check: exitCode == 0
+  check: outp.processOutput.inLines("desc: \"Test package for dump command\"")
+
+test "can dump for installed package":
+  cd "testdump":
+    check: execNimble("install", "-y").exitCode == 0
+  defer:
+    discard execNimble("remove", "-y", "testdump")
+
+  # Otherwise we might find subdirectory instead
+  cd "..":
+    let (outp, exitCode) = execNimble("dump", "testdump")
+    check: exitCode == 0
+    check: outp.processOutput.inLines("desc: \"Test package for dump command\"")