show cmd used with --verbose (even on success); fix #783 (#781)

* show cmd used with --verbose (even on success)

* honor -p:-u:release

* address comment

* remove --showCmds flag

* remove quoteShell

* fix #783

* Revert "fix #783"

This reverts commit af0ac004c00f17e9983c63ab99e40cd38ba6aaa4.

* fix #783 by fixing the gist instead

* Revert "fix #783 by fixing the gist instead" now that upstream gist was updated with my patch

This reverts commit 8bec86039d8335af152acf238ab14d0268e003e5.

.gitignore

@@ -6,4 +6,29 @@ nimcache/
 # Absolute paths
 /src/babel
 /src/nimble
-/tests/tester
+
+# executables from test and build
+/nimble
+src/nimblepkg/cli
+src/nimblepkg/packageinfo
+src/nimblepkg/packageparser
+src/nimblepkg/reversedeps
+src/nimblepkg/version
+src/nimblepkg/download
+
+# Windows executables
+*.exe
+*.dll
+
+# VCC compiler and linker artifacts
+*.ilk
+*.pdb
+
+# Editors and IDEs project files and folders
+.vscode
+
+# VCS artifacts
+*.orig
+
+# Test procedure artifacts
+nimble_*.nims

.travis.yml

@@ -1,18 +1,24 @@
 os:
+  - windows
   - linux
+  - osx
 
 language: c
 
+env:
+  - BRANCH=0.19.6
+  - BRANCH=0.20.2
+  - BRANCH=1.0.6
+  # This is the latest working Nim version against which Nimble is being tested
+  - BRANCH=#ab525cc48abdbbbed1f772e58e9fe21474f70f07
+
+cache:
+  directories:
+    - "$HOME/.choosenim"
+
 install:
-  - |
+  - curl https://gist.github.com/genotrance/fb53504a4fba88bc5201d3783df5c522/raw/travis.sh -LsSf -o travis.sh
-      git clone https://github.com/nim-lang/Nim.git
+  - source travis.sh
-      cd Nim
-      sh bootstrap.sh
-      cd ..
-before_script:
-  - set -e
-  - set -x
-  - export PATH=`pwd`/Nim/bin:$PATH
 
 script:
   - cd tests
@@ -21,4 +27,4 @@ script:
   - ./src/nimble install -y
 
 notifications:
-  irc: "chat.freenode.net#nimbuild"
+  irc: "chat.freenode.net#nimbuild"

changelog.markdown

@@ -1,5 +1,329 @@
+
+[comment]: # (Before releasing, make sure to follow the steps in https://github.com/nim-lang/nimble/wiki/Releasing-a-new-version)
+
 # Nimble changelog
 
+## 0.11.0 - 22/09/2019
+
+This is a major release containing nearly 60 commits. Most changes are
+bug fixes, but this release also includes a couple new features:
+
+- Binaries can now be built and run using the new ``run`` command.
+- The ``NimblePkgVersion`` is now defined so you can easily get the package
+  version in your source code
+  ([example](https://github.com/nim-lang/nimble/blob/4a2aaa07d/tests/nimbleVersionDefine/src/nimbleVersionDefine.nim)).
+
+Some other highlights:
+
+- Temporary files are now kept when the ``--debug`` flag is used.
+- Fixed dependency resolution issues with "#head" packages (#432 and #672).
+- The `install` command can now take Nim compiler flags via the new
+  ``--passNim`` flag.
+- Command line arguments are now passed properly to tasks (#633).
+- The ``test`` command now respects the specified backend (#631).
+- The ``dump`` command will no longer prompt and now has an implicit ``-y``.
+- Fixed bugs with the new nimscript executor (#665).
+- Fixed multiple downloads and installs of the same package (#678).
+- Nimble init no longer overwrites existing files (#581).
+- Fixed incorrect submodule version being pulled when in a non-master branch (#675).
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.10.2...v0.11.0
+
+## 0.10.2 - 03/06/2019
+
+This is a small release which avoids object variant changes that are now
+treated as runtime errors (Nim #1286). It also adds support for `backend`
+selection during `nimble init`.
+
+Multiple bug fixes are also included:
+- Fixed an issue where failing tasks were not returning a non-zero return
+  value (#655).
+- Error out if `bin` is a Nim source file (#597).
+- Fixed an issue where nimble task would not run if file of same name exists.
+- Fixed an issue that prevented multiple instances of nimble from running on
+  the same package.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.10.0...v0.10.2
+
+## 0.10.0 - 27/05/2019
+
+Nimble now uses the Nim compiler directly via `nim e` to execute nimble
+scripts rather than embedding the Nim VM. This has multiple benefits:
+- Evolve independently from Nim enabling new versions of Nimble to work
+  with multiple versions of Nim.
+- Inherit all nimscript enhancements and bug fixes rather than having to
+  duplicate functionality.
+- Fast build time and smaller binary.
+- No dependency on the compiler package which could cause dependency issues
+  when nimble is used as a package.
+
+Several other features and fixes have been implemented to improve general
+development and test workflows.
+- `nimble test` now sports a `-continue` or `-c` flag that allows tests
+  to continue on failure, removes all created test binaries on completion
+  and warns if no tests found.
+- The `--inclDeps` or `-i` flag enables `nimble uninstall` to remove all
+  dependent packages during uninstall.
+- Added documentation on the usage of a custom `nimbleDir`.
+- Package type interactive prompt is more readable.
+- Save temporary files in a per-user temp dir to enable Nimble on multi-user
+  systems.
+- CTRL-C is now handled correctly in interactive prompts.
+- Fixed issue where empty package list led to error.
+- Fixed issue where file:// was prepended incorrectly.
+- Fixed miscellaneous issues in version parsing, Github auth and briefClone.
+- Miscellaneous cleanup of deprecated procs.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.9.0...v0.10.0
+
+## 0.9.0 - 19/09/2018
+
+This is a major new release which contains at least one breaking change.
+Unfortunately even though it was planned, support for lock files did not
+make it into this release. The release does
+however contain a large number of fixes spread across 57 commits.
+
+The breaking change in this release is to do with the handling of binary
+package. **Any package that specifies a ``bin`` value in it's .nimble file**
+**will no longer install any Nim source code files**, in other words it's not
+going to be a hybrid package by default. This means that so called "hybrid
+packages" now need to specify ``installExt = @["nim"]`` in their metadata,
+otherwise they will become binary packages only.
+
+- **Breaking:** hybrid packages require ``installExt = @["nim"]``
+  ([Commit](https://github.com/nim-lang/nimble/commit/09091792615eacd503e87ca70252c572a4bde2b5))
+- **The ``init`` command can now show a list of choices for information such as**
+  **the license.**
+- **The ``init`` command now creates correct project structures for all package**
+  **types.**
+- **Fatal errors are no longer created when the path inside a .nimble-link file**
+  **doesn't exist.**
+- **The ``develop`` command now always clones HEAD and grabs the full repo history.**
+- **The default ``test`` task no longer executes all tests (only those starting with 't').**
+- Colour is no longer used when `isatty` is false.
+- ``publish`` now shows the URL of the created PR.
+- The ``getPkgDir`` procedure has been fixed in the Nimble file API.
+- Improved handling of proxy environment variables.
+- Codebase has been improved not to rely on `nil` in strings and seqs.
+- The handling of pre- and post-hooks has been improved significantly.
+- Fixed the ``path`` command for packages with a ``srcDir`` and optimised the
+  package look-up.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.8.10...v0.9.0
+
+## 0.8.10 - 23/02/2018
+
+The first release of 2018! Another fairly big release containing 40 commits.
+This release fixes many
+issues, with most being fixed by our brilliant contributors. Thanks a lot
+everyone!
+
+One big new feature is the new support for multiple Nimble packages in a single
+Git/Hg repository. You can now specify ``?subdir=<dir>`` at the end of your
+repo's URL and Nimble will know to look in ``<dir>`` for your package.
+
+* **Implemented support for multi-package repos.** See
+  [#421](https://github.com/nim-lang/nimble/issues/421) for the relevant issue.
+* **Better error message when the user has an outdated stdlib version that confuses Nimble**
+* **The validity of a Nimble package can now be checked using the new ``check`` command**
+* Nimble no longer silently ignores an erroneous '@' in for example
+  ``nimble install compiler@``.
+* Issues with the ``nimble path`` command have been fixed.
+* The ``nimble publish`` command has been improved and stabilised.
+* Messages for the ``NIM_LIB_PREFIX`` env var have been improved.
+* ``before install`` is now called when packages are installed by their name.
+  See [#280](https://github.com/nim-lang/nimble/issues/280).
+* Fixed issue with ``nimble init``. See [#446](https://github.com/nim-lang/nimble/issues/446).
+* Nimble now rejects [reserved names on Windows](https://github.com/nim-lang/nimble/commit/74856a87084b73451254555b2c20ad932cf84270).
+* The ``NIMBLE_DIR`` environment variable is now supported, in addition to the
+  command line flag and config setting.
+* The ``init`` command has been improved significantly.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.8.8...v0.8.10
+
+## 0.8.8 - 03/09/2017
+
+This is a relatively big release containing 57 commits, with multiple new
+features and many bug fixes.
+
+* **Implemented the `develop` command.** See
+  [readme](https://github.com/nim-lang/nimble#nimble-develop) for details.
+* **Implemented a default `test` task** for packages that don't define it.
+* **Lowered the memory consumption** in cases where a package contained many files.
+* Nimble now accepts .nimble symlinks.
+* Locally stored package list files can now be specified in the Nimble config.
+* Fixed branch checkout and handling of branch names with dashes.
+* Improved URL detection in ``publish`` feature.
+* Fixed many issues related to binary management. Packages are now resymlinked
+  when an newer version is removed.
+  ([#331](https://github.com/nim-lang/nimble/issues/331))
+* Fixed issues with CLI arg passing to the Nim compiler.
+  ([#351](https://github.com/nim-lang/nimble/issues/351))
+* Improved performance of ``list -i`` command.
+* Fixed issue where warnings weren't suppressed for some commands.
+  ([#290](https://github.com/nim-lang/nimble/issues/290))
+* Special versions other than `#head` are no longer considered to be newest.
+* Improves the reverse dependency lookup by cross checking it with the
+  installed list of packages.
+  ([#287](https://github.com/nim-lang/nimble/issues/287))
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.8.6...v0.8.8
+
+## 0.8.6 - 05/05/2017
+
+Yet another point release which includes various bug fixes and improvements.
+
+* Improves heuristic for finding Nim standard library to support choosenim
+  installations and adds ability to override it via ``NIM_LIB_PREFIX``
+  environment variable.
+* Implement ``--noColor`` option to remove color from the output.
+* Fixes bug when ``srcDir`` contains trailing slash.
+* Fixes failure when ``-d`` flag is passed to ``c`` command.
+* Show raw output for certain commands.
+* GitHub API token can now be specified via the ``NIMBLE_GITHUB_API_TOKEN``
+  environment variable.
+* GitHub API token is now stored in ``~/.nimble/api_token`` so that it
+  doesn't need to be specified each time.
+* Fixes multiple flags not being passed in Nimble task.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.8.4...v0.8.6
+
+## 0.8.4 - 29/01/2017
+
+Another bug fix release which resolves problems related to stale nimscriptapi
+files in /tmp/, no compilation output when ``nimble build`` fails, and issues
+with the new package validation on Windows.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.8.2...v0.8.4
+
+## 0.8.2 - 08/01/2017
+
+This is a small bug fix release which resolves problems with the installation
+of Aporia (and likely other Nimble packages).
+
+## 0.8.0 - 05/01/2017
+
+This is a large release containing multiple new features and many bug fixes.
+
+* Implemented a completely new output system.
+  * Supports different message types and priorities. Each is differently
+    encoded using a color and a brightness.
+  * The amount of messages shown can be changed by using the new ``--verbose``
+    and ``--debug`` flags, by default only high priority messages are shown.
+  * Duplicate warnings are filtered out to prevent too much noise.
+* Package namespaces are now validated. You will see a warning whenever an
+  incorrectly namespaced package is read by Nimble, this can occur either
+  during installation or when the installed package database is being loaded.
+  The namespacing rules are described in Nimble's
+  [readme](https://github.com/nim-lang/nimble#libraries).
+  **Consider these warnings to be unstable, if you see something that you
+  think is incorrect please report it**.
+* Special version dependencies are now installed into a directory with that
+  special version in its name. For example, ``compiler@#head`` will be installed
+  into ``~/.nimble/pkgs/compiler-#head``. This reduces the amount of redundant
+  installs. See [#88](https://github.com/nim-lang/nimble/issues/88) for
+  more information.
+* External dependencies can now be specified in .nimble files. Nimble doesn't
+  install these, but does instruct the user on how they can be installed.
+  More information about this feature can be found in the
+  [readme](https://github.com/nim-lang/nimble#external-dependencies).
+* Nimble now supports package aliases in the packages.json files.
+* Fixed regression that caused transitive dependencies to not be installed.
+* Fixed problem with ``install`` command when a ``src`` directory is present
+  in the current directory.
+* Improved quoting of process execution arguments.
+* Many improvements to custom ``--nimbleDir`` handling. All commands should now
+  support it correctly.
+* Running ``nimble -v`` will no longer read the Nimble config before displaying
+  the version.
+* Refresh command now supports a package list name as argument.
+* Fixes issues with symlinks not being removed correctly.
+* Changed the way the ``dump`` command locates the .nimble file.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.7.10...v0.8.0
+
+Full list of issues which have been closed: https://github.com/nim-lang/nimble/issues?utf8=%E2%9C%93&q=is%3Aissue+closed%3A%222016-10-09+..+2017-01-05%22+
+
+## 0.7.10 - 09/10/2016
+
+This release includes multiple bug fixes.
+
+* Reverted patch that breaks binary stubs in Git Bash on Windows.
+* The ``nimscriptapi.nim`` file is now statically compiled into the binary.
+  This should fix the "could not find nimscriptapi.nim" errors. The file can
+  still be overriden by placing a file named ``nimscriptapi.nim`` inside a
+  ``nimblepkg`` directory that is placed alongside the Nimble binary, or
+  by a ``nimscriptapi.nim`` file inside ``~/.nimble/pkgs/nimble-ver/nimblepkg/``.
+  For more information see the
+  [code that looks for this file](https://github.com/nim-lang/nimble/blob/v0.7.10/src/nimblepkg/nimscriptsupport.nim#L176).
+* Nim files can now be imported in .nimble nimscript files. (Issue [#186](https://github.com/nim-lang/nimble/issues/186))
+* Requiring a specific git commit hash no longer fails. (Issue [#129](https://github.com/nim-lang/nimble/issues/129))
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.7.8...v0.7.10
+
+## 0.7.8 - 28/09/2016
+
+This is a hotfix release which fixes crashes when Nimble (or Nim) is installed
+to ``C:\Program Files`` or other paths with spaces in them.
+
+## 0.7.6 - 26/09/2016
+
+This is a small release designed to coincide with the release of Nim 0.15.0.
+
+* Fixes ``--depsOnly`` flag ([commit](https://github.com/nim-lang/nimble/commit/f6a19b54e47c7c99f2b473fc02915277273f8c41))
+* Fixes compilation on 0.15.0.
+* Fixes #239.
+* Fixes #215.
+* VCS information is now stored in the Nimble package metadata.
+
+## 0.7.4 - 06/06/2016
+
+This release is mainly a bug fix release. The installation problems
+introduced by v0.7.0 should now be fixed.
+
+* Fixed symlink install issue
+  (Thank you [@yglukhov](https://github.com/yglukhov)).
+* Fixed permission issue when installing packages
+  (Thank you [@SSPkrolik](https://github.com/SSPkrolik)).
+* Work around for issue #204.
+  (Thank you [@Jeff-Ciesielski](https://github.com/Jeff-Ciesielski)).
+* Fixed FD leak.
+  (Thank you [@yglukhov](https://github.com/yglukhov)).
+* Implemented the ``--depsOnly`` option for the ``install`` command.
+* Various fixes to installation/nimscript support problems introduced by
+v0.7.0.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.7.2...v0.7.4
+
+## 0.7.2 - 11/02/2016
+
+This is a hotfix release which alleviates problems when building Nimble.
+
+See Issue [#203](https://github.com/nim-lang/nimble/issues/203) for more
+information.
+
 ## 0.7.0 - 30/12/2015
 
 This is a major release.

nimble.bash-completion

@@ -5,7 +5,10 @@ _nimble()
     local cur=${COMP_WORDS[COMP_CWORD]}
     local prev=${COMP_WORDS[COMP_CWORD-1]}
     COMPREPLY=()
-    _init_completion || return
+
+    if declare -F _init_completions >/dev/null 2>&1; then
+        _init_completion || return
+    fi
 
     case "$prev" in
     init|update|refresh|publish|search|build)
@@ -17,9 +20,12 @@ _nimble()
     install)
         COMPREPLY=( $( nimble list 2> /dev/null | grep "^$cur" | grep -v '^ ' | tr -d ':') )
         ;;
-    path|uninstall)
+    path)
         COMPREPLY=( $( nimble list -i 2> /dev/null | cut -d' ' -f1 | grep "^$cur" ) )
         ;;
+    uninstall)
+        COMPREPLY=( $( nimble list -i 2> /dev/null | awk -F'( |\\[|\\])' '{ f=4; while($f) { l=length($f); if(substr($f, l, l)==",") { $f=substr($f, 0, l-1) }; print $1 "@" $f; f++; }}' | sort -f | grep "^$cur" ) )
+            ;;
     list)
         COMPREPLY=( $( compgen -W '--ver -i --installed' -- $cur ) )
         ;;

nimble.nimble

@@ -1,17 +1,25 @@
 # Package
 
-version       = "0.7.0"
+version       = "0.11.0"
 author        = "Dominik Picheta"
 description   = "Nim package manager."
 license       = "BSD"
 
 bin = @["nimble"]
 srcDir = "src"
+installExt = @["nim"]
 
 # Dependencies
 
-requires "nim >= 0.12.1"
+requires "nim >= 0.13.0"
 
-task tests, "Run the Nimble tester!":
+when defined(nimdistros):
+  import distros
+  if detectOs(Ubuntu):
+    foreignDep "libssl-dev"
+  else:
+    foreignDep "openssl"
+
+task test, "Run the Nimble tester!":
   withDir "tests":
-    exec "nim c -r tester"
+    exec "nim c -r tester"

nimble.zsh-completion

@@ -0,0 +1,51 @@
+#compdef nimble
+
+_nimble() {
+  local line
+
+  _arguments -C \
+    '1: :(install init publish uninstall build c cc js doc doc2 refresh search list tasks path dump develop)' \
+    '*::options:->options' \
+    '(--version)--version[show version]' \
+    '(--help)--help[show help]' \
+    '(-)--help[display help information]' \
+    '(-)--version[display version information]' \
+    '(-y --accept)'{-y,--accept}'[accept all interactive prompts]' \
+                    {-n,--reject}'[reject all interactive prompts]' \
+    '--ver[Query remote server for package version information when searching or listing packages]' \
+    '--nimbleDir dirname[Set the Nimble directory]' \
+    '(-d --depsOnly)'{-d,--depsOnly}'[Install only dependencies]'
+
+  if [ $#line -eq 0 ]; then
+    # if the command line is empty and "nimble tasks" is successfull, add custom tasks
+    tasks=$(nimble tasks)
+    if [ $? -eq 0 ]; then
+      compadd - $(echo $tasks | cut -f1 -d" " | tr '\n' ' ')
+    fi
+  fi
+
+  case $line[1] in
+    install)
+      _nimble_installable_packages
+    ;;
+    uninstall|path|dump)
+      _nimble_installed_packages
+    ;;
+    init|publish|build|refresh|search|tasks)
+      (( ret )) && _message 'no more arguments'
+    ;;
+    *)
+      (( ret )) && _message 'no more arguments'
+    ;;
+  esac
+}
+
+function _nimble_installable_packages {
+  compadd - $(nimble list 2> /dev/null | grep -v '^ ' | tr -d ':')
+}
+
+function _nimble_installed_packages {
+  compadd - $(nimble list -i 2> /dev/null | grep ']$' | cut -d' ' -f1)
+}
+
+_nimble "$@"

src/nimble.nim.cfg

@@ -1,3 +1,6 @@
 --path:"$lib/packages/docutils"
---noNimblePath
+#--noNimblePath
---path:"$nim/"
+--path:"$nim/"
+--path:"./vendor/nim"
+-d:ssl
+-d:nimcore # Enable 'gorge' in Nim's VM. See https://github.com/nim-lang/Nim/issues/8096

src/nimblepkg/cli.nim

@@ -0,0 +1,293 @@
+# Copyright (C) Dominik Picheta. All rights reserved.
+# BSD License. Look at license.txt for more info.
+#
+# Rough rules/philosophy for the messages that Nimble displays are the following:
+#   - Green is only shown when the requested operation is successful.
+#   - Blue can be used to emphasise certain keywords, for example actions such
+#     as "Downloading" or "Reading".
+#   - Red is used when the requested operation fails with an error.
+#   - Yellow is used for warnings.
+#
+#   - Dim for LowPriority.
+#   - Bright for HighPriority.
+#   - Normal for MediumPriority.
+
+import terminal, sets, strutils
+import version
+
+when not declared(initHashSet):
+  import common
+
+type
+  CLI* = ref object
+    level: Priority
+    warnings: HashSet[(string, string)]
+    suppressionCount: int ## Amount of messages which were not shown.
+    showColor: bool ## Whether messages should be colored.
+    suppressMessages: bool ## Whether Warning, Message and Success messages
+                           ## should be suppressed, useful for
+                           ## commands like `dump` whose output should be
+                           ## machine readable.
+
+  Priority* = enum
+    DebugPriority, LowPriority, MediumPriority, HighPriority
+
+  DisplayType* = enum
+    Error, Warning, Message, Success
+
+  ForcePrompt* = enum
+    dontForcePrompt, forcePromptYes, forcePromptNo
+
+const
+  longestCategory = len("Downloading")
+  foregrounds: array[Error .. Success, ForegroundColor] =
+    [fgRed, fgYellow, fgCyan, fgGreen]
+  styles: array[DebugPriority .. HighPriority, set[Style]] =
+    [{styleDim}, {styleDim}, {}, {styleBright}]
+
+
+proc newCLI(): CLI =
+  result = CLI(
+    level: HighPriority,
+    warnings: initHashSet[(string, string)](),
+    suppressionCount: 0,
+    showColor: true,
+    suppressMessages: false
+  )
+
+var globalCLI = newCLI()
+
+
+proc calculateCategoryOffset(category: string): int =
+  assert category.len <= longestCategory
+  return longestCategory - category.len
+
+proc isSuppressed(displayType: DisplayType): bool =
+  # Don't print any Warning, Message or Success messages when suppression of
+  # warnings is enabled. That is, unless the user asked for --verbose output.
+  if globalCLI.suppressMessages and displayType >= Warning and
+     globalCLI.level == HighPriority:
+    return true
+
+proc displayCategory(category: string, displayType: DisplayType,
+                     priority: Priority) =
+  if isSuppressed(displayType):
+    return
+
+  # Calculate how much the `category` must be offset to align along a center
+  # line.
+  let offset = calculateCategoryOffset(category)
+
+  # Display the category.
+  let text = "$1$2 " % [spaces(offset), category]
+  if globalCLI.showColor:
+    if priority != DebugPriority:
+      setForegroundColor(stdout, foregrounds[displayType])
+    writeStyled(text, styles[priority])
+    resetAttributes()
+  else:
+    stdout.write(text)
+
+proc displayLine(category, line: string, displayType: DisplayType,
+                 priority: Priority) =
+  if isSuppressed(displayType):
+    return
+
+  displayCategory(category, displayType, priority)
+
+  # Display the message.
+  echo(line)
+
+proc display*(category, msg: string, displayType = Message,
+              priority = MediumPriority) =
+  # Multiple warnings containing the same messages should not be shown.
+  let warningPair = (category, msg)
+  if displayType == Warning:
+    if warningPair in globalCLI.warnings:
+      return
+    else:
+      globalCLI.warnings.incl(warningPair)
+
+  # Suppress this message if its priority isn't high enough.
+  # TODO: Per-priority suppression counts?
+  if priority < globalCLI.level:
+    if priority != DebugPriority:
+      globalCLI.suppressionCount.inc
+    return
+
+  # Display each line in the message.
+  var i = 0
+  for line in msg.splitLines():
+    if len(line) == 0: continue
+    displayLine(if i == 0: category else: "...", line, displayType, priority)
+    i.inc
+
+proc displayDebug*(category, msg: string) =
+  ## Convenience for displaying debug messages.
+  display(category, msg, priority = DebugPriority)
+
+proc displayDebug*(msg: string) =
+  ## Convenience for displaying debug messages with a default category.
+  displayDebug("Debug:", msg)
+
+proc displayTip*() =
+  ## Called just before Nimble exits. Shows some tips for the user, for example
+  ## the amount of messages that were suppressed and how to show them.
+  if globalCLI.suppressionCount > 0:
+    let msg = "$1 messages have been suppressed, use --verbose to show them." %
+             $globalCLI.suppressionCount
+    display("Tip:", msg, Warning, HighPriority)
+
+proc prompt*(forcePrompts: ForcePrompt, question: string): bool =
+  case forcePrompts
+  of forcePromptYes:
+    display("Prompt:", question & " -> [forced yes]", Warning, HighPriority)
+    return true
+  of forcePromptNo:
+    display("Prompt:", question & " -> [forced no]", Warning, HighPriority)
+    return false
+  of dontForcePrompt:
+    displayLine("Prompt:", question & " [y/N]", Warning, HighPriority)
+    displayCategory("Answer:", Warning, HighPriority)
+    let yn = stdin.readLine()
+    case yn.normalize
+    of "y", "yes":
+      return true
+    of "n", "no":
+      return false
+    else:
+      return false
+
+proc promptCustom*(forcePrompts: ForcePrompt, question, default: string): string =
+  case forcePrompts:
+  of forcePromptYes:
+    display("Prompt:", question & " -> [forced " & default & "]", Warning,
+      HighPriority)
+    return default
+  else:
+    if default == "":
+      display("Prompt:", question, Warning, HighPriority)
+      displayCategory("Answer:", Warning, HighPriority)
+      let user = stdin.readLine()
+      if user.len == 0: return promptCustom(forcePrompts, question, default)
+      else: return user
+    else:
+      display("Prompt:", question & " [" & default & "]", Warning, HighPriority)
+      displayCategory("Answer:", Warning, HighPriority)
+      let user = stdin.readLine()
+      if user == "": return default
+      else: return user
+
+proc promptCustom*(question, default: string): string =
+  return promptCustom(dontForcePrompt, question, default)
+
+proc promptListInteractive(question: string, args: openarray[string]): string =
+  display("Prompt:", question, Warning, HighPriority)
+  display("Select", "Cycle with 'Tab', 'Enter' when done", Message,
+    HighPriority)
+  displayCategory("Choices:", Warning, HighPriority)
+  var
+    current = 0
+    selected = false
+  # Incase the cursor is at the bottom of the terminal
+  for arg in args:
+    stdout.write "\n"
+  # Reset the cursor to the start of the selection prompt
+  cursorUp(stdout, args.len)
+  cursorForward(stdout, longestCategory)
+  hideCursor(stdout)
+
+  # The selection loop
+  while not selected:
+    setForegroundColor(fgDefault)
+    # Loop through the options
+    for i, arg in args:
+      # Check if the option is the current
+      if i == current:
+        writeStyled("> " & arg & " <", {styleBright})
+      else:
+        writeStyled("  " & arg & "  ", {styleDim})
+      # Move the cursor back to the start
+      for s in 0..<(arg.len + 4):
+        cursorBackward(stdout)
+      # Move down for the next item
+      cursorDown(stdout)
+    # Move the cursor back up to the start of the selection prompt
+    for i in 0..<(args.len()):
+      cursorUp(stdout)
+    resetAttributes(stdout)
+
+    # Begin key input
+    while true:
+      case getch():
+      of '\t':
+        current = (current + 1) mod args.len
+        break
+      of '\r':
+        selected = true
+        break
+      of '\3':
+        showCursor(stdout)
+        raise newException(NimbleError, "Keyboard interrupt")
+      else: discard
+
+  # Erase all lines of the selection
+  for i in 0..<args.len:
+    eraseLine(stdout)
+    cursorDown(stdout)
+  # Move the cursor back up to the initial selection line
+  for i in 0..<args.len():
+    cursorUp(stdout)
+  showCursor(stdout)
+  display("Answer:", args[current], Warning,HighPriority)
+  return args[current]
+
+proc promptListFallback(question: string, args: openarray[string]): string =
+  display("Prompt:", question & " [" & join(args, "/") & "]", Warning,
+    HighPriority)
+  displayCategory("Answer:", Warning, HighPriority)
+  result = stdin.readLine()
+  for arg in args:
+    if arg.cmpIgnoreCase(result) == 0:
+      return arg
+
+proc promptList*(forcePrompts: ForcePrompt, question: string, args: openarray[string]): string =
+  case forcePrompts:
+  of forcePromptYes:
+    result = args[0]
+    display("Prompt:", question & " -> [forced " & result & "]", Warning,
+      HighPriority)
+  else:
+    if isatty(stdout):
+      return promptListInteractive(question, args)
+    else:
+      return promptListFallback(question, args)
+
+proc setVerbosity*(level: Priority) =
+  globalCLI.level = level
+
+proc setShowColor*(val: bool) =
+  globalCLI.showColor = val
+
+proc setSuppressMessages*(val: bool) =
+  globalCLI.suppressMessages = val
+
+when isMainModule:
+  display("Reading", "config file at /Users/dom/.config/nimble/nimble.ini",
+          priority = LowPriority)
+
+  display("Reading", "official package list",
+        priority = LowPriority)
+
+  display("Downloading", "daemonize v0.0.2 using Git",
+      priority = HighPriority)
+
+  display("Warning", "dashes in package names will be deprecated", Warning,
+      priority = HighPriority)
+
+  display("Error", """Unable to read package info for /Users/dom/.nimble/pkgs/nimble-0.7.11
+Reading as ini file failed with:
+  Invalid section: .
+Evaluating as NimScript file failed with:
+  Users/dom/.nimble/pkgs/nimble-0.7.11/nimble.nimble(3, 23) Error: cannot open 'src/nimblepkg/common'.
+""", Error, HighPriority)

src/nimblepkg/common.nim

@@ -0,0 +1,78 @@
+# Copyright (C) Dominik Picheta. All rights reserved.
+# BSD License. Look at license.txt for more info.
+#
+# Various miscellaneous common types reside here, to avoid problems with
+# recursive imports
+
+when not defined(nimscript):
+  import sets
+
+  import version
+
+  type
+    BuildFailed* = object of NimbleError
+
+    PackageInfo* = object
+      myPath*: string ## The path of this .nimble file
+      isNimScript*: bool ## Determines if this pkg info was read from a nims file
+      isMinimal*: bool
+      isInstalled*: bool ## Determines if the pkg this info belongs to is installed
+      isLinked*: bool ## Determines if the pkg this info belongs to has been linked via `develop`
+      postHooks*: HashSet[string] ## Useful to know so that Nimble doesn't execHook unnecessarily
+      preHooks*: HashSet[string]
+      name*: string
+      ## The version specified in the .nimble file.Assuming info is non-minimal,
+      ## it will always be a non-special version such as '0.1.4'.
+      ## If in doubt, use `getConcreteVersion` instead.
+      version*: string
+      specialVersion*: string ## Either `myVersion` or a special version such as #head.
+      author*: string
+      description*: string
+      license*: string
+      skipDirs*: seq[string]
+      skipFiles*: seq[string]
+      skipExt*: seq[string]
+      installDirs*: seq[string]
+      installFiles*: seq[string]
+      installExt*: seq[string]
+      requires*: seq[PkgTuple]
+      bin*: seq[string]
+      binDir*: string
+      srcDir*: string
+      backend*: string
+      foreignDeps*: seq[string]
+
+    ## Same as quit(QuitSuccess), but allows cleanup.
+    NimbleQuit* = ref object of Exception
+
+  proc raiseNimbleError*(msg: string, hint = "") =
+    var exc = newException(NimbleError, msg)
+    exc.hint = hint
+    raise exc
+
+  proc getOutputInfo*(err: ref NimbleError): (string, string) =
+    var error = ""
+    var hint = ""
+    error = err.msg
+    when not defined(release):
+      let stackTrace = getStackTrace(err)
+      error = stackTrace & "\n\n" & error
+    if not err.isNil:
+      hint = err.hint
+
+    return (error, hint)
+
+const
+  nimbleVersion* = "0.11.0"
+
+when not declared(initHashSet):
+  import sets
+
+  template initHashSet*[A](initialSize = 64): HashSet[A] =
+    initSet[A](initialSize)
+
+when not declared(toHashSet):
+  import sets
+
+  template toHashSet*[A](keys: openArray[A]): HashSet[A] =
+    toSet(keys)

src/nimblepkg/config.nim

@@ -1,26 +1,25 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
-import parsecfg, streams, strutils, os, tables, Uri
+import parsecfg, streams, strutils, os, tables, uri
 
-import tools, version, nimbletypes
+import version, cli
 
 type
   Config* = object
     nimbleDir*: string
     chcp*: bool # Whether to change the code page in .cmd files on Win.
-    packageLists*: Table[string, PackageList] ## URLs to packages.json files
+    packageLists*: Table[string, PackageList] ## Names -> packages.json files
     cloneUsingHttps*: bool # Whether to replace git:// for https://
     httpProxy*: Uri # Proxy for package list downloads.
+    nimLibPrefix*: string # Nim stdlib prefix.
 
   PackageList* = object
     name*: string
     urls*: seq[string]
+    path*: string
 
 proc initConfig(): Config =
-  if getNimrodVersion() > newVersion("0.9.6"):
+  result.nimbleDir = getHomeDir() / ".nimble"
-    result.nimbleDir = getHomeDir() / ".nimble"
-  else:
-    result.nimbleDir = getHomeDir() / ".babel"
 
   result.httpProxy = initUri()
 
@@ -29,15 +28,18 @@ proc initConfig(): Config =
 
   result.packageLists = initTable[string, PackageList]()
   let defaultPkgList = PackageList(name: "Official", urls: @[
+    "https://github.com/nim-lang/packages/raw/master/packages.json",
     "http://irclogs.nim-lang.org/packages.json",
-    "http://nim-lang.org/nimble/packages.json",
+    "http://nim-lang.org/nimble/packages.json"
-    "https://github.com/nim-lang/packages/raw/master/packages.json"
   ])
   result.packageLists["official"] = defaultPkgList
 
+  result.nimLibPrefix = ""
+
 proc initPackageList(): PackageList =
   result.name = ""
   result.urls = @[]
+  result.path = ""
 
 proc addCurrentPkgList(config: var Config, currentPackageList: PackageList) =
   if currentPackageList.name.len > 0:
@@ -50,13 +52,14 @@ proc parseConfig*(): Config =
   var f = newFileStream(confFile, fmRead)
   if f == nil:
     # Try the old deprecated babel.ini
+    # TODO: This can be removed.
     confFile = getConfigDir() / "babel" / "babel.ini"
     f = newFileStream(confFile, fmRead)
     if f != nil:
-      echo("[Warning] Using deprecated config file at ", confFile)
+      display("Warning", "Using deprecated config file at " & confFile,
-  
+              Warning, HighPriority)
   if f != nil:
-    echo("Reading from config file at ", confFile)
+    display("Reading", "config file at " & confFile, priority = LowPriority)
     var p: CfgParser
     open(p, f, confFile)
     var currentSection = ""
@@ -65,7 +68,12 @@ proc parseConfig*(): Config =
       var e = next(p)
       case e.kind
       of cfgEof:
-        addCurrentPkgList(result, currentPackageList)
+        if currentSection.len > 0:
+          if currentPackageList.urls.len == 0 and currentPackageList.path == "":
+            raise newException(NimbleError, "Package list '$1' requires either url or path" % currentPackageList.name)
+          if currentPackageList.urls.len > 0 and currentPackageList.path != "":
+            raise newException(NimbleError, "Attempted to specify `url` and `path` for the same package list '$1'" % currentPackageList.name)
+          addCurrentPkgList(result, currentPackageList)
         break
       of cfgSectionStart:
         addCurrentPkgList(result, currentPackageList)
@@ -98,6 +106,16 @@ proc parseConfig*(): Config =
           of "packagelist":
             currentPackageList.urls.add(e.value)
           else: assert false
+        of "path":
+          case currentSection.normalize
+          of "packagelist":
+            if currentPackageList.path != "":
+              raise newException(NimbleError, "Attempted to specify more than one `path` for the same package list.")
+            else:
+              currentPackageList.path = e.value
+          else: assert false
+        of "nimlibprefix":
+          result.nimLibPrefix = e.value
         else:
           raise newException(NimbleError, "Unable to parse config file:" &
                                      " Unknown key: " & e.key)

src/nimblepkg/download.nim

@@ -1,15 +1,16 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
 
-import parseutils, os, osproc, strutils, tables, pegs
+import parseutils, os, osproc, strutils, tables, pegs, uri
-
+import packageinfo, packageparser, version, tools, common, options, cli
-import packageinfo, packageparser, version, tools, nimbletypes, options
+from algorithm import SortOrder, sorted
+from sequtils import toSeq, filterIt, map
 
 type
   DownloadMethod* {.pure.} = enum
     git = "git", hg = "hg"
 
-proc getSpecificDir(meth: DownloadMethod): string =
+proc getSpecificDir(meth: DownloadMethod): string {.used.} =
   case meth
   of DownloadMethod.git:
     ".git"
@@ -24,11 +25,12 @@ proc doCheckout(meth: DownloadMethod, downloadDir, branch: string) =
       # clone has happened. Like in the case of git on Windows where it
       # messes up the damn line endings.
       doCmd("git checkout --force " & branch)
+      doCmd("git submodule update --recursive")
   of DownloadMethod.hg:
     cd downloadDir:
       doCmd("hg checkout " & branch)
 
-proc doPull(meth: DownloadMethod, downloadDir: string) =
+proc doPull(meth: DownloadMethod, downloadDir: string) {.used.} =
   case meth
   of DownloadMethod.git:
     doCheckout(meth, downloadDir, "")
@@ -42,17 +44,17 @@ proc doPull(meth: DownloadMethod, downloadDir: string) =
       doCmd("hg pull")
 
 proc doClone(meth: DownloadMethod, url, downloadDir: string, branch = "",
-            tip = true) =
+             onlyTip = true) =
   case meth
   of DownloadMethod.git:
     let
-      depthArg = if tip: "--depth 1 " else: ""
+      depthArg = if onlyTip: "--depth 1 " else: ""
       branchArg = if branch == "": "" else: "-b " & branch & " "
     doCmd("git clone --recursive " & depthArg & branchArg & url &
           " " & downloadDir)
   of DownloadMethod.hg:
     let
-      tipArg = if tip: "-r tip " else: ""
+      tipArg = if onlyTip: "-r tip " else: ""
       branchArg = if branch == "": "" else: "-b " & branch & " "
     doCmd("hg clone " & tipArg & branchArg & url & " " & downloadDir)
 
@@ -91,8 +93,10 @@ proc getTagsListRemote*(url: string, meth: DownloadMethod): seq[string] =
       raise newException(OSError, "Unable to query remote tags for " & url &
           ". Git returned: " & output)
     for i in output.splitLines():
-      if i == "": continue
+      let refStart = i.find("refs/tags/")
-      let start = i.find("refs/tags/")+"refs/tags/".len
+      # git outputs warnings, empty lines, etc
+      if refStart == -1: continue
+      let start = refStart+"refs/tags/".len
       let tag = i[start .. i.len-1]
       if not tag.endswith("^{}"): result.add(tag)
 
@@ -100,14 +104,21 @@ proc getTagsListRemote*(url: string, meth: DownloadMethod): seq[string] =
     # http://stackoverflow.com/questions/2039150/show-tags-for-remote-hg-repository
     raise newException(ValueError, "Hg doesn't support remote tag querying.")
 
-proc getVersionList*(tags: seq[string]): Table[Version, string] =
+proc getVersionList*(tags: seq[string]): OrderedTable[Version, string] =
-  # Returns: TTable of version -> git tag name
+  ## Return an ordered table of Version -> git tag label.  Ordering is
-  result = initTable[Version, string]()
+  ## in descending order with the most recent version first.
-  for tag in tags:
+  let taggedVers: seq[tuple[ver: Version, tag: string]] =
-    if tag != "":
+    tags
-      let i = skipUntil(tag, Digits) # skip any chars before the version
+      .filterIt(it != "")
-      # TODO: Better checking, tags can have any names. Add warnings and such.
+      .map(proc(s: string): tuple[ver: Version, tag: string] =
-      result[newVersion(tag[i .. tag.len-1])] = tag
+        # skip any chars before the version
+        let i = skipUntil(s, Digits)
+        # TODO: Better checking, tags can have any
+        # names. Add warnings and such.
+        result = (newVersion(s[i .. s.len-1]), s))
+      .sorted(proc(a, b: (Version, string)): int = cmp(a[0], b[0]),
+              SortOrder.Descending)
+  result = toOrderedTable[Version, string](taggedVers)
 
 proc getDownloadMethod*(meth: string): DownloadMethod =
   case meth
@@ -116,12 +127,12 @@ proc getDownloadMethod*(meth: string): DownloadMethod =
   else:
     raise newException(NimbleError, "Invalid download method: " & meth)
 
-proc getHeadName*(meth: DownloadMethod): string =
+proc getHeadName*(meth: DownloadMethod): Version =
   ## Returns the name of the download method specific head. i.e. for git
   ## it's ``head`` for hg it's ``tip``.
   case meth
-  of DownloadMethod.git: "head"
+  of DownloadMethod.git: newVersion("#head")
-  of DownloadMethod.hg: "tip"
+  of DownloadMethod.hg: newVersion("#tip")
 
 proc checkUrlType*(url: string): DownloadMethod =
   ## Determines the download method based on the URL.
@@ -130,19 +141,29 @@ proc checkUrlType*(url: string): DownloadMethod =
   elif doCmdEx("hg identify " & url).exitCode == QuitSuccess:
     return DownloadMethod.hg
   else:
-    raise newException(NimbleError, "Unable to identify url.")
+    raise newException(NimbleError, "Unable to identify url: " & url)
+
+proc getUrlData*(url: string): (string, Table[string, string]) =
+  var uri = parseUri(url)
+  # TODO: use uri.parseQuery once it lands... this code is quick and dirty.
+  var subdir = ""
+  if uri.query.startsWith("subdir="):
+    subdir = uri.query[7 .. ^1]
+
+  uri.query = ""
+  return ($uri, {"subdir": subdir}.toTable())
 
 proc isURL*(name: string): bool =
   name.startsWith(peg" @'://' ")
 
-proc doDownload*(url: string, downloadDir: string, verRange: VersionRange,
+proc doDownload(url: string, downloadDir: string, verRange: VersionRange,
-                 downMethod: DownloadMethod, options: Options): VersionRange =
+                 downMethod: DownloadMethod,
+                 options: Options): Version =
   ## Downloads the repository specified by ``url`` using the specified download
   ## method.
   ##
   ## Returns the version of the repository which has been downloaded.
-  template getLatestByTag(meth: stmt): stmt {.dirty, immediate.} =
+  template getLatestByTag(meth: untyped) {.dirty.} =
-    echo("Found tags...")
     # Find latest version that fits our ``verRange``.
     var latest = findLatest(verRange, versions)
     ## Note: HEAD is not used when verRange.kind is verAny. This is
@@ -150,65 +171,103 @@ proc doDownload*(url: string, downloadDir: string, verRange: VersionRange,
 
     # If no tagged versions satisfy our range latest.tag will be "".
     # We still clone in that scenario because we want to try HEAD in that case.
-    # https://github.com/nimrod-code/nimble/issues/22
+    # https://github.com/nim-lang/nimble/issues/22
     meth
     if $latest.ver != "":
-      result = parseVersionRange($latest.ver)
+      result = latest.ver
-    else:
-      # Result should already be set to #head here.
-      assert(not result.isNil)
-
-  proc verifyClone() =
-    ## Makes sure that the downloaded package's version satisfies the requested
-    ## version range.
-    let pkginfo = getPkgInfo(downloadDir, options)
-    if pkginfo.version.newVersion notin verRange:
-      raise newException(NimbleError,
-        "Downloaded package's version does not satisfy requested version " &
-        "range: wanted $1 got $2." %
-        [$verRange, $pkginfo.version])
 
   removeDir(downloadDir)
   if verRange.kind == verSpecial:
     # We want a specific commit/branch/tag here.
-    if verRange.spe == newSpecial(getHeadName(downMethod)):
+    if verRange.spe == getHeadName(downMethod):
-      doClone(downMethod, url, downloadDir) # Grab HEAD.
+       # Grab HEAD.
+      doClone(downMethod, url, downloadDir, onlyTip = not options.forceFullClone)
     else:
-      # Mercurial requies a clone and checkout. The git clone operation is
+      # Grab the full repo.
-      # already fragmented into multiple steps so we just call doClone().
+      doClone(downMethod, url, downloadDir, onlyTip = false)
-      if downMethod == DownloadMethod.git:
+      # Then perform a checkout operation to get the specified branch/commit.
-        doClone(downMethod, url, downloadDir, $verRange.spe)
+      # `spe` starts with '#', trim it.
-      else:
+      doAssert(($verRange.spe)[0] == '#')
-        doClone(downMethod, url, downloadDir, tip = false)
+      doCheckout(downMethod, downloadDir, substr($verRange.spe, 1))
-        doCheckout(downMethod, downloadDir, $verRange.spe)
+    result = verRange.spe
-    result = verRange
   else:
     case downMethod
     of DownloadMethod.git:
       # For Git we have to query the repo remotely for its tags. This is
       # necessary as cloning with a --depth of 1 removes all tag info.
-      result = parseVersionRange("#head")
+      result = getHeadName(downMethod)
       let versions = getTagsListRemote(url, downMethod).getVersionList()
       if versions.len > 0:
         getLatestByTag:
-          echo("Cloning latest tagged version: ", latest.tag)
+          display("Cloning", "latest tagged version: " & latest.tag,
-          doClone(downMethod, url, downloadDir, latest.tag)
+                  priority = MediumPriority)
+          doClone(downMethod, url, downloadDir, latest.tag,
+                  onlyTip = not options.forceFullClone)
       else:
         # If no commits have been tagged on the repo we just clone HEAD.
         doClone(downMethod, url, downloadDir) # Grab HEAD.
-
-      verifyClone()
     of DownloadMethod.hg:
-      doClone(downMethod, url, downloadDir)
+      doClone(downMethod, url, downloadDir, onlyTip = not options.forceFullClone)
-      result = parseVersionRange("#tip")
+      result = getHeadName(downMethod)
       let versions = getTagsList(downloadDir, downMethod).getVersionList()
 
       if versions.len > 0:
         getLatestByTag:
-          echo("Switching to latest tagged version: ", latest.tag)
+          display("Switching", "to latest tagged version: " & latest.tag,
+                  priority = MediumPriority)
           doCheckout(downMethod, downloadDir, latest.tag)
 
-      verifyClone()
+proc downloadPkg*(url: string, verRange: VersionRange,
+                 downMethod: DownloadMethod,
+                 subdir: string,
+                 options: Options,
+                 downloadPath = ""): (string, Version) =
+  ## Downloads the repository as specified by ``url`` and ``verRange`` using
+  ## the download method specified.
+  ##
+  ## If `downloadPath` isn't specified a location in /tmp/ will be used.
+  ##
+  ## Returns the directory where it was downloaded (subdir is appended) and
+  ## the concrete version  which was downloaded.
+  let downloadDir =
+    if downloadPath == "":
+      (getNimbleTempDir() / getDownloadDirName(url, verRange))
+    else:
+      downloadPath
+
+  createDir(downloadDir)
+  var modUrl =
+    if url.startsWith("git://") and options.config.cloneUsingHttps:
+      "https://" & url[6 .. ^1]
+    else: url
+
+  # Fixes issue #204
+  # github + https + trailing url slash causes a
+  # checkout/ls-remote to fail with Repository not found
+  if modUrl.contains("github.com") and modUrl.endswith("/"):
+    modUrl = modUrl[0 .. ^2]
+
+  if subdir.len > 0:
+    display("Downloading", "$1 using $2 (subdir is '$3')" %
+                           [modUrl, $downMethod, subdir],
+            priority = HighPriority)
+  else:
+    display("Downloading", "$1 using $2" % [modUrl, $downMethod],
+            priority = HighPriority)
+  result = (
+    downloadDir / subdir,
+    doDownload(modUrl, downloadDir, verRange, downMethod, options)
+  )
+
+  if verRange.kind != verSpecial:
+    ## Makes sure that the downloaded package's version satisfies the requested
+    ## version range.
+    let pkginfo = getPkgInfo(result[0], options)
+    if pkginfo.version.newVersion notin verRange:
+      raise newException(NimbleError,
+        "Downloaded package's version does not satisfy requested version " &
+        "range: wanted $1 got $2." %
+        [$verRange, $pkginfo.version])
 
 proc echoPackageVersions*(pkg: Package) =
   let downMethod = pkg.downloadMethod.getDownloadMethod()
@@ -217,14 +276,8 @@ proc echoPackageVersions*(pkg: Package) =
     try:
       let versions = getTagsListRemote(pkg.url, downMethod).getVersionList()
       if versions.len > 0:
-        var vstr = ""
+        let sortedVersions = toSeq(values(versions))
-        var i = 0
+        echo("  versions:    " & join(sortedVersions, ", "))
-        for v in values(versions):
-          if i != 0:
-            vstr.add(", ")
-          vstr.add(v)
-          i.inc
-        echo("  versions:    " & vstr)
       else:
         echo("  versions:    (No versions tagged in the remote repository)")
     except OSError:
@@ -232,3 +285,32 @@ proc echoPackageVersions*(pkg: Package) =
   of DownloadMethod.hg:
     echo("  versions:    (Remote tag retrieval not supported by " &
         pkg.downloadMethod & ")")
+
+when isMainModule:
+  # Test version sorting
+  block:
+    let data = @["v9.0.0-taeyeon", "v9.0.1-jessica", "v9.2.0-sunny",
+                 "v9.4.0-tiffany", "v9.4.2-hyoyeon"]
+    let expected = toOrderedTable[Version, string]({
+      newVersion("9.4.2-hyoyeon"): "v9.4.2-hyoyeon",
+      newVersion("9.4.0-tiffany"): "v9.4.0-tiffany",
+      newVersion("9.2.0-sunny"): "v9.2.0-sunny",
+      newVersion("9.0.1-jessica"): "v9.0.1-jessica",
+      newVersion("9.0.0-taeyeon"): "v9.0.0-taeyeon"
+    })
+    doAssert expected == getVersionList(data)
+
+
+  block:
+    let data2 = @["v0.1.0", "v0.1.1", "v0.2.0",
+                 "0.4.0", "v0.4.2"]
+    let expected2 = toOrderedTable[Version, string]({
+      newVersion("0.4.2"): "v0.4.2",
+      newVersion("0.4.0"): "0.4.0",
+      newVersion("0.2.0"): "v0.2.0",
+      newVersion("0.1.1"): "v0.1.1",
+      newVersion("0.1.0"): "v0.1.0",
+    })
+    doAssert expected2 == getVersionList(data2)
+
+  echo("Everything works!")

src/nimblepkg/init.nim

@@ -0,0 +1,184 @@
+import os, strutils
+
+import ./cli, ./tools
+
+type
+  PkgInitInfo* = tuple
+    pkgName: string
+    pkgVersion: string
+    pkgAuthor: string
+    pkgDesc: string
+    pkgLicense: string
+    pkgBackend: string
+    pkgSrcDir: string
+    pkgNimDep: string
+    pkgType: string
+
+proc writeExampleIfNonExistent(file: string, content: string) =
+  if not existsFile(file):
+    writeFile(file, content)
+  else:
+    display("Info:", "File " & file & " already exists, did not write " &
+            "example code", priority = HighPriority)
+
+proc createPkgStructure*(info: PkgInitInfo, pkgRoot: string) =
+  # Create source directory
+  createDirD(pkgRoot / info.pkgSrcDir)
+
+  # Initialise the source code directories and create some example code.
+  var nimbleFileOptions = ""
+  case info.pkgType
+  of "binary":
+    let mainFile = pkgRoot / info.pkgSrcDir / info.pkgName.changeFileExt("nim")
+    writeExampleIfNonExistent(mainFile,
+"""
+# This is just an example to get you started. A typical binary package
+# uses this file as the main entry point of the application.
+
+when isMainModule:
+  echo("Hello, World!")
+"""
+    )
+    nimbleFileOptions.add("bin           = @[\"$1\"]\n" % info.pkgName)
+  of "library":
+    let mainFile = pkgRoot / info.pkgSrcDir / info.pkgName.changeFileExt("nim")
+    writeExampleIfNonExistent(mainFile,
+"""
+# This is just an example to get you started. A typical library package
+# exports the main API in this file. Note that you cannot rename this file
+# but you can remove it if you wish.
+
+proc add*(x, y: int): int =
+  ## Adds two files together.
+  return x + y
+"""
+    )
+
+    createDirD(pkgRoot / info.pkgSrcDir / info.pkgName)
+    let submodule = pkgRoot / info.pkgSrcDir / info.pkgName /
+        "submodule".addFileExt("nim")
+    writeExampleIfNonExistent(submodule,
+"""
+# This is just an example to get you started. Users of your library will
+# import this file by writing ``import $1/submodule``. Feel free to rename or
+# remove this file altogether. You may create additional modules alongside
+# this file as required.
+
+type
+  Submodule* = object
+    name*: string
+
+proc initSubmodule*(): Submodule =
+  ## Initialises a new ``Submodule`` object.
+  Submodule(name: "Anonymous")
+""" % info.pkgName
+    )
+  of "hybrid":
+    let mainFile = pkgRoot / info.pkgSrcDir / info.pkgName.changeFileExt("nim")
+    writeExampleIfNonExistent(mainFile,
+"""
+# This is just an example to get you started. A typical hybrid package
+# uses this file as the main entry point of the application.
+
+import $1pkg/submodule
+
+when isMainModule:
+  echo(getWelcomeMessage())
+""" % info.pkgName
+    )
+
+    let pkgSubDir = pkgRoot / info.pkgSrcDir / info.pkgName & "pkg"
+    createDirD(pkgSubDir)
+    let submodule = pkgSubDir / "submodule".addFileExt("nim")
+    writeExampleIfNonExistent(submodule,
+"""
+# This is just an example to get you started. Users of your hybrid library will
+# import this file by writing ``import $1pkg/submodule``. Feel free to rename or
+# remove this file altogether. You may create additional modules alongside
+# this file as required.
+
+proc getWelcomeMessage*(): string = "Hello, World!"
+""" % info.pkgName
+    )
+    nimbleFileOptions.add("installExt    = @[\"nim\"]\n")
+    nimbleFileOptions.add("bin           = @[\"$1\"]\n" % info.pkgName)
+  else:
+    assert false, "Invalid package type specified."
+
+  let pkgTestDir = "tests"
+  # Create test directory
+  case info.pkgType
+  of "binary":
+    discard
+  of "hybrid", "library":
+    let pkgTestPath = pkgRoot / pkgTestDir
+    createDirD(pkgTestPath)
+
+    writeFile(pkgTestPath / "config".addFileExt("nims"),
+      "switch(\"path\", \"$$projectDir/../$#\")" % info.pkgSrcDir
+    )
+
+    if info.pkgType == "library":
+      writeExampleIfNonExistent(pkgTestPath / "test1".addFileExt("nim"),
+"""
+# This is just an example to get you started. You may wish to put all of your
+# tests into a single file, or separate them into multiple `test1`, `test2`
+# etc. files (better names are recommended, just make sure the name starts with
+# the letter 't').
+#
+# To run these tests, simply execute `nimble test`.
+
+import unittest
+
+import $1
+test "can add":
+  check add(5, 5) == 10
+""" % info.pkgName
+      )
+    else:
+      writeExampleIfNonExistent(pkgTestPath / "test1".addFileExt("nim"),
+"""
+# This is just an example to get you started. You may wish to put all of your
+# tests into a single file, or separate them into multiple `test1`, `test2`
+# etc. files (better names are recommended, just make sure the name starts with
+# the letter 't').
+#
+# To run these tests, simply execute `nimble test`.
+
+import unittest
+
+import $1pkg/submodule
+test "correct welcome":
+  check getWelcomeMessage() == "Hello, World!"
+""" % info.pkgName
+      )
+  else:
+    assert false, "Invalid package type specified."
+
+  # Write the nimble file
+  let nimbleFile = pkgRoot / info.pkgName.changeFileExt("nimble")
+  # Only write backend if it isn't "c"
+  var pkgBackend = ""
+  if (info.pkgBackend != "c"):
+    pkgBackend = "backend       = " & info.pkgbackend.escape()
+  writeFile(nimbleFile, """# Package
+
+version       = $#
+author        = "$#"
+description   = "$#"
+license       = $#
+srcDir        = $#
+$#
+$#
+
+# Dependencies
+
+requires "nim >= $#"
+""" % [
+      info.pkgVersion.escape(), info.pkgAuthor.replace("\"", "\\\""), info.pkgDesc.replace("\"", "\\\""),
+      info.pkgLicense.escape(), info.pkgSrcDir.escape(), nimbleFileOptions,
+      pkgBackend, info.pkgNimDep
+    ]
+  )
+
+  display("Info:", "Nimble file created successfully", priority=MediumPriority)

src/nimblepkg/nimbletypes.nim

@@ -1,36 +0,0 @@
-# BSD License. Look at license.txt for more info.
-#
-# Various miscellaneous common types reside here, to avoid problems with
-# recursive imports
-
-import sets
-
-import version
-export version.NimbleError
-
-type
-  BuildFailed* = object of NimbleError
-
-  PackageInfo* = object
-    mypath*: string ## The path of this .nimble file
-    isNimScript*: bool ## Determines if this pkg info was read from a nims file
-    isMinimal*: bool
-    isInstalled*: bool ## Determines if the pkg this info belongs to is installed
-    postHooks*: HashSet[string] ## Useful to know so that Nimble doesn't execHook unnecessarily
-    preHooks*: HashSet[string]
-    name*: string
-    version*: string
-    author*: string
-    description*: string
-    license*: string
-    skipDirs*: seq[string]
-    skipFiles*: seq[string]
-    skipExt*: seq[string]
-    installDirs*: seq[string]
-    installFiles*: seq[string]
-    installExt*: seq[string]
-    requires*: seq[PkgTuple]
-    bin*: seq[string]
-    binDir*: string
-    srcDir*: string
-    backend*: string

src/nimblepkg/nimscriptapi.nim

@@ -3,6 +3,12 @@
 
 ## This module is implicitly imported in NimScript .nimble files.
 
+import system except getCommand, setCommand, switch, `--`
+import strformat, strutils, tables
+
+when not defined(nimscript):
+  import os
+
 var
   packageName* = ""    ## Set this to the package name. It
                        ## is usually not required to do that, nims' filename is
@@ -11,7 +17,7 @@ var
   author*: string      ## The package's author.
   description*: string ## The package's description.
   license*: string     ## The package's license.
-  srcdir*: string      ## The package's source directory.
+  srcDir*: string      ## The package's source directory.
   binDir*: string      ## The package's binary directory.
   backend*: string     ## The package's backend.
 
@@ -19,26 +25,172 @@ var
     installExt*, bin*: seq[string] = @[] ## Nimble metadata.
   requiresData*: seq[string] = @[] ## The package's dependencies.
 
+  foreignDeps*: seq[string] = @[] ## The foreign dependencies. Only
+                                  ## exported for 'distros.nim'.
+
+  beforeHooks: seq[string] = @[]
+  afterHooks: seq[string] = @[]
+  commandLineParams: seq[string] = @[]
+  flags: TableRef[string, seq[string]]
+
+  command = "e"
+  project = ""
+  success = false
+  retVal = true
+  projectFile = ""
+  outFile = ""
+
 proc requires*(deps: varargs[string]) =
   ## Call this to set the list of requirements of your Nimble
   ## package.
   for d in deps: requiresData.add(d)
 
+proc getParams() =
+  # Called by nimscriptwrapper.nim:execNimscript()
+  #   nim e --flags /full/path/to/file.nims /full/path/to/file.out action
+  for i in 2 .. paramCount():
+    let
+      param = paramStr(i)
+    if param[0] != '-':
+      if projectFile.len == 0:
+        projectFile = param
+      elif outFile.len == 0:
+        outFile = param
+      else:
+        commandLineParams.add param.normalize
+
+proc getCommand*(): string =
+  return command
+
+proc setCommand*(cmd: string, prj = "") =
+  command = cmd
+  if prj.len != 0:
+    project = prj
+
+proc switch*(key: string, value="") =
+  if flags.isNil:
+    flags = newTable[string, seq[string]]()
+
+  if flags.hasKey(key):
+    flags[key].add(value)
+  else:
+    flags[key] = @[value]
+
+template `--`*(key, val: untyped) =
+  switch(astToStr(key), strip astToStr(val))
+
+template `--`*(key: untyped) =
+  switch(astToStr(key), "")
+
+template printIfLen(varName) =
+  if varName.len != 0:
+    result &= astToStr(varName) & ": \"\"\"" & varName & "\"\"\"\n"
+
+template printSeqIfLen(varName) =
+  if varName.len != 0:
+    result &= astToStr(varName) & ": \"" & varName.join(", ") & "\"\n"
+
+proc printPkgInfo(): string =
+  if backend.len == 0:
+    backend = "c"
+
+  result = "[Package]\n"
+  if packageName.len != 0:
+    result &= "name: \"" & packageName & "\"\n"
+  printIfLen version
+  printIfLen author
+  printIfLen description
+  printIfLen license
+  printIfLen srcDir
+  printIfLen binDir
+  printIfLen backend
+
+  printSeqIfLen skipDirs
+  printSeqIfLen skipFiles
+  printSeqIfLen skipExt
+  printSeqIfLen installDirs
+  printSeqIfLen installFiles
+  printSeqIfLen installExt
+  printSeqIfLen bin
+  printSeqIfLen beforeHooks
+  printSeqIfLen afterHooks
+
+  if requiresData.len != 0:
+    result &= "\n[Deps]\n"
+    result &= &"requires: \"{requiresData.join(\", \")}\"\n"
+
+proc onExit*() =
+  if "printPkgInfo".normalize in commandLineParams:
+    if outFile.len != 0:
+      writeFile(outFile, printPkgInfo())
+  else:
+    var
+      output = ""
+    output &= "\"success\": " & $success & ", "
+    output &= "\"command\": \"" & command & "\", "
+    if project.len != 0:
+      output &= "\"project\": \"" & project & "\", "
+    if not flags.isNil and flags.len != 0:
+      output &= "\"flags\": {"
+      for key, val in flags.pairs:
+        output &= "\"" & key & "\": ["
+        for v in val:
+          let v = if v.len > 0 and v[0] == '"': strutils.unescape(v)
+                  else: v
+          output &= v.escape & ", "
+        output = output[0 .. ^3] & "], "
+      output = output[0 .. ^3] & "}, "
+
+    output &= "\"retVal\": " & $retVal
+
+    if outFile.len != 0:
+      writeFile(outFile, "{" & output & "}")
+
+# TODO: New release of Nim will move this `task` template under a
+# `when not defined(nimble)`. This will allow us to override it in the future.
+template task*(name: untyped; description: string; body: untyped): untyped =
+  ## Defines a task. Hidden tasks are supported via an empty description.
+  ## Example:
+  ##
+  ## .. code-block:: nim
+  ##  task build, "default build is via the C backend":
+  ##    setCommand "c"
+  proc `name Task`*() = body
+
+  if commandLineParams.len == 0 or "help" in commandLineParams:
+    success = true
+    echo(astToStr(name), "        ", description)
+  elif astToStr(name).normalize in commandLineParams:
+    success = true
+    `name Task`()
+
 template before*(action: untyped, body: untyped): untyped =
   ## Defines a block of code which is evaluated before ``action`` is executed.
   proc `action Before`*(): bool =
     result = true
     body
 
+  beforeHooks.add astToStr(action)
+
+  if (astToStr(action) & "Before").normalize in commandLineParams:
+    success = true
+    retVal = `action Before`()
+
 template after*(action: untyped, body: untyped): untyped =
   ## Defines a block of code which is evaluated after ``action`` is executed.
   proc `action After`*(): bool =
     result = true
     body
 
-template builtin = discard
+  afterHooks.add astToStr(action)
+
+  if (astToStr(action) & "After").normalize in commandLineParams:
+    success = true
+    retVal = `action After`()
 
 proc getPkgDir*(): string =
   ## Returns the package directory containing the .nimble file currently
   ## being evaluated.
-  builtin
+  result = projectFile.rsplit(seps={'/', '\\', ':'}, maxsplit=1)[0]
+
+getParams()

src/nimblepkg/nimscriptexecutor.nim

@@ -0,0 +1,76 @@
+# Copyright (C) Dominik Picheta. All rights reserved.
+# BSD License. Look at license.txt for more info.
+
+import os, strutils, sets
+
+import packageparser, common, packageinfo, options, nimscriptwrapper, cli,
+       version
+
+proc execHook*(options: Options, hookAction: ActionType, before: bool): bool =
+  ## Returns whether to continue.
+  result = true
+
+  # For certain commands hooks should not be evaluated.
+  if hookAction in noHookActions:
+    return
+
+  var nimbleFile = ""
+  try:
+    nimbleFile = findNimbleFile(getCurrentDir(), true)
+  except NimbleError: return true
+  # PackageInfos are cached so we can read them as many times as we want.
+  let pkgInfo = getPkgInfoFromFile(nimbleFile, options)
+  let actionName =
+    if hookAction == actionCustom: options.action.command
+    else: ($hookAction)[6 .. ^1]
+  let hookExists =
+    if before: actionName.normalize in pkgInfo.preHooks
+    else: actionName.normalize in pkgInfo.postHooks
+  if pkgInfo.isNimScript and hookExists:
+    let res = execHook(nimbleFile, actionName, before, options)
+    if res.success:
+      result = res.retVal
+
+proc execCustom*(options: Options,
+                 execResult: var ExecutionResult[bool],
+                 failFast = true): bool =
+  ## Executes the custom command using the nimscript backend.
+  ##
+  ## If failFast is true then exceptions will be raised when something is wrong.
+  ## Otherwise this function will just return false.
+
+  # Custom command. Attempt to call a NimScript task.
+  let nimbleFile = findNimbleFile(getCurrentDir(), true)
+  if not nimbleFile.isNimScript(options) and failFast:
+    writeHelp()
+
+  execResult = execTask(nimbleFile, options.action.command, options)
+  if not execResult.success:
+    if not failFast:
+      return
+    raiseNimbleError(msg = "Could not find task $1 in $2" %
+                           [options.action.command, nimbleFile],
+                     hint = "Run `nimble --help` and/or `nimble tasks` for" &
+                            " a list of possible commands.")
+
+  if execResult.command.normalize == "nop":
+    display("Warning:", "Using `setCommand 'nop'` is not necessary.", Warning,
+            HighPriority)
+    return
+
+  if not execHook(options, actionCustom, false):
+    return
+
+  return true
+
+proc getOptionsForCommand*(execResult: ExecutionResult,
+                           options: Options): Options =
+  ## Creates an Options object for the requested command.
+  var newOptions = options.briefClone()
+  parseCommand(execResult.command, newOptions)
+  for arg in execResult.arguments:
+    parseArgument(arg, newOptions)
+  for flag, vals in execResult.flags:
+    for val in vals:
+      parseFlag(flag, val, newOptions)
+  return newOptions

src/nimblepkg/nimscriptsupport.nim

@@ -1,421 +0,0 @@
-# Copyright (C) Andreas Rumpf. All rights reserved.
-# BSD License. Look at license.txt for more info.
-
-## Implements the new configuration system for Nimble. Uses Nim as a
-## scripting language.
-
-import
-  compiler/ast, compiler/modules, compiler/passes, compiler/passaux,
-  compiler/condsyms, compiler/sem, compiler/semdata,
-  compiler/llstream, compiler/vm, compiler/vmdef, compiler/commands,
-  compiler/msgs, compiler/magicsys, compiler/lists
-
-from compiler/scriptconfig import setupVM
-from compiler/idents import getIdent
-from compiler/astalgo import strTableGet
-import compiler/options as compiler_options
-
-import nimbletypes, version, options, packageinfo
-import os, strutils, strtabs, times, osproc, sets
-
-type
-  ExecutionResult*[T] = object
-    success*: bool
-    command*: string
-    arguments*: seq[string]
-    flags*: StringTableRef
-    retVal*: T
-
-const
-  internalCmd = "NimbleInternal"
-
-proc raiseVariableError(ident, typ: string) {.noinline.} =
-  raise newException(NimbleError,
-    "NimScript's variable '" & ident & "' needs a value of type '" & typ & "'.")
-
-proc isStrLit(n: PNode): bool = n.kind in {nkStrLit..nkTripleStrLit}
-
-proc getGlobal(ident: PSym): string =
-  let n = vm.globalCtx.getGlobalValue(ident)
-  if n.isStrLit:
-    result = if n.strVal.isNil: "" else: n.strVal
-  else:
-    raiseVariableError(ident.name.s, "string")
-
-proc getGlobalAsSeq(ident: PSym): seq[string] =
-  let n = vm.globalCtx.getGlobalValue(ident)
-  result = @[]
-  if n.kind == nkBracket:
-    for x in n:
-      if x.isStrLit:
-        result.add x.strVal
-      else:
-        raiseVariableError(ident.name.s, "seq[string]")
-  else:
-    raiseVariableError(ident.name.s, "seq[string]")
-
-proc extractRequires(ident: PSym, result: var seq[PkgTuple]) =
-  let n = vm.globalCtx.getGlobalValue(ident)
-  if n.kind == nkBracket:
-    for x in n:
-      if x.kind == nkPar and x.len == 2 and x[0].isStrLit and x[1].isStrLit:
-        result.add(parseRequires(x[0].strVal & x[1].strVal))
-      elif x.isStrLit:
-        result.add(parseRequires(x.strVal))
-      else:
-        raiseVariableError("requiresData", "seq[(string, VersionReq)]")
-  else:
-    raiseVariableError("requiresData", "seq[(string, VersionReq)]")
-
-proc setupVM(module: PSym; scriptName: string,
-    flags: StringTableRef): PEvalContext =
-  ## This procedure is exported in the compiler sources, but its implementation
-  ## is too Nim-specific to be used by Nimble.
-  ## Specifically, the implementation of ``switch`` is problematic. Sooo
-  ## I simply copied it here and edited it :)
-
-  result = newCtx(module)
-  result.mode = emRepl
-  registerAdditionalOps(result)
-
-  # captured vars:
-  var errorMsg: string
-  var vthisDir = scriptName.splitFile.dir
-
-  proc listDirs(a: VmArgs, filter: set[PathComponent]) =
-    let dir = getString(a, 0)
-    var result: seq[string] = @[]
-    for kind, path in walkDir(dir):
-      if kind in filter: result.add path
-    setResult(a, result)
-
-  template cbconf(name, body) {.dirty.} =
-    result.registerCallback "stdlib.system." & astToStr(name),
-      proc (a: VmArgs) =
-        body
-
-  template cbos(name, body) {.dirty.} =
-    result.registerCallback "stdlib.system." & astToStr(name),
-      proc (a: VmArgs) =
-        try:
-          body
-        except OSError:
-          errorMsg = getCurrentExceptionMsg()
-
-  # Idea: Treat link to file as a file, but ignore link to directory to prevent
-  # endless recursions out of the box.
-  cbos listFiles:
-    listDirs(a, {pcFile, pcLinkToFile})
-  cbos listDirs:
-    listDirs(a, {pcDir})
-  cbos removeDir:
-    os.removeDir getString(a, 0)
-  cbos removeFile:
-    os.removeFile getString(a, 0)
-  cbos createDir:
-    os.createDir getString(a, 0)
-  cbos getOsError:
-    setResult(a, errorMsg)
-  cbos setCurrentDir:
-    os.setCurrentDir getString(a, 0)
-  cbos getCurrentDir:
-    setResult(a, os.getCurrentDir())
-  cbos moveFile:
-    os.moveFile(getString(a, 0), getString(a, 1))
-  cbos copyFile:
-    os.copyFile(getString(a, 0), getString(a, 1))
-  cbos getLastModificationTime:
-    setResult(a, toSeconds(getLastModificationTime(getString(a, 0))))
-
-  cbos rawExec:
-    setResult(a, osproc.execCmd getString(a, 0))
-
-  cbconf getEnv:
-    setResult(a, os.getEnv(a.getString 0))
-  cbconf existsEnv:
-    setResult(a, os.existsEnv(a.getString 0))
-  cbconf dirExists:
-    setResult(a, os.dirExists(a.getString 0))
-  cbconf fileExists:
-    setResult(a, os.fileExists(a.getString 0))
-
-  cbconf thisDir:
-    setResult(a, vthisDir)
-  cbconf put:
-    compiler_options.setConfigVar(getString(a, 0), getString(a, 1))
-  cbconf get:
-    setResult(a, compiler_options.getConfigVar(a.getString 0))
-  cbconf exists:
-    setResult(a, compiler_options.existsConfigVar(a.getString 0))
-  cbconf nimcacheDir:
-    setResult(a, compiler_options.getNimcacheDir())
-  cbconf paramStr:
-    setResult(a, os.paramStr(int a.getInt 0))
-  cbconf paramCount:
-    setResult(a, os.paramCount())
-  cbconf cmpIgnoreStyle:
-    setResult(a, strutils.cmpIgnoreStyle(a.getString 0, a.getString 1))
-  cbconf cmpIgnoreCase:
-    setResult(a, strutils.cmpIgnoreCase(a.getString 0, a.getString 1))
-  cbconf setCommand:
-    compiler_options.command = a.getString 0
-    let arg = a.getString 1
-    if arg.len > 0:
-      gProjectName = arg
-      try:
-        gProjectFull = canonicalizePath(gProjectPath / gProjectName)
-      except OSError:
-        gProjectFull = gProjectName
-  cbconf getCommand:
-    setResult(a, compiler_options.command)
-  cbconf switch:
-    if not flags.isNil:
-      flags[a.getString 0] = a.getString 1
-
-proc findNimscriptApi(options: Options): string =
-  ## Returns the directory containing ``nimscriptapi.nim``
-  var inPath = false
-  # Try finding it in exe's path
-  if fileExists(getAppDir() / "nimblepkg" / "nimscriptapi.nim"):
-    result = getAppDir()
-    inPath = true
-
-  if not inPath:
-    let pkgs = getInstalledPkgsMin(options.getPkgsDir(), options)
-    var pkg: PackageInfo
-    if pkgs.findPkg(("nimble", newVRAny()), pkg):
-      let pkgDir = pkg.getRealDir()
-      if fileExists(pkgDir / "nimblepkg" / "nimscriptapi.nim"):
-        result = pkgDir
-        inPath = true
-
-  if not inPath:
-    raise newException(NimbleError, "Cannot find nimscriptapi.nim")
-
-proc execScript(scriptName: string, flags: StringTableRef, options: Options) =
-  ## Executes the specified script.
-  ##
-  ## No clean up is performed and must be done manually!
-  if "nimblepkg/nimscriptapi" notin compiler_options.implicitIncludes:
-    compiler_options.implicitIncludes.add("nimblepkg/nimscriptapi")
-
-  let pkgName = scriptName.splitFile.name
-
-  # Ensure that "nimblepkg/nimscriptapi" is in the PATH.
-  let nimscriptApiPath = findNimscriptApi(options)
-  appendStr(searchPaths, nimscriptApiPath)
-
-  setDefaultLibpath()
-  passes.gIncludeFile = includeModule
-  passes.gImportModule = importModule
-  initDefines()
-
-  defineSymbol("nimscript")
-  defineSymbol("nimconfig")
-  defineSymbol("nimble")
-  registerPass(semPass)
-  registerPass(evalPass)
-
-  appendStr(searchPaths, compiler_options.libpath)
-
-  var m = makeModule(scriptName)
-  incl(m.flags, sfMainModule)
-  vm.globalCtx = setupVM(m, scriptName, flags)
-
-  # Setup builtins defined in nimscriptapi.nim
-  template cbApi(name, body) {.dirty.} =
-    vm.globalCtx.registerCallback pkgName & "." & astToStr(name),
-      proc (a: VmArgs) =
-        body
-
-  cbApi getPkgDir:
-    setResult(a, scriptName.splitFile.dir)
-
-  compileSystemModule()
-  processModule(m, llStreamOpen(scriptName, fmRead), nil)
-
-proc cleanup() =
-  # ensure everything can be called again:
-  compiler_options.gProjectName = ""
-  compiler_options.command = ""
-  resetAllModulesHard()
-  clearPasses()
-  msgs.gErrorMax = 1
-  msgs.writeLnHook = nil
-  vm.globalCtx = nil
-  initDefines()
-
-proc readPackageInfoFromNims*(scriptName: string, options: Options,
-    result: var PackageInfo) =
-  ## Executes the `scriptName` nimscript file. Reads the package information
-  ## that it populates.
-
-  # Setup custom error handling.
-  msgs.gErrorMax = high(int)
-  var previousMsg = ""
-  msgs.writeLnHook =
-    proc (output: string) =
-      # The error counter is incremented after the writeLnHook is invoked.
-      if msgs.gErrorCounter > 0:
-        raise newException(NimbleError, previousMsg)
-      elif previousMsg.len > 0:
-        echo(previousMsg)
-      if output.normalize.startsWith("error"):
-        raise newException(NimbleError, output)
-      previousMsg = output
-
-  compiler_options.command = internalCmd
-
-  # Execute the nimscript file.
-  execScript(scriptName, nil, options)
-
-  # Check whether an error has occurred.
-  if msgs.gErrorCounter > 0:
-    raise newException(NimbleError, previousMsg)
-
-  # Extract all the necessary fields populated by the nimscript file.
-  proc getSym(thisModule: PSym, ident: string): PSym =
-    result = thisModule.tab.strTableGet(getIdent(ident))
-    if result.isNil:
-      raise newException(NimbleError, "Ident not found: " & ident)
-
-  template trivialField(field) =
-    result.field = getGlobal(getSym(thisModule, astToStr field))
-
-  template trivialFieldSeq(field) =
-    result.field.add getGlobalAsSeq(getSym(thisModule, astToStr field))
-
-  # Grab the module Sym for .nimble file (nimscriptapi is included in it).
-  let idx = fileInfoIdx(scriptName)
-  let thisModule = getModule(idx)
-  assert(not thisModule.isNil)
-  assert thisModule.kind == skModule
-
-  # keep reasonable default:
-  let name = getGlobal(thisModule.tab.strTableGet(getIdent"packageName"))
-  if name.len > 0: result.name = name
-
-  trivialField version
-  trivialField author
-  trivialField description
-  trivialField license
-  trivialField srcdir
-  trivialField bindir
-  trivialFieldSeq skipDirs
-  trivialFieldSeq skipFiles
-  trivialFieldSeq skipExt
-  trivialFieldSeq installDirs
-  trivialFieldSeq installFiles
-  trivialFieldSeq installExt
-
-  extractRequires(getSym(thisModule, "requiresData"), result.requires)
-
-  let binSeq = getGlobalAsSeq(getSym(thisModule, "bin"))
-  for i in binSeq:
-    result.bin.add(i.addFileExt(ExeExt))
-
-  let backend = getGlobal(getSym(thisModule, "backend"))
-  if backend.len == 0:
-    result.backend = "c"
-  elif cmpIgnoreStyle(backend, "javascript") == 0:
-    result.backend = "js"
-  else:
-    result.backend = backend.toLower()
-
-  # Grab all the global procs
-  for i in thisModule.tab.data:
-    if not i.isNil():
-      let name = i.name.s.normalize()
-      if name.endsWith("before"):
-        result.preHooks.incl(name[0 .. ^7])
-      if name.endsWith("after"):
-        result.postHooks.incl(name[0 .. ^6])
-
-  cleanup()
-
-proc execTask*(scriptName, taskName: string,
-    options: Options): ExecutionResult[void] =
-  ## Executes the specified task in the specified script.
-  ##
-  ## `scriptName` should be a filename pointing to the nimscript file.
-  result.success = true
-  result.flags = newStringTable()
-  compiler_options.command = internalCmd
-  echo("Executing task ", taskName, " in ", scriptName)
-
-  execScript(scriptName, result.flags, options)
-  # Explicitly execute the task procedure, instead of relying on hack.
-  let idx = fileInfoIdx(scriptName)
-  let thisModule = getModule(idx)
-  assert thisModule.kind == skModule
-  let prc = thisModule.tab.strTableGet(getIdent(taskName & "Task"))
-  if prc.isNil:
-    # Procedure not defined in the NimScript module.
-    result.success = false
-    return
-  discard vm.globalCtx.execProc(prc, [])
-
-  # Read the command, arguments and flags set by the executed task.
-  result.command = compiler_options.command
-  result.arguments = @[]
-  for arg in compiler_options.gProjectName.split():
-    result.arguments.add(arg)
-
-  cleanup()
-
-proc execHook*(scriptName, actionName: string, before: bool,
-    options: Options): ExecutionResult[bool] =
-  ## Executes the specified action's hook. Depending on ``before``, either
-  ## the "before" or the "after" hook.
-  ##
-  ## `scriptName` should be a filename pointing to the nimscript file.
-  result.success = true
-  result.flags = newStringTable()
-  compiler_options.command = internalCmd
-  let hookName =
-    if before: actionName.toLower & "Before"
-    else: actionName.toLower & "After"
-  echo("Attempting to execute hook ", hookName, " in ", scriptName)
-
-  execScript(scriptName, result.flags, options)
-  # Explicitly execute the task procedure, instead of relying on hack.
-  let idx = fileInfoIdx(scriptName)
-  let thisModule = getModule(idx)
-  assert thisModule.kind == skModule
-  let prc = thisModule.tab.strTableGet(getIdent(hookName))
-  if prc.isNil:
-    # Procedure not defined in the NimScript module.
-    result.success = false
-    cleanup()
-    return
-  let returnVal = vm.globalCtx.execProc(prc, [])
-  case returnVal.kind
-  of nkCharLit..nkUInt64Lit:
-    result.retVal = returnVal.intVal == 1
-  else: assert false
-
-  # Read the command, arguments and flags set by the executed task.
-  result.command = compiler_options.command
-  result.arguments = @[]
-  for arg in compiler_options.gProjectName.split():
-    result.arguments.add(arg)
-
-  cleanup()
-
-proc getNimScriptCommand(): string =
-  compiler_options.command
-
-proc setNimScriptCommand(command: string) =
-  compiler_options.command = command
-
-proc hasTaskRequestedCommand*(execResult: ExecutionResult): bool =
-  ## Determines whether the last executed task used ``setCommand``
-  return execResult.command != internalCmd
-
-proc listTasks*(scriptName: string, options: Options) =
-  setNimScriptCommand("help")
-
-  execScript(scriptName, nil, options)
-  # TODO: Make the 'task' template generate explicit data structure containing
-  # all the task names + descriptions.
-  cleanup()

src/nimblepkg/nimscriptwrapper.nim

@@ -0,0 +1,216 @@
+# Copyright (C) Andreas Rumpf. All rights reserved.
+# BSD License. Look at license.txt for more info.
+
+## Implements the new configuration system for Nimble. Uses Nim as a
+## scripting language.
+
+import hashes, json, os, strutils, tables, times, osproc, strtabs
+
+import version, options, cli, tools
+
+type
+  Flags = TableRef[string, seq[string]]
+  ExecutionResult*[T] = object
+    success*: bool
+    command*: string
+    arguments*: seq[string]
+    flags*: Flags
+    retVal*: T
+    stdout*: string
+
+const
+  internalCmd = "e"
+  nimscriptApi = staticRead("nimscriptapi.nim")
+  printPkgInfo = "printPkgInfo"
+
+proc isCustomTask(actionName: string, options: Options): bool =
+  options.action.typ == actionCustom and actionName != printPkgInfo
+
+proc needsLiveOutput(actionName: string, options: Options, isHook: bool): bool =
+  let isCustomTask = isCustomTask(actionName, options)
+  return isCustomTask or isHook or actionName == ""
+
+proc writeExecutionOutput(data: string) =
+  # TODO: in the future we will likely want this to be live, users will
+  # undoubtedly be doing loops and other crazy things in their top-level
+  # Nimble files.
+  display("Info", data)
+
+proc execNimscript(
+  nimsFile, projectDir, actionName: string, options: Options, isHook: bool
+): tuple[output: string, exitCode: int, stdout: string] =
+  let
+    nimsFileCopied = projectDir / nimsFile.splitFile().name & "_" & getProcessId() & ".nims"
+    outFile = getNimbleTempDir() & ".out"
+
+  let
+    isScriptResultCopied =
+      nimsFileCopied.fileExists() and
+      nimsFileCopied.getLastModificationTime() >= nimsFile.getLastModificationTime()
+
+  if not isScriptResultCopied:
+    nimsFile.copyFile(nimsFileCopied)
+
+  defer:
+    # Only if copied in this invocation, allows recursive calls of nimble
+    if not isScriptResultCopied and options.shouldRemoveTmp(nimsFileCopied):
+        nimsFileCopied.removeFile()
+
+  var cmd = (
+    "nim e $# -p:$# $# $# $#" % [
+      "--hints:off --verbosity:0",
+      (getTempDir() / "nimblecache").quoteShell,
+      nimsFileCopied.quoteShell,
+      outFile.quoteShell,
+      actionName
+    ]
+  ).strip()
+
+  let isCustomTask = isCustomTask(actionName, options)
+  if isCustomTask:
+    for i in options.action.arguments:
+      cmd &= " " & i.quoteShell()
+    for key, val in options.action.flags.pairs():
+      cmd &= " $#$#" % [if key.len == 1: "-" else: "--", key]
+      if val.len != 0:
+        cmd &= ":" & val.quoteShell()
+
+  displayDebug("Executing " & cmd)
+
+  if needsLiveOutput(actionName, options, isHook):
+    result.exitCode = execCmd(cmd)
+  else:
+    # We want to capture any possible errors when parsing a .nimble
+    # file's metadata. See #710.
+    (result.stdout, result.exitCode) = execCmdEx(cmd)
+  if outFile.fileExists():
+    result.output = outFile.readFile()
+    if options.shouldRemoveTmp(outFile):
+      discard outFile.tryRemoveFile()
+
+proc getNimsFile(scriptName: string, options: Options): string =
+  let
+    cacheDir = getTempDir() / "nimblecache"
+    shash = $scriptName.parentDir().hash().abs()
+    prjCacheDir = cacheDir / scriptName.splitFile().name & "_" & shash
+    nimscriptApiFile = cacheDir / "nimscriptapi.nim"
+
+  result = prjCacheDir / scriptName.extractFilename().changeFileExt ".nims"
+
+  let
+    iniFile = result.changeFileExt(".ini")
+
+    isNimscriptApiCached =
+      nimscriptApiFile.fileExists() and nimscriptApiFile.getLastModificationTime() > 
+      getAppFilename().getLastModificationTime()
+    
+    isScriptResultCached =
+      isNimscriptApiCached and result.fileExists() and result.getLastModificationTime() >
+      scriptName.getLastModificationTime()
+
+  if not isNimscriptApiCached:
+    createDir(cacheDir)
+    writeFile(nimscriptApiFile, nimscriptApi)
+
+  if not isScriptResultCached:
+    createDir(result.parentDir())
+    writeFile(result, """
+import system except getCommand, setCommand, switch, `--`,
+  packageName, version, author, description, license, srcDir, binDir, backend,
+  skipDirs, skipFiles, skipExt, installDirs, installFiles, installExt, bin, foreignDeps,
+  requires, task, packageName
+""" &
+      "import nimscriptapi, strutils\n" & scriptName.readFile() & "\nonExit()\n")
+    discard tryRemoveFile(iniFile)
+
+proc getIniFile*(scriptName: string, options: Options): string =
+  let
+    nimsFile = getNimsFile(scriptName, options)
+
+  result = nimsFile.changeFileExt(".ini")
+
+  let
+    isIniResultCached =
+      result.fileExists() and result.getLastModificationTime() >
+      scriptName.getLastModificationTime()
+
+  if not isIniResultCached:
+    let (output, exitCode, stdout) = execNimscript(
+      nimsFile, scriptName.parentDir(), printPkgInfo, options, isHook=false
+    )
+
+    if exitCode == 0 and output.len != 0:
+      result.writeFile(output)
+      stdout.writeExecutionOutput()
+    else:
+      raise newException(NimbleError, stdout & "\nprintPkgInfo() failed")
+
+proc execScript(
+  scriptName, actionName: string, options: Options, isHook: bool
+): ExecutionResult[bool] =
+  let nimsFile = getNimsFile(scriptName, options)
+
+  let (output, exitCode, stdout) =
+    execNimscript(
+      nimsFile, scriptName.parentDir(), actionName, options, isHook
+    )
+
+  if exitCode != 0:
+    let errMsg =
+      if stdout.len != 0:
+        stdout
+      else:
+        "Exception raised during nimble script execution"
+    raise newException(NimbleError, errMsg)
+
+  let
+    j =
+      if output.len != 0:
+        parseJson(output)
+      else:
+        parseJson("{}")
+
+  result.flags = newTable[string, seq[string]]()
+  result.success = j{"success"}.getBool()
+  result.command = j{"command"}.getStr()
+  if "project" in j:
+    result.arguments.add j["project"].getStr()
+  if "flags" in j:
+    for flag, vals in j["flags"].pairs:
+      result.flags[flag] = @[]
+      for val in vals.items():
+        result.flags[flag].add val.getStr()
+  result.retVal = j{"retVal"}.getBool()
+
+  stdout.writeExecutionOutput()
+
+proc execTask*(scriptName, taskName: string,
+    options: Options): ExecutionResult[bool] =
+  ## Executes the specified task in the specified script.
+  ##
+  ## `scriptName` should be a filename pointing to the nimscript file.
+  display("Executing",  "task $# in $#" % [taskName, scriptName],
+          priority = HighPriority)
+
+  result = execScript(scriptName, taskName, options, isHook=false)
+
+proc execHook*(scriptName, actionName: string, before: bool,
+    options: Options): ExecutionResult[bool] =
+  ## Executes the specified action's hook. Depending on ``before``, either
+  ## the "before" or the "after" hook.
+  ##
+  ## `scriptName` should be a filename pointing to the nimscript file.
+  let hookName =
+    if before: actionName.toLowerAscii & "Before"
+    else: actionName.toLowerAscii & "After"
+  display("Attempting", "to execute hook $# in $#" % [hookName, scriptName],
+          priority = MediumPriority)
+
+  result = execScript(scriptName, hookName, options, isHook=true)
+
+proc hasTaskRequestedCommand*(execResult: ExecutionResult): bool =
+  ## Determines whether the last executed task used ``setCommand``
+  return execResult.command != internalCmd
+
+proc listTasks*(scriptName: string, options: Options) =
+  discard execScript(scriptName, "", options, isHook=false)

src/nimblepkg/options.nim

@@ -1,79 +1,122 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
 
-import json, strutils, os, parseopt, strtabs, uri, tables
+import json, strutils, os, parseopt, strtabs, uri, tables, terminal
+import sequtils, sugar
+import std/options as std_opt
 from httpclient import Proxy, newProxy
 
-import nimblepkg/config, nimblepkg/version,
+import config, version, common, cli
-       nimblepkg/tools, nimblepkg/nimbletypes
 
 type
   Options* = object
     forcePrompts*: ForcePrompt
+    depsOnly*: bool
+    uninstallRevDeps*: bool
     queryVersions*: bool
     queryInstalled*: bool
+    nimbleDir*: string
+    verbosity*: cli.Priority
     action*: Action
     config*: Config
     nimbleData*: JsonNode ## Nimbledata.json
     pkgInfoCache*: TableRef[string, PackageInfo]
+    showHelp*: bool
+    showVersion*: bool
+    noColor*: bool
+    disableValidation*: bool
+    continueTestsOnFailure*: bool
+    ## Whether packages' repos should always be downloaded with their history.
+    forceFullClone*: bool
+    # Temporary storage of flags that have not been captured by any specific Action.
+    unknownFlags*: seq[(CmdLineKind, string, string)]
 
   ActionType* = enum
-    actionNil, actionUpdate, actionInit, actionDump, actionPublish,
+    actionNil, actionRefresh, actionInit, actionDump, actionPublish,
     actionInstall, actionSearch,
     actionList, actionBuild, actionPath, actionUninstall, actionCompile,
-    actionCustom, actionTasks, actionVersion
+    actionDoc, actionCustom, actionTasks, actionDevelop, actionCheck,
+    actionRun
 
   Action* = object
     case typ*: ActionType
-    of actionNil, actionList, actionBuild, actionPublish, actionTasks,
+    of actionNil, actionList, actionPublish, actionTasks, actionCheck: nil
-       actionVersion: nil
+    of actionRefresh:
-    of actionUpdate:
       optionalURL*: string # Overrides default package list.
-    of actionInstall, actionPath, actionUninstall:
+    of actionInstall, actionPath, actionUninstall, actionDevelop:
-      packages*: seq[PkgTuple] # Optional only for actionInstall.
+      packages*: seq[PkgTuple] # Optional only for actionInstall
+                               # and actionDevelop.
+      passNimFlags*: seq[string]
     of actionSearch:
       search*: seq[string] # Search string.
     of actionInit, actionDump:
       projName*: string
-    of actionCompile:
+      vcsOption*: string
+    of actionCompile, actionDoc, actionBuild:
       file*: string
       backend*: string
-      compileOptions*: seq[string]
+      compileOptions: seq[string]
+    of actionRun:
+      runFile: Option[string]
+      compileFlags: seq[string]
+      runFlags*: seq[string]
     of actionCustom:
       command*: string
       arguments*: seq[string]
       flags*: StringTableRef
 
-  ForcePrompt* = enum
-    dontForcePrompt, forcePromptYes, forcePromptNo
-
-
 const
   help* = """
 Usage: nimble COMMAND [opts]
 
 Commands:
   install      [pkgname, ...]     Installs a list of packages.
-  init         [pkgname]          Initializes a new Nimble project.
+               [-d, --depsOnly]   Install only dependencies.
+               [-p, --passNim]    Forward specified flag to compiler.
+  develop      [pkgname, ...]     Clones a list of packages for development.
+                                  Symlinks the cloned packages or any package
+                                  in the current working directory.
+  check                           Verifies the validity of a package in the
+                                  current working directory.
+  init         [pkgname]          Initializes a new Nimble project in the
+                                  current directory or if a name is provided a
+                                  new directory of the same name.
+               --git
+               --hg               Create a git or hg repo in the new nimble project.
   publish                         Publishes a package on nim-lang/packages.
                                   The current working directory needs to be the
                                   toplevel directory of the Nimble package.
   uninstall    [pkgname, ...]     Uninstalls a list of packages.
-  build                           Builds a package.
+               [-i, --inclDeps]   Uninstall package and dependent package(s).
+  build        [opts, ...] [bin]  Builds a package.
+  run          [opts, ...] [bin]  Builds and runs a package.
+                                  Binary needs to be specified after any
+                                  compilation options if there are several
+                                  binaries defined, any flags after the binary
+                                  or -- arg are passed to the binary when it is run.
   c, cc, js    [opts, ...] f.nim  Builds a file inside a package. Passes options
                                   to the Nim compiler.
+  test                            Compiles and executes tests
+               [-c, --continue]   Don't stop execution on a failed test.
+  doc, doc2    [opts, ...] f.nim  Builds documentation for a file inside a
+                                  package. Passes options to the Nim compiler.
   refresh      [url]              Refreshes the package list. A package list URL
                                   can be optionally specified.
-  search       [--ver] pkg/tag    Searches for a specified package. Search is
+  search       pkg/tag            Searches for a specified package. Search is
                                   performed by tag and by name.
-  list         [--ver]            Lists all packages.
+               [--ver]            Query remote server for package version.
+  list                            Lists all packages.
+               [--ver]            Query remote server for package version.
                [-i, --installed]  Lists all installed packages.
   tasks                           Lists the tasks specified in the Nimble
                                   package's Nimble file.
   path         pkgname ...        Shows absolute path to the installed packages
                                   specified.
   dump         [pkgname]          Outputs Nimble package information for
-                                  external tools.
+                                  external tools. The argument can be a
+                                  .nimble file, a project directory or
+                                  the name of an installed package.
+
 
 Options:
   -h, --help                      Print this help message.
@@ -82,36 +125,53 @@ Options:
   -n, --reject                    Reject all interactive prompts.
       --ver                       Query remote server for package version
                                   information when searching or listing packages
-      --nimbleDir dirname         Set the Nimble directory.
+      --nimbleDir:dirname         Set the Nimble directory.
+      --verbose                   Show all non-debug output.
+      --debug                     Show all output including debug messages.
+      --noColor                   Don't colorise output.
 
 For more information read the Github readme:
   https://github.com/nim-lang/nimble#readme
 """
 
-proc writeHelp*() =
+const noHookActions* = {actionCheck}
+
+proc writeHelp*(quit=true) =
   echo(help)
-  quit(QuitSuccess)
+  if quit:
+    raise NimbleQuit(msg: "")
+
+proc writeVersion*() =
+  echo("nimble v$# compiled at $# $#" %
+      [nimbleVersion, CompileDate, CompileTime])
+  const execResult = gorgeEx("git rev-parse HEAD")
+  when execResult[0].len > 0 and execResult[1] == QuitSuccess:
+    echo "git hash: ", execResult[0]
+  else:
+    {.warning: "Couldn't determine GIT hash: " & execResult[0].}
+    echo "git hash: couldn't determine git hash"
+  raise NimbleQuit(msg: "")
 
 proc parseActionType*(action: string): ActionType =
   case action.normalize()
-  of "install", "path":
+  of "install":
-    case action.normalize()
+    result = actionInstall
-    of "install":
+  of "path":
-      result = actionInstall
+    result = actionPath
-    of "path":
-      result = actionPath
-    else:
-      discard
   of "build":
     result = actionBuild
+  of "run":
+    result = actionRun
   of "c", "compile", "js", "cpp", "cc":
     result = actionCompile
+  of "doc", "doc2":
+    result = actionDoc
   of "init":
     result = actionInit
   of "dump":
     result = actionDump
   of "update", "refresh":
-    result = actionUpdate
+    result = actionRefresh
   of "search":
     result = actionSearch
   of "list":
@@ -122,6 +182,10 @@ proc parseActionType*(action: string): ActionType =
     result = actionPublish
   of "tasks":
     result = actionTasks
+  of "develop":
+    result = actionDevelop
+  of "check":
+    result = actionCheck
   else:
     result = actionCustom
 
@@ -130,154 +194,248 @@ proc initAction*(options: var Options, key: string) =
   ## `key`.
   let keyNorm = key.normalize()
   case options.action.typ
-  of actionInstall, actionPath:
+  of actionInstall, actionPath, actionDevelop, actionUninstall:
     options.action.packages = @[]
-  of actionCompile:
+    options.action.passNimFlags = @[]
+  of actionCompile, actionDoc, actionBuild:
     options.action.compileOptions = @[]
     options.action.file = ""
     if keyNorm == "c" or keyNorm == "compile": options.action.backend = ""
     else: options.action.backend = keyNorm
   of actionInit:
     options.action.projName = ""
+    options.action.vcsOption = ""
   of actionDump:
     options.action.projName = ""
-  of actionUpdate:
+    options.action.vcsOption = ""
+    options.forcePrompts = forcePromptYes
+  of actionRefresh:
     options.action.optionalURL = ""
   of actionSearch:
     options.action.search = @[]
-  of actionUninstall:
-    options.action.packages = @[]
   of actionCustom:
     options.action.command = key
     options.action.arguments = @[]
     options.action.flags = newStringTable()
-  of actionBuild, actionPublish, actionList, actionTasks,
+  of actionPublish, actionList, actionTasks, actionCheck, actionRun,
-     actionNil, actionVersion: discard
+     actionNil: discard
 
 proc prompt*(options: Options, question: string): bool =
   ## Asks an interactive question and returns the result.
   ##
   ## The proc will return immediately without asking the user if the global
   ## forcePrompts has a value different than dontForcePrompt.
-  case options.forcePrompts
+  return prompt(options.forcePrompts, question)
-  of forcePromptYes:
-    echo(question & " -> [forced yes]")
-    return true
-  of forcePromptNo:
-    echo(question & " -> [forced no]")
-    return false
-  of dontForcePrompt:
-    echo(question & " [y/N]")
-    let yn = stdin.readLine()
-    case yn.normalize
-    of "y", "yes":
-      return true
-    of "n", "no":
-      return false
-    else:
-      return false
 
-proc renameBabelToNimble(options: Options) {.deprecated.} =
+proc promptCustom*(options: Options, question, default: string): string =
-  let babelDir = getHomeDir() / ".babel"
+  ## Asks an interactive question and returns the result.
-  let nimbleDir = getHomeDir() / ".nimble"
+  ##
-  if dirExists(babelDir):
+  ## The proc will return "default" without asking the user if the global
-    if options.prompt("Found deprecated babel package directory, would you " &
+  ## forcePrompts is forcePromptYes.
-        "like to rename it to nimble?"):
+  return promptCustom(options.forcePrompts, question, default)
-      copyDir(babelDir, nimbleDir)
-      copyFile(babelDir / "babeldata.json", nimbleDir / "nimbledata.json")
 
-      removeDir(babelDir)
+proc promptList*(options: Options, question: string, args: openarray[string]): string =
-      removeFile(nimbleDir / "babeldata.json")
+  ## Asks an interactive question and returns the result.
+  ##
+  ## The proc will return one of the provided args. If not prompting the first
+  ## options is selected.
+  return promptList(options.forcePrompts, question, args)
 
 proc getNimbleDir*(options: Options): string =
-  options.config.nimbleDir
+  result = options.config.nimbleDir
+  if options.nimbleDir.len != 0:
+    # --nimbleDir:<dir> takes priority...
+    result = options.nimbleDir
+  else:
+    # ...followed by the environment variable.
+    let env = getEnv("NIMBLE_DIR")
+    if env.len != 0:
+      display("Warning:", "Using the environment variable: NIMBLE_DIR='" &
+                        env & "'", Warning)
+      result = env
+
+  return expandTilde(result)
 
 proc getPkgsDir*(options: Options): string =
-  options.config.nimbleDir / "pkgs"
+  options.getNimbleDir() / "pkgs"
 
 proc getBinDir*(options: Options): string =
-  options.config.nimbleDir / "bin"
+  options.getNimbleDir() / "bin"
 
 proc parseCommand*(key: string, result: var Options) =
-  result.action.typ = parseActionType(key)
+  result.action = Action(typ: parseActionType(key))
   initAction(result, key)
 
+proc setRunOptions(result: var Options, key, val: string, isArg: bool) =
+  if result.action.runFile.isNone() and (isArg or val == "--"):
+    result.action.runFile = some(key)
+  else:
+    result.action.runFlags.add(val)
+
 proc parseArgument*(key: string, result: var Options) =
   case result.action.typ
   of actionNil:
     assert false
-  of actionInstall, actionPath, actionUninstall:
+  of actionInstall, actionPath, actionDevelop, actionUninstall:
     # Parse pkg@verRange
     if '@' in key:
       let i = find(key, '@')
-      let pkgTup = (key[0 .. i-1],
+      let (pkgName, pkgVer) = (key[0 .. i-1], key[i+1 .. key.len-1])
-        key[i+1 .. key.len-1].parseVersionRange())
+      if pkgVer.len == 0:
-      result.action.packages.add(pkgTup)
+        raise newException(NimbleError, "Version range expected after '@'.")
+      result.action.packages.add((pkgName, pkgVer.parseVersionRange()))
     else:
       result.action.packages.add((key, VersionRange(kind: verAny)))
-  of actionUpdate:
+  of actionRefresh:
     result.action.optionalURL = key
   of actionSearch:
     result.action.search.add(key)
   of actionInit, actionDump:
     if result.action.projName != "":
-      raise newException(NimbleError,
+      raise newException(
-          "Can only initialize one package at a time.")
+        NimbleError, "Can only perform this action on one package at a time."
+      )
     result.action.projName = key
-  of actionCompile:
+  of actionCompile, actionDoc:
     result.action.file = key
-  of actionList, actionBuild, actionPublish:
+  of actionList, actionPublish:
-    writeHelp()
+    result.showHelp = true
+  of actionBuild:
+    result.action.file = key
+  of actionRun:
+    result.setRunOptions(key, key, true)
   of actionCustom:
     result.action.arguments.add(key)
   else:
     discard
 
-proc parseFlag*(flag, val: string, result: var Options) =
+proc getFlagString(kind: CmdLineKind, flag, val: string): string =
+  let prefix =
+    case kind
+    of cmdShortOption: "-"
+    of cmdLongOption: "--"
+    else: ""
+  if val == "":
+    return prefix & flag
+  else:
+    return prefix & flag & ":" & val
+
+proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) =
+
+  let f = flag.normalize()
+
+  # Global flags.
+  var isGlobalFlag = true
+  case f
+  of "help", "h": result.showHelp = true
+  of "version", "v": result.showVersion = true
+  of "accept", "y": result.forcePrompts = forcePromptYes
+  of "reject", "n": result.forcePrompts = forcePromptNo
+  of "nimbledir": result.nimbleDir = val
+  of "verbose": result.verbosity = LowPriority
+  of "debug": result.verbosity = DebugPriority
+  of "nocolor": result.noColor = true
+  of "disablevalidation": result.disableValidation = true
+  else: isGlobalFlag = false
+
+  var wasFlagHandled = true
+  # Action-specific flags.
   case result.action.typ
-  of actionCompile:
+  of actionSearch, actionList:
-    if val == "":
+    case f
-      result.action.compileOptions.add("--" & flag)
+    of "installed", "i":
+      result.queryInstalled = true
+    of "ver":
+      result.queryVersions = true
     else:
-      result.action.compileOptions.add("--" & flag & ":" & val)
+      wasFlagHandled = false
+  of actionInstall:
+    case f
+    of "depsonly", "d":
+      result.depsOnly = true
+    of "passnim", "p":
+      result.action.passNimFlags.add(val)
+    else:
+      wasFlagHandled = false
+  of actionInit:
+    case f
+    of "git", "hg":
+      result.action.vcsOption = f
+    else:
+      wasFlagHandled = false
+  of actionUninstall:
+    case f
+    of "incldeps", "i":
+      result.uninstallRevDeps = true
+    else:
+      wasFlagHandled = false
+  of actionCompile, actionDoc, actionBuild:
+    if not isGlobalFlag:
+      result.action.compileOptions.add(getFlagString(kind, flag, val))
+  of actionRun:
+    result.showHelp = false
+    result.setRunOptions(flag, getFlagString(kind, flag, val), false)
   of actionCustom:
+    if result.action.command.normalize == "test":
+      if f == "continue" or f == "c":
+        result.continueTestsOnFailure = true
     result.action.flags[flag] = val
   else:
-    # TODO: These should not be global.
+    wasFlagHandled = false
-    case flag.normalize()
+
-    of "help", "h": writeHelp()
+  if not wasFlagHandled and not isGlobalFlag:
-    of "version", "v":
+    result.unknownFlags.add((kind, flag, val))
-      assert result.action.typ == actionNil
-      result.action.typ = actionVersion
-    of "accept", "y": result.forcePrompts = forcePromptYes
-    of "reject", "n": result.forcePrompts = forcePromptNo
-    of "ver": result.queryVersions = true
-    of "nimbledir": result.config.nimbleDir = val # overrides option from file
-    of "installed", "i": result.queryInstalled = true
-    else:
-      raise newException(NimbleError, "Unknown option: --" & flag)
 
 proc initOptions*(): Options =
-  result.action.typ = actionNil
+  # Exported for choosenim
-  result.pkgInfoCache = newTable[string, PackageInfo]()
+  Options(
-
+    action: Action(typ: actionNil),
-proc parseMisc(): Options =
+    pkgInfoCache: newTable[string, PackageInfo](),
-  result = initOptions()
+    verbosity: HighPriority,
-  result.config = parseConfig()
+    noColor: not isatty(stdout)
+  )
 
+proc parseMisc(options: var Options) =
   # Load nimbledata.json
-  let nimbledataFilename = result.getNimbleDir() / "nimbledata.json"
+  let nimbledataFilename = options.getNimbleDir() / "nimbledata.json"
+
   if fileExists(nimbledataFilename):
     try:
-      result.nimbleData = parseFile(nimbledataFilename)
+      options.nimbleData = parseFile(nimbledataFilename)
     except:
       raise newException(NimbleError, "Couldn't parse nimbledata.json file " &
           "located at " & nimbledataFilename)
   else:
-    result.nimbleData = %{"reverseDeps": newJObject()}
+    options.nimbleData = %{"reverseDeps": newJObject()}
+
+proc handleUnknownFlags(options: var Options) =
+  if options.action.typ == actionRun:
+    # ActionRun uses flags that come before the command as compilation flags
+    # and flags that come after as run flags.
+    options.action.compileFlags =
+      map(options.unknownFlags, x => getFlagString(x[0], x[1], x[2]))
+    options.unknownFlags = @[]
+  else:
+    # For everything else, handle the flags that came before the command
+    # normally.
+    let unknownFlags = options.unknownFlags
+    options.unknownFlags = @[]
+    for flag in unknownFlags:
+      parseFlag(flag[1], flag[2], options, flag[0])
+
+  # Any unhandled flags?
+  if options.unknownFlags.len > 0:
+    let flag = options.unknownFlags[0]
+    raise newException(
+      NimbleError,
+      "Unknown option: " & getFlagString(flag[0], flag[1], flag[2])
+    )
 
 proc parseCmdLine*(): Options =
-  result = parseMisc()
+  result = initOptions()
+
+  # Parse command line params first. A simple `--version` shouldn't require
+  # a config to be parsed.
   for kind, key, val in getOpt():
     case kind
     of cmdArgument:
@@ -286,15 +444,30 @@ proc parseCmdLine*(): Options =
       else:
         parseArgument(key, result)
     of cmdLongOption, cmdShortOption:
-      parseFlag(key, val, result)
+        parseFlag(key, val, result, kind)
     of cmdEnd: assert(false) # cannot happen
-  if result.action.typ == actionNil:
-    writeHelp()
 
-  # TODO: Remove this after a couple of versions.
+  handleUnknownFlags(result)
-  if getNimrodVersion() > newVersion("0.9.6"):
+
-    # Rename deprecated babel dir.
+  # Set verbosity level.
-    renameBabelToNimble(result)
+  setVerbosity(result.verbosity)
+
+  # Set whether color should be shown.
+  setShowColor(not result.noColor)
+
+  # Parse config.
+  result.config = parseConfig()
+
+  # Parse other things, for example the nimbledata.json file.
+  parseMisc(result)
+
+  if result.action.typ == actionNil and not result.showVersion:
+    result.showHelp = true
+
+  if result.action.typ != actionNil and result.showVersion:
+    # We've got another command that should be handled. For example:
+    # nimble run foobar -v
+    result.showVersion = false
 
 proc getProxy*(options: Options): Proxy =
   ## Returns ``nil`` if no proxy is specified.
@@ -307,9 +480,13 @@ proc getProxy*(options: Options): Proxy =
         url = getEnv("http_proxy")
       elif existsEnv("https_proxy"):
         url = getEnv("https_proxy")
+      elif existsEnv("HTTP_PROXY"):
+        url = getEnv("HTTP_PROXY")
+      elif existsEnv("HTTPS_PROXY"):
+        url = getEnv("HTTPS_PROXY")
     except ValueError:
-      echo("WARNING: Unable to parse proxy from environment: ",
+      display("Warning:", "Unable to parse proxy from environment: " &
-          getCurrentExceptionMsg())
+          getCurrentExceptionMsg(), Warning, HighPriority)
 
   if url.len > 0:
     var parsed = parseUri(url)
@@ -320,4 +497,55 @@ proc getProxy*(options: Options): Proxy =
       else: ""
     return newProxy($parsed, auth)
   else:
-    return nil
+    return nil
+
+proc briefClone*(options: Options): Options =
+  ## Clones the few important fields and creates a new Options object.
+  var newOptions = initOptions()
+  newOptions.config = options.config
+  newOptions.nimbleData = options.nimbleData
+  newOptions.nimbleDir = options.nimbleDir
+  newOptions.forcePrompts = options.forcePrompts
+  newOptions.pkgInfoCache = options.pkgInfoCache
+  return newOptions
+
+proc shouldRemoveTmp*(options: Options, file: string): bool =
+  result = true
+  if options.verbosity <= DebugPriority:
+    let msg = "Not removing temporary path because of debug verbosity: " & file
+    display("Warning:", msg, Warning, MediumPriority)
+    return false
+
+proc getCompilationFlags*(options: var Options): var seq[string] =
+  case options.action.typ
+  of actionBuild, actionDoc, actionCompile:
+    return options.action.compileOptions
+  of actionRun:
+    return options.action.compileFlags
+  else:
+    assert false
+
+proc getCompilationFlags*(options: Options): seq[string] =
+  var opt = options
+  return opt.getCompilationFlags()
+
+proc getCompilationBinary*(options: Options, pkgInfo: PackageInfo): Option[string] =
+  case options.action.typ
+  of actionBuild, actionDoc, actionCompile:
+    let file = options.action.file.changeFileExt("")
+    if file.len > 0:
+      return some(file)
+  of actionRun:
+    let optRunFile = options.action.runFile
+    let runFile =
+      if optRunFile.get("").len > 0:
+        optRunFile.get()
+      elif pkgInfo.bin.len == 1:
+        pkgInfo.bin[0]
+      else:
+        ""
+
+    if runFile.len > 0:
+      return some(runFile.changeFileExt(ExeExt))
+  else:
+    discard

src/nimblepkg/packageinfo.nim

@@ -1,10 +1,15 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
-import parsecfg, json, streams, strutils, parseutils, os, sets, tables
+
-import version, tools, nimbletypes, options
+# Stdlib imports
+import system except TResult
+import hashes, json, strutils, os, sets, tables, httpclient
+
+# Local imports
+import version, tools, common, options, cli, config
 
 type
-  Package* = object
+  Package* = object ## Definition of package from packages.json.
     # Required fields in a package.
     name*: string
     url*: string # Download location.
@@ -16,12 +21,18 @@ type
     version*: string
     dvcsTag*: string
     web*: string # Info url for humans.
+    alias*: string ## A name of another package, that this package aliases.
 
   MetaData* = object
     url*: string
 
+  NimbleLink* = object
+    nimbleFilePath*: string
+    packageDir*: string
+
 proc initPackageInfo*(path: string): PackageInfo =
-  result.mypath = path
+  result.myPath = path
+  result.specialVersion = ""
   result.preHooks.init()
   result.postHooks.init()
   # reasonable default:
@@ -37,6 +48,7 @@ proc initPackageInfo*(path: string): PackageInfo =
   result.installFiles = @[]
   result.installExt = @[]
   result.requires = @[]
+  result.foreignDeps = @[]
   result.bin = @[]
   result.srcDir = ""
   result.binDir = ""
@@ -57,22 +69,24 @@ proc getNameVersion*(pkgpath: string): tuple[name, version: string] =
   ##
   ## Also works for file paths like:
   ##   ``/home/user/.nimble/pkgs/package-0.1/package.nimble``
-
+  if pkgPath.splitFile.ext in [".nimble", ".nimble-link", ".babel"]:
-  if pkgPath.splitFile.ext == ".nimble" or pkgPath.splitFile.ext == ".babel":
     return getNameVersion(pkgPath.splitPath.head)
 
   result.name = ""
   result.version = ""
   let tail = pkgpath.splitPath.tail
-  if '-' notin tail:
+
+  const specialSeparator = "-#"
+  var sepIdx = tail.find(specialSeparator)
+  if sepIdx == -1:
+    sepIdx = tail.rfind('-')
+
+  if sepIdx == -1:
     result.name = tail
     return
 
-  for i in countdown(tail.len-1, 0):
+  result.name = tail[0 .. sepIdx - 1]
-    if tail[i] == '-':
+  result.version = tail.substr(sepIdx + 1)
-      result.name = tail[0 .. i-1]
-      result.version = tail[i+1 .. tail.len-1]
-      break
 
 proc optionalField(obj: JsonNode, name: string, default = ""): string =
   ## Queries ``obj`` for the optional ``name`` string.
@@ -92,8 +106,8 @@ proc requiredField(obj: JsonNode, name: string): string =
   ## Queries ``obj`` for the required ``name`` string.
   ##
   ## Aborts execution if the field does not exist or is of invalid json type.
-  result = optionalField(obj, name, nil)
+  result = optionalField(obj, name)
-  if result == nil:
+  if result.len == 0:
     raise newException(NimbleError,
         "Package in packages.json file does not contain a " & name & " field.")
 
@@ -102,57 +116,170 @@ proc fromJson(obj: JSonNode): Package =
   ##
   ## Aborts execution if the JSON node doesn't contain the required fields.
   result.name = obj.requiredField("name")
-  result.version = obj.optionalField("version")
+  if obj.hasKey("alias"):
-  result.url = obj.requiredField("url")
+    result.alias = obj.requiredField("alias")
-  result.downloadMethod = obj.requiredField("method")
+  else:
-  result.dvcsTag = obj.optionalField("dvcs-tag")
+    result.alias = ""
-  result.license = obj.requiredField("license")
+    result.version = obj.optionalField("version")
-  result.tags = @[]
+    result.url = obj.requiredField("url")
-  for t in obj["tags"]:
+    result.downloadMethod = obj.requiredField("method")
-    result.tags.add(t.str)
+    result.dvcsTag = obj.optionalField("dvcs-tag")
-  result.description = obj.requiredField("description")
+    result.license = obj.requiredField("license")
-  result.web = obj.optionalField("web")
+    result.tags = @[]
+    for t in obj["tags"]:
+      result.tags.add(t.str)
+    result.description = obj.requiredField("description")
+    result.web = obj.optionalField("web")
 
 proc readMetaData*(path: string): MetaData =
   ## Reads the metadata present in ``~/.nimble/pkgs/pkg-0.1/nimblemeta.json``
   var bmeta = path / "nimblemeta.json"
-  if not existsFile(bmeta):
-    bmeta = path / "babelmeta.json"
-    if existsFile(bmeta):
-      echo("WARNING: using deprecated babelmeta.json file in " & path)
   if not existsFile(bmeta):
     result.url = ""
-    echo("WARNING: No nimblemeta.json file found in " & path)
+    display("Warning:", "No nimblemeta.json file found in " & path,
+            Warning, HighPriority)
     return
     # TODO: Make this an error.
   let cont = readFile(bmeta)
   let jsonmeta = parseJson(cont)
   result.url = jsonmeta["url"].str
 
-proc getPackage*(pkg: string, options: Options,
+proc readNimbleLink*(nimbleLinkPath: string): NimbleLink =
-    resPkg: var Package): bool =
+  let s = readFile(nimbleLinkPath).splitLines()
+  result.nimbleFilePath = s[0]
+  result.packageDir = s[1]
+
+proc writeNimbleLink*(nimbleLinkPath: string, contents: NimbleLink) =
+  let c = contents.nimbleFilePath & "\n" & contents.packageDir
+  writeFile(nimbleLinkPath, c)
+
+proc needsRefresh*(options: Options): bool =
+  ## Determines whether a ``nimble refresh`` is needed.
+  ##
+  ## In the future this will check a stored time stamp to determine how long
+  ## ago the package list was refreshed.
+  result = true
+  for name, list in options.config.packageLists:
+    if fileExists(options.getNimbleDir() / "packages_" & name & ".json"):
+      result = false
+
+proc validatePackagesList(path: string): bool =
+  ## Determines whether package list at ``path`` is valid.
+  try:
+    let pkgList = parseFile(path)
+    if pkgList.kind == JArray:
+      if pkgList.len == 0:
+        display("Warning:", path & " contains no packages.", Warning,
+                HighPriority)
+      return true
+  except ValueError, JsonParsingError:
+    return false
+
+proc fetchList*(list: PackageList, options: Options) =
+  ## Downloads or copies the specified package list and saves it in $nimbleDir.
+  let verb = if list.urls.len > 0: "Downloading" else: "Copying"
+  display(verb, list.name & " package list", priority = HighPriority)
+
+  var
+    lastError = ""
+    copyFromPath = ""
+  if list.urls.len > 0:
+    for i in 0 ..< list.urls.len:
+      let url = list.urls[i]
+      display("Trying", url)
+      let tempPath = options.getNimbleDir() / "packages_temp.json"
+
+      # Grab the proxy
+      let proxy = getProxy(options)
+      if not proxy.isNil:
+        var maskedUrl = proxy.url
+        if maskedUrl.password.len > 0: maskedUrl.password = "***"
+        display("Connecting", "to proxy at " & $maskedUrl,
+                priority = LowPriority)
+
+      try:
+        let client = newHttpClient(proxy = proxy)
+        client.downloadFile(url, tempPath)
+      except:
+        let message = "Could not download: " & getCurrentExceptionMsg()
+        display("Warning:", message, Warning)
+        lastError = message
+        continue
+
+      if not validatePackagesList(tempPath):
+        lastError = "Downloaded packages.json file is invalid"
+        display("Warning:", lastError & ", discarding.", Warning)
+        continue
+
+      copyFromPath = tempPath
+      display("Success", "Package list downloaded.", Success, HighPriority)
+      lastError = ""
+      break
+
+  elif list.path != "":
+    if not validatePackagesList(list.path):
+      lastError = "Copied packages.json file is invalid"
+      display("Warning:", lastError & ", discarding.", Warning)
+    else:
+      copyFromPath = list.path
+      display("Success", "Package list copied.", Success, HighPriority)
+
+  if lastError.len != 0:
+    raise newException(NimbleError, "Refresh failed\n" & lastError)
+
+  if copyFromPath.len > 0:
+    copyFile(copyFromPath,
+        options.getNimbleDir() / "packages_$1.json" % list.name.toLowerAscii())
+
+proc readPackageList(name: string, options: Options): JsonNode =
+  # If packages.json is not present ask the user if they want to download it.
+  if needsRefresh(options):
+    if options.prompt("No local packages.json found, download it from " &
+            "internet?"):
+      for name, list in options.config.packageLists:
+        fetchList(list, options)
+    else:
+      # The user might not need a package list for now. So let's try
+      # going further.
+      return newJArray()
+  return parseFile(options.getNimbleDir() / "packages_" &
+                   name.toLowerAscii() & ".json")
+
+proc getPackage*(pkg: string, options: Options, resPkg: var Package): bool
+proc resolveAlias(pkg: Package, options: Options): Package =
+  result = pkg
+  # Resolve alias.
+  if pkg.alias.len > 0:
+    display("Warning:", "The $1 package has been renamed to $2" %
+            [pkg.name, pkg.alias], Warning, HighPriority)
+    if not getPackage(pkg.alias, options, result):
+      raise newException(NimbleError, "Alias for package not found: " &
+                         pkg.alias)
+
+proc getPackage*(pkg: string, options: Options, resPkg: var Package): bool =
   ## Searches any packages.json files defined in ``options.config.packageLists``
   ## Saves the found package into ``resPkg``.
   ##
   ## Pass in ``pkg`` the name of the package you are searching for. As
   ## convenience the proc returns a boolean specifying if the ``resPkg`` was
   ## successfully filled with good data.
+  ##
+  ## Aliases are handled and resolved.
   for name, list in options.config.packageLists:
-    echo("Searching in \"", name, "\" package list...")
+    display("Reading", "$1 package list" % name, priority = LowPriority)
-    let packages = parseFile(options.getNimbleDir() /
+    let packages = readPackageList(name, options)
-        "packages_" & name.toLower() & ".json")
     for p in packages:
-      if p["name"].str == pkg:
+      if normalize(p["name"].str) == normalize(pkg):
         resPkg = p.fromJson()
+        resPkg = resolveAlias(resPkg, options)
         return true
 
 proc getPackageList*(options: Options): seq[Package] =
   ## Returns the list of packages found in the downloaded packages.json files.
   result = @[]
-  var namesAdded = initSet[string]()
+  var namesAdded = initHashSet[string]()
   for name, list in options.config.packageLists:
-    let packages = parseFile(options.getNimbleDir() /
+    let packages = readPackageList(name, options)
-        "packages_" & name.toLower() & ".json")
     for p in packages:
       let pkg: Package = p.fromJson()
       if pkg.name notin namesAdded:
@@ -163,10 +290,10 @@ proc findNimbleFile*(dir: string; error: bool): string =
   result = ""
   var hits = 0
   for kind, path in walkDir(dir):
-    if kind == pcFile:
+    if kind in {pcFile, pcLinkToFile}:
       let ext = path.splitFile.ext
       case ext
-      of ".babel", ".nimble":
+      of ".babel", ".nimble", ".nimble-link":
         result = path
         inc hits
       else: discard
@@ -176,33 +303,73 @@ proc findNimbleFile*(dir: string; error: bool): string =
   elif hits == 0:
     if error:
       raise newException(NimbleError,
-          "Specified directory does not contain a .nimble file.")
+          "Specified directory ($1) does not contain a .nimble file." % dir)
     else:
-      # TODO: Abstract logging.
+      display("Warning:", "No .nimble or .nimble-link file found for " &
-      echo("WARNING: No .nimble file found for ", dir)
+              dir, Warning, HighPriority)
+
+  if result.splitFile.ext == ".nimble-link":
+    # Return the path of the real .nimble file.
+    result = readNimbleLink(result).nimbleFilePath
+    if not fileExists(result):
+      let msg = "The .nimble-link file is pointing to a missing file: " & result
+      let hintMsg =
+        "Remove '$1' or restore the file it points to." % dir
+      display("Warning:", msg, Warning, HighPriority)
+      display("Hint:", hintMsg, Warning, HighPriority)
 
 proc getInstalledPkgsMin*(libsDir: string, options: Options):
         seq[tuple[pkginfo: PackageInfo, meta: MetaData]] =
   ## Gets a list of installed packages. The resulting package info is
   ## minimal. This has the advantage that it does not depend on the
-  ## ``packageparser`` module, and so can be used by ``nimscriptsupport``.
+  ## ``packageparser`` module, and so can be used by ``nimscriptwrapper``.
   ##
-  ## ``libsDir`` is in most cases: ~/.nimble/pkgs/
+  ## ``libsDir`` is in most cases: ~/.nimble/pkgs/ (options.getPkgsDir)
   result = @[]
   for kind, path in walkDir(libsDir):
     if kind == pcDir:
       let nimbleFile = findNimbleFile(path, false)
       if nimbleFile != "":
         let meta = readMetaData(path)
-        let (name, version) = getNameVersion(nimbleFile)
+        let (name, version) = getNameVersion(path)
         var pkg = initPackageInfo(nimbleFile)
         pkg.name = name
         pkg.version = version
+        pkg.specialVersion = version
         pkg.isMinimal = true
         pkg.isInstalled = true
+        let nimbleFileDir = nimbleFile.splitFile().dir
+        pkg.isLinked = cmpPaths(nimbleFileDir, path) != 0
+
+        # Read the package's 'srcDir' (this is stored in the .nimble-link so
+        # we can easily grab it)
+        if pkg.isLinked:
+          let nimbleLinkPath = path / name.addFileExt("nimble-link")
+          let realSrcPath = readNimbleLink(nimbleLinkPath).packageDir
+          assert realSrcPath.startsWith(nimbleFileDir)
+          pkg.srcDir = realSrcPath.replace(nimbleFileDir)
+          pkg.srcDir.removePrefix(DirSep)
         result.add((pkg, meta))
 
-proc findPkg*(pkglist: seq[tuple[pkginfo: PackageInfo, meta: MetaData]],
+proc withinRange*(pkgInfo: PackageInfo, verRange: VersionRange): bool =
+  ## Determines whether the specified package's version is within the
+  ## specified range. The check works with ordinary versions as well as
+  ## special ones.
+  return withinRange(newVersion(pkgInfo.version), verRange) or
+         withinRange(newVersion(pkgInfo.specialVersion), verRange)
+
+proc resolveAlias*(dep: PkgTuple, options: Options): PkgTuple =
+  ## Looks up the specified ``dep.name`` in the packages.json files to resolve
+  ## a potential alias into the package's real name.
+  result = dep
+  var pkg: Package
+  # TODO: This needs better caching.
+  if getPackage(dep.name, options, pkg):
+    # The resulting ``pkg`` will contain the resolved name or the original if
+    # no alias is present.
+    result.name = pkg.name
+
+proc findPkg*(pkglist: seq[tuple[pkgInfo: PackageInfo, meta: MetaData]],
              dep: PkgTuple,
              r: var PackageInfo): bool =
   ## Searches ``pkglist`` for a package of which version is within the range
@@ -214,27 +381,27 @@ proc findPkg*(pkglist: seq[tuple[pkginfo: PackageInfo, meta: MetaData]],
   for pkg in pkglist:
     if cmpIgnoreStyle(pkg.pkginfo.name, dep.name) != 0 and
        cmpIgnoreStyle(pkg.meta.url, dep.name) != 0: continue
-    if withinRange(newVersion(pkg.pkginfo.version), dep.ver):
+    if withinRange(pkg.pkgInfo, dep.ver):
-      if not result or newVersion(r.version) < newVersion(pkg.pkginfo.version):
+      let isNewer = newVersion(r.version) < newVersion(pkg.pkginfo.version)
+      if not result or isNewer:
         r = pkg.pkginfo
         result = true
 
-proc findAllPkgs*(pkglist: seq[tuple[pkginfo: PackageInfo, meta: MetaData]],
+proc findAllPkgs*(pkglist: seq[tuple[pkgInfo: PackageInfo, meta: MetaData]],
                   dep: PkgTuple): seq[PackageInfo] =
   ## Searches ``pkglist`` for packages of which version is within the range
   ## of ``dep.ver``. This is similar to ``findPkg`` but returns multiple
   ## packages if multiple are found.
   result = @[]
   for pkg in pkglist:
-    if cmpIgnoreStyle(pkg.pkginfo.name, dep.name) != 0 and
+    if cmpIgnoreStyle(pkg.pkgInfo.name, dep.name) != 0 and
        cmpIgnoreStyle(pkg.meta.url, dep.name) != 0: continue
-    if withinRange(newVersion(pkg.pkginfo.version), dep.ver):
+    if withinRange(pkg.pkgInfo, dep.ver):
       result.add pkg.pkginfo
 
 proc getRealDir*(pkgInfo: PackageInfo): string =
-  ## Returns the ``pkgInfo.srcDir`` or the .mypath directory if package does
+  ## Returns the directory containing the package source files.
-  ## not specify the src dir.
+  if pkgInfo.srcDir != "" and (not pkgInfo.isInstalled or pkgInfo.isLinked):
-  if pkgInfo.srcDir != "" and not pkgInfo.isInstalled:
     result = pkgInfo.mypath.splitFile.dir / pkgInfo.srcDir
   else:
     result = pkgInfo.mypath.splitFile.dir
@@ -248,12 +415,15 @@ proc getOutputDir*(pkgInfo: PackageInfo, bin: string): string =
 
 proc echoPackage*(pkg: Package) =
   echo(pkg.name & ":")
-  echo("  url:         " & pkg.url & " (" & pkg.downloadMethod & ")")
+  if pkg.alias.len > 0:
-  echo("  tags:        " & pkg.tags.join(", "))
+    echo("  Alias for ", pkg.alias)
-  echo("  description: " & pkg.description)
+  else:
-  echo("  license:     " & pkg.license)
+    echo("  url:         " & pkg.url & " (" & pkg.downloadMethod & ")")
-  if pkg.web.len > 0:
+    echo("  tags:        " & pkg.tags.join(", "))
-    echo("  website:     " & pkg.web)
+    echo("  description: " & pkg.description)
+    echo("  license:     " & pkg.license)
+    if pkg.web.len > 0:
+      echo("  website:     " & pkg.web)
 
 proc getDownloadDirName*(pkg: Package, verRange: VersionRange): string =
   result = pkg.name
@@ -262,26 +432,118 @@ proc getDownloadDirName*(pkg: Package, verRange: VersionRange): string =
     result.add "_"
     result.add verSimple
 
-proc needsRefresh*(options: Options): bool =
+proc checkInstallFile(pkgInfo: PackageInfo,
-  ## Determines whether a ``nimble refresh`` is needed.
+                      origDir, file: string): bool =
-  ##
+  ## Checks whether ``file`` should be installed.
-  ## In the future this will check a stored time stamp to determine how long
+  ## ``True`` means file should be skipped.
-  ## ago the package list was refreshed.
-  result = true
-  for name, list in options.config.packageLists:
-    if fileExists(options.getNimbleDir() / "packages_" & name & ".json"):
-      result = false
 
-proc validatePackagesList*(path: string): bool =
+  for ignoreFile in pkgInfo.skipFiles:
-  ## Determines whether package list at ``path`` is valid.
+    if ignoreFile.endswith("nimble"):
-  try:
+      raise newException(NimbleError, ignoreFile & " must be installed.")
-    let pkgList = parseFile(path)
+    if samePaths(file, origDir / ignoreFile):
-    if pkgList.kind == JArray:
+      result = true
-      if pkgList.len == 0:
+      break
-        echo("WARNING: ", path, " contains no packages.")
+
-      return true
+  for ignoreExt in pkgInfo.skipExt:
-  except ValueError, JsonParsingError:
+    if file.splitFile.ext == ('.' & ignoreExt):
-    return false
+      result = true
+      break
+
+  if file.splitFile().name[0] == '.': result = true
+
+proc checkInstallDir(pkgInfo: PackageInfo,
+                     origDir, dir: string): bool =
+  ## Determines whether ``dir`` should be installed.
+  ## ``True`` means dir should be skipped.
+  for ignoreDir in pkgInfo.skipDirs:
+    if samePaths(dir, origDir / ignoreDir):
+      result = true
+      break
+
+  let thisDir = splitPath(dir).tail
+  assert thisDir != ""
+  if thisDir[0] == '.': result = true
+  if thisDir == "nimcache": result = true
+
+proc iterFilesWithExt(dir: string, pkgInfo: PackageInfo,
+                      action: proc (f: string)) =
+  ## Runs `action` for each filename of the files that have a whitelisted
+  ## file extension.
+  for kind, path in walkDir(dir):
+    if kind == pcDir:
+      iterFilesWithExt(path, pkgInfo, action)
+    else:
+      if path.splitFile.ext.substr(1) in pkgInfo.installExt:
+        action(path)
+
+proc iterFilesInDir(dir: string, action: proc (f: string)) =
+  ## Runs `action` for each file in ``dir`` and any
+  ## subdirectories that are in it.
+  for kind, path in walkDir(dir):
+    if kind == pcDir:
+      iterFilesInDir(path, action)
+    else:
+      action(path)
+
+proc iterInstallFiles*(realDir: string, pkgInfo: PackageInfo,
+                       options: Options, action: proc (f: string)) =
+  ## Runs `action` for each file within the ``realDir`` that should be
+  ## installed.
+  let whitelistMode =
+          pkgInfo.installDirs.len != 0 or
+          pkgInfo.installFiles.len != 0 or
+          pkgInfo.installExt.len != 0
+  if whitelistMode:
+    for file in pkgInfo.installFiles:
+      let src = realDir / file
+      if not src.existsFile():
+        if options.prompt("Missing file " & src & ". Continue?"):
+          continue
+        else:
+          raise NimbleQuit(msg: "")
+
+      action(src)
+
+    for dir in pkgInfo.installDirs:
+      # TODO: Allow skipping files inside dirs?
+      let src = realDir / dir
+      if not src.existsDir():
+        if options.prompt("Missing directory " & src & ". Continue?"):
+          continue
+        else:
+          raise NimbleQuit(msg: "")
+
+      iterFilesInDir(src, action)
+
+    iterFilesWithExt(realDir, pkgInfo, action)
+  else:
+    for kind, file in walkDir(realDir):
+      if kind == pcDir:
+        let skip = pkgInfo.checkInstallDir(realDir, file)
+        if skip: continue
+        # we also have to stop recursing if we reach an in-place nimbleDir
+        if file == options.getNimbleDir().expandFilename(): continue
+
+        iterInstallFiles(file, pkgInfo, options, action)
+      else:
+        let skip = pkgInfo.checkInstallFile(realDir, file)
+        if skip: continue
+
+        action(file)
+
+proc getPkgDest*(pkgInfo: PackageInfo, options: Options): string =
+  let versionStr = '-' & pkgInfo.specialVersion
+  let pkgDestDir = options.getPkgsDir() / (pkgInfo.name & versionStr)
+  return pkgDestDir
+
+proc `==`*(pkg1: PackageInfo, pkg2: PackageInfo): bool =
+  if pkg1.name == pkg2.name and pkg1.myPath == pkg2.myPath:
+    return true
+
+proc hash*(x: PackageInfo): Hash =
+  var h: Hash = 0
+  h = h !& hash(x.myPath)
+  result = !$h
 
 when isMainModule:
   doAssert getNameVersion("/home/user/.nimble/libs/packagea-0.1") ==
@@ -290,16 +552,14 @@ when isMainModule:
       ("package-a", "0.1")
   doAssert getNameVersion("/home/user/.nimble/libs/package-a-0.1/package.nimble") ==
       ("package-a", "0.1")
-
+  doAssert getNameVersion("/home/user/.nimble/libs/package-#head") ==
-  validatePackageName("foo_bar")
+      ("package", "#head")
-  validatePackageName("f_oo_b_a_r")
+  doAssert getNameVersion("/home/user/.nimble/libs/package-#branch-with-dashes") ==
-  try:
+      ("package", "#branch-with-dashes")
-    validatePackageName("foo__bar")
+  # readPackageInfo (and possibly more) depends on this not raising.
-    assert false
+  doAssert getNameVersion("/home/user/.nimble/libs/package") == ("package", "")
-  except NimbleError:
-    assert true
 
   doAssert toValidPackageName("foo__bar") == "foo_bar"
   doAssert toValidPackageName("jhbasdh!£$@%#^_&*_()qwe") == "jhbasdh_qwe"
 
-  echo("All tests passed!")
+  echo("All tests passed!")

src/nimblepkg/packageinstaller.nim

@@ -0,0 +1,112 @@
+# Copyright (C) Dominik Picheta. All rights reserved.
+# BSD License. Look at license.txt for more info.
+import os, strutils, sets, json
+
+# Local imports
+import cli, options, tools
+
+when defined(windows):
+  import version
+
+when not declared(initHashSet) or not declared(toHashSet):
+  import common
+
+when defined(windows):
+  # This is just for Win XP support.
+  # TODO: Drop XP support?
+  from winlean import WINBOOL, DWORD
+  type
+    OSVERSIONINFO* {.final, pure.} = object
+      dwOSVersionInfoSize*: DWORD
+      dwMajorVersion*: DWORD
+      dwMinorVersion*: DWORD
+      dwBuildNumber*: DWORD
+      dwPlatformId*: DWORD
+      szCSDVersion*: array[0..127, char]
+
+  proc GetVersionExA*(VersionInformation: var OSVERSIONINFO): WINBOOL{.stdcall,
+    dynlib: "kernel32", importc: "GetVersionExA".}
+
+proc setupBinSymlink*(symlinkDest, symlinkFilename: string,
+                      options: Options): seq[string] =
+  result = @[]
+  let currentPerms = getFilePermissions(symlinkDest)
+  setFilePermissions(symlinkDest, currentPerms + {fpUserExec})
+  when defined(unix):
+    display("Creating", "symlink: $1 -> $2" %
+            [symlinkDest, symlinkFilename], priority = MediumPriority)
+    if existsFile(symlinkFilename):
+      let msg = "Symlink already exists in $1. Replacing." % symlinkFilename
+      display("Warning:", msg, Warning, HighPriority)
+      removeFile(symlinkFilename)
+
+    createSymlink(symlinkDest, symlinkFilename)
+    result.add symlinkFilename.extractFilename
+  elif defined(windows):
+    # There is a bug on XP, described here:
+    # http://stackoverflow.com/questions/2182568/batch-script-is-not-executed-if-chcp-was-called
+    # But this workaround brakes code page on newer systems, so we need to detect OS version
+    var osver = OSVERSIONINFO()
+    osver.dwOSVersionInfoSize = cast[DWORD](sizeof(OSVERSIONINFO))
+    if GetVersionExA(osver) == WINBOOL(0):
+      raise newException(NimbleError,
+        "Can't detect OS version: GetVersionExA call failed")
+    let fixChcp = osver.dwMajorVersion <= 5
+
+    # Create cmd.exe/powershell stub.
+    let dest = symlinkFilename.changeFileExt("cmd")
+    display("Creating", "stub: $1 -> $2" % [symlinkDest, dest],
+            priority = MediumPriority)
+    var contents = "@"
+    if options.config.chcp:
+      if fixChcp:
+        contents.add "chcp 65001 > nul && "
+      else: contents.add "chcp 65001 > nul\n@"
+    contents.add "\"" & symlinkDest & "\" %*\n"
+    writeFile(dest, contents)
+    result.add dest.extractFilename
+    # For bash on Windows (Cygwin/Git bash).
+    let bashDest = dest.changeFileExt("")
+    display("Creating", "Cygwin stub: $1 -> $2" %
+            [symlinkDest, bashDest], priority = MediumPriority)
+    writeFile(bashDest, "\"" & symlinkDest & "\" \"$@\"\n")
+    result.add bashDest.extractFilename
+  else:
+    {.error: "Sorry, your platform is not supported.".}
+
+proc saveNimbleMeta*(pkgDestDir, url, vcsRevision: string,
+                    filesInstalled, bins: HashSet[string],
+                    isLink: bool = false) =
+  ## Saves the specified data into a ``nimblemeta.json`` file inside
+  ## ``pkgDestDir``.
+  ##
+  ## filesInstalled - A list of absolute paths to files which have been
+  ##                  installed.
+  ## bins - A list of binary filenames which have been installed for this
+  ##        package.
+  ##
+  ## isLink - Determines whether the installed package is a .nimble-link.
+  var nimblemeta = %{"url": %url}
+  if vcsRevision.len > 0:
+    nimblemeta["vcsRevision"] = %vcsRevision
+  let files = newJArray()
+  nimblemeta["files"] = files
+  for file in filesInstalled:
+    files.add(%changeRoot(pkgDestDir, "", file))
+  let binaries = newJArray()
+  nimblemeta["binaries"] = binaries
+  for bin in bins:
+    binaries.add(%bin)
+  nimblemeta["isLink"] = %isLink
+  writeFile(pkgDestDir / "nimblemeta.json", $nimblemeta)
+
+proc saveNimbleMeta*(pkgDestDir, pkgDir, vcsRevision, nimbleLinkPath: string) =
+  ## Overload of saveNimbleMeta for linked (.nimble-link) packages.
+  ##
+  ## pkgDestDir - The directory where the package has been installed.
+  ##              For example: ~/.nimble/pkgs/jester-#head/
+  ##
+  ## pkgDir - The directory where the original package files are.
+  ##          For example: ~/projects/jester/
+  saveNimbleMeta(pkgDestDir, "file://" & pkgDir, vcsRevision,
+                 toHashSet[string]([nimbleLinkPath]), initHashSet[string](), true)

src/nimblepkg/packageparser.nim

@@ -1,31 +1,56 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
-import parsecfg, json, streams, strutils, parseutils, os, tables
+import parsecfg, sets, streams, strutils, os, tables, sugar
-import version, tools, nimbletypes, nimscriptsupport, options, packageinfo
+from sequtils import apply, map
+
+import version, tools, common, nimscriptwrapper, options, packageinfo, cli
 
 ## Contains procedures for parsing .nimble files. Moved here from ``packageinfo``
-## because it depends on ``nimscriptsupport`` (``nimscriptsupport`` also
+## because it depends on ``nimscriptwrapper`` (``nimscriptwrapper`` also
 ## depends on other procedures in ``packageinfo``.
 
-when not declared(system.map):
-  from sequtils import map
-
 type
   NimbleFile* = string
 
   ValidationError* = object of NimbleError
     warnInstalled*: bool # Determines whether to show a warning for installed pkgs
+    warnAll*: bool
 
-proc newValidationError(msg: string, warnInstalled: bool): ref ValidationError =
+const reservedNames = [
+  "CON",
+  "PRN",
+  "AUX",
+  "NUL",
+  "COM1",
+  "COM2",
+  "COM3",
+  "COM4",
+  "COM5",
+  "COM6",
+  "COM7",
+  "COM8",
+  "COM9",
+  "LPT1",
+  "LPT2",
+  "LPT3",
+  "LPT4",
+  "LPT5",
+  "LPT6",
+  "LPT7",
+  "LPT8",
+  "LPT9",
+]
+
+proc newValidationError(msg: string, warnInstalled: bool,
+                        hint: string, warnAll: bool): ref ValidationError =
   result = newException(ValidationError, msg)
   result.warnInstalled = warnInstalled
+  result.warnAll = warnAll
+  result.hint = hint
 
-proc raiseNewValidationError(msg: string, warnInstalled: bool) =
+proc raiseNewValidationError(msg: string, warnInstalled: bool,
-  if warnInstalled:
+                             hint: string = "", warnAll = false) =
-    # TODO: We warn everywhere for now. Raise the error in the next version.
+  raise newValidationError(msg, warnInstalled, hint, warnAll)
-    echo("WARNING: ", msg, ". Will be an error in next version!")
-  else:
-    raiseNewValidationError(msg, warnInstalled)
 
 proc validatePackageName*(name: string) =
   ## Raises an error if specified package name contains invalid characters.
@@ -54,6 +79,13 @@ proc validatePackageName*(name: string) =
     else:
       prevWasUnderscore = false
 
+  if name.endsWith("pkg"):
+    raiseNewValidationError("\"$1\" is an invalid package name: cannot end" &
+                            " with \"pkg\"" % name, false)
+  if name.toUpperAscii() in reservedNames:
+    raiseNewValidationError(
+      "\"$1\" is an invalid package name: reserved name" % name, false)
+
 proc validateVersion*(ver: string) =
   for c in ver:
     if c notin ({'.'} + Digits):
@@ -61,7 +93,74 @@ proc validateVersion*(ver: string) =
           "Version may only consist of numbers and the '.' character " &
           "but found '" & c & "'.", false)
 
-proc validatePackageInfo(pkgInfo: PackageInfo, path: string) =
+proc validatePackageStructure(pkgInfo: PackageInfo, options: Options) =
+  ## This ensures that a package's source code does not leak into
+  ## another package's namespace.
+  ## https://github.com/nim-lang/nimble/issues/144
+  let
+    realDir = pkgInfo.getRealDir()
+    normalizedBinNames = pkgInfo.bin.map(
+      (x) => x.changeFileExt("").toLowerAscii()
+    )
+    correctDir =
+      if pkgInfo.name.toLowerAscii() in normalizedBinNames:
+        pkgInfo.name & "pkg"
+      else:
+        pkgInfo.name
+
+  proc onFile(path: string) =
+    # Remove the root to leave only the package subdirectories.
+    # ~/package-0.1/package/utils.nim -> package/utils.nim.
+    var trailPath = changeRoot(realDir, "", path)
+    if trailPath.startsWith(DirSep): trailPath = trailPath[1 .. ^1]
+    let (dir, file, ext) = trailPath.splitFile
+    # We're only interested in nim files, because only they can pollute our
+    # namespace.
+    if ext != (ExtSep & "nim"):
+      return
+
+    if dir.len == 0:
+      if file != pkgInfo.name:
+        # A source file was found in the top level of srcDir that doesn't share
+        # a name with the package.
+        let
+          msg = ("Package '$1' has an incorrect structure. " &
+                "The top level of the package source directory " &
+                "should contain at most one module, " &
+                "named '$2', but a file named '$3' was found. This " &
+                "will be an error in the future.") %
+                [pkgInfo.name, pkgInfo.name & ext, file & ext]
+          hint = ("If this is the primary source file in the package, " &
+                  "rename it to '$1'. If it's a source file required by " &
+                  "the main module, or if it is one of several " &
+                  "modules exposed by '$4', then move it into a '$2' subdirectory. " &
+                  "If it's a test file or otherwise not required " &
+                  "to build the the package '$1', prevent its installation " &
+                  "by adding `skipFiles = @[\"$3\"]` to the .nimble file. See " &
+                  "https://github.com/nim-lang/nimble#libraries for more info.") %
+                  [pkgInfo.name & ext, correctDir & DirSep, file & ext, pkgInfo.name]
+        raiseNewValidationError(msg, true, hint, true)
+    else:
+      assert(not pkgInfo.isMinimal)
+      # On Windows `pkgInfo.bin` has a .exe extension, so we need to normalize.
+      if not (dir.startsWith(correctDir & DirSep) or dir == correctDir):
+        let
+          msg = ("Package '$2' has an incorrect structure. " &
+                "It should contain a single directory hierarchy " &
+                "for source files, named '$3', but file '$1' " &
+                "is in a directory named '$4' instead. " &
+                "This will be an error in the future.") %
+              [file & ext, pkgInfo.name, correctDir, dir]
+          hint = ("If '$1' contains source files for building '$2', rename it " &
+                  "to '$3'. Otherwise, prevent its installation " &
+                  "by adding `skipDirs = @[\"$1\"]` to the .nimble file.") %
+              [dir, pkgInfo.name, correctDir]
+        raiseNewValidationError(msg, true, hint, true)
+
+  iterInstallFiles(realDir, pkgInfo, options, onFile)
+
+proc validatePackageInfo(pkgInfo: PackageInfo, options: Options) =
+  let path = pkgInfo.myPath
   if pkgInfo.name == "":
     raiseNewValidationError("Incorrect .nimble file: " & path &
         " does not contain a name field.", false)
@@ -88,14 +187,16 @@ proc validatePackageInfo(pkgInfo: PackageInfo, path: string) =
       raiseNewValidationError("'" & pkgInfo.backend &
           "' is an invalid backend.", false)
 
-  validateVersion(pkgInfo.version)
+  validatePackageStructure(pkginfo, options)
+
 
 proc nimScriptHint*(pkgInfo: PackageInfo) =
   if not pkgInfo.isNimScript:
-    # TODO: Turn this into a warning.
+    display("Warning:", "The .nimble file for this project could make use of " &
-    # TODO: Add a URL explaining more.
+            "additional features, if converted into the new NimScript format." &
-    echo("NOTE: The .nimble file for this project could make use of " &
+            "\nFor more details see:" &
-         "additional features, if converted into the new NimScript format.")
+            "https://github.com/nim-lang/nimble#creating-packages",
+            Warning, HighPriority)
 
 proc multiSplit(s: string): seq[string] =
   ## Returns ``s`` split by newline and comma characters.
@@ -105,19 +206,23 @@ proc multiSplit(s: string): seq[string] =
   ## done no entries are found in the list, the proc returns a sequence with
   ## the original string as the only entry.
   result = split(s, {char(0x0A), char(0x0D), ','})
-  map(result, proc(x: var string) = x = x.strip())
+  apply(result, proc(x: var string) = x = x.strip())
   for i in countdown(result.len()-1, 0):
     if len(result[i]) < 1:
       result.del(i)
   # Huh, nothing to return? Return given input.
   if len(result) < 1:
-    return @[s]
+    if s.strip().len != 0:
+      return @[s]
+    else:
+      return @[]
 
 proc readPackageInfoFromNimble(path: string; result: var PackageInfo) =
   var fs = newFileStream(path, fmRead)
   if fs != nil:
     var p: CfgParser
     open(p, fs, path)
+    defer: close(p)
     var currentSection = ""
     while true:
       var ev = next(p)
@@ -151,12 +256,20 @@ proc readPackageInfoFromNimble(path: string; result: var PackageInfo) =
             result.installExt.add(ev.value.multiSplit)
           of "bin":
             for i in ev.value.multiSplit:
+              if i.splitFile().ext == ".nim":
+                raise newException(NimbleError, "`bin` entry should not be a source file: " & i)
               result.bin.add(i.addFileExt(ExeExt))
           of "backend":
-            result.backend = ev.value.toLower()
+            result.backend = ev.value.toLowerAscii()
             case result.backend.normalize
             of "javascript": result.backend = "js"
             else: discard
+          of "beforehooks":
+            for i in ev.value.multiSplit:
+              result.preHooks.incl(i.normalize)
+          of "afterhooks":
+            for i in ev.value.multiSplit:
+              result.postHooks.incl(i.normalize)
           else:
             raise newException(NimbleError, "Invalid field: " & ev.key)
         of "deps", "dependencies":
@@ -172,11 +285,37 @@ proc readPackageInfoFromNimble(path: string; result: var PackageInfo) =
             "Invalid package info, should not contain --" & ev.value)
       of cfgError:
         raise newException(NimbleError, "Error parsing .nimble file: " & ev.msg)
-    close(p)
   else:
     raise newException(ValueError, "Cannot open package info: " & path)
 
-proc readPackageInfo*(nf: NimbleFile, options: Options,
+proc readPackageInfoFromNims(scriptName: string, options: Options,
+    result: var PackageInfo) =
+  let
+    iniFile = getIniFile(scriptName, options)
+
+  if iniFile.fileExists():
+    readPackageInfoFromNimble(iniFile, result)
+
+proc inferInstallRules(pkgInfo: var PackageInfo, options: Options) =
+  # Binary packages shouldn't install .nim files by default.
+  # (As long as the package info doesn't explicitly specify what should be
+  # installed.)
+  let installInstructions =
+    pkgInfo.installDirs.len + pkgInfo.installExt.len + pkgInfo.installFiles.len
+  if installInstructions == 0 and pkgInfo.bin.len > 0:
+    pkgInfo.skipExt.add("nim")
+
+  # When a package doesn't specify a `srcDir` it's fair to assume that
+  # the .nim files are in the root of the package. So we can explicitly select
+  # them and prevent the installation of anything else. The user can always
+  # override this with `installFiles`.
+  if pkgInfo.srcDir == "":
+    if dirExists(pkgInfo.getRealDir() / pkgInfo.name):
+      pkgInfo.installDirs.add(pkgInfo.name)
+    if fileExists(pkgInfo.getRealDir() / pkgInfo.name.addFileExt("nim")):
+      pkgInfo.installFiles.add(pkgInfo.name.addFileExt("nim"))
+
+proc readPackageInfo(nf: NimbleFile, options: Options,
     onlyMinimalInfo=false): PackageInfo =
   ## Reads package info from the specified Nimble file.
   ##
@@ -186,17 +325,20 @@ proc readPackageInfo*(nf: NimbleFile, options: Options,
   ## If both fail then returns an error.
   ##
   ## When ``onlyMinimalInfo`` is true, only the `name` and `version` fields are
-  ## populated. The isNimScript field can also be relied on.
+  ## populated. The ``isNimScript`` field can also be relied on.
   ##
   ## This version uses a cache stored in ``options``, so calling it multiple
   ## times on the same ``nf`` shouldn't require re-evaluation of the Nimble
   ## file.
 
+  assert fileExists(nf)
+
   # Check the cache.
   if options.pkgInfoCache.hasKey(nf):
     return options.pkgInfoCache[nf]
 
   result = initPackageInfo(nf)
+  let minimalInfo = getNameVersion(nf)
 
   validatePackageName(nf.splitFile.name)
 
@@ -212,61 +354,155 @@ proc readPackageInfo*(nf: NimbleFile, options: Options,
 
   if not success:
     if onlyMinimalInfo:
-      let tmp = getNameVersion(nf)
+      result.name = minimalInfo.name
-      result.name = tmp.name
+      result.version = minimalInfo.version
-      result.version = tmp.version
       result.isNimScript = true
       result.isMinimal = true
+
+      # It's possible this proc will receive a .nimble-link file eventually,
+      # I added this assert to hopefully make this error clear for everyone.
+      let msg = "No version detected. Received nimble-link?"
+      assert result.version.len > 0, msg
     else:
       try:
         readPackageInfoFromNims(nf, options, result)
         result.isNimScript = true
-      except NimbleError:
+      except NimbleError as exc:
+        if exc.hint.len > 0:
+          raise
         let msg = "Could not read package info file in " & nf & ";\n" &
                   "  Reading as ini file failed with: \n" &
                   "    " & iniError.msg & ".\n" &
                   "  Evaluating as NimScript file failed with: \n" &
-                  "    " & getCurrentExceptionMsg() & "."
+                  "    " & exc.msg & "."
         raise newException(NimbleError, msg)
 
-  validatePackageInfo(result, nf)
+  # By default specialVersion is the same as version.
+  result.specialVersion = result.version
+
+  # Only attempt to read a special version if `nf` is inside the $nimbleDir.
+  if nf.startsWith(options.getNimbleDir()):
+    # The package directory name may include a "special" version
+    # (example #head). If so, it is given higher priority and therefore
+    # overwrites the .nimble file's version.
+    let version = parseVersionRange(minimalInfo.version)
+    if version.kind == verSpecial:
+      result.specialVersion = minimalInfo.version
+
+  # Apply rules to infer which files should/shouldn't be installed. See #469.
+  inferInstallRules(result, options)
+
   if not result.isMinimal:
     options.pkgInfoCache[nf] = result
 
+  # Validate the rest of the package info last.
+  if not options.disableValidation:
+    validateVersion(result.version)
+    validatePackageInfo(result, options)
+
+proc validate*(file: NimbleFile, options: Options,
+               error: var ValidationError, pkgInfo: var PackageInfo): bool =
+  try:
+    pkgInfo = readPackageInfo(file, options)
+  except ValidationError as exc:
+    error = exc[]
+    return false
+
+  return true
+
+proc getPkgInfoFromFile*(file: NimbleFile, options: Options): PackageInfo =
+  ## Reads the specified .nimble file and returns its data as a PackageInfo
+  ## object. Any validation errors are handled and displayed as warnings.
+  try:
+    result = readPackageInfo(file, options)
+  except ValidationError:
+    let exc = (ref ValidationError)(getCurrentException())
+    if exc.warnAll:
+      display("Warning:", exc.msg, Warning, HighPriority)
+      display("Hint:", exc.hint, Warning, HighPriority)
+    else:
+      raise
+
 proc getPkgInfo*(dir: string, options: Options): PackageInfo =
   ## Find the .nimble file in ``dir`` and parses it, returning a PackageInfo.
   let nimbleFile = findNimbleFile(dir, true)
-  result = readPackageInfo(nimbleFile, options)
+  return getPkgInfoFromFile(nimbleFile, options)
 
 proc getInstalledPkgs*(libsDir: string, options: Options):
         seq[tuple[pkginfo: PackageInfo, meta: MetaData]] =
   ## Gets a list of installed packages.
   ##
   ## ``libsDir`` is in most cases: ~/.nimble/pkgs/
+  const
+    readErrorMsg = "Installed package '$1@$2' is outdated or corrupt."
+    validationErrorMsg = readErrorMsg & "\nPackage did not pass validation: $3"
+    hintMsg = "The corrupted package will need to be removed manually. To fix" &
+              " this error message, remove $1."
+
+  proc createErrorMsg(tmplt, path, msg: string): string =
+    let (name, version) = getNameVersion(path)
+    return tmplt % [name, version, msg]
+
+  display("Loading", "list of installed packages", priority = MediumPriority)
+
   result = @[]
   for kind, path in walkDir(libsDir):
     if kind == pcDir:
       let nimbleFile = findNimbleFile(path, false)
       if nimbleFile != "":
         let meta = readMetaData(path)
+        var pkg: PackageInfo
         try:
-          var pkg = readPackageInfo(nimbleFile, options, true)
+          pkg = readPackageInfo(nimbleFile, options, onlyMinimalInfo=false)
-          pkg.isInstalled = true
-          result.add((pkg, meta))
         except ValidationError:
           let exc = (ref ValidationError)(getCurrentException())
-          if exc.warnInstalled:
+          exc.msg = createErrorMsg(validationErrorMsg, path, exc.msg)
-            echo("WARNING: Unable to read package info for " & path & "\n" &
+          exc.hint = hintMsg % path
-                "  Package did not pass validation: " & exc.msg)
+          if exc.warnInstalled or exc.warnAll:
+            display("Warning:", exc.msg, Warning, HighPriority)
+            # Don't show hints here because they are only useful for package
+            # owners.
           else:
-            exc.msg = "Unable to read package info for " & path & "\n" &
-                "  Package did not pass validation: " & exc.msg
             raise exc
         except:
-          let exc = getCurrentException()
+          let tmplt = readErrorMsg & "\nMore info: $3"
-          exc.msg = "Unable to read package info for " & path & "\n" &
+          let msg = createErrorMsg(tmplt, path, getCurrentException().msg)
-              "  Error: " & exc.msg
+          var exc = newException(NimbleError, msg)
+          exc.hint = hintMsg % path
           raise exc
 
+        pkg.isInstalled = true
+        pkg.isLinked =
+          cmpPaths(nimbleFile.splitFile().dir, path) != 0
+        result.add((pkg, meta))
+
 proc isNimScript*(nf: string, options: Options): bool =
-  result = readPackageInfo(nf, options).isNimScript
+  result = readPackageInfo(nf, options).isNimScript
+
+proc toFullInfo*(pkg: PackageInfo, options: Options): PackageInfo =
+  if pkg.isMinimal:
+    result = getPkgInfoFromFile(pkg.mypath, options)
+    result.isInstalled = pkg.isInstalled
+    result.isLinked = pkg.isLinked
+  else:
+    return pkg
+
+proc getConcreteVersion*(pkgInfo: PackageInfo, options: Options): string =
+  ## Returns a non-special version from the specified ``pkgInfo``. If the
+  ## ``pkgInfo`` is minimal it looks it up and retrieves the concrete version.
+  result = pkgInfo.version
+  if pkgInfo.isMinimal:
+    let pkgInfo = pkgInfo.toFullInfo(options)
+    result = pkgInfo.version
+  assert(not newVersion(result).isSpecial)
+
+when isMainModule:
+  validatePackageName("foo_bar")
+  validatePackageName("f_oo_b_a_r")
+  try:
+    validatePackageName("foo__bar")
+    assert false
+  except NimbleError:
+    assert true
+
+  echo("Everything passed!")

src/nimblepkg/publish.nim

@@ -4,63 +4,103 @@
 ## Implements 'nimble publish' to create a pull request against
 ## nim-lang/packages automatically.
 
-import httpclient, base64, strutils, rdstdin, json, os, browsers, times, uri
+import system except TResult
-import tools, nimbletypes
+import httpclient, strutils, json, os, browsers, times, uri
+import version, tools, common, cli, config, options
 
 type
   Auth = object
     user: string
-    pw: string
+    token: string  ## Github access token
-    token: string  ## base64 encoding of user:pw
+    http: HttpClient ## http client for doing API requests
+
+const
+  ApiKeyFile = "github_api_token"
+  ApiTokenEnvironmentVariable = "NIMBLE_GITHUB_API_TOKEN"
+  ReposUrl = "https://api.github.com/repos/"
 
 proc userAborted() =
   raise newException(NimbleError, "User aborted the process.")
 
-proc createHeaders(a: Auth): string =
+proc createHeaders(a: Auth) =
-  (("Authorization: token $1\c\L" % a.token) &
+  a.http.headers = newHttpHeaders({
-          "Content-Type: application/x-www-form-urlencoded\c\L" &
+    "Authorization": "token $1" % a.token,
-          "Accept: */*\c\L")
+    "Content-Type": "application/x-www-form-urlencoded",
+    "Accept": "*/*"
+  })
 
-proc getGithubAuth(): Auth =
+proc requestNewToken(cfg: Config): string =
-  echo("Please create a new personal access token on Github in order to " &
+  display("Info:", "Please create a new personal access token on Github in" &
-      "allow Nimble to fork the packages repository.")
+          " order to allow Nimble to fork the packages repository.",
+          priority = HighPriority)
+  display("Hint:", "Make sure to give the access token access to public repos" &
+          " (public_repo scope)!", Warning, HighPriority)
   sleep(5000)
-  echo("Your default browser should open with the following URL: " &
+  display("Info:", "Your default browser should open with the following URL: " &
-      "https://github.com/settings/tokens/new")
+          "https://github.com/settings/tokens/new", priority = HighPriority)
   sleep(3000)
   openDefaultBrowser("https://github.com/settings/tokens/new")
-  result.token = readLineFromStdin("Personal access token: ").strip()
+  let token = promptCustom("Personal access token?", "").strip()
-  let resp = getContent("https://api.github.com/user",
+  # inform the user that their token will be written to disk
-        extraHeaders=createHeaders(result)).parseJson()
+  let tokenWritePath = cfg.nimbleDir / ApiKeyFile
+  display("Info:", "Writing access token to file:" & tokenWritePath, 
+          priority = HighPriority)
+  writeFile(tokenWritePath, token)
+  sleep(3000)
+  return token
+
+proc getGithubAuth(o: Options): Auth =
+  let cfg = o.config
+  result.http = newHttpClient(proxy = getProxy(o))
+  # always prefer the environment variable to asking for a new one
+  if existsEnv(ApiTokenEnvironmentVariable):
+    result.token = getEnv(ApiTokenEnvironmentVariable)
+    display("Info:", "Using the '" & ApiTokenEnvironmentVariable & 
+            "' environment variable for the GitHub API Token.",
+            priority = HighPriority)
+  else:
+    # try to read from disk, if it cannot be found write a new one
+    try:
+      let apiTokenFilePath = cfg.nimbleDir / ApiKeyFile
+      result.token = readFile(apiTokenFilePath).strip()
+      display("Info:", "Using GitHub API Token in file: " & apiTokenFilePath,
+              priority = HighPriority)
+    except IOError:
+      result.token = requestNewToken(cfg)
+  createHeaders(result)
+  let resp = result.http.getContent("https://api.github.com/user").parseJson()
 
   result.user = resp["login"].str
-  echo("Successfully verified as ", result.user)
+  display("Success:", "Verified as " & result.user, Success, HighPriority)
 
 proc isCorrectFork(j: JsonNode): bool =
   # Check whether this is a fork of the nimble packages repo.
   result = false
-  if j{"fork"}.getBVal():
+  if j{"fork"}.getBool():
     result = j{"parent"}{"full_name"}.getStr() == "nim-lang/packages"
 
 proc forkExists(a: Auth): bool =
   try:
-    let x = getContent("https://api.github.com/repos/" & a.user & "/packages",
+    let x = a.http.getContent(ReposUrl & a.user & "/packages")
-        extraHeaders=createHeaders(a))
     let j = parseJson(x)
     result = isCorrectFork(j)
   except JsonParsingError, IOError:
     result = false
 
 proc createFork(a: Auth) =
-  discard postContent("https://api.github.com/repos/nim-lang/packages/forks",
+  try:
-      extraHeaders=createHeaders(a))
+    discard a.http.postContent(ReposUrl & "nim-lang/packages/forks")
+  except HttpRequestError:
+    raise newException(NimbleError, "Unable to create fork. Access token" &
+                       " might not have enough permissions.")
 
-proc createPullRequest(a: Auth, packageName, branch: string) =
+proc createPullRequest(a: Auth, packageName, branch: string): string =
-  echo("Creating PR")
+  display("Info", "Creating PR", priority = HighPriority)
-  discard postContent("https://api.github.com/repos/nim-lang/packages/pulls",
+  var body = a.http.postContent(ReposUrl & "nim-lang/packages/pulls",
-      extraHeaders=createHeaders(a),
       body="""{"title": "Add package $1", "head": "$2:$3",
                "base": "master"}""" % [packageName, a.user, branch])
+  var pr = parseJson(body)
+  return pr{"html_url"}.getStr()
 
 proc `%`(s: openArray[string]): JsonNode =
   result = newJArray()
@@ -101,48 +141,43 @@ proc cleanupWhitespace(s: string): string =
 proc editJson(p: PackageInfo; url, tags, downloadMethod: string) =
   var contents = parseFile("packages.json")
   doAssert contents.kind == JArray
-  contents.add(%{
+  contents.add(%*{
-    "name": %p.name,
+    "name": p.name,
-    "url": %url,
+    "url": url,
-    "method": %downloadMethod,
+    "method": downloadMethod,
-    "tags": %tags.split(),
+    "tags": tags.split(),
-    "description": %p.description,
+    "description": p.description,
-    "license": %p.license,
+    "license": p.license,
-    "web": %url})
+    "web": url
+  })
   writeFile("packages.json", contents.pretty.cleanupWhitespace)
 
-proc getPackageOriginUrl(a: Auth): string =
+proc publish*(p: PackageInfo, o: Options) =
-  ## Adds 'user:pw' to the URL so that the user is not asked *again* for it.
-  ## We need this for 'git push'.
-  let (output, exitCode) = doCmdEx("git config --get remote.origin.url")
-  result = "origin"
-  if exitCode == 0:
-    result = output.string.strip
-    if result.endsWith(".git"): result.setLen(result.len - 4)
-    if result.startsWith("https://"):
-      result = "https://" & a.user & ':' & a.pw & '@' &
-          result["https://".len .. ^1]
-
-proc publish*(p: PackageInfo) =
   ## Publishes the package p.
-  let auth = getGithubAuth()
+  let auth = getGithubAuth(o)
-  var pkgsDir = getTempDir() / "nimble-packages-fork"
+  var pkgsDir = getNimbleUserTempDir() / "nimble-packages-fork"
   if not forkExists(auth):
     createFork(auth)
-    echo "waiting 10s to let Github create a fork ..."
+    display("Info:", "Waiting 10s to let Github create a fork",
+            priority = HighPriority)
     os.sleep(10_000)
 
-    echo "... done"
+    display("Info:", "Finished waiting", priority = LowPriority)
   if dirExists(pkgsDir):
-    echo("Removing old packages fork git directory.")
+    display("Removing", "old packages fork git directory.",
+            priority = LowPriority)
     removeDir(pkgsDir)
-  echo "Cloning packages into: ", pkgsDir
+  createDir(pkgsDir)
-  doCmd("git clone git@github.com:" & auth.user & "/packages " & pkgsDir)
-  # Make sure to update the clone.
-  echo("Updating the fork...")
   cd pkgsDir:
+    # Avoid git clone to prevent token from being stored in repo
+    # https://github.com/blog/1270-easier-builds-and-deployments-using-git-over-https-and-oauth
+    display("Copying", "packages fork into: " & pkgsDir, priority = HighPriority)
+    doCmd("git init")
+    doCmd("git pull https://github.com/" & auth.user & "/packages")
+    # Make sure to update the fork
+    display("Updating", "the fork", priority = HighPriority)
     doCmd("git pull https://github.com/nim-lang/packages.git master")
-    doCmd("git push origin master")
+    doCmd("git push https://" & auth.token & "@github.com/" & auth.user & "/packages master")
 
   if not dirExists(pkgsDir):
     raise newException(NimbleError,
@@ -157,7 +192,7 @@ proc publish*(p: PackageInfo) =
   var url = ""
   var downloadMethod = ""
   if dirExists(os.getCurrentDir() / ".git"):
-    let (output, exitCode) = doCmdEx("git config --get remote.origin.url")
+    let (output, exitCode) = doCmdEx("git ls-remote --get-url")
     if exitCode == 0:
       url = output.string.strip
       if url.endsWith(".git"): url.setLen(url.len - 4)
@@ -166,7 +201,7 @@ proc publish*(p: PackageInfo) =
     if parsed.scheme == "":
       # Assuming that we got an ssh write/read URL.
       let sshUrl = parseUri("ssh://" & url)
-      url = "https://github.com/" & sshUrl.port & sshUrl.path
+      url = "https://" & sshUrl.hostname & "/" & sshUrl.port & sshUrl.path
   elif dirExists(os.getCurrentDir() / ".hg"):
     downloadMethod = "hg"
     # TODO: Retrieve URL from hg.
@@ -175,22 +210,20 @@ proc publish*(p: PackageInfo) =
          "No .git nor .hg directory found. Stopping.")
 
   if url.len == 0:
-    url = readLineFromStdin("Github URL of " & p.name & ": ")
+    url = promptCustom("Github URL of " & p.name & "?", "")
     if url.len == 0: userAborted()
 
-  let tags = readLineFromStdin("Please enter a whitespace separated list of tags: ")
+  let tags = promptCustom(
+    "Whitespace separated list of tags? (For example: web library wrapper)",
+    ""
+  )
 
   cd pkgsDir:
     editJson(p, url, tags, downloadMethod)
-    let branchName = "add-" & p.name & getTime().getGMTime().format("HHmm")
+    let branchName = "add-" & p.name & getTime().utc.format("HHmm")
     doCmd("git checkout -B " & branchName)
     doCmd("git commit packages.json -m \"Added package " & p.name & "\"")
-    echo("Pushing to remote of fork.")
+    display("Pushing", "to remote of fork.", priority = HighPriority)
-    doCmd("git push " & getPackageOriginUrl(auth) & " " & branchName)
+    doCmd("git push https://" & auth.token & "@github.com/" & auth.user & "/packages " & branchName)
-    createPullRequest(auth, p.name, branchName)
+    let prUrl = createPullRequest(auth, p.name, branchName)
-  echo "Pull request successful."
+  display("Success:", "Pull request successful, check at " & prUrl , Success, HighPriority)
-
-when isMainModule:
-  import packageinfo
-  var p = getPkgInfo(getCurrentDir())
-  publish(p)

src/nimblepkg/reversedeps.nim

@@ -0,0 +1,135 @@
+# Copyright (C) Dominik Picheta. All rights reserved.
+# BSD License. Look at license.txt for more info.
+
+import os, json, sets
+
+import options, common, version, download, packageinfo
+
+proc saveNimbleData*(options: Options) =
+  # TODO: This file should probably be locked.
+  writeFile(options.getNimbleDir() / "nimbledata.json",
+            pretty(options.nimbleData))
+
+proc addRevDep*(nimbleData: JsonNode, dep: tuple[name, version: string],
+                pkg: PackageInfo) =
+  # Add a record which specifies that `pkg` has a dependency on `dep`, i.e.
+  # the reverse dependency of `dep` is `pkg`.
+  if not nimbleData["reverseDeps"].hasKey(dep.name):
+    nimbleData["reverseDeps"][dep.name] = newJObject()
+  if not nimbleData["reverseDeps"][dep.name].hasKey(dep.version):
+    nimbleData["reverseDeps"][dep.name][dep.version] = newJArray()
+  let revDep = %{ "name": %pkg.name, "version": %pkg.specialVersion}
+  let thisDep = nimbleData["reverseDeps"][dep.name][dep.version]
+  if revDep notin thisDep:
+    thisDep.add revDep
+
+proc removeRevDep*(nimbleData: JsonNode, pkg: PackageInfo) =
+  ## Removes ``pkg`` from the reverse dependencies of every package.
+  assert(not pkg.isMinimal)
+  proc remove(pkg: PackageInfo, depTup: PkgTuple, thisDep: JsonNode) =
+    for ver, val in thisDep:
+      if ver.newVersion in depTup.ver:
+        var newVal = newJArray()
+        for revDep in val:
+          if not (revDep["name"].str == pkg.name and
+                  revDep["version"].str == pkg.specialVersion):
+            newVal.add revDep
+        thisDep[ver] = newVal
+
+  for depTup in pkg.requires:
+    if depTup.name.isURL():
+      # We sadly must go through everything in this case...
+      for key, val in nimbleData["reverseDeps"]:
+        remove(pkg, depTup, val)
+    else:
+      let thisDep = nimbleData{"reverseDeps", depTup.name}
+      if thisDep.isNil: continue
+      remove(pkg, depTup, thisDep)
+
+  # Clean up empty objects/arrays
+  var newData = newJObject()
+  for key, val in nimbleData["reverseDeps"]:
+    if val.len != 0:
+      var newVal = newJObject()
+      for ver, elem in val:
+        if elem.len != 0:
+          newVal[ver] = elem
+      if newVal.len != 0:
+        newData[key] = newVal
+  nimbleData["reverseDeps"] = newData
+
+proc getRevDepTups*(options: Options, pkg: PackageInfo): seq[PkgTuple] =
+  ## Returns a list of *currently installed* reverse dependencies for `pkg`.
+  result = @[]
+  let thisPkgsDep =
+    options.nimbleData["reverseDeps"]{pkg.name}{pkg.specialVersion}
+  if not thisPkgsDep.isNil:
+    let pkgList = getInstalledPkgsMin(options.getPkgsDir(), options)
+    for pkg in thisPkgsDep:
+      let pkgTup = (
+        name: pkg["name"].getStr(),
+        ver: parseVersionRange(pkg["version"].getStr())
+      )
+      var pkgInfo: PackageInfo
+      if not findPkg(pkgList, pkgTup, pkgInfo):
+        continue
+
+      result.add(pkgTup)
+
+proc getRevDeps*(options: Options, pkg: PackageInfo): HashSet[PackageInfo] =
+  result.init()
+  let installedPkgs = getInstalledPkgsMin(options.getPkgsDir(), options)
+  for rdepTup in getRevDepTups(options, pkg):
+    for rdepInfo in findAllPkgs(installedPkgs, rdepTup):
+      result.incl rdepInfo
+
+proc getAllRevDeps*(options: Options, pkg: PackageInfo, result: var HashSet[PackageInfo]) =
+  if pkg in result:
+    return
+
+  let installedPkgs = getInstalledPkgsMin(options.getPkgsDir(), options)
+  for rdepTup in getRevDepTups(options, pkg):
+    for rdepInfo in findAllPkgs(installedPkgs, rdepTup):
+      if rdepInfo in result:
+        continue
+
+      getAllRevDeps(options, rdepInfo, result)
+  result.incl pkg
+
+when isMainModule:
+  var nimbleData = %{"reverseDeps": newJObject()}
+
+  let nimforum1 = PackageInfo(
+    isMinimal: false,
+    name: "nimforum",
+    specialVersion: "0.1.0",
+    requires: @[("jester", parseVersionRange("0.1.0")),
+                ("captcha", parseVersionRange("1.0.0")),
+                ("auth", parseVersionRange("#head"))]
+  )
+  let nimforum2 = PackageInfo(isMinimal: false, name: "nimforum", specialVersion: "0.2.0")
+  let play = PackageInfo(isMinimal: false, name: "play", specialVersion: "#head")
+
+  nimbleData.addRevDep(("jester", "0.1.0"), nimforum1)
+  nimbleData.addRevDep(("jester", "0.1.0"), play)
+  nimbleData.addRevDep(("captcha", "1.0.0"), nimforum1)
+  nimbleData.addRevDep(("auth", "#head"), nimforum1)
+  nimbleData.addRevDep(("captcha", "1.0.0"), nimforum2)
+  nimbleData.addRevDep(("auth", "#head"), nimforum2)
+
+  doAssert nimbleData["reverseDeps"]["jester"]["0.1.0"].len == 2
+  doAssert nimbleData["reverseDeps"]["captcha"]["1.0.0"].len == 2
+  doAssert nimbleData["reverseDeps"]["auth"]["#head"].len == 2
+
+  block:
+    nimbleData.removeRevDep(nimforum1)
+    let jester = nimbleData["reverseDeps"]["jester"]["0.1.0"][0]
+    doAssert jester["name"].getStr() == play.name
+    doAssert jester["version"].getStr() == play.specialVersion
+
+    let captcha = nimbleData["reverseDeps"]["captcha"]["1.0.0"][0]
+    doAssert captcha["name"].getStr() == nimforum2.name
+    doAssert captcha["version"].getStr() == nimforum2.specialVersion
+
+  echo("Everything works!")
+

src/nimblepkg/tools.nim

@@ -2,11 +2,17 @@
 # BSD License. Look at license.txt for more info.
 #
 # Various miscellaneous utility functions reside here.
-import osproc, pegs, strutils, os, uri, sets, json
+import osproc, pegs, strutils, os, uri, sets, json, parseutils
-import version, packageinfo, nimbletypes
+import version, cli
 
-proc doCmd*(cmd: string) =
+proc extractBin(cmd: string): string =
-  let bin = cmd.split(' ')[0]
+  if cmd[0] == '"':
+    return cmd.captureBetween('"')
+  else:
+    return cmd.split(' ')[0]
+
+proc doCmd*(cmd: string, showOutput = false, showCmd = false) =
+  let bin = extractBin(cmd)
   if findExe(bin) == "":
     raise newException(NimbleError, "'" & bin & "' not in PATH.")
 
@@ -14,19 +20,34 @@ proc doCmd*(cmd: string) =
   stdout.flushFile()
   stderr.flushFile()
 
-  let exitCode = execCmd(cmd)
+  if showCmd:
+    display("Executing", cmd, priority = MediumPriority)
+  else:
+    displayDebug("Executing", cmd)
+  if showOutput:
+    let exitCode = execCmd(cmd)
+    displayDebug("Finished", "with exit code " & $exitCode)
+    if exitCode != QuitSuccess:
+      raise newException(NimbleError,
+          "Execution failed with exit code $1\nCommand: $2" %
+          [$exitCode, cmd])
+  else:
+    let (output, exitCode) = execCmdEx(cmd)
+    displayDebug("Finished", "with exit code " & $exitCode)
+    displayDebug("Output", output)
 
-  if exitCode != QuitSuccess:
+    if exitCode != QuitSuccess:
-    raise newException(NimbleError,
+      raise newException(NimbleError,
-        "Execution failed with exit code " & $exitCode)
+          "Execution failed with exit code $1\nCommand: $2\nOutput: $3" %
+          [$exitCode, cmd, output])
 
 proc doCmdEx*(cmd: string): tuple[output: TaintedString, exitCode: int] =
-  let bin = cmd.split(' ')[0]
+  let bin = extractBin(cmd)
   if findExe(bin) == "":
     raise newException(NimbleError, "'" & bin & "' not in PATH.")
   return execCmdEx(cmd)
 
-template cd*(dir: string, body: stmt) =
+template cd*(dir: string, body: untyped) =
   ## Sets the current dir to ``dir``, executes ``body`` and restores the
   ## previous working dir.
   let lastDir = getCurrentDir()
@@ -41,10 +62,10 @@ proc getNimBin*: string =
 
 proc getNimrodVersion*: Version =
   let nimBin = getNimBin()
-  let vOutput = doCmdEx(nimBin & " -v").output
+  let vOutput = doCmdEx('"' & nimBin & "\" -v").output
   var matches: array[0..MaxSubpatterns, string]
   if vOutput.find(peg"'Version'\s{(\d+\.)+\d}", matches) == -1:
-    quit("Couldn't find Nim version.", QuitFailure)
+    raise newException(NimbleError, "Couldn't find Nim version.")
   newVersion(matches[0])
 
 proc samePaths*(p1, p2: string): bool =
@@ -61,26 +82,37 @@ proc changeRoot*(origRoot, newRoot, path: string): string =
   ## newRoot:  /home/test/
   ## path:     /home/dom/bar/blah/2/foo.txt
   ## Return value -> /home/test/bar/blah/2/foo.txt
-  if path.startsWith(origRoot):
+
-    return newRoot / path[origRoot.len .. path.len-1]
+  ## The additional check of `path.samePaths(origRoot)` is necessary to prevent
+  ## a regression, where by ending the `srcDir` defintion in a nimble file in a
+  ## trailing separator would cause the `path.startsWith(origRoot)` evaluation to
+  ## fail because of the value of `origRoot` would be longer than `path` due to
+  ## the trailing separator. This would cause this method to throw during package
+  ## installation.
+  if path.startsWith(origRoot) or path.samePaths(origRoot):
+    return newRoot / path.substr(origRoot.len, path.len-1)
   else:
     raise newException(ValueError,
       "Cannot change root of path: Path does not begin with original root.")
 
 proc copyFileD*(fro, to: string): string =
   ## Returns the destination (``to``).
-  echo(fro, " -> ", to)
+  display("Copying", "file $# to $#" % [fro, to], priority = LowPriority)
-  copyFile(fro, to)
+  copyFileWithPermissions(fro, to)
   result = to
 
 proc copyDirD*(fro, to: string): seq[string] =
   ## Returns the filenames of the files in the directory that were copied.
   result = @[]
-  echo("Copying directory: ", fro, " -> ", to)
+  display("Copying", "directory $# to $#" % [fro, to], priority = LowPriority)
   for path in walkDirRec(fro):
     createDir(changeRoot(fro, to, path.splitFile.dir))
     result.add copyFileD(path, changeRoot(fro, to, path))
 
+proc createDirD*(dir: string) =
+  display("Creating", "directory $#" % dir, priority = LowPriority)
+  createDir(dir)
+
 proc getDownloadDirName*(uri: string, verRange: VersionRange): string =
   ## Creates a directory name based on the specified ``uri`` (url)
   result = ""
@@ -106,12 +138,47 @@ proc incl*(s: var HashSet[string], v: seq[string] | HashSet[string]) =
   for i in v:
     s.incl i
 
-proc contains*(j: JsonNode, elem: JsonNode): bool =
+when not declared(json.contains):
-  for i in j:
+  proc contains*(j: JsonNode, elem: JsonNode): bool =
-    if i == elem:
+    for i in j:
-      return true
+      if i == elem:
+        return true
 
 proc contains*(j: JsonNode, elem: tuple[key: string, val: JsonNode]): bool =
   for key, val in pairs(j):
     if key == elem.key and val == elem.val:
       return true
+
+when not defined(windows):
+  from posix import getpid
+
+proc getProcessId*(): string =
+  when defined(windows):
+    proc GetCurrentProcessId(): int32 {.stdcall, dynlib: "kernel32",
+                                        importc: "GetCurrentProcessId".}
+    result = $GetCurrentProcessId()
+  else:
+    result = $getpid()
+
+proc getNimbleTempDir*(): string =
+  ## Returns a path to a temporary directory.
+  ##
+  ## The returned path will be the same for the duration of the process but
+  ## different for different runs of it. You have to make sure to create it
+  ## first. In release builds the directory will be removed when nimble finishes
+  ## its work.
+  result = getTempDir() / "nimble_" & getProcessId()
+
+proc getNimbleUserTempDir*(): string =
+  ## Returns a path to a temporary directory.
+  ##
+  ## The returned path will be the same for the duration of the process but
+  ## different for different runs of it. You have to make sure to create it
+  ## first. In release builds the directory will be removed when nimble finishes
+  ## its work.
+  var tmpdir: string
+  if existsEnv("TMPDIR") and existsEnv("USER"):
+    tmpdir = joinPath(getEnv("TMPDIR"), getEnv("USER"))
+  else:
+    tmpdir = getTempDir()
+  return tmpdir

src/nimblepkg/version.nim

@@ -5,7 +5,6 @@
 import strutils, tables, hashes, parseutils
 type
   Version* = distinct string
-  Special* = distinct string
 
   VersionRangeEnum* = enum
     verLater, # > V
@@ -23,7 +22,7 @@ type
     of verLater, verEarlier, verEqLater, verEqEarlier, verEq:
       ver*: Version
     of verSpecial:
-      spe*: Special
+      spe*: Version
     of verIntersect:
       verILeft, verIRight: VersionRange
     of verAny:
@@ -34,19 +33,32 @@ type
 
   ParseVersionError* = object of ValueError
   NimbleError* = object of Exception
-
+    hint*: string
-proc newVersion*(ver: string): Version = return Version(ver)
-proc newSpecial*(spe: string): Special = return Special(spe)
 
 proc `$`*(ver: Version): string {.borrow.}
 
-proc hash*(ver: Version): THash {.borrow.}
+proc hash*(ver: Version): Hash {.borrow.}
 
-proc `$`*(ver: Special): string {.borrow.}
+proc newVersion*(ver: string): Version =
+  doAssert(ver.len == 0 or ver[0] in {'#', '\0'} + Digits,
+           "Wrong version: " & ver)
+  return Version(ver)
 
-proc hash*(ver: Special): THash {.borrow.}
+proc isSpecial*(ver: Version): bool =
+  return ($ver).len > 0 and ($ver)[0] == '#'
 
 proc `<`*(ver: Version, ver2: Version): bool =
+  # Handling for special versions such as "#head" or "#branch".
+  if ver.isSpecial or ver2.isSpecial:
+    # TODO: This may need to be reverted. See #311.
+    if ver2.isSpecial and ($ver2).normalize == "#head":
+      return ($ver).normalize != "#head"
+
+    if not ver2.isSpecial:
+      # `#aa111 < 1.1`
+      return ($ver).normalize != "#head"
+
+  # Handling for normal versions such as "0.1.0" or "1.0".
   var sVer = string(ver).split('.')
   var sVer2 = string(ver2).split('.')
   for i in 0..max(sVer.len, sVer2.len)-1:
@@ -64,6 +76,9 @@ proc `<`*(ver: Version, ver2: Version): bool =
       return false
 
 proc `==`*(ver: Version, ver2: Version): bool =
+  if ver.isSpecial or ver2.isSpecial:
+    return ($ver).toLowerAscii() == ($ver2).toLowerAscii()
+
   var sVer = string(ver).split('.')
   var sVer2 = string(ver2).split('.')
   for i in 0..max(sVer.len, sVer2.len)-1:
@@ -78,8 +93,10 @@ proc `==`*(ver: Version, ver2: Version): bool =
     else:
       return false
 
-proc `==`*(spe: Special, spe2: Special): bool =
+proc cmp*(a, b: Version): int =
-  return ($spe).toLower() == ($spe2).toLower()
+  if a < b: -1
+  elif a > b: 1
+  else: 0
 
 proc `<=`*(ver: Version, ver2: Version): bool =
   return (ver == ver2) or (ver < ver2)
@@ -108,64 +125,54 @@ proc withinRange*(ver: Version, ran: VersionRange): bool =
   of verEq:
     return ver == ran.ver
   of verSpecial:
-    return false
+    return ver == ran.spe
   of verIntersect:
     return withinRange(ver, ran.verILeft) and withinRange(ver, ran.verIRight)
   of verAny:
     return true
 
-proc withinRange*(spe: Special, ran: VersionRange): bool =
-  case ran.kind
-  of verLater, verEarlier, verEqLater, verEqEarlier, verEq, verIntersect:
-    return false
-  of verSpecial:
-    return spe == ran.spe
-  of verAny:
-    return true
-
 proc contains*(ran: VersionRange, ver: Version): bool =
   return withinRange(ver, ran)
 
-proc contains*(ran: VersionRange, spe: Special): bool =
-  return withinRange(spe, ran)
-
 proc makeRange*(version: string, op: string): VersionRange =
-  new(result)
   if version == "":
     raise newException(ParseVersionError,
         "A version needs to accompany the operator.")
   case op
   of ">":
-    result.kind = verLater
+    result = VersionRange(kind: verLater)
   of "<":
-    result.kind = verEarlier
+    result = VersionRange(kind: verEarlier)
   of ">=":
-    result.kind = verEqLater
+    result = VersionRange(kind: verEqLater)
   of "<=":
-    result.kind = verEqEarlier
+    result = VersionRange(kind: verEqEarlier)
-  of "":
+  of "", "==":
-    result.kind = verEq
+    result = VersionRange(kind: verEq)
   else:
     raise newException(ParseVersionError, "Invalid operator: " & op)
   result.ver = Version(version)
 
 proc parseVersionRange*(s: string): VersionRange =
   # >= 1.5 & <= 1.8
-  new(result)
+  if s.len == 0:
+    result = VersionRange(kind: verAny)
+    return
+
   if s[0] == '#':
-    result.kind = verSpecial
+    result = VersionRange(kind: verSpecial)
-    result.spe = s[1 .. s.len-1].Special
+    result.spe = s.Version
     return
 
   var i = 0
   var op = ""
   var version = ""
-  while true:
+  while i < s.len:
     case s[i]
     of '>', '<', '=':
       op.add(s[i])
     of '&':
-      result.kind = verIntersect
+      result = VersionRange(kind: verIntersect)
       result.verILeft = makeRange(version, op)
 
       # Parse everything after &
@@ -178,27 +185,33 @@ proc parseVersionRange*(s: string): VersionRange =
         raise newException(ParseVersionError,
             "Having more than one `&` in a version range is pointless")
 
-      break
+      return
 
     of '0'..'9', '.':
       version.add(s[i])
 
-    of '\0':
-      result = makeRange(version, op)
-      break
-
     of ' ':
       # Make sure '0.9 8.03' is not allowed.
-      if version != "" and i < s.len:
+      if version != "" and i < s.len - 1:
         if s[i+1] in {'0'..'9', '.'}:
           raise newException(ParseVersionError,
               "Whitespace is not allowed in a version literal.")
 
     else:
       raise newException(ParseVersionError,
-          "Unexpected char in version range: " & s[i])
+          "Unexpected char in version range '" & s & "': " & s[i])
     inc(i)
+  result = makeRange(version, op)
 
+proc toVersionRange*(ver: Version): VersionRange =
+  ## Converts a version to either a verEq or verSpecial VersionRange.
+  new(result)
+  if ver.isSpecial:
+    result = VersionRange(kind: verSpecial)
+    result.spe = ver
+  else:
+    result = VersionRange(kind: verEq)
+    result.ver = ver
 
 proc parseRequires*(req: string): PkgTuple =
   try:
@@ -230,7 +243,7 @@ proc `$`*(verRange: VersionRange): string =
   of verEq:
     result = ""
   of verSpecial:
-    return "#" & $verRange.spe
+    return $verRange.spe
   of verIntersect:
     return $verRange.verILeft & " & " & $verRange.verIRight
   of verAny:
@@ -253,41 +266,42 @@ proc getSimpleString*(verRange: VersionRange): string =
     result = ""
 
 proc newVRAny*(): VersionRange =
-  new(result)
+  result = VersionRange(kind: verAny)
-  result.kind = verAny
 
 proc newVREarlier*(ver: string): VersionRange =
-  new(result)
+  result = VersionRange(kind: verEarlier)
-  result.kind = verEarlier
   result.ver = newVersion(ver)
 
 proc newVREq*(ver: string): VersionRange =
-  new(result)
+  result = VersionRange(kind: verEq)
-  result.kind = verEq
   result.ver = newVersion(ver)
 
 proc findLatest*(verRange: VersionRange,
-        versions: Table[Version, string]): tuple[ver: Version, tag: string] =
+        versions: OrderedTable[Version, string]): tuple[ver: Version, tag: string] =
   result = (newVersion(""), "")
   for ver, tag in versions:
     if not withinRange(ver, verRange): continue
     if ver > result.ver:
       result = (ver, tag)
 
+proc `$`*(dep: PkgTuple): string =
+  return dep.name & "@" & $dep.ver
+
 when isMainModule:
   doAssert(newVersion("1.0") < newVersion("1.4"))
   doAssert(newVersion("1.0.1") > newVersion("1.0"))
   doAssert(newVersion("1.0.6") <= newVersion("1.0.6"))
-  #doAssert(not withinRange(newVersion("0.1.0"), parseVersionRange("> 0.1")))
+  doAssert(not withinRange(newVersion("0.1.0"), parseVersionRange("> 0.1")))
   doAssert(not (newVersion("0.1.0") < newVersion("0.1")))
   doAssert(not (newVersion("0.1.0") > newVersion("0.1")))
   doAssert(newVersion("0.1.0") < newVersion("0.1.0.0.1"))
   doAssert(newVersion("0.1.0") <= newVersion("0.1"))
 
   var inter1 = parseVersionRange(">= 1.0 & <= 1.5")
+  doAssert(inter1.kind == verIntersect)
   var inter2 = parseVersionRange("1.0")
   doAssert(inter2.kind == verEq)
-  #echo(parseVersionRange(">= 0.8 0.9"))
+  doAssert(parseVersionRange("== 3.4.2") == parseVersionRange("3.4.2"))
 
   doAssert(not withinRange(newVersion("1.5.1"), inter1))
   doAssert(withinRange(newVersion("1.0.2.3.4.5.6.7.8.9.10.11.12"), inter1))
@@ -301,8 +315,11 @@ when isMainModule:
   doAssert(newVersion("") < newVersion("1.0.0"))
   doAssert(newVersion("") < newVersion("0.1.0"))
 
-  var versions = toTable[Version, string]({newVersion("0.1.1"): "v0.1.1",
+  var versions = toOrderedTable[Version, string]({
-      newVersion("0.2.3"): "v0.2.3", newVersion("0.5"): "v0.5"})
+    newVersion("0.1.1"): "v0.1.1",
+    newVersion("0.2.3"): "v0.2.3",
+    newVersion("0.5"): "v0.5"
+  })
   doAssert findLatest(parseVersionRange(">= 0.1 & <= 0.4"), versions) ==
       (newVersion("0.2.3"), "v0.2.3")
 
@@ -311,13 +328,34 @@ when isMainModule:
   #doAssert newVersion("0.1-rc1") < newVersion("0.1")
 
   # Special tests
-  doAssert newSpecial("ab26sgdt362") != newSpecial("ab26saggdt362")
+  doAssert newVersion("#ab26sgdt362") != newVersion("#qwersaggdt362")
-  doAssert newSpecial("ab26saggdt362") == newSpecial("ab26saggdt362")
+  doAssert newVersion("#ab26saggdt362") == newVersion("#ab26saggdt362")
-  doAssert newSpecial("head") == newSpecial("HEAD")
+  doAssert newVersion("#head") == newVersion("#HEAD")
-  doAssert newSpecial("head") == newSpecial("head")
+  doAssert newVersion("#head") == newVersion("#head")
 
   var sp = parseVersionRange("#ab26sgdt362")
-  doAssert newSpecial("ab26sgdt362") in sp
+  doAssert newVersion("#ab26sgdt362") in sp
-  doAssert newSpecial("ab26saggdt362") notin sp
+  doAssert newVersion("#ab26saggdt362") notin sp
+
+  doAssert newVersion("#head") in parseVersionRange("#head")
+
+  # We assume that #head > 0.1.0, in practice this shouldn't be a problem.
+  doAssert(newVersion("#head") > newVersion("0.1.0"))
+  doAssert(not(newVersion("#head") > newVersion("#head")))
+  doAssert(withinRange(newVersion("#head"), parseVersionRange(">= 0.5.0")))
+  doAssert newVersion("#a111") < newVersion("#head")
+  # We assume that all other special versions are not higher than a normal
+  # version.
+  doAssert newVersion("#a111") < newVersion("1.1")
+
+  # An empty version range should give verAny
+  doAssert parseVersionRange("").kind == verAny
+
+  # toVersionRange tests
+  doAssert toVersionRange(newVersion("#head")).kind == verSpecial
+  doAssert toVersionRange(newVersion("0.2.0")).kind == verEq
+
+  # Something raised on IRC
+  doAssert newVersion("1") == newVersion("1.0")
 
   echo("Everything works!")

tests/.gitignore

@@ -0,0 +1,23 @@
+tester
+/nimble-test
+/buildDir
+/binaryPackage/v1/binaryPackage
+/binaryPackage/v2/binaryPackage
+/develop/dependent/src/dependent
+/issue27/issue27
+/issue206/issue/issue206bin
+/issue289/issue289
+/issue428/nimbleDir/
+/nimbleDir/
+/packageStructure/c/c
+/packageStructure/y/y
+/testCommand/testOverride/myTester
+/testCommand/testsFail/tests/a
+/testCommand/testsFail/tests/b
+/testCommand/testsPass/tests/one
+/testCommand/testsPass/tests/three
+/testCommand/testsPass/tests/two
+/nimscript/nimscript
+/packageStructure/validBinary/y
+/testCommand/testsFail/tests/t2
+/passNimFlags/passNimFlags

tests/binaryPackage/v1/binaryPackage.nim

@@ -0,0 +1 @@
+echo("v1")

tests/binaryPackage/v1/binaryPackage.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "1.0"
+author        = "Dominik Picheta"
+description   = "binary"
+license       = "MIT"
+
+bin = @["binaryPackage"]
+
+# Dependencies
+
+requires "nim >= 0.15.3"

tests/binaryPackage/v2/binaryPackage.nim

@@ -0,0 +1 @@
+echo("v2")

tests/binaryPackage/v2/binaryPackage.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "2.0"
+author        = "Dominik Picheta"
+description   = "binary"
+license       = "MIT"
+
+bin = @["binaryPackage"]
+
+# Dependencies
+
+requires "nim >= 0.15.3"

tests/caching/caching.nimble

@@ -0,0 +1,10 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Test package"
+license       = "BSD"
+
+# Dependencies
+
+requires "nim >= 0.12.1"

tests/develop/binary/binary.nim

@@ -0,0 +1 @@
+echo("hello")

tests/develop/binary/binary.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "1.0"
+author        = "Dominik Picheta"
+description   = "binary"
+license       = "MIT"
+
+bin = @["binary"]
+
+skipExt = @["nim"]
+
+# Dependencies
+
+requires "nim >= 0.16.0"

tests/develop/dependent/dependent.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "1.0"
+author        = "Dominik Picheta"
+description   = "dependent"
+license       = "MIT"
+
+srcDir = "src"
+
+# Dependencies
+
+requires "nim >= 0.16.0", "srcdirtest"

tests/develop/dependent/src/dependent.nim

@@ -0,0 +1,3 @@
+import srcdirtest
+
+doAssert foo() == "correct"

tests/develop/hybrid/hybrid.nim

@@ -0,0 +1 @@
+echo("hello")

tests/develop/hybrid/hybrid.nimble

@@ -0,0 +1,13 @@
+# Package
+
+version       = "1.0"
+author        = "Dominik Picheta"
+description   = "hybrid"
+license       = "MIT"
+
+bin = @["hybrid"]
+installExt = @["nim"]
+
+# Dependencies
+
+requires "nim >= 0.16.0"

tests/develop/srcdirtest/src/srcdirtest.nim

@@ -0,0 +1,4 @@
+proc foo*(): string =
+  return "correct"
+
+echo("hello")

tests/develop/srcdirtest/srcdirtest.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "1.0"
+author        = "Dominik Picheta"
+description   = "srcdir"
+license       = "MIT"
+
+srcDir = "src"
+
+# Dependencies
+
+requires "nim >= 0.16.0"

tests/diamond_deps/a/a.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "a"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.3", "b", "c"
+
+
+task test, "test":
+  echo("hello")

tests/diamond_deps/b/b.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "b"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.3", "d"
+

tests/diamond_deps/c/c.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "c"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.3", "d"
+

tests/diamond_deps/d/d.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "d"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.3"
+

tests/invalidPackage/invalidPackage.nimble

@@ -0,0 +1,13 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+
+thisFieldDoesNotExist = "hello"
+
+# Dependencies
+
+requires "nim >= 0.20.0"

tests/issue206/issue/issue206bin.nim

@@ -0,0 +1,2 @@
+
+echo "Hello"

tests/issue206/issue206.nimble

@@ -0,0 +1,10 @@
+# Package
+version       = "0.1.0"
+author        = "Yuriy Glukhov"
+description   = "Test package for Issue 206"
+license       = "BSD"
+
+bin = @["issue/issue206bin"]
+
+# Dependencies
+requires "nimrod >= 0.9.3"

tests/issue289/issue289.nim

@@ -0,0 +1 @@
+echo 42

tests/issue289/issue289.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Package reproducing issues depending on #head and concrete version of the same package."
+license       = "MIT"
+
+bin = @["issue289"]
+
+# Dependencies
+
+requires "nim >= 0.15.0", "https://github.com/nimble-test/packagea.git 0.6.0"
+requires "https://github.com/nimble-test/packagea.git#head"
+

tests/issue304/package-test/package_test.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Package for ensuring that issue #304 is resolved."
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.3"
+

tests/issue338/issue338.nimble

@@ -0,0 +1,7 @@
+# Package
+version = "0.1.0"
+author = "Samantha Marshall"
+description = "test case to validate successful install when `srcDir` value ends in a directory separator"
+license = "MIT"
+
+srcDir = "src/"

tests/issue368/packages.json

@@ -0,0 +1,14 @@
+[
+  {
+    "name": "discordnim",
+    "url": "https://github.com/Krognol/discordnim",
+    "method": "git",
+    "tags": [
+      "library",
+      "discord"
+    ],
+    "description": "Discord library for Nim",
+    "license": "MIT",
+    "web": "https://github.com/Krognol/discordnim"
+  }
+]

tests/issue428/dummy.nimble

@@ -0,0 +1,10 @@
+# Package
+
+version       = "0.1.0"
+author        = "Author"
+description   = "dummy"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.17.0"

tests/issue432/issue432.nimble

@@ -0,0 +1,15 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+
+
+
+# Dependencies
+
+requires "nim >= 0.16.0"
+requires "https://github.com/nimble-test/packagea#head",
+         "https://github.com/nimble-test/packagebin2"

tests/issue432/src/issue432.nim

@@ -0,0 +1,7 @@
+# This is just an example to get you started. A typical library package
+# exports the main API in this file. Note that you cannot rename this file
+# but you can remove it if you wish.
+
+proc add*(x, y: int): int =
+  ## Adds two files together.
+  return x + y

tests/issue564/issue564.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir = "src"
+bin           = @["issue564/issue564build"]
+
+
+
+# Dependencies
+
+requires "nim >= 0.16.0"

tests/issue564/src/issue564/issue564build.nim

@@ -0,0 +1,5 @@
+# This is just an example to get you started. A typical binary package
+# uses this file as the main entry point of the application.
+
+when isMainModule:
+  echo("Hello, World!")

tests/issue597/dummy.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "0.1.0"
+author        = "Author"
+description   = "dummy"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.17.0"
+
+bin = @["test.nim"]

tests/issue633/issue633.nimble

@@ -0,0 +1,16 @@
+# Package
+
+version     = "0.1.0"
+author      = "GT"
+description = "Package for ensuring that issue #633 is resolved."
+license     = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.19.6"
+# to reproduce dependency 2 must be before 1
+
+task testTask, "Test":
+  for i in 0 .. paramCount():
+    if paramStr(i) == "--testTask":
+      echo "Got it"

tests/issue678/issue678.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version     = "0.1.0"
+author      = "Ivan Bobev"
+description = "Package for ensuring that issue #678 is resolved."
+license     = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.19.6"
+# to reproduce dependency 2 must be before 1
+requires "issue678_dependency_2", "issue678_dependency_1"

tests/issue678/packages.json

@@ -0,0 +1,19 @@
+[
+  {
+    "name": "issue678_dependency_1",
+    "url": "https://github.com/nimble-test/issue678?subdir=dependency_1",
+    "method": "git",
+    "tags": [ "test" ], 
+    "description":
+      "Both first and second level dependency of the issue678 package.",
+    "license": "MIT"
+  },
+  {
+    "name": "issue678_dependency_2",
+    "url": "https://github.com/nimble-test/issue678?subdir=dependency_2",
+    "method": "git",
+    "tags": [ "test" ],
+    "description": "First level dependency of the issue678 package.",
+    "license": "MIT"
+  }
+]

tests/issue708/issue708.nimble

@@ -0,0 +1,17 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+
+
+
+# Dependencies
+
+requires "nim >= 0.16.0"
+
+
+echo "hello"
+echo "hello2"

tests/issue708/src/issue708.nim

@@ -0,0 +1,7 @@
+# This is just an example to get you started. A typical library package
+# exports the main API in this file. Note that you cannot rename this file
+# but you can remove it if you wish.
+
+proc add*(x, y: int): int =
+  ## Adds two files together.
+  return x + y

tests/nimbleVersionDefine/nimbleVersionDefine.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+bin           = @["nimbleVersionDefine"]
+
+
+
+# Dependencies
+
+requires "nim >= 0.16.0"

tests/nimbleVersionDefine/src/nimbleVersionDefine.nim

@@ -0,0 +1,3 @@
+when isMainModule:
+  const NimblePkgVersion {.strdefine.} = "Unknown"
+  echo(NimblePkgVersion)

tests/nimscript/nimscript.nimble

@@ -2,7 +2,9 @@
 
 version       = "0.1.0"
 author        = "Dominik Picheta"
-description   = "Test package"
+description   = """Test package
+with multi-line description
+"""
 license       = "BSD"
 
 bin = @["nimscript"]
@@ -11,7 +13,7 @@ bin = @["nimscript"]
 
 requires "nim >= 0.12.1"
 
-task test, "test description":
+task work, "test description":
   echo(5+5)
 
 task c_test, "Testing `setCommand \"c\", \"nimscript.nim\"`":
@@ -21,8 +23,16 @@ task cr, "Testing `nimble c -r nimscript.nim` via setCommand":
   --r
   setCommand "c", "nimscript.nim"
 
+task repeated, "Testing `nimble c nimscript.nim` with repeated flags":
+  --define: foo
+  --define: bar
+  --define: "quoted"
+  --define: "quoted\\\"with\\\"quotes"
+  setCommand "c", "nimscript.nim"
+
 task api, "Testing nimscriptapi module functionality":
-  echo(getPkgDir())
+  doAssert(findExe("nim").len != 0)
+  echo("PKG_DIR: ", getPkgDir())
 
 before hooks:
   echo("First")
@@ -37,4 +47,16 @@ before hooks2:
   return false
 
 task hooks2, "Testing the hooks again":
-  echo("Shouldn't happen")
+  echo("Shouldn't happen")
+
+before install:
+  echo("Before PkgDir: ", getPkgDir())
+
+after install:
+  echo("After PkgDir: ", getPkgDir())
+
+before build:
+  echo("Before build")
+
+after build:
+  echo("After build")

tests/packageStructure/a/a.nim

@@ -0,0 +1 @@
+

tests/packageStructure/a/a.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package A"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/b/b.nim

@@ -0,0 +1 @@
+

tests/packageStructure/b/b.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package B"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/b/b/foobar.nim

@@ -0,0 +1 @@
+

tests/packageStructure/c/c.nim

@@ -0,0 +1 @@
+

tests/packageStructure/c/c.nimble

@@ -0,0 +1,13 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package C"
+license       = "MIT"
+
+bin = @["c"]
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/c/cpkg/foobar.nim

@@ -0,0 +1 @@
+

tests/packageStructure/softened/myPkg.nimble

@@ -0,0 +1,15 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package myPkg. See #469."
+license       = "MIT"
+
+# This is inferred implicitly by Nimble:
+# installDirs = @["myPkg"]
+# installFiles = @["myPkg.nim"]
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/validBinary/y.nimble

@@ -0,0 +1,13 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package Y"
+license       = "MIT"
+
+bin = @["y"]
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/validBinary/yWrong/foobar.nim

@@ -0,0 +1 @@
+

tests/packageStructure/x/x.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Incorrectly structured package X."
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/y/y.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Incorrectly structured package Y"
+license       = "MIT"
+
+installExt = @["nim"]
+bin = @["y"]
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/y/yWrong/foobar.nim

@@ -0,0 +1 @@
+

tests/packageStructure/z/z.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Incorrect package structure Z."
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/passNimFlags/passNimFlags.nim

@@ -0,0 +1 @@
+when not defined(passNimIsWorking): {.error: "-d:passNimIsWorking wasn't passed to the compiler"}

tests/passNimFlags/passNimFlags.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "SolitudeSF"
+description   = "Test nimble install flag forwarding"
+license       = "BSD"
+bin           = @["passNimFlags"]
+
+# Dependencies
+
+requires "nim >= 0.13.0"

tests/recursive/recursive.nimble

@@ -0,0 +1,25 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Test package"
+license       = "BSD"
+
+# Dependencies
+
+requires "nim >= 0.12.1"
+
+let
+  callNimble = getEnv("NIMBLE_TEST_BINARY_PATH")
+doAssert callNimble.len != 0, "NIMBLE_TEST_BINARY_PATH not set"
+
+task recurse, "Level 1":
+  echo 1
+  exec callNimble & " recurse2"
+
+task recurse2, "Level 2":
+  echo 2
+  exec callNimble & " recurse3"
+
+task recurse3, "Level 3":
+  echo 3

tests/revdep/mydep/mydep.nimble

@@ -0,0 +1,10 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Random dep"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0"

tests/revdep/pkgNoDep/pkgA.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package A"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/revdep/pkgWithDep/pkgA.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package A"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0", "mydep"
+

tests/run/run.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+bin           = @["run"]
+
+
+
+# Dependencies
+
+requires "nim >= 0.19.0"

tests/run/src/run.nim

@@ -0,0 +1,4 @@
+import os
+
+when isMainModule:
+  echo("Testing `nimble run`: ", commandLineParams())