show cmd used with --verbose (even on success); fix #783 (#781)

* show cmd used with --verbose (even on success)

* honor -p:-u:release

* address comment

* remove --showCmds flag

* remove quoteShell

* fix #783

* Revert "fix #783"

This reverts commit af0ac004c00f17e9983c63ab99e40cd38ba6aaa4.

* fix #783 by fixing the gist instead

* Revert "fix #783 by fixing the gist instead" now that upstream gist was updated with my patch

This reverts commit 8bec86039d8335af152acf238ab14d0268e003e5.

.gitignore

@@ -6,9 +6,29 @@ nimcache/
 # Absolute paths
 /src/babel
 /src/nimble
-/tests/tester
 
 # executables from test and build
 /nimble
-/tests/nimscript/nimscript
-/tests/issue27/issue27
+src/nimblepkg/cli
+src/nimblepkg/packageinfo
+src/nimblepkg/packageparser
+src/nimblepkg/reversedeps
+src/nimblepkg/version
+src/nimblepkg/download
+
+# Windows executables
+*.exe
+*.dll
+
+# VCC compiler and linker artifacts
+*.ilk
+*.pdb
+
+# Editors and IDEs project files and folders
+.vscode
+
+# VCS artifacts
+*.orig
+
+# Test procedure artifacts
+nimble_*.nims

.travis.yml

@@ -1,20 +1,24 @@
 os:
+  - windows
   - linux
+  - osx
 
 language: c
 
-install:
-  - |
-      wget http://nim-lang.org/download/nim-0.15.2.tar.xz
-      tar -xf nim-0.15.2.tar.xz
-      cd nim-0.15.2
-      LDFLAGS=-lrt sh build.sh
-      cd ..
+env:
+  - BRANCH=0.19.6
+  - BRANCH=0.20.2
+  - BRANCH=1.0.6
+  # This is the latest working Nim version against which Nimble is being tested
+  - BRANCH=#ab525cc48abdbbbed1f772e58e9fe21474f70f07
 
-before_script:
-  - set -e
-  - set -x
-  - export PATH=`pwd`/nim-0.15.2/bin:$PATH
+cache:
+  directories:
+    - "$HOME/.choosenim"
+
+install:
+  - curl https://gist.github.com/genotrance/fb53504a4fba88bc5201d3783df5c522/raw/travis.sh -LsSf -o travis.sh
+  - source travis.sh
 
 script:
   - cd tests

changelog.markdown

@@ -3,6 +3,220 @@
 
 # Nimble changelog
 
+## 0.11.0 - 22/09/2019
+
+This is a major release containing nearly 60 commits. Most changes are
+bug fixes, but this release also includes a couple new features:
+
+- Binaries can now be built and run using the new ``run`` command.
+- The ``NimblePkgVersion`` is now defined so you can easily get the package
+  version in your source code
+  ([example](https://github.com/nim-lang/nimble/blob/4a2aaa07d/tests/nimbleVersionDefine/src/nimbleVersionDefine.nim)).
+
+Some other highlights:
+
+- Temporary files are now kept when the ``--debug`` flag is used.
+- Fixed dependency resolution issues with "#head" packages (#432 and #672).
+- The `install` command can now take Nim compiler flags via the new
+  ``--passNim`` flag.
+- Command line arguments are now passed properly to tasks (#633).
+- The ``test`` command now respects the specified backend (#631).
+- The ``dump`` command will no longer prompt and now has an implicit ``-y``.
+- Fixed bugs with the new nimscript executor (#665).
+- Fixed multiple downloads and installs of the same package (#678).
+- Nimble init no longer overwrites existing files (#581).
+- Fixed incorrect submodule version being pulled when in a non-master branch (#675).
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.10.2...v0.11.0
+
+## 0.10.2 - 03/06/2019
+
+This is a small release which avoids object variant changes that are now
+treated as runtime errors (Nim #1286). It also adds support for `backend`
+selection during `nimble init`.
+
+Multiple bug fixes are also included:
+- Fixed an issue where failing tasks were not returning a non-zero return
+  value (#655).
+- Error out if `bin` is a Nim source file (#597).
+- Fixed an issue where nimble task would not run if file of same name exists.
+- Fixed an issue that prevented multiple instances of nimble from running on
+  the same package.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.10.0...v0.10.2
+
+## 0.10.0 - 27/05/2019
+
+Nimble now uses the Nim compiler directly via `nim e` to execute nimble
+scripts rather than embedding the Nim VM. This has multiple benefits:
+- Evolve independently from Nim enabling new versions of Nimble to work
+  with multiple versions of Nim.
+- Inherit all nimscript enhancements and bug fixes rather than having to
+  duplicate functionality.
+- Fast build time and smaller binary.
+- No dependency on the compiler package which could cause dependency issues
+  when nimble is used as a package.
+
+Several other features and fixes have been implemented to improve general
+development and test workflows.
+- `nimble test` now sports a `-continue` or `-c` flag that allows tests
+  to continue on failure, removes all created test binaries on completion
+  and warns if no tests found.
+- The `--inclDeps` or `-i` flag enables `nimble uninstall` to remove all
+  dependent packages during uninstall.
+- Added documentation on the usage of a custom `nimbleDir`.
+- Package type interactive prompt is more readable.
+- Save temporary files in a per-user temp dir to enable Nimble on multi-user
+  systems.
+- CTRL-C is now handled correctly in interactive prompts.
+- Fixed issue where empty package list led to error.
+- Fixed issue where file:// was prepended incorrectly.
+- Fixed miscellaneous issues in version parsing, Github auth and briefClone.
+- Miscellaneous cleanup of deprecated procs.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.9.0...v0.10.0
+
+## 0.9.0 - 19/09/2018
+
+This is a major new release which contains at least one breaking change.
+Unfortunately even though it was planned, support for lock files did not
+make it into this release. The release does
+however contain a large number of fixes spread across 57 commits.
+
+The breaking change in this release is to do with the handling of binary
+package. **Any package that specifies a ``bin`` value in it's .nimble file**
+**will no longer install any Nim source code files**, in other words it's not
+going to be a hybrid package by default. This means that so called "hybrid
+packages" now need to specify ``installExt = @["nim"]`` in their metadata,
+otherwise they will become binary packages only.
+
+- **Breaking:** hybrid packages require ``installExt = @["nim"]``
+  ([Commit](https://github.com/nim-lang/nimble/commit/09091792615eacd503e87ca70252c572a4bde2b5))
+- **The ``init`` command can now show a list of choices for information such as**
+  **the license.**
+- **The ``init`` command now creates correct project structures for all package**
+  **types.**
+- **Fatal errors are no longer created when the path inside a .nimble-link file**
+  **doesn't exist.**
+- **The ``develop`` command now always clones HEAD and grabs the full repo history.**
+- **The default ``test`` task no longer executes all tests (only those starting with 't').**
+- Colour is no longer used when `isatty` is false.
+- ``publish`` now shows the URL of the created PR.
+- The ``getPkgDir`` procedure has been fixed in the Nimble file API.
+- Improved handling of proxy environment variables.
+- Codebase has been improved not to rely on `nil` in strings and seqs.
+- The handling of pre- and post-hooks has been improved significantly.
+- Fixed the ``path`` command for packages with a ``srcDir`` and optimised the
+  package look-up.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.8.10...v0.9.0
+
+## 0.8.10 - 23/02/2018
+
+The first release of 2018! Another fairly big release containing 40 commits.
+This release fixes many
+issues, with most being fixed by our brilliant contributors. Thanks a lot
+everyone!
+
+One big new feature is the new support for multiple Nimble packages in a single
+Git/Hg repository. You can now specify ``?subdir=<dir>`` at the end of your
+repo's URL and Nimble will know to look in ``<dir>`` for your package.
+
+* **Implemented support for multi-package repos.** See
+  [#421](https://github.com/nim-lang/nimble/issues/421) for the relevant issue.
+* **Better error message when the user has an outdated stdlib version that confuses Nimble**
+* **The validity of a Nimble package can now be checked using the new ``check`` command**
+* Nimble no longer silently ignores an erroneous '@' in for example
+  ``nimble install compiler@``.
+* Issues with the ``nimble path`` command have been fixed.
+* The ``nimble publish`` command has been improved and stabilised.
+* Messages for the ``NIM_LIB_PREFIX`` env var have been improved.
+* ``before install`` is now called when packages are installed by their name.
+  See [#280](https://github.com/nim-lang/nimble/issues/280).
+* Fixed issue with ``nimble init``. See [#446](https://github.com/nim-lang/nimble/issues/446).
+* Nimble now rejects [reserved names on Windows](https://github.com/nim-lang/nimble/commit/74856a87084b73451254555b2c20ad932cf84270).
+* The ``NIMBLE_DIR`` environment variable is now supported, in addition to the
+  command line flag and config setting.
+* The ``init`` command has been improved significantly.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.8.8...v0.8.10
+
+## 0.8.8 - 03/09/2017
+
+This is a relatively big release containing 57 commits, with multiple new
+features and many bug fixes.
+
+* **Implemented the `develop` command.** See
+  [readme](https://github.com/nim-lang/nimble#nimble-develop) for details.
+* **Implemented a default `test` task** for packages that don't define it.
+* **Lowered the memory consumption** in cases where a package contained many files.
+* Nimble now accepts .nimble symlinks.
+* Locally stored package list files can now be specified in the Nimble config.
+* Fixed branch checkout and handling of branch names with dashes.
+* Improved URL detection in ``publish`` feature.
+* Fixed many issues related to binary management. Packages are now resymlinked
+  when an newer version is removed.
+  ([#331](https://github.com/nim-lang/nimble/issues/331))
+* Fixed issues with CLI arg passing to the Nim compiler.
+  ([#351](https://github.com/nim-lang/nimble/issues/351))
+* Improved performance of ``list -i`` command.
+* Fixed issue where warnings weren't suppressed for some commands.
+  ([#290](https://github.com/nim-lang/nimble/issues/290))
+* Special versions other than `#head` are no longer considered to be newest.
+* Improves the reverse dependency lookup by cross checking it with the
+  installed list of packages.
+  ([#287](https://github.com/nim-lang/nimble/issues/287))
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.8.6...v0.8.8
+
+## 0.8.6 - 05/05/2017
+
+Yet another point release which includes various bug fixes and improvements.
+
+* Improves heuristic for finding Nim standard library to support choosenim
+  installations and adds ability to override it via ``NIM_LIB_PREFIX``
+  environment variable.
+* Implement ``--noColor`` option to remove color from the output.
+* Fixes bug when ``srcDir`` contains trailing slash.
+* Fixes failure when ``-d`` flag is passed to ``c`` command.
+* Show raw output for certain commands.
+* GitHub API token can now be specified via the ``NIMBLE_GITHUB_API_TOKEN``
+  environment variable.
+* GitHub API token is now stored in ``~/.nimble/api_token`` so that it
+  doesn't need to be specified each time.
+* Fixes multiple flags not being passed in Nimble task.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.8.4...v0.8.6
+
+## 0.8.4 - 29/01/2017
+
+Another bug fix release which resolves problems related to stale nimscriptapi
+files in /tmp/, no compilation output when ``nimble build`` fails, and issues
+with the new package validation on Windows.
+
+----
+
+Full changelog: https://github.com/nim-lang/nimble/compare/v0.8.2...v0.8.4
+
+## 0.8.2 - 08/01/2017
+
+This is a small bug fix release which resolves problems with the installation
+of Aporia (and likely other Nimble packages).
+
 ## 0.8.0 - 05/01/2017
 
 This is a large release containing multiple new features and many bug fixes.
@@ -45,6 +259,7 @@ This is a large release containing multiple new features and many bug fixes.
 ----
 
 Full changelog: https://github.com/nim-lang/nimble/compare/v0.7.10...v0.8.0
+
 Full list of issues which have been closed: https://github.com/nim-lang/nimble/issues?utf8=%E2%9C%93&q=is%3Aissue+closed%3A%222016-10-09+..+2017-01-05%22+
 
 ## 0.7.10 - 09/10/2016

nimble.nimble

@@ -1,26 +1,17 @@
-import ospaths
-template thisModuleFile: string = instantiationInfo(fullPaths = true).filename
-
-when fileExists(thisModuleFile.parentDir / "src/nimblepkg/common.nim"):
-  # In the git repository the Nimble sources are in a ``src`` directory.
-  import src/nimblepkg/common
-else:
-  # When the package is installed, the ``src`` directory disappears.
-  import nimblepkg/common
-
 # Package
 
-version       = nimbleVersion
+version       = "0.11.0"
 author        = "Dominik Picheta"
 description   = "Nim package manager."
 license       = "BSD"
 
 bin = @["nimble"]
 srcDir = "src"
+installExt = @["nim"]
 
 # Dependencies
 
-requires "nim >= 0.13.0", "compiler#head"
+requires "nim >= 0.13.0"
 
 when defined(nimdistros):
   import distros

nimble.zsh-completion

@@ -0,0 +1,51 @@
+#compdef nimble
+
+_nimble() {
+  local line
+
+  _arguments -C \
+    '1: :(install init publish uninstall build c cc js doc doc2 refresh search list tasks path dump develop)' \
+    '*::options:->options' \
+    '(--version)--version[show version]' \
+    '(--help)--help[show help]' \
+    '(-)--help[display help information]' \
+    '(-)--version[display version information]' \
+    '(-y --accept)'{-y,--accept}'[accept all interactive prompts]' \
+                    {-n,--reject}'[reject all interactive prompts]' \
+    '--ver[Query remote server for package version information when searching or listing packages]' \
+    '--nimbleDir dirname[Set the Nimble directory]' \
+    '(-d --depsOnly)'{-d,--depsOnly}'[Install only dependencies]'
+
+  if [ $#line -eq 0 ]; then
+    # if the command line is empty and "nimble tasks" is successfull, add custom tasks
+    tasks=$(nimble tasks)
+    if [ $? -eq 0 ]; then
+      compadd - $(echo $tasks | cut -f1 -d" " | tr '\n' ' ')
+    fi
+  fi
+
+  case $line[1] in
+    install)
+      _nimble_installable_packages
+    ;;
+    uninstall|path|dump)
+      _nimble_installed_packages
+    ;;
+    init|publish|build|refresh|search|tasks)
+      (( ret )) && _message 'no more arguments'
+    ;;
+    *)
+      (( ret )) && _message 'no more arguments'
+    ;;
+  esac
+}
+
+function _nimble_installable_packages {
+  compadd - $(nimble list 2> /dev/null | grep -v '^ ' | tr -d ':')
+}
+
+function _nimble_installed_packages {
+  compadd - $(nimble list -i 2> /dev/null | grep ']$' | cut -d' ' -f1)
+}
+
+_nimble "$@"

readme.markdown

@@ -1,7 +1,7 @@
 # Nimble [![Build Status](https://travis-ci.org/nim-lang/nimble.svg?branch=master)](https://travis-ci.org/nim-lang/nimble)
 
 Nimble is a *beta*-grade *package manager* for the [Nim programming
-language](http://nim-lang.org).
+language](https://nim-lang.org).
 
 Interested in learning **how to create a package**? Skip directly to that section
 [here](#creating-packages).
@@ -15,6 +15,7 @@ Interested in learning **how to create a package**? Skip directly to that sectio
   - [nimble install](#nimble-install)
   - [nimble uninstall](#nimble-uninstall)
   - [nimble build](#nimble-build)
+  - [nimble run](#nimble-run)
   - [nimble c](#nimble-c)
   - [nimble list](#nimble-list)
   - [nimble search](#nimble-search)
@@ -50,23 +51,22 @@ Interested in learning **how to create a package**? Skip directly to that sectio
 
 ## Requirements
 
-Nimble has some runtime dependencies on external tools, these tools are
-used to download Nimble packages.
-For
-instance, if a package is hosted on [Github](https://github.com) you require to
-have [git](http://www.git-scm.com) installed and added to your environment
-``PATH``. Same goes for [Mercurial](http://mercurial.selenic.com) repositories
-on [Bitbucket](https://bitbucket.org). Nimble packages are typically hosted in
-Git repositories so you may be able to get away without installing Mercurial.
+Nimble has some runtime dependencies on external tools, these tools are used to
+download Nimble packages. For instance, if a package is hosted on
+[GitHub](https://github.com), you need to have [git](https://www.git-scm.com)
+installed and added to your environment ``PATH``. Same goes for
+[Mercurial](http://mercurial.selenic.com) repositories on
+[Bitbucket](https://bitbucket.org). Nimble packages are typically hosted in Git
+repositories so you may be able to get away without installing Mercurial.
 
 **Warning:** Ensure that you have a fairly recent version of Git installed.
-If the version is less recent than 1.9.0 then Nimble may have trouble using it.
+If the version is older than 1.9.0, then Nimble may have trouble using it.
 See [this issue](https://github.com/nim-lang/nimble/issues/105) for more
-info.
+information.
 
 ## Installation
 
-Nimble is now bundled with [Nim](http://nim-lang.org)
+Nimble is now bundled with [Nim](https://nim-lang.org)
 (since Nim version 0.15.0).
 This means that you should have Nimble installed already, as long as you have
 the latest version of Nim installed as well. Because of this **you likely do
@@ -75,35 +75,36 @@ not need to install Nimble manually**.
 But in case you still want to install Nimble manually, you can follow the
 following instructions.
 
-There are two ways to install Nimble manually. The first is using the
-``install_nimble.nims`` script included in the Nim distribution and
-[repository](https://github.com/nim-lang/Nim/blob/devel/install_nimble.nims).
-Simply execute this to install Nimble.
+There are two ways to install Nimble manually. Using ``koch`` and using Nimble
+itself.
+
+### Using koch
+
+The ``koch`` tool is included in the Nim distribution and
+[repository](https://github.com/nim-lang/Nim/blob/devel/koch.nim).
+Simply navigate to the location of your Nim installation and execute the
+following command to compile and install Nimble.
 
 ```
-nim e install_nimble.nims
+./koch nimble
 ```
 
 This will clone the Nimble repository, compile Nimble and copy it into
 Nim's bin directory.
 
-The second approach is to install Nimble as a Nimble package. You can do this
-by compiling Nimble, then running ``nimble install`` in Nimble's directory.
+### Using Nimble
+
+In most cases you will already have Nimble installed, you can install a newer
+version of Nimble by simply running the following command:
 
 ```
-git clone https://github.com/nim-lang/nimble.git
-cd nimble
-nim c src/nimble
-src/nimble install
+nimble install nimble
 ```
 
-**Note for Windows users**: You will need to rename ``nimble.exe`` after
-compilation to something else like ``nimble1.exe``, then run
-``src\nimble1.exe install``.
+This will download the latest release of Nimble and install it on your system.
 
-This will install Nimble to the default Nimble packages location:
-``~/.nimble/pkgs``. The binary will be installed to ``~/.nimble/bin``, so you
-will need to add this directory to your PATH.
+Note that you must have `~/.nimble/bin` in your PATH for this to work, if you're
+using choosenim then you likely already have this set up correctly.
 
 ## Nimble usage
 
@@ -130,6 +131,20 @@ a third-party package list.
 Package lists can be specified in Nimble's config. Take a look at the
 config section below to see how to do this.
 
+### nimble check
+
+The ``check`` command will read your package's .nimble file. It will then
+verify that the package's structure is valid.
+
+Example:
+
+    $ nimble check
+        Error: Package 'x' has an incorrect structure. It should contain a single directory hierarchy for source files, named 'x', but file 'foobar.nim' is in a directory named 'incorrect' instead. This will be an error in the future.
+         Hint: If 'incorrect' contains source files for building 'x', rename it to 'x'. Otherwise, prevent its installation by adding `skipDirs = @["incorrect"]` to the .nimble file.
+      Failure: Validation failed
+
+
+
 ### nimble install
 
 The ``install`` command will download and install a package. You need to pass
@@ -144,9 +159,9 @@ Example:
     nake installed successfully
 
 Nimble always fetches and installs the latest version of a package. Note that
-latest version is defined as the latest tagged version in the git (or hg)
+latest version is defined as the latest tagged version in the Git (or Mercurial)
 repository, if the package has no tagged versions then the latest commit in the
-remote repository will be installed. If you already have that version installed
+remote repository will be installed. If you already have that version installed,
 Nimble will ask you whether you wish it to overwrite your local copy.
 
 You can force Nimble to download the latest commit from the package's repo, for
@@ -154,15 +169,16 @@ example:
 
     $ nimble install nimgame@#head
 
-This is of course git specific, for hg use ``tip`` instead of ``head``. A
+This is of course Git-specific, for Mercurial, use ``tip`` instead of ``head``. A
 branch, tag, or commit hash may also be specified in the place of ``head``.
 
-Instead of specifying a VCS branch you may also specify a version range, for
-example:
+Instead of specifying a VCS branch, you may also specify a concrete version or a
+version range, for example:
 
+    $ nimble install nimgame@0.5
     $ nimble install nimgame@"> 0.5"
 
-In this case a version which is greater than ``0.5`` will be installed.
+The latter command will install a version which is greater than ``0.5``.
 
 If you don't specify a parameter and there is a ``package.nimble`` file in your
 current working directory then Nimble will install the package residing in
@@ -170,14 +186,46 @@ the current working directory. This can be useful for developers who are testing
 locally their ``.nimble`` files before submitting them to the official package
 list. See the [Creating Packages](#creating-packages) section for more info on this.
 
-A URL to a repository can also be specified, Nimble will automatically detect
-the type of the repository that the url points to and install it.
+#### Package URLs
+
+A valid URL to a Git or Merurial repository can also be specified, Nimble will
+automatically detect the type of the repository that the url points to and
+install it.
+
+For repositories containing the Nimble package in a subdirectory, you can
+instruct Nimble about the location of your package using the ``?subdir=<path>``
+query parameter. For example:
+
+    $ nimble install https://github.com/nimble-test/multi?subdir=alpha
+
+### nimble develop
+
+The ``develop`` command allows you to link an existing copy of a package into
+your installation directory. This is so that when developing a package you
+don't need to keep reinstalling it for every single change.
+
+    $ cd ~/projects/jester
+    $ nimble develop
+
+Any packages depending on ``jester`` will now use the code in
+``~/projects/jester``.
+
+If you specify a package name to this command, Nimble will clone it into the
+current working directory.
+
+    $ nimble develop jester
+
+The ``jester`` package will be cloned into ``./jester`` and it will be linked
+to your installation directory.
+
+Just as with the ``install`` command, a package URL may also be specified
+instead of a name.
 
 ### nimble uninstall
 
 The ``uninstall`` command will remove an installed package. Attempting to remove
-a package which other packages depend on is disallowed and will result in an
-error. You must currently manually remove the reverse dependencies first.
+a package which other packages depend on will result in an error. You can use the
+``--inclDeps`` or ``-i`` flag to remove all dependent packages along with the package.
 
 Similar to the ``install`` command you can specify a version range, for example:
 
@@ -188,8 +236,15 @@ Similar to the ``install`` command you can specify a version range, for example:
 The ``build`` command is mostly used by developers who want to test building
 their ``.nimble`` package. This command will build the package with default
 flags, i.e. a debug build which includes stack traces but no GDB debug
-information.
-The ``install`` command will build the package in release mode instead.
+information. The ``install`` command will build the package in release mode
+instead.
+
+### nimble run
+
+The ``run`` command can be used to build and run any binary specified in your
+package's ``bin`` list. You can pass any compilation flags you wish by specifying
+them before the ``run`` command, and you can specify arguments for your binary
+by specifying them after the ``run`` command.
 
 ### nimble c
 
@@ -205,7 +260,7 @@ the command ``c`` or ``compile`` is specified. The more specific ``js``, ``cc``,
 
 The ``list`` command will display the known list of packages available for
 Nimble. An optional ``--ver`` parameter can be specified to tell Nimble to
-query remote git repositories for the list of versions of the packages and to
+query remote Git repositories for the list of versions of the packages and to
 then print the versions. Please note however that this can be slow as each
 package must be queried separately.
 
@@ -233,8 +288,8 @@ substrings). Example:
 Searches are case insensitive.
 
 An optional ``--ver`` parameter can be specified to tell Nimble to
-query remote git repositories for the list of versions of the packages and to
-then print the versions. Please note however that this can be slow as each
+query remote Git repositories for the list of versions of the packages and
+then print the versions. However, please note that this can be slow as each
 package must be queried separately.
 
 ### nimble path
@@ -258,7 +313,7 @@ which can be useful to read the bundled documentation. Example:
 ### nimble init
 
 The nimble ``init`` command will start a simple wizard which will create
-a quick ``.nimble`` file for your project.
+a quick ``.nimble`` file for your project in the current directory.
 
 As of version 0.7.0, the ``.nimble`` file that this command creates will
 use the new NimScript format.
@@ -268,11 +323,11 @@ Check out the [Creating Packages](#creating-packages) section for more info.
 
 Publishes your Nimble package to the official Nimble package repository.
 
-**Note:** Requires a valid Github account.
+**Note:** Requires a valid GitHub account with an SSH key attached to it. To upload your public key onto your GitHub account, follow [this link](https://github.com/settings/keys).
 
 ### nimble tasks
 
-For a nimble package in the current working directory, list the tasks which that
+For a Nimble package in the current working directory, list the tasks which that
 package defines. This is only supported for packages utilising the new
 nimscript .nimble files.
 
@@ -297,22 +352,27 @@ nimbleDir = r"C:\Nimble\"
 [PackageList]
 name = "CustomPackages"
 url = "http://mydomain.org/packages.json"
+
+[PackageList]
+name = "Local project packages"
+path = r"C:\Projects\Nim\packages.json"
 ```
 
 You can currently configure the following in this file:
 
-* ``nimbleDir`` - The directory which nimble uses for package installation.
+* ``nimbleDir`` - The directory which Nimble uses for package installation.
   **Default:** ``~/.nimble/``
 * ``chcp`` - Whether to change the current code page when executing Nim
   application packages. If ``true`` this will add ``chcp 65001`` to the
   .cmd stubs generated in ``~/.nimble/bin/``.
   **Default:** ``true``
-* ``[PackageList]`` + ``name`` + ``url`` - You can use this section to specify
+* ``[PackageList]`` + ``name`` + (``url``|``path``) - You can use this section to specify
   a new custom package list. Multiple package lists can be specified. Nimble
   defaults to the "Official" package list, you can override it by specifying
   a ``[PackageList]`` section named "official". Multiple URLs can be specified
   under each section, Nimble will try each in succession if
-  downloading from the first fails.
+  downloading from the first fails. Alternately, ``path`` can specify a
+  local file path to copy a package list .json file from.
 * ``cloneUsingHttps`` - Whether to replace any ``git://`` inside URLs with
   ``https://``.
   **Default: true**
@@ -320,25 +380,28 @@ You can currently configure the following in this file:
   Nimble will also attempt to read the ``http_proxy`` and ``https_proxy``
   environment variables.
   **Default: ""**
+* ``nimLibPrefix`` - Specifies the Nim standard library prefix to help Nimble
+  find the Nim standard library.
+  **Default: ""**
 
 ## Creating Packages
 
-Nimble works on git repositories as its primary source of packages. Its list of
+Nimble works on Git repositories as its primary source of packages. Its list of
 packages is stored in a JSON file which is freely accessible in the
 [nim-lang/packages repository](https://github.com/nim-lang/packages).
-This JSON file provides nimble with the required Git URL to clone the package
+This JSON file provides Nimble with the required Git URL to clone the package
 and install it. Installation and build instructions are contained inside a
-file with the ``.nimble`` file extension. The nimble file shares the
+file with the ``.nimble`` file extension. The Nimble file shares the
 package's name, i.e. a package
 named "foobar" should have a corresponding ``foobar.nimble`` file.
 
-These files specify information about the package including its the author,
-license, dependencies and more. Without one Nimble is not able to install
+These files specify information about the package including its author,
+license, dependencies and more. Without one, Nimble is not able to install
 a package.
 
 A .nimble file can be created easily using Nimble's ``init`` command. This
 command will ask you a bunch of questions about your package, then generate a
-.nimble file for you.
+.nimble file for you in the current directory.
 
 A bare minimum .nimble file follows:
 
@@ -365,10 +428,11 @@ You can also specify multiple dependencies like so:
 
 requires "nim >= 0.10.0", "foobar >= 0.1.0"
 requires "fizzbuzz >= 1.0"
+requires "https://github.com/user/pkg#5a54b5e"
 ```
 
 Nimble currently supports installation of packages from a local directory, a
-git repository and a mercurial repository. The .nimble file must be present in
+Git repository and a mercurial repository. The .nimble file must be present in
 the root of the directory or repository being installed.
 
 The .nimble file is very flexible because it is interpreted using NimScript.
@@ -396,7 +460,7 @@ Hello World!
 
 You can place any Nim code inside these tasks. As long as that code does not
 access the FFI. The ``nimscript``
-[module](http://nim-lang.org/docs/nimscript.html) in Nim's standard library defines
+[module](https://nim-lang.org/docs/nimscript.html) in Nim's standard library defines
 additional functionality such as the ability to execute external processes
 which makes this feature very powerful.
 
@@ -431,15 +495,63 @@ which are also useful. Take a look at it for more information.
 
 ### Project structure
 
-There is nothing surprising about the recommended project structure. The advice
-resembles that of many other package managers.
+For a package named "foobar", the recommended project structure is the following:
 
-| Directory     | Purpose                                |
-| ------------- | -------------------------------------- |
-| ``.``         | Root directory containing .nimble file.|
-| ``./src/``    | Project source code                    |
-| ``./tests/``  | Project test files                     |
-| ``./docs/``   | Project documentation                  |
+```
+.                   # The root directory of the project
+├── LICENSE
+├── README.md
+├── foobar.nimble   # The project .nimble file
+└── src
+    └── foobar.nim  # Imported via `import foobar`
+└── tests           # Contains the tests
+    ├── config.nims
+    ├── tfoo1.nim   # First test
+    └── tfoo2.nim   # Second test
+
+```
+
+Note that the .nimble file needs to be in the project's root directory. This
+directory structure will be created if you run ``nimble init`` inside a
+``foobar`` directory.
+
+**Warning:** When source files are placed in a ``src`` directory, the
+.nimble file must contain a ``srcDir = "src"`` directive. The ``nimble init``
+command takes care of that for you.
+
+When introducing more modules into your package, you should place them in a
+separate directory named ``foobar`` (i.e. your package's name). For example:
+
+```
+.                   # The root directory of the project
+├── ...
+├── foobar.nimble   # The project .nimble file
+├── src
+│   ├── foobar
+│   │   ├── utils.nim   # Imported via `import foobar/utils`
+│   │   └── common.nim  # Imported via `import foobar/common`
+│   └── foobar.nim      # Imported via `import foobar`
+└── ...
+```
+
+#### Private modules
+
+You may wish to hide certain modules in your package from the users. Create a
+``private`` directory for that purpose. For example:
+
+```
+.                   # The root directory of the project
+├── ...
+├── foobar.nimble   # The project .nimble file
+├── src
+│   ├── foobar
+│   │   ├── private
+│   │   │   └── hidden.nim  # Imported via `import foobar/private/hidden`
+│   │   ├── utils.nim       # Imported via `import foobar/utils`
+│   │   └── common.nim      # Imported via `import foobar/common`
+│   └── foobar.nim          # Imported via `import foobar`
+└── ...
+```
 
 #### Tests
 
@@ -461,43 +573,55 @@ your ``tests`` directory with the following contents:
 --path:"../src/"
 ```
 
-To make testing even more convenient, you may wish to define a ``test`` task
-in your ``.nimble`` file. Like so:
+Nimble offers a pre-defined ``test`` task which compiles and runs all files
+in the ``tests`` directory beginning with 't' in their filename.
+
+You may wish to override this ``test`` task in your ``.nimble`` file. This
+is particularly useful when you have a single test suite program. Just add
+the following to your ``.nimble`` file to override the default ``test`` task.
 
 ```nim
 task test, "Runs the test suite":
   exec "nim c -r tests/tester"
 ```
 
-You can compile and run a single tester application or multiple test files.
+Running ``nimble test`` will now use the ``test`` task you have defined.
 
 ### Libraries
 
 Library packages are likely the most popular form of Nimble packages. They are
 meant to be used by other library or binary packages.
 
-When nimble installs a library it will copy all of its files
+When Nimble installs a library, it will copy all of its files
 into ``$nimbleDir/pkgs/pkgname-ver``. It's up to the package creator to make sure
 that the package directory layout is correct, this is so that users of the
 package can correctly import the package.
 
-By convention, it is suggested that the layout be as follows. The directory
-layout is determined by the nature of your package, that is, whether your
-package exposes only one module or multiple modules.
+It is suggested that the layout be as follows. The directory layout is
+determined by the nature of your package, that is, whether your package exposes
+only one module or multiple modules.
 
 If your package exposes only a single module, then that module should be
-present in the root directory (the directory with the .nimble file) of your git
-repository, it is recommended that in this case you name that module whatever
-your package's name is. A good example of this is the
-[jester](https://github.com/dom96/jester) package which exposes the ``jester``
-module. In this case the jester package is imported with ``import jester``.
+present in the root directory (the directory with the .nimble file) of your Git
+repository, and should be named whatever your package's name is. A good example
+of this is the [jester](https://github.com/dom96/jester) package which exposes
+the ``jester`` module. In this case the jester package is imported with
+``import jester``.
 
 If your package exposes multiple modules then the modules should be in a
 ``PackageName`` directory. This will allow for a certain measure of isolation
 from other packages which expose modules with the same names. In this case
 the package's modules will be imported with ``import PackageName/module``.
 
-You are free to combine the two approaches described.
+Here's a simple example multi-module library package called `kool`:
+
+```
+.
+├── kool
+│   ├── useful.nim
+│   └── also_useful.nim
+└── kool.nimble
+```
 
 In regards to modules which you do **not** wish to be exposed. You should place
 them in a ``PackageName/private`` directory. Your modules may then import these
@@ -521,33 +645,36 @@ A package is automatically a binary package as soon as it sets at least one
 bin = @["main"]
 ```
 
-In this case when ``nimble install`` is invoked, nimble will build the ``main.nim``
+In this case when ``nimble install`` is invoked, Nimble will build the ``main.nim``
 file, copy it into ``$nimbleDir/pkgs/pkgname-ver/`` and subsequently create a
 symlink to the binary in ``$nimbleDir/bin/``. On Windows a stub .cmd file is
 created instead.
 
 Other files will be copied in the same way as they are for library packages.
 
-Binary packages should not install .nim files so include
-``skipExt = @["nim"]`` in your .nimble file, unless you intend for your package to
-be a binary/library combo which is fine.
+Binary packages should not install .nim files so include ``skipExt = @["nim"]``
+in your .nimble file, unless you intend for your package to be a binary/library
+combo.
 
 Dependencies are automatically installed before building.
 It's a good idea to test that the dependencies you specified are correct by
-running by running ``nimble build`` or ``nimble install`` in the directory
+running ``nimble build`` or ``nimble install`` in the directory
 of your package.
 
 ### Hybrids
 
-One thing to note about library and binary package hybrids is that your binary
-may share the name of the package. This will mean that you will
-not be able to put your .nim files in a ``pkgname`` directory. The reason you
-will not be able to do this is because binaries on some operating systems
-do not have an extension so they will clash with a directory of the same name.
+One thing to note about binary packages that contain source files aside from
+the one(s) specified in `bin` (or that also expose multiple library modules, as
+above) is that a binary may share the name of the package: this will mean
+that you will not be able to put your additional .nim files in a ``pkgname``
+directory. The reason for this is that binaries on some operating systems do
+not have an extension, so they will clash with a directory of the same name.
 
-The current
-convention to get around this problem is to append ``pkg`` to the name as is
-done for nimble.
+If this is the case, you should place your additional .nim files in a directory
+with `pkg` appended after the name of the project. For instance, if you were
+building a binary named `project`, you would put any additional source files in
+a directory called `projectpkg`. From within project.nim you would then import
+those modules namespaced with `projectpkg/`.
 
 ### Dependencies
 
@@ -561,16 +688,16 @@ requires "nim >= 0.10.0", "jester > 0.1 & <= 0.5"
 Dependency lists support version ranges. These versions may either be a concrete
 version like ``0.1``, or they may contain any of the less-than (``<``),
 greater-than (``>``), less-than-or-equal-to (``<=``) and greater-than-or-equal-to
-(``>=``) oeprators.
+(``>=``) operators.
 Two version ranges may be combined using the ``&`` operator, for example
 ``> 0.2 & < 1.0``, which will install a package with the version greater than 0.2
 and less than 1.0.
 
 Specifying a concrete version as a dependency is not a good idea because your
 package may end up depending on two different versions of the same package.
-If this happens Nimble will refuse to install the package.
+If this happens, Nimble will refuse to install the package.
 
-In addition to versions you may also specify git/hg tags, branches and commits.
+In addition to versions you may also specify Git/Mercurial tags, branches and commits.
 Although these have to be specific; ranges of commits are not supported.
 This is done with the ``#`` character,
 for example: ``jester#head``. Which will make your package depend on the
@@ -601,7 +728,7 @@ when defined(nimdistros):
 The ``when`` branch is important to support installation using older versions
 of Nimble.
 
-The [distros module](nim-lang.org/docs/distros.html) in Nim's
+The [distros module](https://nim-lang.org/docs/distros.html) in Nim's
 standard library contains a list of all the supported Operating Systems and
 Linux distributions.
 
@@ -611,13 +738,13 @@ installing your package (on macOS):
 ```
   Hint: This package requires some external dependencies.
   Hint: To install them you may be able to run:
-  Hint:   sudo brew install openssl
+  Hint:   brew install openssl
 ```
 
 ### Nim compiler
 
 The Nim compiler cannot read .nimble files. Its knowledge of Nimble is
-limited to the ``nimblePaths`` feature which allows it to use packages installed
+limited to the ``nimblePath`` feature which allows it to use packages installed
 in Nimble's package directory when compiling your software. This means that
 it cannot resolve dependencies, and it can only use the latest version of a
 package when compiling.
@@ -627,18 +754,39 @@ It resolves the dependencies and feeds the path of each package to
 the compiler so that it knows precisely which version to use.
 
 This means that you can safely compile using the compiler when developing your
-software, but you should use nimble to build the package before publishing it
+software, but you should use Nimble to build the package before publishing it
 to ensure that the dependencies you specified are correct.
 
+### Compile with `nim` after changing the nimble directory
+
+The Nim compiler has been preconfigured to look at the default nimble directory while compiling,
+so no extra step is required to use nimble managed packages in your code.
+However, if you are using a custom `nimbleDir`, you need to specify the
+`--nimblePath:PATH` option. For example,
+if your `nimble` directory is located at `/some/custom/path/nimble`, this should work:
+
+```
+nim c --nimblePath:/some/custom/path/nimble/pkgs main.nim
+```
+
+Some code editors rely on `nim check` to check for errors under the hood (e.g. VScode),
+and the editor extension may not allow users to pass custom option to `nim check`, which
+will cause `nim check` to scream `Error: cannot open file:<the_package>`. In this case,
+you will have to use [Nim compiler's configuration files](https://nim-lang.org/docs/nimc.html#compiler-usage-configuration-files). Simply add the line:
+```
+nimblePath = "/some/custom/path/nimble/pkgs"
+```
+to the `nim.cfg` located in any directory listed in the [documentation](https://nim-lang.org/docs/nimc.html#compiler-usage-configuration-files), this should resolve the problem.
+
 ### Versions
 
-Versions of cloned packages via git or mercurial are determined through the
+Versions of cloned packages via Git or Mercurial are determined through the
 repository's *tags*.
 
 When installing a package which needs to be downloaded, after the download is
-complete and if the package is distributed through a VCS, nimble will check the
-cloned repository's tags list. If no tags exist, nimble will simply install the
-HEAD (or tip in mercurial) of the repository. If tags exist, nimble will attempt
+complete and if the package is distributed through a VCS, Nimble will check the
+cloned repository's tags list. If no tags exist, Nimble will simply install the
+HEAD (or tip in Mercurial) of the repository. If tags exist, Nimble will attempt
 to look for tags which resemble versions (e.g. v0.1) and will then find the
 latest version out of the available tags, once it does so it will install the
 package after checking out the latest version.
@@ -665,7 +813,20 @@ To summarise, the steps for release are:
 Once the new tag is in the remote repository, Nimble will be able to detect
 the new version.
 
-##<a name="submitting-your-package-to-the-package-list"> Publishing packages
+##### Git Version Tagging
+
+Use dot separated numbers to represent the release version in the git
+tag label.  Nimble will parse these git tag labels to know which
+versions of a package are published.
+
+``` text
+v0.2.0        # 0.2.0
+v1            # 1
+v1.2.3-zuzu   # 1.2.3
+foo-1.2.3.4   # 1.2.3.4
+```
+
+## Publishing packages
 
 Publishing packages isn't a requirement. But doing so allows people to associate
 a specific name to a URL pointing to your package. This mapping is stored
@@ -728,7 +889,7 @@ Nimble includes a ``publish`` command which does this for you automatically.
   root dir of the package.
 * ``bin`` - A list of files which should be built separated by commas with
   no file extension required. This option turns your package into a *binary
-  package*, nimble will build the files specified and install them appropriately.
+  package*, Nimble will build the files specified and install them appropriately.
 * ``backend`` - Specifies the backend which will be used to build the files
   listed in ``bin``. Possible values include: ``c``, ``cc``, ``cpp``, ``objc``,
   ``js``.
@@ -760,11 +921,27 @@ work properly and you won't be able to run them.
 
 * ```SSL support is not available. Cannot connect over SSL. [HttpRequestError]```
 
-Make sure that nimble is configured to run with SSL, adding a ```-d:ssl```
+Make sure that Nimble is configured to run with SSL, adding a ```-d:ssl```
 flag to the file ```src/nimble.nim.cfg```.
 After that, you can run ```src/nimble install``` and overwrite the existing
 installation.
 
+* ``Could not download: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure``
+
+If you are on macOS, you need to set and export the ```DYLD_LIBRARY_PATH``` environment variable to the directory where your OpenSSL libraries are. For example, if you use OpenSSL, you have to set ```export DYLD_LIBRARY_PATH=/usr/local/opt/openssl/lib``` in your ```$HOME/.bashrc``` file.
+
+* ``Error: ambiguous identifier: 'version' --use nimscriptapi.version or system.version``
+
+Make sure that you are running at least version 0.16.0 of Nim (or the latest nightly).
+
+* ``Error: cannot open '/home/user/.nimble/lib/system.nim'.``
+
+Nimble cannot find the Nim standard library. This is considered a bug so
+please report it. As a workaround you can set the ``NIM_LIB_PREFIX`` environment
+variable to the directory where ``lib/system.nim`` (and other standard library
+files) are found. Alternatively you can also configure this in Nimble's
+config file.
+
 ## Repository information
 
 This repository has two main branches: ``master`` and ``stable``.
@@ -773,7 +950,7 @@ The ``master`` branch is...
 
 * default
 * bleeding edge
-* tested to compile with the latest Nim version
+* tested to compile with a pinned (close to HEAD) commit of Nim
 
 The ``stable`` branch is...
 
@@ -784,21 +961,37 @@ The ``stable`` branch is...
 Note: The travis build only tests whether Nimble works with the latest Nim
 version.
 
-A new Nim release (via ``koch xz``) will always bundle the latest tagged
-Nimble release.
+A new Nim release (via ``koch xz``) will always bundle the ``stable`` branch.
 
 ## Contribution
 
 If you would like to help, feel free to fork and make any additions you see fit
 and then send a pull request.
 
-If you have any questions about the project you can ask me directly on github,
-ask on the Nim [forum](http://forum.nim-lang.org), or ask on Freenode in
+If you have any questions about the project, you can ask me directly on GitHub,
+ask on the Nim [forum](https://forum.nim-lang.org), or ask on Freenode in
 the #nim channel.
 
+## Implementation details
+
+### .nimble-link
+
+These files are created by Nimble when using the ``develop`` command. They
+are very simple and contain two lines.
+
+**The first line:** Always a path to the `.nimble` file.
+
+**The second line:** Always a path to the Nimble package's source code. Usually
+``$pkgDir/src``, depending on what ``srcDir`` is set to.
+
+The paths written by Nimble are **always** absolute. But Nimble (and the
+Nim compiler) also supports relative paths, which will be read relative to
+the `.nimble-link` file.
+
 ## About
 
-Nimble has been written by [Dominik Picheta](http://picheta.me/) with help from
+Nimble has been written by [Dominik Picheta](https://picheta.me/) with help from
 a number of
 [contributors](https://github.com/nim-lang/nimble/graphs/contributors).
-It is licensed under the BSD license (Look at license.txt for more info).
+It is licensed under the 3-clause BSD license, see [license.txt](license.txt)
+for more information.

src/nimble.nim.cfg

@@ -3,3 +3,4 @@
 --path:"$nim/"
 --path:"./vendor/nim"
 -d:ssl
+-d:nimcore # Enable 'gorge' in Nim's VM. See https://github.com/nim-lang/Nim/issues/8096

src/nimblepkg/cli.nim

@@ -12,13 +12,22 @@
 #   - Bright for HighPriority.
 #   - Normal for MediumPriority.
 
-import logging, terminal, sets, strutils
+import terminal, sets, strutils
+import version
+
+when not declared(initHashSet):
+  import common
 
 type
   CLI* = ref object
     level: Priority
     warnings: HashSet[(string, string)]
     suppressionCount: int ## Amount of messages which were not shown.
+    showColor: bool ## Whether messages should be colored.
+    suppressMessages: bool ## Whether Warning, Message and Success messages
+                           ## should be suppressed, useful for
+                           ## commands like `dump` whose output should be
+                           ## machine readable.
 
   Priority* = enum
     DebugPriority, LowPriority, MediumPriority, HighPriority
@@ -36,11 +45,14 @@ const
   styles: array[DebugPriority .. HighPriority, set[Style]] =
     [{styleDim}, {styleDim}, {}, {styleBright}]
 
+
 proc newCLI(): CLI =
   result = CLI(
     level: HighPriority,
-    warnings: initSet[(string, string)](),
-    suppressionCount: 0
+    warnings: initHashSet[(string, string)](),
+    suppressionCount: 0,
+    showColor: true,
+    suppressMessages: false
   )
 
 var globalCLI = newCLI()
@@ -50,19 +62,37 @@ proc calculateCategoryOffset(category: string): int =
   assert category.len <= longestCategory
   return longestCategory - category.len
 
+proc isSuppressed(displayType: DisplayType): bool =
+  # Don't print any Warning, Message or Success messages when suppression of
+  # warnings is enabled. That is, unless the user asked for --verbose output.
+  if globalCLI.suppressMessages and displayType >= Warning and
+     globalCLI.level == HighPriority:
+    return true
+
 proc displayCategory(category: string, displayType: DisplayType,
                      priority: Priority) =
+  if isSuppressed(displayType):
+    return
+
   # Calculate how much the `category` must be offset to align along a center
   # line.
   let offset = calculateCategoryOffset(category)
+
   # Display the category.
-  if priority != DebugPriority:
-    setForegroundColor(stdout, foregrounds[displayType])
-  writeStyled("$1$2 " % [repeatChar(offset), category], styles[priority])
-  resetAttributes()
+  let text = "$1$2 " % [spaces(offset), category]
+  if globalCLI.showColor:
+    if priority != DebugPriority:
+      setForegroundColor(stdout, foregrounds[displayType])
+    writeStyled(text, styles[priority])
+    resetAttributes()
+  else:
+    stdout.write(text)
 
 proc displayLine(category, line: string, displayType: DisplayType,
                  priority: Priority) =
+  if isSuppressed(displayType):
+    return
+
   displayCategory(category, displayType, priority)
 
   # Display the message.
@@ -117,7 +147,7 @@ proc prompt*(forcePrompts: ForcePrompt, question: string): bool =
     display("Prompt:", question & " -> [forced no]", Warning, HighPriority)
     return false
   of dontForcePrompt:
-    display("Prompt:", question & " [y/N]", Warning, HighPriority)
+    displayLine("Prompt:", question & " [y/N]", Warning, HighPriority)
     displayCategory("Answer:", Warning, HighPriority)
     let yn = stdin.readLine()
     case yn.normalize
@@ -128,23 +158,120 @@ proc prompt*(forcePrompts: ForcePrompt, question: string): bool =
     else:
       return false
 
-proc promptCustom*(question, default: string): string =
-  if default == "":
-    display("Prompt:", question, Warning, HighPriority)
-    displayCategory("Answer:", Warning, HighPriority)
-    let user = stdin.readLine()
-    if user.len == 0: return promptCustom(question, default)
-    else: return user
+proc promptCustom*(forcePrompts: ForcePrompt, question, default: string): string =
+  case forcePrompts:
+  of forcePromptYes:
+    display("Prompt:", question & " -> [forced " & default & "]", Warning,
+      HighPriority)
+    return default
   else:
-    display("Prompt:", question & " [" & default & "]", Warning, HighPriority)
-    displayCategory("Answer:", Warning, HighPriority)
-    let user = stdin.readLine()
-    if user == "": return default
-    else: return user
+    if default == "":
+      display("Prompt:", question, Warning, HighPriority)
+      displayCategory("Answer:", Warning, HighPriority)
+      let user = stdin.readLine()
+      if user.len == 0: return promptCustom(forcePrompts, question, default)
+      else: return user
+    else:
+      display("Prompt:", question & " [" & default & "]", Warning, HighPriority)
+      displayCategory("Answer:", Warning, HighPriority)
+      let user = stdin.readLine()
+      if user == "": return default
+      else: return user
+
+proc promptCustom*(question, default: string): string =
+  return promptCustom(dontForcePrompt, question, default)
+
+proc promptListInteractive(question: string, args: openarray[string]): string =
+  display("Prompt:", question, Warning, HighPriority)
+  display("Select", "Cycle with 'Tab', 'Enter' when done", Message,
+    HighPriority)
+  displayCategory("Choices:", Warning, HighPriority)
+  var
+    current = 0
+    selected = false
+  # Incase the cursor is at the bottom of the terminal
+  for arg in args:
+    stdout.write "\n"
+  # Reset the cursor to the start of the selection prompt
+  cursorUp(stdout, args.len)
+  cursorForward(stdout, longestCategory)
+  hideCursor(stdout)
+
+  # The selection loop
+  while not selected:
+    setForegroundColor(fgDefault)
+    # Loop through the options
+    for i, arg in args:
+      # Check if the option is the current
+      if i == current:
+        writeStyled("> " & arg & " <", {styleBright})
+      else:
+        writeStyled("  " & arg & "  ", {styleDim})
+      # Move the cursor back to the start
+      for s in 0..<(arg.len + 4):
+        cursorBackward(stdout)
+      # Move down for the next item
+      cursorDown(stdout)
+    # Move the cursor back up to the start of the selection prompt
+    for i in 0..<(args.len()):
+      cursorUp(stdout)
+    resetAttributes(stdout)
+
+    # Begin key input
+    while true:
+      case getch():
+      of '\t':
+        current = (current + 1) mod args.len
+        break
+      of '\r':
+        selected = true
+        break
+      of '\3':
+        showCursor(stdout)
+        raise newException(NimbleError, "Keyboard interrupt")
+      else: discard
+
+  # Erase all lines of the selection
+  for i in 0..<args.len:
+    eraseLine(stdout)
+    cursorDown(stdout)
+  # Move the cursor back up to the initial selection line
+  for i in 0..<args.len():
+    cursorUp(stdout)
+  showCursor(stdout)
+  display("Answer:", args[current], Warning,HighPriority)
+  return args[current]
+
+proc promptListFallback(question: string, args: openarray[string]): string =
+  display("Prompt:", question & " [" & join(args, "/") & "]", Warning,
+    HighPriority)
+  displayCategory("Answer:", Warning, HighPriority)
+  result = stdin.readLine()
+  for arg in args:
+    if arg.cmpIgnoreCase(result) == 0:
+      return arg
+
+proc promptList*(forcePrompts: ForcePrompt, question: string, args: openarray[string]): string =
+  case forcePrompts:
+  of forcePromptYes:
+    result = args[0]
+    display("Prompt:", question & " -> [forced " & result & "]", Warning,
+      HighPriority)
+  else:
+    if isatty(stdout):
+      return promptListInteractive(question, args)
+    else:
+      return promptListFallback(question, args)
 
 proc setVerbosity*(level: Priority) =
   globalCLI.level = level
 
+proc setShowColor*(val: bool) =
+  globalCLI.showColor = val
+
+proc setSuppressMessages*(val: bool) =
+  globalCLI.suppressMessages = val
+
 when isMainModule:
   display("Reading", "config file at /Users/dom/.config/nimble/nimble.ini",
           priority = LowPriority)

src/nimblepkg/common.nim

@@ -8,7 +8,6 @@ when not defined(nimscript):
   import sets
 
   import version
-  export version.NimbleError # TODO: Surely there is a better way?
 
   type
     BuildFailed* = object of NimbleError
@@ -18,11 +17,13 @@ when not defined(nimscript):
       isNimScript*: bool ## Determines if this pkg info was read from a nims file
       isMinimal*: bool
       isInstalled*: bool ## Determines if the pkg this info belongs to is installed
+      isLinked*: bool ## Determines if the pkg this info belongs to has been linked via `develop`
       postHooks*: HashSet[string] ## Useful to know so that Nimble doesn't execHook unnecessarily
       preHooks*: HashSet[string]
       name*: string
       ## The version specified in the .nimble file.Assuming info is non-minimal,
-      ## it will always be a non-special version such as '0.1.4'
+      ## it will always be a non-special version such as '0.1.4'.
+      ## If in doubt, use `getConcreteVersion` instead.
       version*: string
       specialVersion*: string ## Either `myVersion` or a special version such as #head.
       author*: string
@@ -49,5 +50,29 @@ when not defined(nimscript):
     exc.hint = hint
     raise exc
 
+  proc getOutputInfo*(err: ref NimbleError): (string, string) =
+    var error = ""
+    var hint = ""
+    error = err.msg
+    when not defined(release):
+      let stackTrace = getStackTrace(err)
+      error = stackTrace & "\n\n" & error
+    if not err.isNil:
+      hint = err.hint
+
+    return (error, hint)
+
 const
-  nimbleVersion* = "0.8.0"
+  nimbleVersion* = "0.11.0"
+
+when not declared(initHashSet):
+  import sets
+
+  template initHashSet*[A](initialSize = 64): HashSet[A] =
+    initSet[A](initialSize)
+
+when not declared(toHashSet):
+  import sets
+
+  template toHashSet*[A](keys: openArray[A]): HashSet[A] =
+    toSet(keys)

src/nimblepkg/config.nim

@@ -1,8 +1,8 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
-import parsecfg, streams, strutils, os, tables, Uri
+import parsecfg, streams, strutils, os, tables, uri
 
-import tools, version, common, cli
+import version, cli
 
 type
   Config* = object
@@ -11,16 +11,15 @@ type
     packageLists*: Table[string, PackageList] ## Names -> packages.json files
     cloneUsingHttps*: bool # Whether to replace git:// for https://
     httpProxy*: Uri # Proxy for package list downloads.
+    nimLibPrefix*: string # Nim stdlib prefix.
 
   PackageList* = object
     name*: string
     urls*: seq[string]
+    path*: string
 
 proc initConfig(): Config =
-  if getNimrodVersion() > newVersion("0.9.6"):
-    result.nimbleDir = getHomeDir() / ".nimble"
-  else:
-    result.nimbleDir = getHomeDir() / ".babel"
+  result.nimbleDir = getHomeDir() / ".nimble"
 
   result.httpProxy = initUri()
 
@@ -35,9 +34,12 @@ proc initConfig(): Config =
   ])
   result.packageLists["official"] = defaultPkgList
 
+  result.nimLibPrefix = ""
+
 proc initPackageList(): PackageList =
   result.name = ""
   result.urls = @[]
+  result.path = ""
 
 proc addCurrentPkgList(config: var Config, currentPackageList: PackageList) =
   if currentPackageList.name.len > 0:
@@ -56,7 +58,6 @@ proc parseConfig*(): Config =
     if f != nil:
       display("Warning", "Using deprecated config file at " & confFile,
               Warning, HighPriority)
-
   if f != nil:
     display("Reading", "config file at " & confFile, priority = LowPriority)
     var p: CfgParser
@@ -67,7 +68,12 @@ proc parseConfig*(): Config =
       var e = next(p)
       case e.kind
       of cfgEof:
-        addCurrentPkgList(result, currentPackageList)
+        if currentSection.len > 0:
+          if currentPackageList.urls.len == 0 and currentPackageList.path == "":
+            raise newException(NimbleError, "Package list '$1' requires either url or path" % currentPackageList.name)
+          if currentPackageList.urls.len > 0 and currentPackageList.path != "":
+            raise newException(NimbleError, "Attempted to specify `url` and `path` for the same package list '$1'" % currentPackageList.name)
+          addCurrentPkgList(result, currentPackageList)
         break
       of cfgSectionStart:
         addCurrentPkgList(result, currentPackageList)
@@ -100,6 +106,16 @@ proc parseConfig*(): Config =
           of "packagelist":
             currentPackageList.urls.add(e.value)
           else: assert false
+        of "path":
+          case currentSection.normalize
+          of "packagelist":
+            if currentPackageList.path != "":
+              raise newException(NimbleError, "Attempted to specify more than one `path` for the same package list.")
+            else:
+              currentPackageList.path = e.value
+          else: assert false
+        of "nimlibprefix":
+          result.nimLibPrefix = e.value
         else:
           raise newException(NimbleError, "Unable to parse config file:" &
                                      " Unknown key: " & e.key)

src/nimblepkg/download.nim

@@ -1,15 +1,16 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
 
-import parseutils, os, osproc, strutils, tables, pegs
-
+import parseutils, os, osproc, strutils, tables, pegs, uri
 import packageinfo, packageparser, version, tools, common, options, cli
+from algorithm import SortOrder, sorted
+from sequtils import toSeq, filterIt, map
 
 type
   DownloadMethod* {.pure.} = enum
     git = "git", hg = "hg"
 
-proc getSpecificDir(meth: DownloadMethod): string =
+proc getSpecificDir(meth: DownloadMethod): string {.used.} =
   case meth
   of DownloadMethod.git:
     ".git"
@@ -24,11 +25,12 @@ proc doCheckout(meth: DownloadMethod, downloadDir, branch: string) =
       # clone has happened. Like in the case of git on Windows where it
       # messes up the damn line endings.
       doCmd("git checkout --force " & branch)
+      doCmd("git submodule update --recursive")
   of DownloadMethod.hg:
     cd downloadDir:
       doCmd("hg checkout " & branch)
 
-proc doPull(meth: DownloadMethod, downloadDir: string) =
+proc doPull(meth: DownloadMethod, downloadDir: string) {.used.} =
   case meth
   of DownloadMethod.git:
     doCheckout(meth, downloadDir, "")
@@ -42,17 +44,17 @@ proc doPull(meth: DownloadMethod, downloadDir: string) =
       doCmd("hg pull")
 
 proc doClone(meth: DownloadMethod, url, downloadDir: string, branch = "",
-            tip = true) =
+             onlyTip = true) =
   case meth
   of DownloadMethod.git:
     let
-      depthArg = if tip: "--depth 1 " else: ""
+      depthArg = if onlyTip: "--depth 1 " else: ""
       branchArg = if branch == "": "" else: "-b " & branch & " "
     doCmd("git clone --recursive " & depthArg & branchArg & url &
           " " & downloadDir)
   of DownloadMethod.hg:
     let
-      tipArg = if tip: "-r tip " else: ""
+      tipArg = if onlyTip: "-r tip " else: ""
       branchArg = if branch == "": "" else: "-b " & branch & " "
     doCmd("hg clone " & tipArg & branchArg & url & " " & downloadDir)
 
@@ -102,14 +104,21 @@ proc getTagsListRemote*(url: string, meth: DownloadMethod): seq[string] =
     # http://stackoverflow.com/questions/2039150/show-tags-for-remote-hg-repository
     raise newException(ValueError, "Hg doesn't support remote tag querying.")
 
-proc getVersionList*(tags: seq[string]): Table[Version, string] =
-  # Returns: TTable of version -> git tag name
-  result = initTable[Version, string]()
-  for tag in tags:
-    if tag != "":
-      let i = skipUntil(tag, Digits) # skip any chars before the version
-      # TODO: Better checking, tags can have any names. Add warnings and such.
-      result[newVersion(tag[i .. tag.len-1])] = tag
+proc getVersionList*(tags: seq[string]): OrderedTable[Version, string] =
+  ## Return an ordered table of Version -> git tag label.  Ordering is
+  ## in descending order with the most recent version first.
+  let taggedVers: seq[tuple[ver: Version, tag: string]] =
+    tags
+      .filterIt(it != "")
+      .map(proc(s: string): tuple[ver: Version, tag: string] =
+        # skip any chars before the version
+        let i = skipUntil(s, Digits)
+        # TODO: Better checking, tags can have any
+        # names. Add warnings and such.
+        result = (newVersion(s[i .. s.len-1]), s))
+      .sorted(proc(a, b: (Version, string)): int = cmp(a[0], b[0]),
+              SortOrder.Descending)
+  result = toOrderedTable[Version, string](taggedVers)
 
 proc getDownloadMethod*(meth: string): DownloadMethod =
   case meth
@@ -132,13 +141,24 @@ proc checkUrlType*(url: string): DownloadMethod =
   elif doCmdEx("hg identify " & url).exitCode == QuitSuccess:
     return DownloadMethod.hg
   else:
-    raise newException(NimbleError, "Unable to identify url.")
+    raise newException(NimbleError, "Unable to identify url: " & url)
+
+proc getUrlData*(url: string): (string, Table[string, string]) =
+  var uri = parseUri(url)
+  # TODO: use uri.parseQuery once it lands... this code is quick and dirty.
+  var subdir = ""
+  if uri.query.startsWith("subdir="):
+    subdir = uri.query[7 .. ^1]
+
+  uri.query = ""
+  return ($uri, {"subdir": subdir}.toTable())
 
 proc isURL*(name: string): bool =
   name.startsWith(peg" @'://' ")
 
-proc doDownload*(url: string, downloadDir: string, verRange: VersionRange,
-                 downMethod: DownloadMethod, options: Options): Version =
+proc doDownload(url: string, downloadDir: string, verRange: VersionRange,
+                 downMethod: DownloadMethod,
+                 options: Options): Version =
   ## Downloads the repository specified by ``url`` using the specified download
   ## method.
   ##
@@ -155,30 +175,20 @@ proc doDownload*(url: string, downloadDir: string, verRange: VersionRange,
     meth
     if $latest.ver != "":
       result = latest.ver
-    else:
-      # Result should already be set to #head here.
-      assert(not result.isNil)
-
-  proc verifyClone() =
-    ## Makes sure that the downloaded package's version satisfies the requested
-    ## version range.
-    let pkginfo = getPkgInfo(downloadDir, options)
-    if pkginfo.version.newVersion notin verRange:
-      raise newException(NimbleError,
-        "Downloaded package's version does not satisfy requested version " &
-        "range: wanted $1 got $2." %
-        [$verRange, $pkginfo.version])
 
   removeDir(downloadDir)
   if verRange.kind == verSpecial:
     # We want a specific commit/branch/tag here.
     if verRange.spe == getHeadName(downMethod):
-      doClone(downMethod, url, downloadDir) # Grab HEAD.
+       # Grab HEAD.
+      doClone(downMethod, url, downloadDir, onlyTip = not options.forceFullClone)
     else:
       # Grab the full repo.
-      doClone(downMethod, url, downloadDir, tip = false)
+      doClone(downMethod, url, downloadDir, onlyTip = false)
       # Then perform a checkout operation to get the specified branch/commit.
-      doCheckout(downMethod, downloadDir, $verRange.spe)
+      # `spe` starts with '#', trim it.
+      doAssert(($verRange.spe)[0] == '#')
+      doCheckout(downMethod, downloadDir, substr($verRange.spe, 1))
     result = verRange.spe
   else:
     case downMethod
@@ -191,14 +201,13 @@ proc doDownload*(url: string, downloadDir: string, verRange: VersionRange,
         getLatestByTag:
           display("Cloning", "latest tagged version: " & latest.tag,
                   priority = MediumPriority)
-          doClone(downMethod, url, downloadDir, latest.tag)
+          doClone(downMethod, url, downloadDir, latest.tag,
+                  onlyTip = not options.forceFullClone)
       else:
         # If no commits have been tagged on the repo we just clone HEAD.
         doClone(downMethod, url, downloadDir) # Grab HEAD.
-
-      verifyClone()
     of DownloadMethod.hg:
-      doClone(downMethod, url, downloadDir)
+      doClone(downMethod, url, downloadDir, onlyTip = not options.forceFullClone)
       result = getHeadName(downMethod)
       let versions = getTagsList(downloadDir, downMethod).getVersionList()
 
@@ -208,7 +217,57 @@ proc doDownload*(url: string, downloadDir: string, verRange: VersionRange,
                   priority = MediumPriority)
           doCheckout(downMethod, downloadDir, latest.tag)
 
-      verifyClone()
+proc downloadPkg*(url: string, verRange: VersionRange,
+                 downMethod: DownloadMethod,
+                 subdir: string,
+                 options: Options,
+                 downloadPath = ""): (string, Version) =
+  ## Downloads the repository as specified by ``url`` and ``verRange`` using
+  ## the download method specified.
+  ##
+  ## If `downloadPath` isn't specified a location in /tmp/ will be used.
+  ##
+  ## Returns the directory where it was downloaded (subdir is appended) and
+  ## the concrete version  which was downloaded.
+  let downloadDir =
+    if downloadPath == "":
+      (getNimbleTempDir() / getDownloadDirName(url, verRange))
+    else:
+      downloadPath
+
+  createDir(downloadDir)
+  var modUrl =
+    if url.startsWith("git://") and options.config.cloneUsingHttps:
+      "https://" & url[6 .. ^1]
+    else: url
+
+  # Fixes issue #204
+  # github + https + trailing url slash causes a
+  # checkout/ls-remote to fail with Repository not found
+  if modUrl.contains("github.com") and modUrl.endswith("/"):
+    modUrl = modUrl[0 .. ^2]
+
+  if subdir.len > 0:
+    display("Downloading", "$1 using $2 (subdir is '$3')" %
+                           [modUrl, $downMethod, subdir],
+            priority = HighPriority)
+  else:
+    display("Downloading", "$1 using $2" % [modUrl, $downMethod],
+            priority = HighPriority)
+  result = (
+    downloadDir / subdir,
+    doDownload(modUrl, downloadDir, verRange, downMethod, options)
+  )
+
+  if verRange.kind != verSpecial:
+    ## Makes sure that the downloaded package's version satisfies the requested
+    ## version range.
+    let pkginfo = getPkgInfo(result[0], options)
+    if pkginfo.version.newVersion notin verRange:
+      raise newException(NimbleError,
+        "Downloaded package's version does not satisfy requested version " &
+        "range: wanted $1 got $2." %
+        [$verRange, $pkginfo.version])
 
 proc echoPackageVersions*(pkg: Package) =
   let downMethod = pkg.downloadMethod.getDownloadMethod()
@@ -217,14 +276,8 @@ proc echoPackageVersions*(pkg: Package) =
     try:
       let versions = getTagsListRemote(pkg.url, downMethod).getVersionList()
       if versions.len > 0:
-        var vstr = ""
-        var i = 0
-        for v in values(versions):
-          if i != 0:
-            vstr.add(", ")
-          vstr.add(v)
-          i.inc
-        echo("  versions:    " & vstr)
+        let sortedVersions = toSeq(values(versions))
+        echo("  versions:    " & join(sortedVersions, ", "))
       else:
         echo("  versions:    (No versions tagged in the remote repository)")
     except OSError:
@@ -232,3 +285,32 @@ proc echoPackageVersions*(pkg: Package) =
   of DownloadMethod.hg:
     echo("  versions:    (Remote tag retrieval not supported by " &
         pkg.downloadMethod & ")")
+
+when isMainModule:
+  # Test version sorting
+  block:
+    let data = @["v9.0.0-taeyeon", "v9.0.1-jessica", "v9.2.0-sunny",
+                 "v9.4.0-tiffany", "v9.4.2-hyoyeon"]
+    let expected = toOrderedTable[Version, string]({
+      newVersion("9.4.2-hyoyeon"): "v9.4.2-hyoyeon",
+      newVersion("9.4.0-tiffany"): "v9.4.0-tiffany",
+      newVersion("9.2.0-sunny"): "v9.2.0-sunny",
+      newVersion("9.0.1-jessica"): "v9.0.1-jessica",
+      newVersion("9.0.0-taeyeon"): "v9.0.0-taeyeon"
+    })
+    doAssert expected == getVersionList(data)
+
+
+  block:
+    let data2 = @["v0.1.0", "v0.1.1", "v0.2.0",
+                 "0.4.0", "v0.4.2"]
+    let expected2 = toOrderedTable[Version, string]({
+      newVersion("0.4.2"): "v0.4.2",
+      newVersion("0.4.0"): "0.4.0",
+      newVersion("0.2.0"): "v0.2.0",
+      newVersion("0.1.1"): "v0.1.1",
+      newVersion("0.1.0"): "v0.1.0",
+    })
+    doAssert expected2 == getVersionList(data2)
+
+  echo("Everything works!")

src/nimblepkg/init.nim

@@ -0,0 +1,184 @@
+import os, strutils
+
+import ./cli, ./tools
+
+type
+  PkgInitInfo* = tuple
+    pkgName: string
+    pkgVersion: string
+    pkgAuthor: string
+    pkgDesc: string
+    pkgLicense: string
+    pkgBackend: string
+    pkgSrcDir: string
+    pkgNimDep: string
+    pkgType: string
+
+proc writeExampleIfNonExistent(file: string, content: string) =
+  if not existsFile(file):
+    writeFile(file, content)
+  else:
+    display("Info:", "File " & file & " already exists, did not write " &
+            "example code", priority = HighPriority)
+
+proc createPkgStructure*(info: PkgInitInfo, pkgRoot: string) =
+  # Create source directory
+  createDirD(pkgRoot / info.pkgSrcDir)
+
+  # Initialise the source code directories and create some example code.
+  var nimbleFileOptions = ""
+  case info.pkgType
+  of "binary":
+    let mainFile = pkgRoot / info.pkgSrcDir / info.pkgName.changeFileExt("nim")
+    writeExampleIfNonExistent(mainFile,
+"""
+# This is just an example to get you started. A typical binary package
+# uses this file as the main entry point of the application.
+
+when isMainModule:
+  echo("Hello, World!")
+"""
+    )
+    nimbleFileOptions.add("bin           = @[\"$1\"]\n" % info.pkgName)
+  of "library":
+    let mainFile = pkgRoot / info.pkgSrcDir / info.pkgName.changeFileExt("nim")
+    writeExampleIfNonExistent(mainFile,
+"""
+# This is just an example to get you started. A typical library package
+# exports the main API in this file. Note that you cannot rename this file
+# but you can remove it if you wish.
+
+proc add*(x, y: int): int =
+  ## Adds two files together.
+  return x + y
+"""
+    )
+
+    createDirD(pkgRoot / info.pkgSrcDir / info.pkgName)
+    let submodule = pkgRoot / info.pkgSrcDir / info.pkgName /
+        "submodule".addFileExt("nim")
+    writeExampleIfNonExistent(submodule,
+"""
+# This is just an example to get you started. Users of your library will
+# import this file by writing ``import $1/submodule``. Feel free to rename or
+# remove this file altogether. You may create additional modules alongside
+# this file as required.
+
+type
+  Submodule* = object
+    name*: string
+
+proc initSubmodule*(): Submodule =
+  ## Initialises a new ``Submodule`` object.
+  Submodule(name: "Anonymous")
+""" % info.pkgName
+    )
+  of "hybrid":
+    let mainFile = pkgRoot / info.pkgSrcDir / info.pkgName.changeFileExt("nim")
+    writeExampleIfNonExistent(mainFile,
+"""
+# This is just an example to get you started. A typical hybrid package
+# uses this file as the main entry point of the application.
+
+import $1pkg/submodule
+
+when isMainModule:
+  echo(getWelcomeMessage())
+""" % info.pkgName
+    )
+
+    let pkgSubDir = pkgRoot / info.pkgSrcDir / info.pkgName & "pkg"
+    createDirD(pkgSubDir)
+    let submodule = pkgSubDir / "submodule".addFileExt("nim")
+    writeExampleIfNonExistent(submodule,
+"""
+# This is just an example to get you started. Users of your hybrid library will
+# import this file by writing ``import $1pkg/submodule``. Feel free to rename or
+# remove this file altogether. You may create additional modules alongside
+# this file as required.
+
+proc getWelcomeMessage*(): string = "Hello, World!"
+""" % info.pkgName
+    )
+    nimbleFileOptions.add("installExt    = @[\"nim\"]\n")
+    nimbleFileOptions.add("bin           = @[\"$1\"]\n" % info.pkgName)
+  else:
+    assert false, "Invalid package type specified."
+
+  let pkgTestDir = "tests"
+  # Create test directory
+  case info.pkgType
+  of "binary":
+    discard
+  of "hybrid", "library":
+    let pkgTestPath = pkgRoot / pkgTestDir
+    createDirD(pkgTestPath)
+
+    writeFile(pkgTestPath / "config".addFileExt("nims"),
+      "switch(\"path\", \"$$projectDir/../$#\")" % info.pkgSrcDir
+    )
+
+    if info.pkgType == "library":
+      writeExampleIfNonExistent(pkgTestPath / "test1".addFileExt("nim"),
+"""
+# This is just an example to get you started. You may wish to put all of your
+# tests into a single file, or separate them into multiple `test1`, `test2`
+# etc. files (better names are recommended, just make sure the name starts with
+# the letter 't').
+#
+# To run these tests, simply execute `nimble test`.
+
+import unittest
+
+import $1
+test "can add":
+  check add(5, 5) == 10
+""" % info.pkgName
+      )
+    else:
+      writeExampleIfNonExistent(pkgTestPath / "test1".addFileExt("nim"),
+"""
+# This is just an example to get you started. You may wish to put all of your
+# tests into a single file, or separate them into multiple `test1`, `test2`
+# etc. files (better names are recommended, just make sure the name starts with
+# the letter 't').
+#
+# To run these tests, simply execute `nimble test`.
+
+import unittest
+
+import $1pkg/submodule
+test "correct welcome":
+  check getWelcomeMessage() == "Hello, World!"
+""" % info.pkgName
+      )
+  else:
+    assert false, "Invalid package type specified."
+
+  # Write the nimble file
+  let nimbleFile = pkgRoot / info.pkgName.changeFileExt("nimble")
+  # Only write backend if it isn't "c"
+  var pkgBackend = ""
+  if (info.pkgBackend != "c"):
+    pkgBackend = "backend       = " & info.pkgbackend.escape()
+  writeFile(nimbleFile, """# Package
+
+version       = $#
+author        = "$#"
+description   = "$#"
+license       = $#
+srcDir        = $#
+$#
+$#
+
+# Dependencies
+
+requires "nim >= $#"
+""" % [
+      info.pkgVersion.escape(), info.pkgAuthor.replace("\"", "\\\""), info.pkgDesc.replace("\"", "\\\""),
+      info.pkgLicense.escape(), info.pkgSrcDir.escape(), nimbleFileOptions,
+      pkgBackend, info.pkgNimDep
+    ]
+  )
+
+  display("Info:", "Nimble file created successfully", priority=MediumPriority)

src/nimblepkg/nimscriptapi.nim

@@ -3,6 +3,12 @@
 
 ## This module is implicitly imported in NimScript .nimble files.
 
+import system except getCommand, setCommand, switch, `--`
+import strformat, strutils, tables
+
+when not defined(nimscript):
+  import os
+
 var
   packageName* = ""    ## Set this to the package name. It
                        ## is usually not required to do that, nims' filename is
@@ -11,7 +17,7 @@ var
   author*: string      ## The package's author.
   description*: string ## The package's description.
   license*: string     ## The package's license.
-  srcdir*: string      ## The package's source directory.
+  srcDir*: string      ## The package's source directory.
   binDir*: string      ## The package's binary directory.
   backend*: string     ## The package's backend.
 
@@ -22,26 +28,169 @@ var
   foreignDeps*: seq[string] = @[] ## The foreign dependencies. Only
                                   ## exported for 'distros.nim'.
 
+  beforeHooks: seq[string] = @[]
+  afterHooks: seq[string] = @[]
+  commandLineParams: seq[string] = @[]
+  flags: TableRef[string, seq[string]]
+
+  command = "e"
+  project = ""
+  success = false
+  retVal = true
+  projectFile = ""
+  outFile = ""
+
 proc requires*(deps: varargs[string]) =
   ## Call this to set the list of requirements of your Nimble
   ## package.
   for d in deps: requiresData.add(d)
 
+proc getParams() =
+  # Called by nimscriptwrapper.nim:execNimscript()
+  #   nim e --flags /full/path/to/file.nims /full/path/to/file.out action
+  for i in 2 .. paramCount():
+    let
+      param = paramStr(i)
+    if param[0] != '-':
+      if projectFile.len == 0:
+        projectFile = param
+      elif outFile.len == 0:
+        outFile = param
+      else:
+        commandLineParams.add param.normalize
+
+proc getCommand*(): string =
+  return command
+
+proc setCommand*(cmd: string, prj = "") =
+  command = cmd
+  if prj.len != 0:
+    project = prj
+
+proc switch*(key: string, value="") =
+  if flags.isNil:
+    flags = newTable[string, seq[string]]()
+
+  if flags.hasKey(key):
+    flags[key].add(value)
+  else:
+    flags[key] = @[value]
+
+template `--`*(key, val: untyped) =
+  switch(astToStr(key), strip astToStr(val))
+
+template `--`*(key: untyped) =
+  switch(astToStr(key), "")
+
+template printIfLen(varName) =
+  if varName.len != 0:
+    result &= astToStr(varName) & ": \"\"\"" & varName & "\"\"\"\n"
+
+template printSeqIfLen(varName) =
+  if varName.len != 0:
+    result &= astToStr(varName) & ": \"" & varName.join(", ") & "\"\n"
+
+proc printPkgInfo(): string =
+  if backend.len == 0:
+    backend = "c"
+
+  result = "[Package]\n"
+  if packageName.len != 0:
+    result &= "name: \"" & packageName & "\"\n"
+  printIfLen version
+  printIfLen author
+  printIfLen description
+  printIfLen license
+  printIfLen srcDir
+  printIfLen binDir
+  printIfLen backend
+
+  printSeqIfLen skipDirs
+  printSeqIfLen skipFiles
+  printSeqIfLen skipExt
+  printSeqIfLen installDirs
+  printSeqIfLen installFiles
+  printSeqIfLen installExt
+  printSeqIfLen bin
+  printSeqIfLen beforeHooks
+  printSeqIfLen afterHooks
+
+  if requiresData.len != 0:
+    result &= "\n[Deps]\n"
+    result &= &"requires: \"{requiresData.join(\", \")}\"\n"
+
+proc onExit*() =
+  if "printPkgInfo".normalize in commandLineParams:
+    if outFile.len != 0:
+      writeFile(outFile, printPkgInfo())
+  else:
+    var
+      output = ""
+    output &= "\"success\": " & $success & ", "
+    output &= "\"command\": \"" & command & "\", "
+    if project.len != 0:
+      output &= "\"project\": \"" & project & "\", "
+    if not flags.isNil and flags.len != 0:
+      output &= "\"flags\": {"
+      for key, val in flags.pairs:
+        output &= "\"" & key & "\": ["
+        for v in val:
+          let v = if v.len > 0 and v[0] == '"': strutils.unescape(v)
+                  else: v
+          output &= v.escape & ", "
+        output = output[0 .. ^3] & "], "
+      output = output[0 .. ^3] & "}, "
+
+    output &= "\"retVal\": " & $retVal
+
+    if outFile.len != 0:
+      writeFile(outFile, "{" & output & "}")
+
+# TODO: New release of Nim will move this `task` template under a
+# `when not defined(nimble)`. This will allow us to override it in the future.
+template task*(name: untyped; description: string; body: untyped): untyped =
+  ## Defines a task. Hidden tasks are supported via an empty description.
+  ## Example:
+  ##
+  ## .. code-block:: nim
+  ##  task build, "default build is via the C backend":
+  ##    setCommand "c"
+  proc `name Task`*() = body
+
+  if commandLineParams.len == 0 or "help" in commandLineParams:
+    success = true
+    echo(astToStr(name), "        ", description)
+  elif astToStr(name).normalize in commandLineParams:
+    success = true
+    `name Task`()
+
 template before*(action: untyped, body: untyped): untyped =
   ## Defines a block of code which is evaluated before ``action`` is executed.
   proc `action Before`*(): bool =
     result = true
     body
 
+  beforeHooks.add astToStr(action)
+
+  if (astToStr(action) & "Before").normalize in commandLineParams:
+    success = true
+    retVal = `action Before`()
+
 template after*(action: untyped, body: untyped): untyped =
   ## Defines a block of code which is evaluated after ``action`` is executed.
   proc `action After`*(): bool =
     result = true
     body
 
-template builtin = discard
+  afterHooks.add astToStr(action)
+
+  if (astToStr(action) & "After").normalize in commandLineParams:
+    success = true
+    retVal = `action After`()
 
 proc getPkgDir*(): string =
   ## Returns the package directory containing the .nimble file currently
   ## being evaluated.
-  builtin
+  result = projectFile.rsplit(seps={'/', '\\', ':'}, maxsplit=1)[0]
+
+getParams()

src/nimblepkg/nimscriptexecutor.nim

@@ -0,0 +1,76 @@
+# Copyright (C) Dominik Picheta. All rights reserved.
+# BSD License. Look at license.txt for more info.
+
+import os, strutils, sets
+
+import packageparser, common, packageinfo, options, nimscriptwrapper, cli,
+       version
+
+proc execHook*(options: Options, hookAction: ActionType, before: bool): bool =
+  ## Returns whether to continue.
+  result = true
+
+  # For certain commands hooks should not be evaluated.
+  if hookAction in noHookActions:
+    return
+
+  var nimbleFile = ""
+  try:
+    nimbleFile = findNimbleFile(getCurrentDir(), true)
+  except NimbleError: return true
+  # PackageInfos are cached so we can read them as many times as we want.
+  let pkgInfo = getPkgInfoFromFile(nimbleFile, options)
+  let actionName =
+    if hookAction == actionCustom: options.action.command
+    else: ($hookAction)[6 .. ^1]
+  let hookExists =
+    if before: actionName.normalize in pkgInfo.preHooks
+    else: actionName.normalize in pkgInfo.postHooks
+  if pkgInfo.isNimScript and hookExists:
+    let res = execHook(nimbleFile, actionName, before, options)
+    if res.success:
+      result = res.retVal
+
+proc execCustom*(options: Options,
+                 execResult: var ExecutionResult[bool],
+                 failFast = true): bool =
+  ## Executes the custom command using the nimscript backend.
+  ##
+  ## If failFast is true then exceptions will be raised when something is wrong.
+  ## Otherwise this function will just return false.
+
+  # Custom command. Attempt to call a NimScript task.
+  let nimbleFile = findNimbleFile(getCurrentDir(), true)
+  if not nimbleFile.isNimScript(options) and failFast:
+    writeHelp()
+
+  execResult = execTask(nimbleFile, options.action.command, options)
+  if not execResult.success:
+    if not failFast:
+      return
+    raiseNimbleError(msg = "Could not find task $1 in $2" %
+                           [options.action.command, nimbleFile],
+                     hint = "Run `nimble --help` and/or `nimble tasks` for" &
+                            " a list of possible commands.")
+
+  if execResult.command.normalize == "nop":
+    display("Warning:", "Using `setCommand 'nop'` is not necessary.", Warning,
+            HighPriority)
+    return
+
+  if not execHook(options, actionCustom, false):
+    return
+
+  return true
+
+proc getOptionsForCommand*(execResult: ExecutionResult,
+                           options: Options): Options =
+  ## Creates an Options object for the requested command.
+  var newOptions = options.briefClone()
+  parseCommand(execResult.command, newOptions)
+  for arg in execResult.arguments:
+    parseArgument(arg, newOptions)
+  for flag, vals in execResult.flags:
+    for val in vals:
+      parseFlag(flag, val, newOptions)
+  return newOptions

src/nimblepkg/nimscriptsupport.nim

@@ -1,442 +0,0 @@
-# Copyright (C) Andreas Rumpf. All rights reserved.
-# BSD License. Look at license.txt for more info.
-
-## Implements the new configuration system for Nimble. Uses Nim as a
-## scripting language.
-
-import
-  compiler/ast, compiler/modules, compiler/passes, compiler/passaux,
-  compiler/condsyms, compiler/sem, compiler/semdata,
-  compiler/llstream, compiler/vm, compiler/vmdef, compiler/commands,
-  compiler/msgs, compiler/magicsys, compiler/lists, compiler/idents,
-  compiler/nimconf
-
-from compiler/scriptconfig import setupVM
-from compiler/astalgo import strTableGet
-import compiler/options as compiler_options
-
-import common, version, options, packageinfo, cli
-import os, strutils, strtabs, times, osproc, sets
-
-when not declared(resetAllModulesHard):
-  import compiler/modulegraphs
-
-type
-  ExecutionResult*[T] = object
-    success*: bool
-    command*: string
-    arguments*: seq[string]
-    flags*: StringTableRef
-    retVal*: T
-
-const
-  internalCmd = "NimbleInternal"
-  nimscriptApi = staticRead("nimscriptapi.nim")
-
-proc raiseVariableError(ident, typ: string) {.noinline.} =
-  raise newException(NimbleError,
-    "NimScript's variable '" & ident & "' needs a value of type '" & typ & "'.")
-
-proc isStrLit(n: PNode): bool = n.kind in {nkStrLit..nkTripleStrLit}
-
-proc getGlobal(ident: PSym): string =
-  let n = vm.globalCtx.getGlobalValue(ident)
-  if n.isStrLit:
-    result = if n.strVal.isNil: "" else: n.strVal
-  else:
-    raiseVariableError(ident.name.s, "string")
-
-proc getGlobalAsSeq(ident: PSym): seq[string] =
-  let n = vm.globalCtx.getGlobalValue(ident)
-  result = @[]
-  if n.kind == nkBracket:
-    for x in n:
-      if x.isStrLit:
-        result.add x.strVal
-      else:
-        raiseVariableError(ident.name.s, "seq[string]")
-  else:
-    raiseVariableError(ident.name.s, "seq[string]")
-
-proc extractRequires(ident: PSym, result: var seq[PkgTuple]) =
-  let n = vm.globalCtx.getGlobalValue(ident)
-  if n.kind == nkBracket:
-    for x in n:
-      if x.kind == nkPar and x.len == 2 and x[0].isStrLit and x[1].isStrLit:
-        result.add(parseRequires(x[0].strVal & x[1].strVal))
-      elif x.isStrLit:
-        result.add(parseRequires(x.strVal))
-      else:
-        raiseVariableError("requiresData", "seq[(string, VersionReq)]")
-  else:
-    raiseVariableError("requiresData", "seq[(string, VersionReq)]")
-
-when declared(newIdentCache):
-  var identCache = newIdentCache()
-
-proc setupVM(module: PSym; scriptName: string,
-    flags: StringTableRef): PEvalContext =
-  ## This procedure is exported in the compiler sources, but its implementation
-  ## is too Nim-specific to be used by Nimble.
-  ## Specifically, the implementation of ``switch`` is problematic. Sooo
-  ## I simply copied it here and edited it :)
-
-  when declared(newIdentCache):
-    result = newCtx(module, identCache)
-  else:
-    result = newCtx(module)
-  result.mode = emRepl
-  registerAdditionalOps(result)
-
-  # captured vars:
-  var errorMsg: string
-  var vthisDir = scriptName.splitFile.dir
-
-  proc listDirs(a: VmArgs, filter: set[PathComponent]) =
-    let dir = getString(a, 0)
-    var res: seq[string] = @[]
-    for kind, path in walkDir(dir):
-      if kind in filter: res.add path
-    setResult(a, res)
-
-  template cbconf(name, body) {.dirty.} =
-    result.registerCallback "stdlib.system." & astToStr(name),
-      proc (a: VmArgs) =
-        body
-
-  template cbos(name, body) {.dirty.} =
-    result.registerCallback "stdlib.system." & astToStr(name),
-      proc (a: VmArgs) =
-        try:
-          body
-        except OSError:
-          errorMsg = getCurrentExceptionMsg()
-
-  # Idea: Treat link to file as a file, but ignore link to directory to prevent
-  # endless recursions out of the box.
-  cbos listFiles:
-    listDirs(a, {pcFile, pcLinkToFile})
-  cbos listDirs:
-    listDirs(a, {pcDir})
-  cbos removeDir:
-    os.removeDir getString(a, 0)
-  cbos removeFile:
-    os.removeFile getString(a, 0)
-  cbos createDir:
-    os.createDir getString(a, 0)
-  cbos getOsError:
-    setResult(a, errorMsg)
-  cbos setCurrentDir:
-    os.setCurrentDir getString(a, 0)
-  cbos getCurrentDir:
-    setResult(a, os.getCurrentDir())
-  cbos moveFile:
-    os.moveFile(getString(a, 0), getString(a, 1))
-  cbos copyFile:
-    os.copyFile(getString(a, 0), getString(a, 1))
-  cbos getLastModificationTime:
-    setResult(a, toSeconds(getLastModificationTime(getString(a, 0))))
-
-  cbos rawExec:
-    setResult(a, osproc.execCmd getString(a, 0))
-
-  cbconf getEnv:
-    setResult(a, os.getEnv(a.getString 0))
-  cbconf existsEnv:
-    setResult(a, os.existsEnv(a.getString 0))
-  cbconf dirExists:
-    setResult(a, os.dirExists(a.getString 0))
-  cbconf fileExists:
-    setResult(a, os.fileExists(a.getString 0))
-
-  cbconf thisDir:
-    setResult(a, vthisDir)
-  cbconf put:
-    compiler_options.setConfigVar(getString(a, 0), getString(a, 1))
-  cbconf get:
-    setResult(a, compiler_options.getConfigVar(a.getString 0))
-  cbconf exists:
-    setResult(a, compiler_options.existsConfigVar(a.getString 0))
-  cbconf nimcacheDir:
-    setResult(a, compiler_options.getNimcacheDir())
-  cbconf paramStr:
-    setResult(a, os.paramStr(int a.getInt 0))
-  cbconf paramCount:
-    setResult(a, os.paramCount())
-  cbconf cmpIgnoreStyle:
-    setResult(a, strutils.cmpIgnoreStyle(a.getString 0, a.getString 1))
-  cbconf cmpIgnoreCase:
-    setResult(a, strutils.cmpIgnoreCase(a.getString 0, a.getString 1))
-  cbconf setCommand:
-    compiler_options.command = a.getString 0
-    let arg = a.getString 1
-    if arg.len > 0:
-      gProjectName = arg
-      try:
-        gProjectFull = canonicalizePath(gProjectPath / gProjectName)
-      except OSError:
-        gProjectFull = gProjectName
-  cbconf getCommand:
-    setResult(a, compiler_options.command)
-  cbconf switch:
-    if not flags.isNil:
-      flags[a.getString 0] = a.getString 1
-
-proc getNimPrefixDir(): string = splitPath(findExe("nim")).head.parentDir
-
-when declared(ModuleGraph):
-  var graph: ModuleGraph
-
-proc execScript(scriptName: string, flags: StringTableRef,
-                options: Options): PSym =
-  ## Executes the specified script. Returns the script's module symbol.
-  ##
-  ## No clean up is performed and must be done manually!
-  when declared(resetAllModulesHard):
-    # for compatibility with older Nim versions:
-    if "nimblepkg/nimscriptapi" notin compiler_options.implicitIncludes:
-      compiler_options.implicitIncludes.add("nimblepkg/nimscriptapi")
-  else:
-    if "nimblepkg/nimscriptapi" notin compiler_options.implicitImports:
-      compiler_options.implicitImports.add("nimblepkg/nimscriptapi")
-
-  # Ensure the compiler can find its standard library #220.
-  compiler_options.gPrefixDir = getNimPrefixDir()
-
-  let pkgName = scriptName.splitFile.name
-
-  # Ensure that "nimblepkg/nimscriptapi" is in the PATH.
-  # TODO: put this in a more isolated directory.
-  let tmpNimscriptApiPath = getTempDir() / "nimblepkg" / "nimscriptapi.nim"
-  createDir(tmpNimscriptApiPath.splitFile.dir)
-  if not existsFile(tmpNimscriptApiPath):
-    writeFile(tmpNimscriptApiPath, nimscriptApi)
-  appendStr(searchPaths, getTempDir())
-
-  initDefines()
-  loadConfigs(DefaultConfig)
-  passes.gIncludeFile = includeModule
-  passes.gImportModule = importModule
-
-  defineSymbol("nimscript")
-  defineSymbol("nimconfig")
-  defineSymbol("nimble")
-  registerPass(semPass)
-  registerPass(evalPass)
-
-  appendStr(searchPaths, compiler_options.libpath)
-
-  when declared(resetAllModulesHard):
-    result = makeModule(scriptName)
-  else:
-    graph = newModuleGraph()
-    result = graph.makeModule(scriptName)
-
-  incl(result.flags, sfMainModule)
-  vm.globalCtx = setupVM(result, scriptName, flags)
-
-  # Setup builtins defined in nimscriptapi.nim
-  template cbApi(name, body) {.dirty.} =
-    vm.globalCtx.registerCallback pkgName & "." & astToStr(name),
-      proc (a: VmArgs) =
-        body
-
-  cbApi getPkgDir:
-    setResult(a, scriptName.splitFile.dir)
-
-  when declared(newIdentCache):
-    graph.compileSystemModule(identCache)
-    graph.processModule(result, llStreamOpen(scriptName, fmRead), nil, identCache)
-  else:
-    compileSystemModule()
-    processModule(result, llStreamOpen(scriptName, fmRead), nil)
-
-proc cleanup() =
-  # ensure everything can be called again:
-  compiler_options.gProjectName = ""
-  compiler_options.command = ""
-  when declared(resetAllModulesHard):
-    resetAllModulesHard()
-  else:
-    resetSystemArtifacts()
-  clearPasses()
-  msgs.gErrorMax = 1
-  msgs.writeLnHook = nil
-  vm.globalCtx = nil
-  initDefines()
-
-proc readPackageInfoFromNims*(scriptName: string, options: Options,
-    result: var PackageInfo) =
-  ## Executes the `scriptName` nimscript file. Reads the package information
-  ## that it populates.
-
-  # Setup custom error handling.
-  msgs.gErrorMax = high(int)
-  var previousMsg = ""
-  msgs.writeLnHook =
-    proc (output: string) =
-      # The error counter is incremented after the writeLnHook is invoked.
-      if msgs.gErrorCounter > 0:
-        raise newException(NimbleError, previousMsg)
-      elif previousMsg.len > 0:
-        display("Info", previousMsg, priority = HighPriority)
-      if output.normalize.startsWith("error"):
-        raise newException(NimbleError, output)
-      previousMsg = output
-
-  compiler_options.command = internalCmd
-
-  # Execute the nimscript file.
-  let thisModule = execScript(scriptName, nil, options)
-
-  when declared(resetAllModulesHard):
-    let apiModule = thisModule
-  else:
-    var apiModule: PSym
-    for i in 0..<graph.modules.len:
-      if graph.modules[i] != nil and
-          graph.modules[i].name.s == "nimscriptapi":
-        apiModule = graph.modules[i]
-        break
-    doAssert apiModule != nil
-
-  # Check whether an error has occurred.
-  if msgs.gErrorCounter > 0:
-    raise newException(NimbleError, previousMsg)
-
-  # Extract all the necessary fields populated by the nimscript file.
-  proc getSym(apiModule: PSym, ident: string): PSym =
-    result = apiModule.tab.strTableGet(getIdent(ident))
-    if result.isNil:
-      raise newException(NimbleError, "Ident not found: " & ident)
-
-  template trivialField(field) =
-    result.field = getGlobal(getSym(apiModule, astToStr field))
-
-  template trivialFieldSeq(field) =
-    result.field.add getGlobalAsSeq(getSym(apiModule, astToStr field))
-
-  # keep reasonable default:
-  let name = getGlobal(apiModule.tab.strTableGet(getIdent"packageName"))
-  if name.len > 0: result.name = name
-
-  trivialField version
-  trivialField author
-  trivialField description
-  trivialField license
-  trivialField srcdir
-  trivialField bindir
-  trivialFieldSeq skipDirs
-  trivialFieldSeq skipFiles
-  trivialFieldSeq skipExt
-  trivialFieldSeq installDirs
-  trivialFieldSeq installFiles
-  trivialFieldSeq installExt
-  trivialFieldSeq foreignDeps
-
-  extractRequires(getSym(apiModule, "requiresData"), result.requires)
-
-  let binSeq = getGlobalAsSeq(getSym(apiModule, "bin"))
-  for i in binSeq:
-    result.bin.add(i.addFileExt(ExeExt))
-
-  let backend = getGlobal(getSym(apiModule, "backend"))
-  if backend.len == 0:
-    result.backend = "c"
-  elif cmpIgnoreStyle(backend, "javascript") == 0:
-    result.backend = "js"
-  else:
-    result.backend = backend.toLowerAscii()
-
-  # Grab all the global procs
-  for i in thisModule.tab.data:
-    if not i.isNil():
-      let name = i.name.s.normalize()
-      if name.endsWith("before"):
-        result.preHooks.incl(name[0 .. ^7])
-      if name.endsWith("after"):
-        result.postHooks.incl(name[0 .. ^6])
-
-  cleanup()
-
-proc execTask*(scriptName, taskName: string,
-    options: Options): ExecutionResult[void] =
-  ## Executes the specified task in the specified script.
-  ##
-  ## `scriptName` should be a filename pointing to the nimscript file.
-  result.success = true
-  result.flags = newStringTable()
-  compiler_options.command = internalCmd
-  display("Executing",  "task $# in $#" % [taskName, scriptName],
-          priority = HighPriority)
-
-  let thisModule = execScript(scriptName, result.flags, options)
-  let prc = thisModule.tab.strTableGet(getIdent(taskName & "Task"))
-  if prc.isNil:
-    # Procedure not defined in the NimScript module.
-    result.success = false
-    return
-  discard vm.globalCtx.execProc(prc, [])
-
-  # Read the command, arguments and flags set by the executed task.
-  result.command = compiler_options.command
-  result.arguments = @[]
-  for arg in compiler_options.gProjectName.split():
-    result.arguments.add(arg)
-
-  cleanup()
-
-proc execHook*(scriptName, actionName: string, before: bool,
-    options: Options): ExecutionResult[bool] =
-  ## Executes the specified action's hook. Depending on ``before``, either
-  ## the "before" or the "after" hook.
-  ##
-  ## `scriptName` should be a filename pointing to the nimscript file.
-  result.success = true
-  result.flags = newStringTable()
-  compiler_options.command = internalCmd
-  let hookName =
-    if before: actionName.toLowerAscii & "Before"
-    else: actionName.toLowerAscii & "After"
-  display("Attempting", "to execute hook $# in $#" % [hookName, scriptName],
-          priority = MediumPriority)
-
-  let thisModule = execScript(scriptName, result.flags, options)
-  # Explicitly execute the task procedure, instead of relying on hack.
-  let prc = thisModule.tab.strTableGet(getIdent(hookName))
-  if prc.isNil:
-    # Procedure not defined in the NimScript module.
-    result.success = false
-    cleanup()
-    return
-  let returnVal = vm.globalCtx.execProc(prc, [])
-  case returnVal.kind
-  of nkCharLit..nkUInt64Lit:
-    result.retVal = returnVal.intVal == 1
-  else: assert false
-
-  # Read the command, arguments and flags set by the executed task.
-  result.command = compiler_options.command
-  result.arguments = @[]
-  for arg in compiler_options.gProjectName.split():
-    result.arguments.add(arg)
-
-  cleanup()
-
-proc getNimScriptCommand(): string =
-  compiler_options.command
-
-proc setNimScriptCommand(command: string) =
-  compiler_options.command = command
-
-proc hasTaskRequestedCommand*(execResult: ExecutionResult): bool =
-  ## Determines whether the last executed task used ``setCommand``
-  return execResult.command != internalCmd
-
-proc listTasks*(scriptName: string, options: Options) =
-  setNimScriptCommand("help")
-
-  discard execScript(scriptName, nil, options)
-  # TODO: Make the 'task' template generate explicit data structure containing
-  # all the task names + descriptions.
-  cleanup()

src/nimblepkg/nimscriptwrapper.nim

@@ -0,0 +1,216 @@
+# Copyright (C) Andreas Rumpf. All rights reserved.
+# BSD License. Look at license.txt for more info.
+
+## Implements the new configuration system for Nimble. Uses Nim as a
+## scripting language.
+
+import hashes, json, os, strutils, tables, times, osproc, strtabs
+
+import version, options, cli, tools
+
+type
+  Flags = TableRef[string, seq[string]]
+  ExecutionResult*[T] = object
+    success*: bool
+    command*: string
+    arguments*: seq[string]
+    flags*: Flags
+    retVal*: T
+    stdout*: string
+
+const
+  internalCmd = "e"
+  nimscriptApi = staticRead("nimscriptapi.nim")
+  printPkgInfo = "printPkgInfo"
+
+proc isCustomTask(actionName: string, options: Options): bool =
+  options.action.typ == actionCustom and actionName != printPkgInfo
+
+proc needsLiveOutput(actionName: string, options: Options, isHook: bool): bool =
+  let isCustomTask = isCustomTask(actionName, options)
+  return isCustomTask or isHook or actionName == ""
+
+proc writeExecutionOutput(data: string) =
+  # TODO: in the future we will likely want this to be live, users will
+  # undoubtedly be doing loops and other crazy things in their top-level
+  # Nimble files.
+  display("Info", data)
+
+proc execNimscript(
+  nimsFile, projectDir, actionName: string, options: Options, isHook: bool
+): tuple[output: string, exitCode: int, stdout: string] =
+  let
+    nimsFileCopied = projectDir / nimsFile.splitFile().name & "_" & getProcessId() & ".nims"
+    outFile = getNimbleTempDir() & ".out"
+
+  let
+    isScriptResultCopied =
+      nimsFileCopied.fileExists() and
+      nimsFileCopied.getLastModificationTime() >= nimsFile.getLastModificationTime()
+
+  if not isScriptResultCopied:
+    nimsFile.copyFile(nimsFileCopied)
+
+  defer:
+    # Only if copied in this invocation, allows recursive calls of nimble
+    if not isScriptResultCopied and options.shouldRemoveTmp(nimsFileCopied):
+        nimsFileCopied.removeFile()
+
+  var cmd = (
+    "nim e $# -p:$# $# $# $#" % [
+      "--hints:off --verbosity:0",
+      (getTempDir() / "nimblecache").quoteShell,
+      nimsFileCopied.quoteShell,
+      outFile.quoteShell,
+      actionName
+    ]
+  ).strip()
+
+  let isCustomTask = isCustomTask(actionName, options)
+  if isCustomTask:
+    for i in options.action.arguments:
+      cmd &= " " & i.quoteShell()
+    for key, val in options.action.flags.pairs():
+      cmd &= " $#$#" % [if key.len == 1: "-" else: "--", key]
+      if val.len != 0:
+        cmd &= ":" & val.quoteShell()
+
+  displayDebug("Executing " & cmd)
+
+  if needsLiveOutput(actionName, options, isHook):
+    result.exitCode = execCmd(cmd)
+  else:
+    # We want to capture any possible errors when parsing a .nimble
+    # file's metadata. See #710.
+    (result.stdout, result.exitCode) = execCmdEx(cmd)
+  if outFile.fileExists():
+    result.output = outFile.readFile()
+    if options.shouldRemoveTmp(outFile):
+      discard outFile.tryRemoveFile()
+
+proc getNimsFile(scriptName: string, options: Options): string =
+  let
+    cacheDir = getTempDir() / "nimblecache"
+    shash = $scriptName.parentDir().hash().abs()
+    prjCacheDir = cacheDir / scriptName.splitFile().name & "_" & shash
+    nimscriptApiFile = cacheDir / "nimscriptapi.nim"
+
+  result = prjCacheDir / scriptName.extractFilename().changeFileExt ".nims"
+
+  let
+    iniFile = result.changeFileExt(".ini")
+
+    isNimscriptApiCached =
+      nimscriptApiFile.fileExists() and nimscriptApiFile.getLastModificationTime() > 
+      getAppFilename().getLastModificationTime()
+    
+    isScriptResultCached =
+      isNimscriptApiCached and result.fileExists() and result.getLastModificationTime() >
+      scriptName.getLastModificationTime()
+
+  if not isNimscriptApiCached:
+    createDir(cacheDir)
+    writeFile(nimscriptApiFile, nimscriptApi)
+
+  if not isScriptResultCached:
+    createDir(result.parentDir())
+    writeFile(result, """
+import system except getCommand, setCommand, switch, `--`,
+  packageName, version, author, description, license, srcDir, binDir, backend,
+  skipDirs, skipFiles, skipExt, installDirs, installFiles, installExt, bin, foreignDeps,
+  requires, task, packageName
+""" &
+      "import nimscriptapi, strutils\n" & scriptName.readFile() & "\nonExit()\n")
+    discard tryRemoveFile(iniFile)
+
+proc getIniFile*(scriptName: string, options: Options): string =
+  let
+    nimsFile = getNimsFile(scriptName, options)
+
+  result = nimsFile.changeFileExt(".ini")
+
+  let
+    isIniResultCached =
+      result.fileExists() and result.getLastModificationTime() >
+      scriptName.getLastModificationTime()
+
+  if not isIniResultCached:
+    let (output, exitCode, stdout) = execNimscript(
+      nimsFile, scriptName.parentDir(), printPkgInfo, options, isHook=false
+    )
+
+    if exitCode == 0 and output.len != 0:
+      result.writeFile(output)
+      stdout.writeExecutionOutput()
+    else:
+      raise newException(NimbleError, stdout & "\nprintPkgInfo() failed")
+
+proc execScript(
+  scriptName, actionName: string, options: Options, isHook: bool
+): ExecutionResult[bool] =
+  let nimsFile = getNimsFile(scriptName, options)
+
+  let (output, exitCode, stdout) =
+    execNimscript(
+      nimsFile, scriptName.parentDir(), actionName, options, isHook
+    )
+
+  if exitCode != 0:
+    let errMsg =
+      if stdout.len != 0:
+        stdout
+      else:
+        "Exception raised during nimble script execution"
+    raise newException(NimbleError, errMsg)
+
+  let
+    j =
+      if output.len != 0:
+        parseJson(output)
+      else:
+        parseJson("{}")
+
+  result.flags = newTable[string, seq[string]]()
+  result.success = j{"success"}.getBool()
+  result.command = j{"command"}.getStr()
+  if "project" in j:
+    result.arguments.add j["project"].getStr()
+  if "flags" in j:
+    for flag, vals in j["flags"].pairs:
+      result.flags[flag] = @[]
+      for val in vals.items():
+        result.flags[flag].add val.getStr()
+  result.retVal = j{"retVal"}.getBool()
+
+  stdout.writeExecutionOutput()
+
+proc execTask*(scriptName, taskName: string,
+    options: Options): ExecutionResult[bool] =
+  ## Executes the specified task in the specified script.
+  ##
+  ## `scriptName` should be a filename pointing to the nimscript file.
+  display("Executing",  "task $# in $#" % [taskName, scriptName],
+          priority = HighPriority)
+
+  result = execScript(scriptName, taskName, options, isHook=false)
+
+proc execHook*(scriptName, actionName: string, before: bool,
+    options: Options): ExecutionResult[bool] =
+  ## Executes the specified action's hook. Depending on ``before``, either
+  ## the "before" or the "after" hook.
+  ##
+  ## `scriptName` should be a filename pointing to the nimscript file.
+  let hookName =
+    if before: actionName.toLowerAscii & "Before"
+    else: actionName.toLowerAscii & "After"
+  display("Attempting", "to execute hook $# in $#" % [hookName, scriptName],
+          priority = MediumPriority)
+
+  result = execScript(scriptName, hookName, options, isHook=true)
+
+proc hasTaskRequestedCommand*(execResult: ExecutionResult): bool =
+  ## Determines whether the last executed task used ``setCommand``
+  return execResult.command != internalCmd
+
+proc listTasks*(scriptName: string, options: Options) =
+  discard execScript(scriptName, "", options, isHook=false)

src/nimblepkg/options.nim

@@ -1,15 +1,18 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
 
-import json, strutils, os, parseopt, strtabs, uri, tables
+import json, strutils, os, parseopt, strtabs, uri, tables, terminal
+import sequtils, sugar
+import std/options as std_opt
 from httpclient import Proxy, newProxy
 
-import config, version, tools, common, cli
+import config, version, common, cli
 
 type
   Options* = object
     forcePrompts*: ForcePrompt
     depsOnly*: bool
+    uninstallRevDeps*: bool
     queryVersions*: bool
     queryInstalled*: bool
     nimbleDir*: string
@@ -18,34 +21,50 @@ type
     config*: Config
     nimbleData*: JsonNode ## Nimbledata.json
     pkgInfoCache*: TableRef[string, PackageInfo]
+    showHelp*: bool
+    showVersion*: bool
+    noColor*: bool
+    disableValidation*: bool
+    continueTestsOnFailure*: bool
+    ## Whether packages' repos should always be downloaded with their history.
+    forceFullClone*: bool
+    # Temporary storage of flags that have not been captured by any specific Action.
+    unknownFlags*: seq[(CmdLineKind, string, string)]
 
   ActionType* = enum
     actionNil, actionRefresh, actionInit, actionDump, actionPublish,
     actionInstall, actionSearch,
     actionList, actionBuild, actionPath, actionUninstall, actionCompile,
-    actionDoc, actionCustom, actionTasks
+    actionDoc, actionCustom, actionTasks, actionDevelop, actionCheck,
+    actionRun
 
   Action* = object
     case typ*: ActionType
-    of actionNil, actionList, actionBuild, actionPublish, actionTasks: nil
+    of actionNil, actionList, actionPublish, actionTasks, actionCheck: nil
     of actionRefresh:
       optionalURL*: string # Overrides default package list.
-    of actionInstall, actionPath, actionUninstall:
-      packages*: seq[PkgTuple] # Optional only for actionInstall.
+    of actionInstall, actionPath, actionUninstall, actionDevelop:
+      packages*: seq[PkgTuple] # Optional only for actionInstall
+                               # and actionDevelop.
+      passNimFlags*: seq[string]
     of actionSearch:
       search*: seq[string] # Search string.
     of actionInit, actionDump:
       projName*: string
-    of actionCompile, actionDoc:
+      vcsOption*: string
+    of actionCompile, actionDoc, actionBuild:
       file*: string
       backend*: string
-      compileOptions*: seq[string]
+      compileOptions: seq[string]
+    of actionRun:
+      runFile: Option[string]
+      compileFlags: seq[string]
+      runFlags*: seq[string]
     of actionCustom:
       command*: string
       arguments*: seq[string]
       flags*: StringTableRef
 
-
 const
   help* = """
 Usage: nimble COMMAND [opts]
@@ -53,14 +72,32 @@ Usage: nimble COMMAND [opts]
 Commands:
   install      [pkgname, ...]     Installs a list of packages.
                [-d, --depsOnly]   Install only dependencies.
-  init         [pkgname]          Initializes a new Nimble project.
+               [-p, --passNim]    Forward specified flag to compiler.
+  develop      [pkgname, ...]     Clones a list of packages for development.
+                                  Symlinks the cloned packages or any package
+                                  in the current working directory.
+  check                           Verifies the validity of a package in the
+                                  current working directory.
+  init         [pkgname]          Initializes a new Nimble project in the
+                                  current directory or if a name is provided a
+                                  new directory of the same name.
+               --git
+               --hg               Create a git or hg repo in the new nimble project.
   publish                         Publishes a package on nim-lang/packages.
                                   The current working directory needs to be the
                                   toplevel directory of the Nimble package.
   uninstall    [pkgname, ...]     Uninstalls a list of packages.
-  build                           Builds a package.
+               [-i, --inclDeps]   Uninstall package and dependent package(s).
+  build        [opts, ...] [bin]  Builds a package.
+  run          [opts, ...] [bin]  Builds and runs a package.
+                                  Binary needs to be specified after any
+                                  compilation options if there are several
+                                  binaries defined, any flags after the binary
+                                  or -- arg are passed to the binary when it is run.
   c, cc, js    [opts, ...] f.nim  Builds a file inside a package. Passes options
                                   to the Nim compiler.
+  test                            Compiles and executes tests
+               [-c, --continue]   Don't stop execution on a failed test.
   doc, doc2    [opts, ...] f.nim  Builds documentation for a file inside a
                                   package. Passes options to the Nim compiler.
   refresh      [url]              Refreshes the package list. A package list URL
@@ -88,22 +125,31 @@ Options:
   -n, --reject                    Reject all interactive prompts.
       --ver                       Query remote server for package version
                                   information when searching or listing packages
-      --nimbleDir dirname         Set the Nimble directory.
+      --nimbleDir:dirname         Set the Nimble directory.
       --verbose                   Show all non-debug output.
       --debug                     Show all output including debug messages.
+      --noColor                   Don't colorise output.
 
 For more information read the Github readme:
   https://github.com/nim-lang/nimble#readme
 """
 
+const noHookActions* = {actionCheck}
+
 proc writeHelp*(quit=true) =
   echo(help)
   if quit:
     raise NimbleQuit(msg: "")
 
-proc writeVersion() =
+proc writeVersion*() =
   echo("nimble v$# compiled at $# $#" %
       [nimbleVersion, CompileDate, CompileTime])
+  const execResult = gorgeEx("git rev-parse HEAD")
+  when execResult[0].len > 0 and execResult[1] == QuitSuccess:
+    echo "git hash: ", execResult[0]
+  else:
+    {.warning: "Couldn't determine GIT hash: " & execResult[0].}
+    echo "git hash: couldn't determine git hash"
   raise NimbleQuit(msg: "")
 
 proc parseActionType*(action: string): ActionType =
@@ -114,6 +160,8 @@ proc parseActionType*(action: string): ActionType =
     result = actionPath
   of "build":
     result = actionBuild
+  of "run":
+    result = actionRun
   of "c", "compile", "js", "cpp", "cc":
     result = actionCompile
   of "doc", "doc2":
@@ -134,6 +182,10 @@ proc parseActionType*(action: string): ActionType =
     result = actionPublish
   of "tasks":
     result = actionTasks
+  of "develop":
+    result = actionDevelop
+  of "check":
+    result = actionCheck
   else:
     result = actionCustom
 
@@ -142,28 +194,30 @@ proc initAction*(options: var Options, key: string) =
   ## `key`.
   let keyNorm = key.normalize()
   case options.action.typ
-  of actionInstall, actionPath:
+  of actionInstall, actionPath, actionDevelop, actionUninstall:
     options.action.packages = @[]
-  of actionCompile, actionDoc:
+    options.action.passNimFlags = @[]
+  of actionCompile, actionDoc, actionBuild:
     options.action.compileOptions = @[]
     options.action.file = ""
     if keyNorm == "c" or keyNorm == "compile": options.action.backend = ""
     else: options.action.backend = keyNorm
   of actionInit:
     options.action.projName = ""
+    options.action.vcsOption = ""
   of actionDump:
     options.action.projName = ""
+    options.action.vcsOption = ""
+    options.forcePrompts = forcePromptYes
   of actionRefresh:
     options.action.optionalURL = ""
   of actionSearch:
     options.action.search = @[]
-  of actionUninstall:
-    options.action.packages = @[]
   of actionCustom:
     options.action.command = key
     options.action.arguments = @[]
     options.action.flags = newStringTable()
-  of actionBuild, actionPublish, actionList, actionTasks,
+  of actionPublish, actionList, actionTasks, actionCheck, actionRun,
      actionNil: discard
 
 proc prompt*(options: Options, question: string): bool =
@@ -173,24 +227,32 @@ proc prompt*(options: Options, question: string): bool =
   ## forcePrompts has a value different than dontForcePrompt.
   return prompt(options.forcePrompts, question)
 
-proc renameBabelToNimble(options: Options) {.deprecated.} =
-  let babelDir = getHomeDir() / ".babel"
-  let nimbleDir = getHomeDir() / ".nimble"
-  if dirExists(babelDir):
-    if options.prompt("Found deprecated babel package directory, would you " &
-        "like to rename it to nimble?"):
-      copyDir(babelDir, nimbleDir)
-      copyFile(babelDir / "babeldata.json", nimbleDir / "nimbledata.json")
+proc promptCustom*(options: Options, question, default: string): string =
+  ## Asks an interactive question and returns the result.
+  ##
+  ## The proc will return "default" without asking the user if the global
+  ## forcePrompts is forcePromptYes.
+  return promptCustom(options.forcePrompts, question, default)
 
-      removeDir(babelDir)
-      removeFile(nimbleDir / "babeldata.json")
+proc promptList*(options: Options, question: string, args: openarray[string]): string =
+  ## Asks an interactive question and returns the result.
+  ##
+  ## The proc will return one of the provided args. If not prompting the first
+  ## options is selected.
+  return promptList(options.forcePrompts, question, args)
 
 proc getNimbleDir*(options: Options): string =
-  result =
-    if options.nimbleDir.len == 0:
-      options.config.nimbleDir
-    else:
-      options.nimbleDir
+  result = options.config.nimbleDir
+  if options.nimbleDir.len != 0:
+    # --nimbleDir:<dir> takes priority...
+    result = options.nimbleDir
+  else:
+    # ...followed by the environment variable.
+    let env = getEnv("NIMBLE_DIR")
+    if env.len != 0:
+      display("Warning:", "Using the environment variable: NIMBLE_DIR='" &
+                        env & "'", Warning)
+      result = env
 
   return expandTilde(result)
 
@@ -201,20 +263,27 @@ proc getBinDir*(options: Options): string =
   options.getNimbleDir() / "bin"
 
 proc parseCommand*(key: string, result: var Options) =
-  result.action.typ = parseActionType(key)
+  result.action = Action(typ: parseActionType(key))
   initAction(result, key)
 
+proc setRunOptions(result: var Options, key, val: string, isArg: bool) =
+  if result.action.runFile.isNone() and (isArg or val == "--"):
+    result.action.runFile = some(key)
+  else:
+    result.action.runFlags.add(val)
+
 proc parseArgument*(key: string, result: var Options) =
   case result.action.typ
   of actionNil:
     assert false
-  of actionInstall, actionPath, actionUninstall:
+  of actionInstall, actionPath, actionDevelop, actionUninstall:
     # Parse pkg@verRange
     if '@' in key:
       let i = find(key, '@')
-      let pkgTup = (key[0 .. i-1],
-        key[i+1 .. key.len-1].parseVersionRange())
-      result.action.packages.add(pkgTup)
+      let (pkgName, pkgVer) = (key[0 .. i-1], key[i+1 .. key.len-1])
+      if pkgVer.len == 0:
+        raise newException(NimbleError, "Version range expected after '@'.")
+      result.action.packages.add((pkgName, pkgVer.parseVersionRange()))
     else:
       result.action.packages.add((key, VersionRange(kind: verAny)))
   of actionRefresh:
@@ -223,66 +292,108 @@ proc parseArgument*(key: string, result: var Options) =
     result.action.search.add(key)
   of actionInit, actionDump:
     if result.action.projName != "":
-      raise newException(NimbleError,
-          "Can only initialize one package at a time.")
+      raise newException(
+        NimbleError, "Can only perform this action on one package at a time."
+      )
     result.action.projName = key
   of actionCompile, actionDoc:
     result.action.file = key
-  of actionList, actionBuild, actionPublish:
-    writeHelp()
+  of actionList, actionPublish:
+    result.showHelp = true
+  of actionBuild:
+    result.action.file = key
+  of actionRun:
+    result.setRunOptions(key, key, true)
   of actionCustom:
     result.action.arguments.add(key)
   else:
     discard
 
-proc parseFlag*(flag, val: string, result: var Options) =
-  var wasFlagHandled = true
+proc getFlagString(kind: CmdLineKind, flag, val: string): string =
+  let prefix =
+    case kind
+    of cmdShortOption: "-"
+    of cmdLongOption: "--"
+    else: ""
+  if val == "":
+    return prefix & flag
+  else:
+    return prefix & flag & ":" & val
+
+proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) =
+
+  let f = flag.normalize()
 
   # Global flags.
-  case flag.normalize()
-  of "help", "h": writeHelp()
-  of "version", "v": writeVersion()
+  var isGlobalFlag = true
+  case f
+  of "help", "h": result.showHelp = true
+  of "version", "v": result.showVersion = true
   of "accept", "y": result.forcePrompts = forcePromptYes
   of "reject", "n": result.forcePrompts = forcePromptNo
   of "nimbledir": result.nimbleDir = val
   of "verbose": result.verbosity = LowPriority
   of "debug": result.verbosity = DebugPriority
+  of "nocolor": result.noColor = true
+  of "disablevalidation": result.disableValidation = true
+  else: isGlobalFlag = false
+
+  var wasFlagHandled = true
   # Action-specific flags.
-  of "installed", "i":
-    if result.action.typ in [actionSearch, actionList]:
+  case result.action.typ
+  of actionSearch, actionList:
+    case f
+    of "installed", "i":
       result.queryInstalled = true
-    else:
-      wasFlagHandled = false
-  of "depsonly", "d":
-    if result.action.typ == actionInstall:
-      result.depsOnly = true
-    else:
-      wasFlagHandled = false
-  of "ver":
-    if result.action.typ in [actionSearch, actionList]:
+    of "ver":
       result.queryVersions = true
     else:
       wasFlagHandled = false
-  else:
-    case result.action.typ
-    of actionCompile, actionDoc:
-      if val == "":
-        result.action.compileOptions.add("--" & flag)
-      else:
-        result.action.compileOptions.add("--" & flag & ":" & val)
-    of actionCustom:
-      result.action.flags[flag] = val
+  of actionInstall:
+    case f
+    of "depsonly", "d":
+      result.depsOnly = true
+    of "passnim", "p":
+      result.action.passNimFlags.add(val)
     else:
       wasFlagHandled = false
+  of actionInit:
+    case f
+    of "git", "hg":
+      result.action.vcsOption = f
+    else:
+      wasFlagHandled = false
+  of actionUninstall:
+    case f
+    of "incldeps", "i":
+      result.uninstallRevDeps = true
+    else:
+      wasFlagHandled = false
+  of actionCompile, actionDoc, actionBuild:
+    if not isGlobalFlag:
+      result.action.compileOptions.add(getFlagString(kind, flag, val))
+  of actionRun:
+    result.showHelp = false
+    result.setRunOptions(flag, getFlagString(kind, flag, val), false)
+  of actionCustom:
+    if result.action.command.normalize == "test":
+      if f == "continue" or f == "c":
+        result.continueTestsOnFailure = true
+    result.action.flags[flag] = val
+  else:
+    wasFlagHandled = false
 
-  if not wasFlagHandled:
-    raise newException(NimbleError, "Unknown option: --" & flag)
+  if not wasFlagHandled and not isGlobalFlag:
+    result.unknownFlags.add((kind, flag, val))
 
 proc initOptions*(): Options =
-  result.action.typ = actionNil
-  result.pkgInfoCache = newTable[string, PackageInfo]()
-  result.nimbleDir = ""
-  result.verbosity = HighPriority
+  # Exported for choosenim
+  Options(
+    action: Action(typ: actionNil),
+    pkgInfoCache: newTable[string, PackageInfo](),
+    verbosity: HighPriority,
+    noColor: not isatty(stdout)
+  )
 
 proc parseMisc(options: var Options) =
   # Load nimbledata.json
@@ -297,6 +408,29 @@ proc parseMisc(options: var Options) =
   else:
     options.nimbleData = %{"reverseDeps": newJObject()}
 
+proc handleUnknownFlags(options: var Options) =
+  if options.action.typ == actionRun:
+    # ActionRun uses flags that come before the command as compilation flags
+    # and flags that come after as run flags.
+    options.action.compileFlags =
+      map(options.unknownFlags, x => getFlagString(x[0], x[1], x[2]))
+    options.unknownFlags = @[]
+  else:
+    # For everything else, handle the flags that came before the command
+    # normally.
+    let unknownFlags = options.unknownFlags
+    options.unknownFlags = @[]
+    for flag in unknownFlags:
+      parseFlag(flag[1], flag[2], options, flag[0])
+
+  # Any unhandled flags?
+  if options.unknownFlags.len > 0:
+    let flag = options.unknownFlags[0]
+    raise newException(
+      NimbleError,
+      "Unknown option: " & getFlagString(flag[0], flag[1], flag[2])
+    )
+
 proc parseCmdLine*(): Options =
   result = initOptions()
 
@@ -310,20 +444,30 @@ proc parseCmdLine*(): Options =
       else:
         parseArgument(key, result)
     of cmdLongOption, cmdShortOption:
-      parseFlag(key, val, result)
+        parseFlag(key, val, result, kind)
     of cmdEnd: assert(false) # cannot happen
 
+  handleUnknownFlags(result)
+
   # Set verbosity level.
   setVerbosity(result.verbosity)
 
+  # Set whether color should be shown.
+  setShowColor(not result.noColor)
+
   # Parse config.
   result.config = parseConfig()
 
   # Parse other things, for example the nimbledata.json file.
   parseMisc(result)
 
-  if result.action.typ == actionNil:
-    writeHelp()
+  if result.action.typ == actionNil and not result.showVersion:
+    result.showHelp = true
+
+  if result.action.typ != actionNil and result.showVersion:
+    # We've got another command that should be handled. For example:
+    # nimble run foobar -v
+    result.showVersion = false
 
 proc getProxy*(options: Options): Proxy =
   ## Returns ``nil`` if no proxy is specified.
@@ -336,6 +480,10 @@ proc getProxy*(options: Options): Proxy =
         url = getEnv("http_proxy")
       elif existsEnv("https_proxy"):
         url = getEnv("https_proxy")
+      elif existsEnv("HTTP_PROXY"):
+        url = getEnv("HTTP_PROXY")
+      elif existsEnv("HTTPS_PROXY"):
+        url = getEnv("HTTPS_PROXY")
     except ValueError:
       display("Warning:", "Unable to parse proxy from environment: " &
           getCurrentExceptionMsg(), Warning, HighPriority)
@@ -350,3 +498,54 @@ proc getProxy*(options: Options): Proxy =
     return newProxy($parsed, auth)
   else:
     return nil
+
+proc briefClone*(options: Options): Options =
+  ## Clones the few important fields and creates a new Options object.
+  var newOptions = initOptions()
+  newOptions.config = options.config
+  newOptions.nimbleData = options.nimbleData
+  newOptions.nimbleDir = options.nimbleDir
+  newOptions.forcePrompts = options.forcePrompts
+  newOptions.pkgInfoCache = options.pkgInfoCache
+  return newOptions
+
+proc shouldRemoveTmp*(options: Options, file: string): bool =
+  result = true
+  if options.verbosity <= DebugPriority:
+    let msg = "Not removing temporary path because of debug verbosity: " & file
+    display("Warning:", msg, Warning, MediumPriority)
+    return false
+
+proc getCompilationFlags*(options: var Options): var seq[string] =
+  case options.action.typ
+  of actionBuild, actionDoc, actionCompile:
+    return options.action.compileOptions
+  of actionRun:
+    return options.action.compileFlags
+  else:
+    assert false
+
+proc getCompilationFlags*(options: Options): seq[string] =
+  var opt = options
+  return opt.getCompilationFlags()
+
+proc getCompilationBinary*(options: Options, pkgInfo: PackageInfo): Option[string] =
+  case options.action.typ
+  of actionBuild, actionDoc, actionCompile:
+    let file = options.action.file.changeFileExt("")
+    if file.len > 0:
+      return some(file)
+  of actionRun:
+    let optRunFile = options.action.runFile
+    let runFile =
+      if optRunFile.get("").len > 0:
+        optRunFile.get()
+      elif pkgInfo.bin.len == 1:
+        pkgInfo.bin[0]
+      else:
+        ""
+
+    if runFile.len > 0:
+      return some(runFile.changeFileExt(ExeExt))
+  else:
+    discard

src/nimblepkg/packageinfo.nim

@@ -3,8 +3,7 @@
 
 # Stdlib imports
 import system except TResult
-import parsecfg, json, streams, strutils, parseutils, os, sets, tables
-import httpclient
+import hashes, json, strutils, os, sets, tables, httpclient
 
 # Local imports
 import version, tools, common, options, cli, config
@@ -27,6 +26,10 @@ type
   MetaData* = object
     url*: string
 
+  NimbleLink* = object
+    nimbleFilePath*: string
+    packageDir*: string
+
 proc initPackageInfo*(path: string): PackageInfo =
   result.myPath = path
   result.specialVersion = ""
@@ -66,21 +69,24 @@ proc getNameVersion*(pkgpath: string): tuple[name, version: string] =
   ##
   ## Also works for file paths like:
   ##   ``/home/user/.nimble/pkgs/package-0.1/package.nimble``
-  if pkgPath.splitFile.ext == ".nimble" or pkgPath.splitFile.ext == ".babel":
+  if pkgPath.splitFile.ext in [".nimble", ".nimble-link", ".babel"]:
     return getNameVersion(pkgPath.splitPath.head)
 
   result.name = ""
   result.version = ""
   let tail = pkgpath.splitPath.tail
-  if '-' notin tail:
+
+  const specialSeparator = "-#"
+  var sepIdx = tail.find(specialSeparator)
+  if sepIdx == -1:
+    sepIdx = tail.rfind('-')
+
+  if sepIdx == -1:
     result.name = tail
     return
 
-  for i in countdown(tail.len-1, 0):
-    if tail[i] == '-':
-      result.name = tail[0 .. i-1]
-      result.version = tail[i+1 .. tail.len-1]
-      break
+  result.name = tail[0 .. sepIdx - 1]
+  result.version = tail.substr(sepIdx + 1)
 
 proc optionalField(obj: JsonNode, name: string, default = ""): string =
   ## Queries ``obj`` for the optional ``name`` string.
@@ -100,8 +106,8 @@ proc requiredField(obj: JsonNode, name: string): string =
   ## Queries ``obj`` for the required ``name`` string.
   ##
   ## Aborts execution if the field does not exist or is of invalid json type.
-  result = optionalField(obj, name, nil)
-  if result == nil:
+  result = optionalField(obj, name)
+  if result.len == 0:
     raise newException(NimbleError,
         "Package in packages.json file does not contain a " & name & " field.")
 
@@ -138,6 +144,15 @@ proc readMetaData*(path: string): MetaData =
   let jsonmeta = parseJson(cont)
   result.url = jsonmeta["url"].str
 
+proc readNimbleLink*(nimbleLinkPath: string): NimbleLink =
+  let s = readFile(nimbleLinkPath).splitLines()
+  result.nimbleFilePath = s[0]
+  result.packageDir = s[1]
+
+proc writeNimbleLink*(nimbleLinkPath: string, contents: NimbleLink) =
+  let c = contents.nimbleFilePath & "\n" & contents.packageDir
+  writeFile(nimbleLinkPath, c)
+
 proc needsRefresh*(options: Options): bool =
   ## Determines whether a ``nimble refresh`` is needed.
   ##
@@ -160,55 +175,73 @@ proc validatePackagesList(path: string): bool =
   except ValueError, JsonParsingError:
     return false
 
-proc downloadList*(list: PackageList, options: Options) =
-  ## Downloads the specified package list and saves it in $nimbleDir.
-  display("Downloading", list.name & " package list", priority = HighPriority)
+proc fetchList*(list: PackageList, options: Options) =
+  ## Downloads or copies the specified package list and saves it in $nimbleDir.
+  let verb = if list.urls.len > 0: "Downloading" else: "Copying"
+  display(verb, list.name & " package list", priority = HighPriority)
 
-  var lastError = ""
-  for i in 0 .. <list.urls.len:
-    let url = list.urls[i]
-    display("Trying", url)
-    let tempPath = options.getNimbleDir() / "packages_temp.json"
-
-    # Grab the proxy
-    let proxy = getProxy(options)
-    if not proxy.isNil:
-      var maskedUrl = proxy.url
-      if maskedUrl.password.len > 0: maskedUrl.password = "***"
-      display("Connecting", "to proxy at " & $maskedUrl,
-              priority = LowPriority)
-
-    try:
-      downloadFile(url, tempPath, proxy = getProxy(options))
-    except:
-      let message = "Could not download: " & getCurrentExceptionMsg()
-      display("Warning:", message, Warning)
-      lastError = message
-      continue
-
-    if not validatePackagesList(tempPath):
-      lastError = "Downloaded packages.json file is invalid"
-      display("Warning:", lastError & ", discarding.", Warning)
-      continue
-
-    copyFile(tempPath,
-        options.getNimbleDir() / "packages_$1.json" % list.name.toLowerAscii())
-    display("Success", "Package list downloaded.", Success, HighPriority)
+  var
     lastError = ""
-    break
+    copyFromPath = ""
+  if list.urls.len > 0:
+    for i in 0 ..< list.urls.len:
+      let url = list.urls[i]
+      display("Trying", url)
+      let tempPath = options.getNimbleDir() / "packages_temp.json"
+
+      # Grab the proxy
+      let proxy = getProxy(options)
+      if not proxy.isNil:
+        var maskedUrl = proxy.url
+        if maskedUrl.password.len > 0: maskedUrl.password = "***"
+        display("Connecting", "to proxy at " & $maskedUrl,
+                priority = LowPriority)
+
+      try:
+        let client = newHttpClient(proxy = proxy)
+        client.downloadFile(url, tempPath)
+      except:
+        let message = "Could not download: " & getCurrentExceptionMsg()
+        display("Warning:", message, Warning)
+        lastError = message
+        continue
+
+      if not validatePackagesList(tempPath):
+        lastError = "Downloaded packages.json file is invalid"
+        display("Warning:", lastError & ", discarding.", Warning)
+        continue
+
+      copyFromPath = tempPath
+      display("Success", "Package list downloaded.", Success, HighPriority)
+      lastError = ""
+      break
+
+  elif list.path != "":
+    if not validatePackagesList(list.path):
+      lastError = "Copied packages.json file is invalid"
+      display("Warning:", lastError & ", discarding.", Warning)
+    else:
+      copyFromPath = list.path
+      display("Success", "Package list copied.", Success, HighPriority)
 
   if lastError.len != 0:
     raise newException(NimbleError, "Refresh failed\n" & lastError)
 
+  if copyFromPath.len > 0:
+    copyFile(copyFromPath,
+        options.getNimbleDir() / "packages_$1.json" % list.name.toLowerAscii())
+
 proc readPackageList(name: string, options: Options): JsonNode =
   # If packages.json is not present ask the user if they want to download it.
   if needsRefresh(options):
     if options.prompt("No local packages.json found, download it from " &
             "internet?"):
       for name, list in options.config.packageLists:
-        downloadList(list, options)
+        fetchList(list, options)
     else:
-      raise newException(NimbleError, "Please run nimble refresh.")
+      # The user might not need a package list for now. So let's try
+      # going further.
+      return newJArray()
   return parseFile(options.getNimbleDir() / "packages_" &
                    name.toLowerAscii() & ".json")
 
@@ -244,7 +277,7 @@ proc getPackage*(pkg: string, options: Options, resPkg: var Package): bool =
 proc getPackageList*(options: Options): seq[Package] =
   ## Returns the list of packages found in the downloaded packages.json files.
   result = @[]
-  var namesAdded = initSet[string]()
+  var namesAdded = initHashSet[string]()
   for name, list in options.config.packageLists:
     let packages = readPackageList(name, options)
     for p in packages:
@@ -257,10 +290,10 @@ proc findNimbleFile*(dir: string; error: bool): string =
   result = ""
   var hits = 0
   for kind, path in walkDir(dir):
-    if kind == pcFile:
+    if kind in {pcFile, pcLinkToFile}:
       let ext = path.splitFile.ext
       case ext
-      of ".babel", ".nimble":
+      of ".babel", ".nimble", ".nimble-link":
         result = path
         inc hits
       else: discard
@@ -270,31 +303,52 @@ proc findNimbleFile*(dir: string; error: bool): string =
   elif hits == 0:
     if error:
       raise newException(NimbleError,
-          "Specified directory does not contain a .nimble file.")
+          "Specified directory ($1) does not contain a .nimble file." % dir)
     else:
-      display("Warning:", "No .nimble file found for " & dir, Warning,
-              HighPriority)
+      display("Warning:", "No .nimble or .nimble-link file found for " &
+              dir, Warning, HighPriority)
+
+  if result.splitFile.ext == ".nimble-link":
+    # Return the path of the real .nimble file.
+    result = readNimbleLink(result).nimbleFilePath
+    if not fileExists(result):
+      let msg = "The .nimble-link file is pointing to a missing file: " & result
+      let hintMsg =
+        "Remove '$1' or restore the file it points to." % dir
+      display("Warning:", msg, Warning, HighPriority)
+      display("Hint:", hintMsg, Warning, HighPriority)
 
 proc getInstalledPkgsMin*(libsDir: string, options: Options):
         seq[tuple[pkginfo: PackageInfo, meta: MetaData]] =
   ## Gets a list of installed packages. The resulting package info is
   ## minimal. This has the advantage that it does not depend on the
-  ## ``packageparser`` module, and so can be used by ``nimscriptsupport``.
+  ## ``packageparser`` module, and so can be used by ``nimscriptwrapper``.
   ##
-  ## ``libsDir`` is in most cases: ~/.nimble/pkgs/
+  ## ``libsDir`` is in most cases: ~/.nimble/pkgs/ (options.getPkgsDir)
   result = @[]
   for kind, path in walkDir(libsDir):
     if kind == pcDir:
       let nimbleFile = findNimbleFile(path, false)
       if nimbleFile != "":
         let meta = readMetaData(path)
-        let (name, version) = getNameVersion(nimbleFile)
+        let (name, version) = getNameVersion(path)
         var pkg = initPackageInfo(nimbleFile)
         pkg.name = name
         pkg.version = version
         pkg.specialVersion = version
         pkg.isMinimal = true
         pkg.isInstalled = true
+        let nimbleFileDir = nimbleFile.splitFile().dir
+        pkg.isLinked = cmpPaths(nimbleFileDir, path) != 0
+
+        # Read the package's 'srcDir' (this is stored in the .nimble-link so
+        # we can easily grab it)
+        if pkg.isLinked:
+          let nimbleLinkPath = path / name.addFileExt("nimble-link")
+          let realSrcPath = readNimbleLink(nimbleLinkPath).packageDir
+          assert realSrcPath.startsWith(nimbleFileDir)
+          pkg.srcDir = realSrcPath.replace(nimbleFileDir)
+          pkg.srcDir.removePrefix(DirSep)
         result.add((pkg, meta))
 
 proc withinRange*(pkgInfo: PackageInfo, verRange: VersionRange): bool =
@@ -328,8 +382,7 @@ proc findPkg*(pkglist: seq[tuple[pkgInfo: PackageInfo, meta: MetaData]],
     if cmpIgnoreStyle(pkg.pkginfo.name, dep.name) != 0 and
        cmpIgnoreStyle(pkg.meta.url, dep.name) != 0: continue
     if withinRange(pkg.pkgInfo, dep.ver):
-      let isNewer = (not r.version.isNil) and
-                    newVersion(r.version) < newVersion(pkg.pkginfo.version)
+      let isNewer = newVersion(r.version) < newVersion(pkg.pkginfo.version)
       if not result or isNewer:
         r = pkg.pkginfo
         result = true
@@ -348,7 +401,7 @@ proc findAllPkgs*(pkglist: seq[tuple[pkgInfo: PackageInfo, meta: MetaData]],
 
 proc getRealDir*(pkgInfo: PackageInfo): string =
   ## Returns the directory containing the package source files.
-  if pkgInfo.srcDir != "" and not pkgInfo.isInstalled:
+  if pkgInfo.srcDir != "" and (not pkgInfo.isInstalled or pkgInfo.isLinked):
     result = pkgInfo.mypath.splitFile.dir / pkgInfo.srcDir
   else:
     result = pkgInfo.mypath.splitFile.dir
@@ -412,30 +465,30 @@ proc checkInstallDir(pkgInfo: PackageInfo,
   if thisDir[0] == '.': result = true
   if thisDir == "nimcache": result = true
 
-proc findWithExt(dir: string, pkgInfo: PackageInfo): seq[string] =
-  ## Returns the filenames of the files that should be copied.
-  result = @[]
+proc iterFilesWithExt(dir: string, pkgInfo: PackageInfo,
+                      action: proc (f: string)) =
+  ## Runs `action` for each filename of the files that have a whitelisted
+  ## file extension.
   for kind, path in walkDir(dir):
     if kind == pcDir:
-      result.add findWithExt(path, pkgInfo)
+      iterFilesWithExt(path, pkgInfo, action)
     else:
-      if path.splitFile.ext[1 .. ^1] in pkgInfo.installExt:
-        result.add path
+      if path.splitFile.ext.substr(1) in pkgInfo.installExt:
+        action(path)
 
-proc getFilesInDir(dir: string): seq[string] =
-  ## Returns a list of paths to files inside the specified directory and any
+proc iterFilesInDir(dir: string, action: proc (f: string)) =
+  ## Runs `action` for each file in ``dir`` and any
   ## subdirectories that are in it.
-  result = @[]
   for kind, path in walkDir(dir):
     if kind == pcDir:
-      result.add getFilesInDir(path)
+      iterFilesInDir(path, action)
     else:
-      result.add path
+      action(path)
 
-proc getInstallFiles*(realDir: string, pkgInfo: PackageInfo,
-                      options: Options): seq[string] =
-  ## Returns a list of files within the ``realDir`` that should be installed.
-  result = @[]
+proc iterInstallFiles*(realDir: string, pkgInfo: PackageInfo,
+                       options: Options, action: proc (f: string)) =
+  ## Runs `action` for each file within the ``realDir`` that should be
+  ## installed.
   let whitelistMode =
           pkgInfo.installDirs.len != 0 or
           pkgInfo.installFiles.len != 0 or
@@ -448,7 +501,8 @@ proc getInstallFiles*(realDir: string, pkgInfo: PackageInfo,
           continue
         else:
           raise NimbleQuit(msg: "")
-      result.add src
+
+      action(src)
 
     for dir in pkgInfo.installDirs:
       # TODO: Allow skipping files inside dirs?
@@ -459,23 +513,37 @@ proc getInstallFiles*(realDir: string, pkgInfo: PackageInfo,
         else:
           raise NimbleQuit(msg: "")
 
-      result.add getFilesInDir(src)
+      iterFilesInDir(src, action)
 
-    result.add findWithExt(realDir, pkgInfo)
+    iterFilesWithExt(realDir, pkgInfo, action)
   else:
     for kind, file in walkDir(realDir):
       if kind == pcDir:
         let skip = pkgInfo.checkInstallDir(realDir, file)
-
         if skip: continue
+        # we also have to stop recursing if we reach an in-place nimbleDir
+        if file == options.getNimbleDir().expandFilename(): continue
 
-        result.add getInstallFiles(file, pkgInfo, options)
+        iterInstallFiles(file, pkgInfo, options, action)
       else:
         let skip = pkgInfo.checkInstallFile(realDir, file)
-
         if skip: continue
 
-        result.add file
+        action(file)
+
+proc getPkgDest*(pkgInfo: PackageInfo, options: Options): string =
+  let versionStr = '-' & pkgInfo.specialVersion
+  let pkgDestDir = options.getPkgsDir() / (pkgInfo.name & versionStr)
+  return pkgDestDir
+
+proc `==`*(pkg1: PackageInfo, pkg2: PackageInfo): bool =
+  if pkg1.name == pkg2.name and pkg1.myPath == pkg2.myPath:
+    return true
+
+proc hash*(x: PackageInfo): Hash =
+  var h: Hash = 0
+  h = h !& hash(x.myPath)
+  result = !$h
 
 when isMainModule:
   doAssert getNameVersion("/home/user/.nimble/libs/packagea-0.1") ==
@@ -486,6 +554,10 @@ when isMainModule:
       ("package-a", "0.1")
   doAssert getNameVersion("/home/user/.nimble/libs/package-#head") ==
       ("package", "#head")
+  doAssert getNameVersion("/home/user/.nimble/libs/package-#branch-with-dashes") ==
+      ("package", "#branch-with-dashes")
+  # readPackageInfo (and possibly more) depends on this not raising.
+  doAssert getNameVersion("/home/user/.nimble/libs/package") == ("package", "")
 
   doAssert toValidPackageName("foo__bar") == "foo_bar"
   doAssert toValidPackageName("jhbasdh!£$@%#^_&*_()qwe") == "jhbasdh_qwe"

src/nimblepkg/packageinstaller.nim

@@ -0,0 +1,112 @@
+# Copyright (C) Dominik Picheta. All rights reserved.
+# BSD License. Look at license.txt for more info.
+import os, strutils, sets, json
+
+# Local imports
+import cli, options, tools
+
+when defined(windows):
+  import version
+
+when not declared(initHashSet) or not declared(toHashSet):
+  import common
+
+when defined(windows):
+  # This is just for Win XP support.
+  # TODO: Drop XP support?
+  from winlean import WINBOOL, DWORD
+  type
+    OSVERSIONINFO* {.final, pure.} = object
+      dwOSVersionInfoSize*: DWORD
+      dwMajorVersion*: DWORD
+      dwMinorVersion*: DWORD
+      dwBuildNumber*: DWORD
+      dwPlatformId*: DWORD
+      szCSDVersion*: array[0..127, char]
+
+  proc GetVersionExA*(VersionInformation: var OSVERSIONINFO): WINBOOL{.stdcall,
+    dynlib: "kernel32", importc: "GetVersionExA".}
+
+proc setupBinSymlink*(symlinkDest, symlinkFilename: string,
+                      options: Options): seq[string] =
+  result = @[]
+  let currentPerms = getFilePermissions(symlinkDest)
+  setFilePermissions(symlinkDest, currentPerms + {fpUserExec})
+  when defined(unix):
+    display("Creating", "symlink: $1 -> $2" %
+            [symlinkDest, symlinkFilename], priority = MediumPriority)
+    if existsFile(symlinkFilename):
+      let msg = "Symlink already exists in $1. Replacing." % symlinkFilename
+      display("Warning:", msg, Warning, HighPriority)
+      removeFile(symlinkFilename)
+
+    createSymlink(symlinkDest, symlinkFilename)
+    result.add symlinkFilename.extractFilename
+  elif defined(windows):
+    # There is a bug on XP, described here:
+    # http://stackoverflow.com/questions/2182568/batch-script-is-not-executed-if-chcp-was-called
+    # But this workaround brakes code page on newer systems, so we need to detect OS version
+    var osver = OSVERSIONINFO()
+    osver.dwOSVersionInfoSize = cast[DWORD](sizeof(OSVERSIONINFO))
+    if GetVersionExA(osver) == WINBOOL(0):
+      raise newException(NimbleError,
+        "Can't detect OS version: GetVersionExA call failed")
+    let fixChcp = osver.dwMajorVersion <= 5
+
+    # Create cmd.exe/powershell stub.
+    let dest = symlinkFilename.changeFileExt("cmd")
+    display("Creating", "stub: $1 -> $2" % [symlinkDest, dest],
+            priority = MediumPriority)
+    var contents = "@"
+    if options.config.chcp:
+      if fixChcp:
+        contents.add "chcp 65001 > nul && "
+      else: contents.add "chcp 65001 > nul\n@"
+    contents.add "\"" & symlinkDest & "\" %*\n"
+    writeFile(dest, contents)
+    result.add dest.extractFilename
+    # For bash on Windows (Cygwin/Git bash).
+    let bashDest = dest.changeFileExt("")
+    display("Creating", "Cygwin stub: $1 -> $2" %
+            [symlinkDest, bashDest], priority = MediumPriority)
+    writeFile(bashDest, "\"" & symlinkDest & "\" \"$@\"\n")
+    result.add bashDest.extractFilename
+  else:
+    {.error: "Sorry, your platform is not supported.".}
+
+proc saveNimbleMeta*(pkgDestDir, url, vcsRevision: string,
+                    filesInstalled, bins: HashSet[string],
+                    isLink: bool = false) =
+  ## Saves the specified data into a ``nimblemeta.json`` file inside
+  ## ``pkgDestDir``.
+  ##
+  ## filesInstalled - A list of absolute paths to files which have been
+  ##                  installed.
+  ## bins - A list of binary filenames which have been installed for this
+  ##        package.
+  ##
+  ## isLink - Determines whether the installed package is a .nimble-link.
+  var nimblemeta = %{"url": %url}
+  if vcsRevision.len > 0:
+    nimblemeta["vcsRevision"] = %vcsRevision
+  let files = newJArray()
+  nimblemeta["files"] = files
+  for file in filesInstalled:
+    files.add(%changeRoot(pkgDestDir, "", file))
+  let binaries = newJArray()
+  nimblemeta["binaries"] = binaries
+  for bin in bins:
+    binaries.add(%bin)
+  nimblemeta["isLink"] = %isLink
+  writeFile(pkgDestDir / "nimblemeta.json", $nimblemeta)
+
+proc saveNimbleMeta*(pkgDestDir, pkgDir, vcsRevision, nimbleLinkPath: string) =
+  ## Overload of saveNimbleMeta for linked (.nimble-link) packages.
+  ##
+  ## pkgDestDir - The directory where the package has been installed.
+  ##              For example: ~/.nimble/pkgs/jester-#head/
+  ##
+  ## pkgDir - The directory where the original package files are.
+  ##          For example: ~/projects/jester/
+  saveNimbleMeta(pkgDestDir, "file://" & pkgDir, vcsRevision,
+                 toHashSet[string]([nimbleLinkPath]), initHashSet[string](), true)

src/nimblepkg/packageparser.nim

@@ -1,14 +1,14 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
-import parsecfg, json, streams, strutils, parseutils, os, tables
-import version, tools, common, nimscriptsupport, options, packageinfo, cli
+import parsecfg, sets, streams, strutils, os, tables, sugar
+from sequtils import apply, map
+
+import version, tools, common, nimscriptwrapper, options, packageinfo, cli
 
 ## Contains procedures for parsing .nimble files. Moved here from ``packageinfo``
-## because it depends on ``nimscriptsupport`` (``nimscriptsupport`` also
+## because it depends on ``nimscriptwrapper`` (``nimscriptwrapper`` also
 ## depends on other procedures in ``packageinfo``.
 
-from sequtils import apply
-
 type
   NimbleFile* = string
 
@@ -16,6 +16,31 @@ type
     warnInstalled*: bool # Determines whether to show a warning for installed pkgs
     warnAll*: bool
 
+const reservedNames = [
+  "CON",
+  "PRN",
+  "AUX",
+  "NUL",
+  "COM1",
+  "COM2",
+  "COM3",
+  "COM4",
+  "COM5",
+  "COM6",
+  "COM7",
+  "COM8",
+  "COM9",
+  "LPT1",
+  "LPT2",
+  "LPT3",
+  "LPT4",
+  "LPT5",
+  "LPT6",
+  "LPT7",
+  "LPT8",
+  "LPT9",
+]
+
 proc newValidationError(msg: string, warnInstalled: bool,
                         hint: string, warnAll: bool): ref ValidationError =
   result = newException(ValidationError, msg)
@@ -57,6 +82,9 @@ proc validatePackageName*(name: string) =
   if name.endsWith("pkg"):
     raiseNewValidationError("\"$1\" is an invalid package name: cannot end" &
                             " with \"pkg\"" % name, false)
+  if name.toUpperAscii() in reservedNames:
+    raiseNewValidationError(
+      "\"$1\" is an invalid package name: reserved name" % name, false)
 
 proc validateVersion*(ver: string) =
   for c in ver:
@@ -69,8 +97,18 @@ proc validatePackageStructure(pkgInfo: PackageInfo, options: Options) =
   ## This ensures that a package's source code does not leak into
   ## another package's namespace.
   ## https://github.com/nim-lang/nimble/issues/144
-  let realDir = pkgInfo.getRealDir()
-  for path in getInstallFiles(realDir, pkgInfo, options):
+  let
+    realDir = pkgInfo.getRealDir()
+    normalizedBinNames = pkgInfo.bin.map(
+      (x) => x.changeFileExt("").toLowerAscii()
+    )
+    correctDir =
+      if pkgInfo.name.toLowerAscii() in normalizedBinNames:
+        pkgInfo.name & "pkg"
+      else:
+        pkgInfo.name
+
+  proc onFile(path: string) =
     # Remove the root to leave only the package subdirectories.
     # ~/package-0.1/package/utils.nim -> package/utils.nim.
     var trailPath = changeRoot(realDir, "", path)
@@ -79,41 +117,48 @@ proc validatePackageStructure(pkgInfo: PackageInfo, options: Options) =
     # We're only interested in nim files, because only they can pollute our
     # namespace.
     if ext != (ExtSep & "nim"):
-      continue
+      return
 
     if dir.len == 0:
       if file != pkgInfo.name:
-        let msg = ("File inside package '$1' is outside of permitted " &
-                   "namespace, should be " &
-                   "named '$2' but was named '$3' instead. This will be an error" &
-                   " in the future.") %
-                   [pkgInfo.name, pkgInfo.name & ext, file & ext]
-        let hint = ("Rename this file to '$1', move it into a '$2' " &
-                "subdirectory, or prevent its installation by adding " &
-                "`skipFiles = @[\"$3\"]` to the .nimble file. See " &
-                "https://github.com/nim-lang/nimble#libraries for more info.") %
-                [pkgInfo.name & ext, pkgInfo.name & DirSep, file & ext]
+        # A source file was found in the top level of srcDir that doesn't share
+        # a name with the package.
+        let
+          msg = ("Package '$1' has an incorrect structure. " &
+                "The top level of the package source directory " &
+                "should contain at most one module, " &
+                "named '$2', but a file named '$3' was found. This " &
+                "will be an error in the future.") %
+                [pkgInfo.name, pkgInfo.name & ext, file & ext]
+          hint = ("If this is the primary source file in the package, " &
+                  "rename it to '$1'. If it's a source file required by " &
+                  "the main module, or if it is one of several " &
+                  "modules exposed by '$4', then move it into a '$2' subdirectory. " &
+                  "If it's a test file or otherwise not required " &
+                  "to build the the package '$1', prevent its installation " &
+                  "by adding `skipFiles = @[\"$3\"]` to the .nimble file. See " &
+                  "https://github.com/nim-lang/nimble#libraries for more info.") %
+                  [pkgInfo.name & ext, correctDir & DirSep, file & ext, pkgInfo.name]
         raiseNewValidationError(msg, true, hint, true)
     else:
       assert(not pkgInfo.isMinimal)
-      let correctDir =
-        if pkgInfo.name in pkgInfo.bin:
-          pkgInfo.name & "pkg"
-        else:
-          pkgInfo.name
-
+      # On Windows `pkgInfo.bin` has a .exe extension, so we need to normalize.
       if not (dir.startsWith(correctDir & DirSep) or dir == correctDir):
-        let msg = ("File '$1' inside package '$2' is outside of the" &
-                " permitted namespace" &
-                ", should be inside a directory named '$3' but is in a" &
-                " directory named '$4' instead. This will be an error in the " &
-                "future.") %
-                [file & ext, pkgInfo.name, correctDir, dir]
-        let hint = ("Rename the directory to '$1' or prevent its " &
-                "installation by adding `skipDirs = @[\"$2\"]` to the " &
-                ".nimble file.") % [correctDir, dir]
+        let
+          msg = ("Package '$2' has an incorrect structure. " &
+                "It should contain a single directory hierarchy " &
+                "for source files, named '$3', but file '$1' " &
+                "is in a directory named '$4' instead. " &
+                "This will be an error in the future.") %
+              [file & ext, pkgInfo.name, correctDir, dir]
+          hint = ("If '$1' contains source files for building '$2', rename it " &
+                  "to '$3'. Otherwise, prevent its installation " &
+                  "by adding `skipDirs = @[\"$1\"]` to the .nimble file.") %
+              [dir, pkgInfo.name, correctDir]
         raiseNewValidationError(msg, true, hint, true)
 
+  iterInstallFiles(realDir, pkgInfo, options, onFile)
+
 proc validatePackageInfo(pkgInfo: PackageInfo, options: Options) =
   let path = pkgInfo.myPath
   if pkgInfo.name == "":
@@ -167,7 +212,10 @@ proc multiSplit(s: string): seq[string] =
       result.del(i)
   # Huh, nothing to return? Return given input.
   if len(result) < 1:
-    return @[s]
+    if s.strip().len != 0:
+      return @[s]
+    else:
+      return @[]
 
 proc readPackageInfoFromNimble(path: string; result: var PackageInfo) =
   var fs = newFileStream(path, fmRead)
@@ -208,12 +256,20 @@ proc readPackageInfoFromNimble(path: string; result: var PackageInfo) =
             result.installExt.add(ev.value.multiSplit)
           of "bin":
             for i in ev.value.multiSplit:
+              if i.splitFile().ext == ".nim":
+                raise newException(NimbleError, "`bin` entry should not be a source file: " & i)
               result.bin.add(i.addFileExt(ExeExt))
           of "backend":
             result.backend = ev.value.toLowerAscii()
             case result.backend.normalize
             of "javascript": result.backend = "js"
             else: discard
+          of "beforehooks":
+            for i in ev.value.multiSplit:
+              result.preHooks.incl(i.normalize)
+          of "afterhooks":
+            for i in ev.value.multiSplit:
+              result.postHooks.incl(i.normalize)
           else:
             raise newException(NimbleError, "Invalid field: " & ev.key)
         of "deps", "dependencies":
@@ -232,6 +288,33 @@ proc readPackageInfoFromNimble(path: string; result: var PackageInfo) =
   else:
     raise newException(ValueError, "Cannot open package info: " & path)
 
+proc readPackageInfoFromNims(scriptName: string, options: Options,
+    result: var PackageInfo) =
+  let
+    iniFile = getIniFile(scriptName, options)
+
+  if iniFile.fileExists():
+    readPackageInfoFromNimble(iniFile, result)
+
+proc inferInstallRules(pkgInfo: var PackageInfo, options: Options) =
+  # Binary packages shouldn't install .nim files by default.
+  # (As long as the package info doesn't explicitly specify what should be
+  # installed.)
+  let installInstructions =
+    pkgInfo.installDirs.len + pkgInfo.installExt.len + pkgInfo.installFiles.len
+  if installInstructions == 0 and pkgInfo.bin.len > 0:
+    pkgInfo.skipExt.add("nim")
+
+  # When a package doesn't specify a `srcDir` it's fair to assume that
+  # the .nim files are in the root of the package. So we can explicitly select
+  # them and prevent the installation of anything else. The user can always
+  # override this with `installFiles`.
+  if pkgInfo.srcDir == "":
+    if dirExists(pkgInfo.getRealDir() / pkgInfo.name):
+      pkgInfo.installDirs.add(pkgInfo.name)
+    if fileExists(pkgInfo.getRealDir() / pkgInfo.name.addFileExt("nim")):
+      pkgInfo.installFiles.add(pkgInfo.name.addFileExt("nim"))
+
 proc readPackageInfo(nf: NimbleFile, options: Options,
     onlyMinimalInfo=false): PackageInfo =
   ## Reads package info from the specified Nimble file.
@@ -275,16 +358,23 @@ proc readPackageInfo(nf: NimbleFile, options: Options,
       result.version = minimalInfo.version
       result.isNimScript = true
       result.isMinimal = true
+
+      # It's possible this proc will receive a .nimble-link file eventually,
+      # I added this assert to hopefully make this error clear for everyone.
+      let msg = "No version detected. Received nimble-link?"
+      assert result.version.len > 0, msg
     else:
       try:
         readPackageInfoFromNims(nf, options, result)
         result.isNimScript = true
-      except NimbleError:
+      except NimbleError as exc:
+        if exc.hint.len > 0:
+          raise
         let msg = "Could not read package info file in " & nf & ";\n" &
                   "  Reading as ini file failed with: \n" &
                   "    " & iniError.msg & ".\n" &
                   "  Evaluating as NimScript file failed with: \n" &
-                  "    " & getCurrentExceptionMsg() & "."
+                  "    " & exc.msg & "."
         raise newException(NimbleError, msg)
 
   # By default specialVersion is the same as version.
@@ -299,12 +389,26 @@ proc readPackageInfo(nf: NimbleFile, options: Options,
     if version.kind == verSpecial:
       result.specialVersion = minimalInfo.version
 
+  # Apply rules to infer which files should/shouldn't be installed. See #469.
+  inferInstallRules(result, options)
+
   if not result.isMinimal:
     options.pkgInfoCache[nf] = result
 
   # Validate the rest of the package info last.
-  validateVersion(result.version)
-  validatePackageInfo(result, options)
+  if not options.disableValidation:
+    validateVersion(result.version)
+    validatePackageInfo(result, options)
+
+proc validate*(file: NimbleFile, options: Options,
+               error: var ValidationError, pkgInfo: var PackageInfo): bool =
+  try:
+    pkgInfo = readPackageInfo(file, options)
+  except ValidationError as exc:
+    error = exc[]
+    return false
+
+  return true
 
 proc getPkgInfoFromFile*(file: NimbleFile, options: Options): PackageInfo =
   ## Reads the specified .nimble file and returns its data as a PackageInfo
@@ -368,6 +472,8 @@ proc getInstalledPkgs*(libsDir: string, options: Options):
           raise exc
 
         pkg.isInstalled = true
+        pkg.isLinked =
+          cmpPaths(nimbleFile.splitFile().dir, path) != 0
         result.add((pkg, meta))
 
 proc isNimScript*(nf: string, options: Options): bool =
@@ -375,10 +481,21 @@ proc isNimScript*(nf: string, options: Options): bool =
 
 proc toFullInfo*(pkg: PackageInfo, options: Options): PackageInfo =
   if pkg.isMinimal:
-    return getPkgInfoFromFile(pkg.mypath, options)
+    result = getPkgInfoFromFile(pkg.mypath, options)
+    result.isInstalled = pkg.isInstalled
+    result.isLinked = pkg.isLinked
   else:
     return pkg
 
+proc getConcreteVersion*(pkgInfo: PackageInfo, options: Options): string =
+  ## Returns a non-special version from the specified ``pkgInfo``. If the
+  ## ``pkgInfo`` is minimal it looks it up and retrieves the concrete version.
+  result = pkgInfo.version
+  if pkgInfo.isMinimal:
+    let pkgInfo = pkgInfo.toFullInfo(options)
+    result = pkgInfo.version
+  assert(not newVersion(result).isSpecial)
+
 when isMainModule:
   validatePackageName("foo_bar")
   validatePackageName("f_oo_b_a_r")

src/nimblepkg/publish.nim

@@ -5,27 +5,34 @@
 ## nim-lang/packages automatically.
 
 import system except TResult
-import httpclient, base64, strutils, rdstdin, json, os, browsers, times, uri
-import tools, common, cli
+import httpclient, strutils, json, os, browsers, times, uri
+import version, tools, common, cli, config, options
 
 type
   Auth = object
     user: string
-    pw: string
-    token: string  ## base64 encoding of user:pw
+    token: string  ## Github access token
+    http: HttpClient ## http client for doing API requests
+
+const
+  ApiKeyFile = "github_api_token"
+  ApiTokenEnvironmentVariable = "NIMBLE_GITHUB_API_TOKEN"
+  ReposUrl = "https://api.github.com/repos/"
 
 proc userAborted() =
   raise newException(NimbleError, "User aborted the process.")
 
-proc createHeaders(a: Auth): string =
-  (("Authorization: token $1\c\L" % a.token) &
-          "Content-Type: application/x-www-form-urlencoded\c\L" &
-          "Accept: */*\c\L")
+proc createHeaders(a: Auth) =
+  a.http.headers = newHttpHeaders({
+    "Authorization": "token $1" % a.token,
+    "Content-Type": "application/x-www-form-urlencoded",
+    "Accept": "*/*"
+  })
 
-proc getGithubAuth(): Auth =
+proc requestNewToken(cfg: Config): string =
   display("Info:", "Please create a new personal access token on Github in" &
-         " order to allow Nimble to fork the packages repository.",
-         priority = HighPriority)
+          " order to allow Nimble to fork the packages repository.",
+          priority = HighPriority)
   display("Hint:", "Make sure to give the access token access to public repos" &
           " (public_repo scope)!", Warning, HighPriority)
   sleep(5000)
@@ -33,9 +40,35 @@ proc getGithubAuth(): Auth =
           "https://github.com/settings/tokens/new", priority = HighPriority)
   sleep(3000)
   openDefaultBrowser("https://github.com/settings/tokens/new")
-  result.token = promptCustom("Personal access token?", "").strip()
-  let resp = getContent("https://api.github.com/user",
-        extraHeaders=createHeaders(result)).parseJson()
+  let token = promptCustom("Personal access token?", "").strip()
+  # inform the user that their token will be written to disk
+  let tokenWritePath = cfg.nimbleDir / ApiKeyFile
+  display("Info:", "Writing access token to file:" & tokenWritePath, 
+          priority = HighPriority)
+  writeFile(tokenWritePath, token)
+  sleep(3000)
+  return token
+
+proc getGithubAuth(o: Options): Auth =
+  let cfg = o.config
+  result.http = newHttpClient(proxy = getProxy(o))
+  # always prefer the environment variable to asking for a new one
+  if existsEnv(ApiTokenEnvironmentVariable):
+    result.token = getEnv(ApiTokenEnvironmentVariable)
+    display("Info:", "Using the '" & ApiTokenEnvironmentVariable & 
+            "' environment variable for the GitHub API Token.",
+            priority = HighPriority)
+  else:
+    # try to read from disk, if it cannot be found write a new one
+    try:
+      let apiTokenFilePath = cfg.nimbleDir / ApiKeyFile
+      result.token = readFile(apiTokenFilePath).strip()
+      display("Info:", "Using GitHub API Token in file: " & apiTokenFilePath,
+              priority = HighPriority)
+    except IOError:
+      result.token = requestNewToken(cfg)
+  createHeaders(result)
+  let resp = result.http.getContent("https://api.github.com/user").parseJson()
 
   result.user = resp["login"].str
   display("Success:", "Verified as " & result.user, Success, HighPriority)
@@ -43,13 +76,12 @@ proc getGithubAuth(): Auth =
 proc isCorrectFork(j: JsonNode): bool =
   # Check whether this is a fork of the nimble packages repo.
   result = false
-  if j{"fork"}.getBVal():
+  if j{"fork"}.getBool():
     result = j{"parent"}{"full_name"}.getStr() == "nim-lang/packages"
 
 proc forkExists(a: Auth): bool =
   try:
-    let x = getContent("https://api.github.com/repos/" & a.user & "/packages",
-        extraHeaders=createHeaders(a))
+    let x = a.http.getContent(ReposUrl & a.user & "/packages")
     let j = parseJson(x)
     result = isCorrectFork(j)
   except JsonParsingError, IOError:
@@ -57,18 +89,18 @@ proc forkExists(a: Auth): bool =
 
 proc createFork(a: Auth) =
   try:
-    discard postContent("https://api.github.com/repos/nim-lang/packages/forks",
-        extraHeaders=createHeaders(a))
+    discard a.http.postContent(ReposUrl & "nim-lang/packages/forks")
   except HttpRequestError:
     raise newException(NimbleError, "Unable to create fork. Access token" &
                        " might not have enough permissions.")
 
-proc createPullRequest(a: Auth, packageName, branch: string) =
+proc createPullRequest(a: Auth, packageName, branch: string): string =
   display("Info", "Creating PR", priority = HighPriority)
-  discard postContent("https://api.github.com/repos/nim-lang/packages/pulls",
-      extraHeaders=createHeaders(a),
+  var body = a.http.postContent(ReposUrl & "nim-lang/packages/pulls",
       body="""{"title": "Add package $1", "head": "$2:$3",
                "base": "master"}""" % [packageName, a.user, branch])
+  var pr = parseJson(body)
+  return pr{"html_url"}.getStr()
 
 proc `%`(s: openArray[string]): JsonNode =
   result = newJArray()
@@ -109,32 +141,21 @@ proc cleanupWhitespace(s: string): string =
 proc editJson(p: PackageInfo; url, tags, downloadMethod: string) =
   var contents = parseFile("packages.json")
   doAssert contents.kind == JArray
-  contents.add(%{
-    "name": %p.name,
-    "url": %url,
-    "method": %downloadMethod,
-    "tags": %tags.split(),
-    "description": %p.description,
-    "license": %p.license,
-    "web": %url})
+  contents.add(%*{
+    "name": p.name,
+    "url": url,
+    "method": downloadMethod,
+    "tags": tags.split(),
+    "description": p.description,
+    "license": p.license,
+    "web": url
+  })
   writeFile("packages.json", contents.pretty.cleanupWhitespace)
 
-proc getPackageOriginUrl(a: Auth): string =
-  ## Adds 'user:pw' to the URL so that the user is not asked *again* for it.
-  ## We need this for 'git push'.
-  let (output, exitCode) = doCmdEx("git config --get remote.origin.url")
-  result = "origin"
-  if exitCode == 0:
-    result = output.string.strip
-    if result.endsWith(".git"): result.setLen(result.len - 4)
-    if result.startsWith("https://"):
-      result = "https://" & a.user & ':' & a.pw & '@' &
-          result["https://".len .. ^1]
-
-proc publish*(p: PackageInfo) =
+proc publish*(p: PackageInfo, o: Options) =
   ## Publishes the package p.
-  let auth = getGithubAuth()
-  var pkgsDir = getTempDir() / "nimble-packages-fork"
+  let auth = getGithubAuth(o)
+  var pkgsDir = getNimbleUserTempDir() / "nimble-packages-fork"
   if not forkExists(auth):
     createFork(auth)
     display("Info:", "Waiting 10s to let Github create a fork",
@@ -146,13 +167,17 @@ proc publish*(p: PackageInfo) =
     display("Removing", "old packages fork git directory.",
             priority = LowPriority)
     removeDir(pkgsDir)
-  display("Cloning", "packages into: " & pkgsDir, priority = HighPriority)
-  doCmd("git clone git@github.com:" & auth.user & "/packages " & pkgsDir)
-  # Make sure to update the clone.
-  display("Updating", "the fork", priority = HighPriority)
+  createDir(pkgsDir)
   cd pkgsDir:
+    # Avoid git clone to prevent token from being stored in repo
+    # https://github.com/blog/1270-easier-builds-and-deployments-using-git-over-https-and-oauth
+    display("Copying", "packages fork into: " & pkgsDir, priority = HighPriority)
+    doCmd("git init")
+    doCmd("git pull https://github.com/" & auth.user & "/packages")
+    # Make sure to update the fork
+    display("Updating", "the fork", priority = HighPriority)
     doCmd("git pull https://github.com/nim-lang/packages.git master")
-    doCmd("git push origin master")
+    doCmd("git push https://" & auth.token & "@github.com/" & auth.user & "/packages master")
 
   if not dirExists(pkgsDir):
     raise newException(NimbleError,
@@ -167,7 +192,7 @@ proc publish*(p: PackageInfo) =
   var url = ""
   var downloadMethod = ""
   if dirExists(os.getCurrentDir() / ".git"):
-    let (output, exitCode) = doCmdEx("git config --get remote.origin.url")
+    let (output, exitCode) = doCmdEx("git ls-remote --get-url")
     if exitCode == 0:
       url = output.string.strip
       if url.endsWith(".git"): url.setLen(url.len - 4)
@@ -176,7 +201,7 @@ proc publish*(p: PackageInfo) =
     if parsed.scheme == "":
       # Assuming that we got an ssh write/read URL.
       let sshUrl = parseUri("ssh://" & url)
-      url = "https://github.com/" & sshUrl.port & sshUrl.path
+      url = "https://" & sshUrl.hostname & "/" & sshUrl.port & sshUrl.path
   elif dirExists(os.getCurrentDir() / ".hg"):
     downloadMethod = "hg"
     # TODO: Retrieve URL from hg.
@@ -188,19 +213,17 @@ proc publish*(p: PackageInfo) =
     url = promptCustom("Github URL of " & p.name & "?", "")
     if url.len == 0: userAborted()
 
-  let tags = promptCustom("Whitespace separated list of tags?", "")
+  let tags = promptCustom(
+    "Whitespace separated list of tags? (For example: web library wrapper)",
+    ""
+  )
 
   cd pkgsDir:
     editJson(p, url, tags, downloadMethod)
-    let branchName = "add-" & p.name & getTime().getGMTime().format("HHmm")
+    let branchName = "add-" & p.name & getTime().utc.format("HHmm")
     doCmd("git checkout -B " & branchName)
     doCmd("git commit packages.json -m \"Added package " & p.name & "\"")
     display("Pushing", "to remote of fork.", priority = HighPriority)
-    doCmd("git push " & getPackageOriginUrl(auth) & " " & branchName)
-    createPullRequest(auth, p.name, branchName)
-  display("Success:", "Pull request successful.", Success, HighPriority)
-
-when isMainModule:
-  import packageinfo
-  var p = getPkgInfo(getCurrentDir())
-  publish(p)
+    doCmd("git push https://" & auth.token & "@github.com/" & auth.user & "/packages " & branchName)
+    let prUrl = createPullRequest(auth, p.name, branchName)
+  display("Success:", "Pull request successful, check at " & prUrl , Success, HighPriority)

src/nimblepkg/reversedeps.nim

@@ -0,0 +1,135 @@
+# Copyright (C) Dominik Picheta. All rights reserved.
+# BSD License. Look at license.txt for more info.
+
+import os, json, sets
+
+import options, common, version, download, packageinfo
+
+proc saveNimbleData*(options: Options) =
+  # TODO: This file should probably be locked.
+  writeFile(options.getNimbleDir() / "nimbledata.json",
+            pretty(options.nimbleData))
+
+proc addRevDep*(nimbleData: JsonNode, dep: tuple[name, version: string],
+                pkg: PackageInfo) =
+  # Add a record which specifies that `pkg` has a dependency on `dep`, i.e.
+  # the reverse dependency of `dep` is `pkg`.
+  if not nimbleData["reverseDeps"].hasKey(dep.name):
+    nimbleData["reverseDeps"][dep.name] = newJObject()
+  if not nimbleData["reverseDeps"][dep.name].hasKey(dep.version):
+    nimbleData["reverseDeps"][dep.name][dep.version] = newJArray()
+  let revDep = %{ "name": %pkg.name, "version": %pkg.specialVersion}
+  let thisDep = nimbleData["reverseDeps"][dep.name][dep.version]
+  if revDep notin thisDep:
+    thisDep.add revDep
+
+proc removeRevDep*(nimbleData: JsonNode, pkg: PackageInfo) =
+  ## Removes ``pkg`` from the reverse dependencies of every package.
+  assert(not pkg.isMinimal)
+  proc remove(pkg: PackageInfo, depTup: PkgTuple, thisDep: JsonNode) =
+    for ver, val in thisDep:
+      if ver.newVersion in depTup.ver:
+        var newVal = newJArray()
+        for revDep in val:
+          if not (revDep["name"].str == pkg.name and
+                  revDep["version"].str == pkg.specialVersion):
+            newVal.add revDep
+        thisDep[ver] = newVal
+
+  for depTup in pkg.requires:
+    if depTup.name.isURL():
+      # We sadly must go through everything in this case...
+      for key, val in nimbleData["reverseDeps"]:
+        remove(pkg, depTup, val)
+    else:
+      let thisDep = nimbleData{"reverseDeps", depTup.name}
+      if thisDep.isNil: continue
+      remove(pkg, depTup, thisDep)
+
+  # Clean up empty objects/arrays
+  var newData = newJObject()
+  for key, val in nimbleData["reverseDeps"]:
+    if val.len != 0:
+      var newVal = newJObject()
+      for ver, elem in val:
+        if elem.len != 0:
+          newVal[ver] = elem
+      if newVal.len != 0:
+        newData[key] = newVal
+  nimbleData["reverseDeps"] = newData
+
+proc getRevDepTups*(options: Options, pkg: PackageInfo): seq[PkgTuple] =
+  ## Returns a list of *currently installed* reverse dependencies for `pkg`.
+  result = @[]
+  let thisPkgsDep =
+    options.nimbleData["reverseDeps"]{pkg.name}{pkg.specialVersion}
+  if not thisPkgsDep.isNil:
+    let pkgList = getInstalledPkgsMin(options.getPkgsDir(), options)
+    for pkg in thisPkgsDep:
+      let pkgTup = (
+        name: pkg["name"].getStr(),
+        ver: parseVersionRange(pkg["version"].getStr())
+      )
+      var pkgInfo: PackageInfo
+      if not findPkg(pkgList, pkgTup, pkgInfo):
+        continue
+
+      result.add(pkgTup)
+
+proc getRevDeps*(options: Options, pkg: PackageInfo): HashSet[PackageInfo] =
+  result.init()
+  let installedPkgs = getInstalledPkgsMin(options.getPkgsDir(), options)
+  for rdepTup in getRevDepTups(options, pkg):
+    for rdepInfo in findAllPkgs(installedPkgs, rdepTup):
+      result.incl rdepInfo
+
+proc getAllRevDeps*(options: Options, pkg: PackageInfo, result: var HashSet[PackageInfo]) =
+  if pkg in result:
+    return
+
+  let installedPkgs = getInstalledPkgsMin(options.getPkgsDir(), options)
+  for rdepTup in getRevDepTups(options, pkg):
+    for rdepInfo in findAllPkgs(installedPkgs, rdepTup):
+      if rdepInfo in result:
+        continue
+
+      getAllRevDeps(options, rdepInfo, result)
+  result.incl pkg
+
+when isMainModule:
+  var nimbleData = %{"reverseDeps": newJObject()}
+
+  let nimforum1 = PackageInfo(
+    isMinimal: false,
+    name: "nimforum",
+    specialVersion: "0.1.0",
+    requires: @[("jester", parseVersionRange("0.1.0")),
+                ("captcha", parseVersionRange("1.0.0")),
+                ("auth", parseVersionRange("#head"))]
+  )
+  let nimforum2 = PackageInfo(isMinimal: false, name: "nimforum", specialVersion: "0.2.0")
+  let play = PackageInfo(isMinimal: false, name: "play", specialVersion: "#head")
+
+  nimbleData.addRevDep(("jester", "0.1.0"), nimforum1)
+  nimbleData.addRevDep(("jester", "0.1.0"), play)
+  nimbleData.addRevDep(("captcha", "1.0.0"), nimforum1)
+  nimbleData.addRevDep(("auth", "#head"), nimforum1)
+  nimbleData.addRevDep(("captcha", "1.0.0"), nimforum2)
+  nimbleData.addRevDep(("auth", "#head"), nimforum2)
+
+  doAssert nimbleData["reverseDeps"]["jester"]["0.1.0"].len == 2
+  doAssert nimbleData["reverseDeps"]["captcha"]["1.0.0"].len == 2
+  doAssert nimbleData["reverseDeps"]["auth"]["#head"].len == 2
+
+  block:
+    nimbleData.removeRevDep(nimforum1)
+    let jester = nimbleData["reverseDeps"]["jester"]["0.1.0"][0]
+    doAssert jester["name"].getStr() == play.name
+    doAssert jester["version"].getStr() == play.specialVersion
+
+    let captcha = nimbleData["reverseDeps"]["captcha"]["1.0.0"][0]
+    doAssert captcha["name"].getStr() == nimforum2.name
+    doAssert captcha["version"].getStr() == nimforum2.specialVersion
+
+  echo("Everything works!")
+

src/nimblepkg/tools.nim

@@ -3,7 +3,7 @@
 #
 # Various miscellaneous utility functions reside here.
 import osproc, pegs, strutils, os, uri, sets, json, parseutils
-import version, common, cli
+import version, cli
 
 proc extractBin(cmd: string): string =
   if cmd[0] == '"':
@@ -11,7 +11,7 @@ proc extractBin(cmd: string): string =
   else:
     return cmd.split(' ')[0]
 
-proc doCmd*(cmd: string, showOutput = false) =
+proc doCmd*(cmd: string, showOutput = false, showCmd = false) =
   let bin = extractBin(cmd)
   if findExe(bin) == "":
     raise newException(NimbleError, "'" & bin & "' not in PATH.")
@@ -20,19 +20,26 @@ proc doCmd*(cmd: string, showOutput = false) =
   stdout.flushFile()
   stderr.flushFile()
 
-  displayDebug("Executing", cmd)
-  let (output, exitCode) = execCmdEx(cmd)
-  displayDebug("Finished", "with exit code " & $exitCode)
-  # TODO: Improve to show output in real-time.
-  if showOutput:
-    display("Output:", output, priority = HighPriority)
+  if showCmd:
+    display("Executing", cmd, priority = MediumPriority)
   else:
+    displayDebug("Executing", cmd)
+  if showOutput:
+    let exitCode = execCmd(cmd)
+    displayDebug("Finished", "with exit code " & $exitCode)
+    if exitCode != QuitSuccess:
+      raise newException(NimbleError,
+          "Execution failed with exit code $1\nCommand: $2" %
+          [$exitCode, cmd])
+  else:
+    let (output, exitCode) = execCmdEx(cmd)
+    displayDebug("Finished", "with exit code " & $exitCode)
     displayDebug("Output", output)
 
-  if exitCode != QuitSuccess:
-    raise newException(NimbleError,
-        "Execution failed with exit code $1\nCommand: $2\nOutput: $3" %
-        [$exitCode, cmd, output])
+    if exitCode != QuitSuccess:
+      raise newException(NimbleError,
+          "Execution failed with exit code $1\nCommand: $2\nOutput: $3" %
+          [$exitCode, cmd, output])
 
 proc doCmdEx*(cmd: string): tuple[output: TaintedString, exitCode: int] =
   let bin = extractBin(cmd)
@@ -75,8 +82,15 @@ proc changeRoot*(origRoot, newRoot, path: string): string =
   ## newRoot:  /home/test/
   ## path:     /home/dom/bar/blah/2/foo.txt
   ## Return value -> /home/test/bar/blah/2/foo.txt
-  if path.startsWith(origRoot):
-    return newRoot / path[origRoot.len .. path.len-1]
+
+  ## The additional check of `path.samePaths(origRoot)` is necessary to prevent
+  ## a regression, where by ending the `srcDir` defintion in a nimble file in a
+  ## trailing separator would cause the `path.startsWith(origRoot)` evaluation to
+  ## fail because of the value of `origRoot` would be longer than `path` due to
+  ## the trailing separator. This would cause this method to throw during package
+  ## installation.
+  if path.startsWith(origRoot) or path.samePaths(origRoot):
+    return newRoot / path.substr(origRoot.len, path.len-1)
   else:
     raise newException(ValueError,
       "Cannot change root of path: Path does not begin with original root.")
@@ -95,6 +109,10 @@ proc copyDirD*(fro, to: string): seq[string] =
     createDir(changeRoot(fro, to, path.splitFile.dir))
     result.add copyFileD(path, changeRoot(fro, to, path))
 
+proc createDirD*(dir: string) =
+  display("Creating", "directory $#" % dir, priority = LowPriority)
+  createDir(dir)
+
 proc getDownloadDirName*(uri: string, verRange: VersionRange): string =
   ## Creates a directory name based on the specified ``uri`` (url)
   result = ""
@@ -130,3 +148,37 @@ proc contains*(j: JsonNode, elem: tuple[key: string, val: JsonNode]): bool =
   for key, val in pairs(j):
     if key == elem.key and val == elem.val:
       return true
+
+when not defined(windows):
+  from posix import getpid
+
+proc getProcessId*(): string =
+  when defined(windows):
+    proc GetCurrentProcessId(): int32 {.stdcall, dynlib: "kernel32",
+                                        importc: "GetCurrentProcessId".}
+    result = $GetCurrentProcessId()
+  else:
+    result = $getpid()
+
+proc getNimbleTempDir*(): string =
+  ## Returns a path to a temporary directory.
+  ##
+  ## The returned path will be the same for the duration of the process but
+  ## different for different runs of it. You have to make sure to create it
+  ## first. In release builds the directory will be removed when nimble finishes
+  ## its work.
+  result = getTempDir() / "nimble_" & getProcessId()
+
+proc getNimbleUserTempDir*(): string =
+  ## Returns a path to a temporary directory.
+  ##
+  ## The returned path will be the same for the duration of the process but
+  ## different for different runs of it. You have to make sure to create it
+  ## first. In release builds the directory will be removed when nimble finishes
+  ## its work.
+  var tmpdir: string
+  if existsEnv("TMPDIR") and existsEnv("USER"):
+    tmpdir = joinPath(getEnv("TMPDIR"), getEnv("USER"))
+  else:
+    tmpdir = getTempDir()
+  return tmpdir

src/nimblepkg/version.nim

@@ -35,23 +35,28 @@ type
   NimbleError* = object of Exception
     hint*: string
 
-proc newVersion*(ver: string): Version =
-  doAssert(ver[0] in {'#', '\0'} + Digits)
-  return Version(ver)
-
 proc `$`*(ver: Version): string {.borrow.}
 
 proc hash*(ver: Version): Hash {.borrow.}
 
-proc isNil*(ver: Version): bool {.borrow.}
+proc newVersion*(ver: string): Version =
+  doAssert(ver.len == 0 or ver[0] in {'#', '\0'} + Digits,
+           "Wrong version: " & ver)
+  return Version(ver)
 
 proc isSpecial*(ver: Version): bool =
-  return ($ver)[0] == '#'
+  return ($ver).len > 0 and ($ver)[0] == '#'
 
 proc `<`*(ver: Version, ver2: Version): bool =
   # Handling for special versions such as "#head" or "#branch".
   if ver.isSpecial or ver2.isSpecial:
-    return false
+    # TODO: This may need to be reverted. See #311.
+    if ver2.isSpecial and ($ver2).normalize == "#head":
+      return ($ver).normalize != "#head"
+
+    if not ver2.isSpecial:
+      # `#aa111 < 1.1`
+      return ($ver).normalize != "#head"
 
   # Handling for normal versions such as "0.1.0" or "1.0".
   var sVer = string(ver).split('.')
@@ -88,6 +93,11 @@ proc `==`*(ver: Version, ver2: Version): bool =
     else:
       return false
 
+proc cmp*(a, b: Version): int =
+  if a < b: -1
+  elif a > b: 1
+  else: 0
+
 proc `<=`*(ver: Version, ver2: Version): bool =
   return (ver == ver2) or (ver < ver2)
 
@@ -103,77 +113,66 @@ proc `==`*(range1: VersionRange, range2: VersionRange): bool =
   of verAny: true
 
 proc withinRange*(ver: Version, ran: VersionRange): bool =
-  if ver.isSpecial:
-    case ran.kind
-    of verLater, verEarlier, verEqLater, verEqEarlier, verEq, verIntersect:
-      return false
-    of verSpecial:
-      return ver == ran.spe
-    of verAny:
-      return true
-  else:
-    case ran.kind
-    of verLater:
-      return ver > ran.ver
-    of verEarlier:
-      return ver < ran.ver
-    of verEqLater:
-      return ver >= ran.ver
-    of verEqEarlier:
-      return ver <= ran.ver
-    of verEq:
-      return ver == ran.ver
-    of verSpecial:
-      return false
-    of verIntersect:
-      return withinRange(ver, ran.verILeft) and withinRange(ver, ran.verIRight)
-    of verAny:
-      return true
+  case ran.kind
+  of verLater:
+    return ver > ran.ver
+  of verEarlier:
+    return ver < ran.ver
+  of verEqLater:
+    return ver >= ran.ver
+  of verEqEarlier:
+    return ver <= ran.ver
+  of verEq:
+    return ver == ran.ver
+  of verSpecial:
+    return ver == ran.spe
+  of verIntersect:
+    return withinRange(ver, ran.verILeft) and withinRange(ver, ran.verIRight)
+  of verAny:
+    return true
 
 proc contains*(ran: VersionRange, ver: Version): bool =
   return withinRange(ver, ran)
 
 proc makeRange*(version: string, op: string): VersionRange =
-  new(result)
   if version == "":
     raise newException(ParseVersionError,
         "A version needs to accompany the operator.")
   case op
   of ">":
-    result.kind = verLater
+    result = VersionRange(kind: verLater)
   of "<":
-    result.kind = verEarlier
+    result = VersionRange(kind: verEarlier)
   of ">=":
-    result.kind = verEqLater
+    result = VersionRange(kind: verEqLater)
   of "<=":
-    result.kind = verEqEarlier
-  of "":
-    result.kind = verEq
+    result = VersionRange(kind: verEqEarlier)
+  of "", "==":
+    result = VersionRange(kind: verEq)
   else:
     raise newException(ParseVersionError, "Invalid operator: " & op)
   result.ver = Version(version)
 
 proc parseVersionRange*(s: string): VersionRange =
   # >= 1.5 & <= 1.8
-  new(result)
   if s.len == 0:
-    result.kind = verAny
+    result = VersionRange(kind: verAny)
     return
 
   if s[0] == '#':
-    result.kind = verSpecial
+    result = VersionRange(kind: verSpecial)
     result.spe = s.Version
     return
 
   var i = 0
   var op = ""
   var version = ""
-  while true:
+  while i < s.len:
     case s[i]
     of '>', '<', '=':
       op.add(s[i])
     of '&':
-      result.kind = verIntersect
+      result = VersionRange(kind: verIntersect)
       result.verILeft = makeRange(version, op)
 
       # Parse everything after &
@@ -186,35 +185,32 @@ proc parseVersionRange*(s: string): VersionRange =
         raise newException(ParseVersionError,
             "Having more than one `&` in a version range is pointless")
 
-      break
+      return
 
     of '0'..'9', '.':
       version.add(s[i])
 
-    of '\0':
-      result = makeRange(version, op)
-      break
-
     of ' ':
       # Make sure '0.9 8.03' is not allowed.
-      if version != "" and i < s.len:
+      if version != "" and i < s.len - 1:
         if s[i+1] in {'0'..'9', '.'}:
           raise newException(ParseVersionError,
               "Whitespace is not allowed in a version literal.")
 
     else:
       raise newException(ParseVersionError,
-          "Unexpected char in version range: " & s[i])
+          "Unexpected char in version range '" & s & "': " & s[i])
     inc(i)
+  result = makeRange(version, op)
 
 proc toVersionRange*(ver: Version): VersionRange =
   ## Converts a version to either a verEq or verSpecial VersionRange.
   new(result)
   if ver.isSpecial:
-    result.kind = verSpecial
+    result = VersionRange(kind: verSpecial)
     result.spe = ver
   else:
-    result.kind = verEq
+    result = VersionRange(kind: verEq)
     result.ver = ver
 
 proc parseRequires*(req: string): PkgTuple =
@@ -270,21 +266,18 @@ proc getSimpleString*(verRange: VersionRange): string =
     result = ""
 
 proc newVRAny*(): VersionRange =
-  new(result)
-  result.kind = verAny
+  result = VersionRange(kind: verAny)
 
 proc newVREarlier*(ver: string): VersionRange =
-  new(result)
-  result.kind = verEarlier
+  result = VersionRange(kind: verEarlier)
   result.ver = newVersion(ver)
 
 proc newVREq*(ver: string): VersionRange =
-  new(result)
-  result.kind = verEq
+  result = VersionRange(kind: verEq)
   result.ver = newVersion(ver)
 
 proc findLatest*(verRange: VersionRange,
-        versions: Table[Version, string]): tuple[ver: Version, tag: string] =
+        versions: OrderedTable[Version, string]): tuple[ver: Version, tag: string] =
   result = (newVersion(""), "")
   for ver, tag in versions:
     if not withinRange(ver, verRange): continue
@@ -298,16 +291,17 @@ when isMainModule:
   doAssert(newVersion("1.0") < newVersion("1.4"))
   doAssert(newVersion("1.0.1") > newVersion("1.0"))
   doAssert(newVersion("1.0.6") <= newVersion("1.0.6"))
-  #doAssert(not withinRange(newVersion("0.1.0"), parseVersionRange("> 0.1")))
+  doAssert(not withinRange(newVersion("0.1.0"), parseVersionRange("> 0.1")))
   doAssert(not (newVersion("0.1.0") < newVersion("0.1")))
   doAssert(not (newVersion("0.1.0") > newVersion("0.1")))
   doAssert(newVersion("0.1.0") < newVersion("0.1.0.0.1"))
   doAssert(newVersion("0.1.0") <= newVersion("0.1"))
 
   var inter1 = parseVersionRange(">= 1.0 & <= 1.5")
+  doAssert(inter1.kind == verIntersect)
   var inter2 = parseVersionRange("1.0")
   doAssert(inter2.kind == verEq)
-  #echo(parseVersionRange(">= 0.8 0.9"))
+  doAssert(parseVersionRange("== 3.4.2") == parseVersionRange("3.4.2"))
 
   doAssert(not withinRange(newVersion("1.5.1"), inter1))
   doAssert(withinRange(newVersion("1.0.2.3.4.5.6.7.8.9.10.11.12"), inter1))
@@ -321,8 +315,11 @@ when isMainModule:
   doAssert(newVersion("") < newVersion("1.0.0"))
   doAssert(newVersion("") < newVersion("0.1.0"))
 
-  var versions = toTable[Version, string]({newVersion("0.1.1"): "v0.1.1",
-      newVersion("0.2.3"): "v0.2.3", newVersion("0.5"): "v0.5"})
+  var versions = toOrderedTable[Version, string]({
+    newVersion("0.1.1"): "v0.1.1",
+    newVersion("0.2.3"): "v0.2.3",
+    newVersion("0.5"): "v0.5"
+  })
   doAssert findLatest(parseVersionRange(">= 0.1 & <= 0.4"), versions) ==
       (newVersion("0.2.3"), "v0.2.3")
 
@@ -342,9 +339,14 @@ when isMainModule:
 
   doAssert newVersion("#head") in parseVersionRange("#head")
 
-  # TODO: It may be worth changing this in the future, although we can't be
-  # certain that #head is in fact newer than v0.1.0.
-  doAssert(not(newVersion("#head") > newVersion("0.1.0")))
+  # We assume that #head > 0.1.0, in practice this shouldn't be a problem.
+  doAssert(newVersion("#head") > newVersion("0.1.0"))
+  doAssert(not(newVersion("#head") > newVersion("#head")))
+  doAssert(withinRange(newVersion("#head"), parseVersionRange(">= 0.5.0")))
+  doAssert newVersion("#a111") < newVersion("#head")
+  # We assume that all other special versions are not higher than a normal
+  # version.
+  doAssert newVersion("#a111") < newVersion("1.1")
 
   # An empty version range should give verAny
   doAssert parseVersionRange("").kind == verAny
@@ -353,4 +355,7 @@ when isMainModule:
   doAssert toVersionRange(newVersion("#head")).kind == verSpecial
   doAssert toVersionRange(newVersion("0.2.0")).kind == verEq
 
+  # Something raised on IRC
+  doAssert newVersion("1") == newVersion("1.0")
+
   echo("Everything works!")

tests/.gitignore

@@ -0,0 +1,23 @@
+tester
+/nimble-test
+/buildDir
+/binaryPackage/v1/binaryPackage
+/binaryPackage/v2/binaryPackage
+/develop/dependent/src/dependent
+/issue27/issue27
+/issue206/issue/issue206bin
+/issue289/issue289
+/issue428/nimbleDir/
+/nimbleDir/
+/packageStructure/c/c
+/packageStructure/y/y
+/testCommand/testOverride/myTester
+/testCommand/testsFail/tests/a
+/testCommand/testsFail/tests/b
+/testCommand/testsPass/tests/one
+/testCommand/testsPass/tests/three
+/testCommand/testsPass/tests/two
+/nimscript/nimscript
+/packageStructure/validBinary/y
+/testCommand/testsFail/tests/t2
+/passNimFlags/passNimFlags

tests/binaryPackage/v1/binaryPackage.nim

@@ -0,0 +1 @@
+echo("v1")

tests/binaryPackage/v1/binaryPackage.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "1.0"
+author        = "Dominik Picheta"
+description   = "binary"
+license       = "MIT"
+
+bin = @["binaryPackage"]
+
+# Dependencies
+
+requires "nim >= 0.15.3"

tests/binaryPackage/v2/binaryPackage.nim

@@ -0,0 +1 @@
+echo("v2")

tests/binaryPackage/v2/binaryPackage.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "2.0"
+author        = "Dominik Picheta"
+description   = "binary"
+license       = "MIT"
+
+bin = @["binaryPackage"]
+
+# Dependencies
+
+requires "nim >= 0.15.3"

tests/caching/caching.nimble

@@ -0,0 +1,10 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Test package"
+license       = "BSD"
+
+# Dependencies
+
+requires "nim >= 0.12.1"

tests/develop/binary/binary.nim

@@ -0,0 +1 @@
+echo("hello")

tests/develop/binary/binary.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "1.0"
+author        = "Dominik Picheta"
+description   = "binary"
+license       = "MIT"
+
+bin = @["binary"]
+
+skipExt = @["nim"]
+
+# Dependencies
+
+requires "nim >= 0.16.0"

tests/develop/dependent/dependent.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "1.0"
+author        = "Dominik Picheta"
+description   = "dependent"
+license       = "MIT"
+
+srcDir = "src"
+
+# Dependencies
+
+requires "nim >= 0.16.0", "srcdirtest"

tests/develop/dependent/src/dependent.nim

@@ -0,0 +1,3 @@
+import srcdirtest
+
+doAssert foo() == "correct"

tests/develop/hybrid/hybrid.nim

@@ -0,0 +1 @@
+echo("hello")

tests/develop/hybrid/hybrid.nimble

@@ -0,0 +1,13 @@
+# Package
+
+version       = "1.0"
+author        = "Dominik Picheta"
+description   = "hybrid"
+license       = "MIT"
+
+bin = @["hybrid"]
+installExt = @["nim"]
+
+# Dependencies
+
+requires "nim >= 0.16.0"

tests/develop/srcdirtest/src/srcdirtest.nim

@@ -0,0 +1,4 @@
+proc foo*(): string =
+  return "correct"
+
+echo("hello")

tests/develop/srcdirtest/srcdirtest.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "1.0"
+author        = "Dominik Picheta"
+description   = "srcdir"
+license       = "MIT"
+
+srcDir = "src"
+
+# Dependencies
+
+requires "nim >= 0.16.0"

tests/diamond_deps/a/a.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "a"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.3", "b", "c"
+
+
+task test, "test":
+  echo("hello")

tests/diamond_deps/b/b.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "b"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.3", "d"
+

tests/diamond_deps/c/c.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "c"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.3", "d"
+

tests/diamond_deps/d/d.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "d"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.3"
+

tests/invalidPackage/invalidPackage.nimble

@@ -0,0 +1,13 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+
+thisFieldDoesNotExist = "hello"
+
+# Dependencies
+
+requires "nim >= 0.20.0"

tests/issue338/issue338.nimble

@@ -0,0 +1,7 @@
+# Package
+version = "0.1.0"
+author = "Samantha Marshall"
+description = "test case to validate successful install when `srcDir` value ends in a directory separator"
+license = "MIT"
+
+srcDir = "src/"

tests/issue368/packages.json

@@ -0,0 +1,14 @@
+[
+  {
+    "name": "discordnim",
+    "url": "https://github.com/Krognol/discordnim",
+    "method": "git",
+    "tags": [
+      "library",
+      "discord"
+    ],
+    "description": "Discord library for Nim",
+    "license": "MIT",
+    "web": "https://github.com/Krognol/discordnim"
+  }
+]

tests/issue428/dummy.nimble

@@ -0,0 +1,10 @@
+# Package
+
+version       = "0.1.0"
+author        = "Author"
+description   = "dummy"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.17.0"

tests/issue432/issue432.nimble

@@ -0,0 +1,15 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+
+
+
+# Dependencies
+
+requires "nim >= 0.16.0"
+requires "https://github.com/nimble-test/packagea#head",
+         "https://github.com/nimble-test/packagebin2"

tests/issue432/src/issue432.nim

@@ -0,0 +1,7 @@
+# This is just an example to get you started. A typical library package
+# exports the main API in this file. Note that you cannot rename this file
+# but you can remove it if you wish.
+
+proc add*(x, y: int): int =
+  ## Adds two files together.
+  return x + y

tests/issue564/issue564.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir = "src"
+bin           = @["issue564/issue564build"]
+
+
+
+# Dependencies
+
+requires "nim >= 0.16.0"

tests/issue564/src/issue564/issue564build.nim

@@ -0,0 +1,5 @@
+# This is just an example to get you started. A typical binary package
+# uses this file as the main entry point of the application.
+
+when isMainModule:
+  echo("Hello, World!")

tests/issue597/dummy.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "0.1.0"
+author        = "Author"
+description   = "dummy"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.17.0"
+
+bin = @["test.nim"]

tests/issue633/issue633.nimble

@@ -0,0 +1,16 @@
+# Package
+
+version     = "0.1.0"
+author      = "GT"
+description = "Package for ensuring that issue #633 is resolved."
+license     = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.19.6"
+# to reproduce dependency 2 must be before 1
+
+task testTask, "Test":
+  for i in 0 .. paramCount():
+    if paramStr(i) == "--testTask":
+      echo "Got it"

tests/issue678/issue678.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version     = "0.1.0"
+author      = "Ivan Bobev"
+description = "Package for ensuring that issue #678 is resolved."
+license     = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.19.6"
+# to reproduce dependency 2 must be before 1
+requires "issue678_dependency_2", "issue678_dependency_1"

tests/issue678/packages.json

@@ -0,0 +1,19 @@
+[
+  {
+    "name": "issue678_dependency_1",
+    "url": "https://github.com/nimble-test/issue678?subdir=dependency_1",
+    "method": "git",
+    "tags": [ "test" ], 
+    "description":
+      "Both first and second level dependency of the issue678 package.",
+    "license": "MIT"
+  },
+  {
+    "name": "issue678_dependency_2",
+    "url": "https://github.com/nimble-test/issue678?subdir=dependency_2",
+    "method": "git",
+    "tags": [ "test" ],
+    "description": "First level dependency of the issue678 package.",
+    "license": "MIT"
+  }
+]

tests/issue708/issue708.nimble

@@ -0,0 +1,17 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+
+
+
+# Dependencies
+
+requires "nim >= 0.16.0"
+
+
+echo "hello"
+echo "hello2"

tests/issue708/src/issue708.nim

@@ -0,0 +1,7 @@
+# This is just an example to get you started. A typical library package
+# exports the main API in this file. Note that you cannot rename this file
+# but you can remove it if you wish.
+
+proc add*(x, y: int): int =
+  ## Adds two files together.
+  return x + y

tests/nimbleVersionDefine/nimbleVersionDefine.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+bin           = @["nimbleVersionDefine"]
+
+
+
+# Dependencies
+
+requires "nim >= 0.16.0"

tests/nimbleVersionDefine/src/nimbleVersionDefine.nim

@@ -0,0 +1,3 @@
+when isMainModule:
+  const NimblePkgVersion {.strdefine.} = "Unknown"
+  echo(NimblePkgVersion)

tests/nimscript/nimscript.nimble

@@ -2,7 +2,9 @@
 
 version       = "0.1.0"
 author        = "Dominik Picheta"
-description   = "Test package"
+description   = """Test package
+with multi-line description
+"""
 license       = "BSD"
 
 bin = @["nimscript"]
@@ -11,7 +13,7 @@ bin = @["nimscript"]
 
 requires "nim >= 0.12.1"
 
-task test, "test description":
+task work, "test description":
   echo(5+5)
 
 task c_test, "Testing `setCommand \"c\", \"nimscript.nim\"`":
@@ -21,8 +23,16 @@ task cr, "Testing `nimble c -r nimscript.nim` via setCommand":
   --r
   setCommand "c", "nimscript.nim"
 
+task repeated, "Testing `nimble c nimscript.nim` with repeated flags":
+  --define: foo
+  --define: bar
+  --define: "quoted"
+  --define: "quoted\\\"with\\\"quotes"
+  setCommand "c", "nimscript.nim"
+
 task api, "Testing nimscriptapi module functionality":
-  echo(getPkgDir())
+  doAssert(findExe("nim").len != 0)
+  echo("PKG_DIR: ", getPkgDir())
 
 before hooks:
   echo("First")
@@ -37,4 +47,16 @@ before hooks2:
   return false
 
 task hooks2, "Testing the hooks again":
-  echo("Shouldn't happen")
+  echo("Shouldn't happen")
+
+before install:
+  echo("Before PkgDir: ", getPkgDir())
+
+after install:
+  echo("After PkgDir: ", getPkgDir())
+
+before build:
+  echo("Before build")
+
+after build:
+  echo("After build")

tests/packageStructure/softened/myPkg.nimble

@@ -0,0 +1,15 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package myPkg. See #469."
+license       = "MIT"
+
+# This is inferred implicitly by Nimble:
+# installDirs = @["myPkg"]
+# installFiles = @["myPkg.nim"]
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/validBinary/y.nimble

@@ -0,0 +1,13 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package Y"
+license       = "MIT"
+
+bin = @["y"]
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/packageStructure/validBinary/yWrong/foobar.nim

@@ -0,0 +1 @@
+

tests/packageStructure/y/y.nimble

@@ -5,6 +5,7 @@ author        = "Dominik Picheta"
 description   = "Incorrectly structured package Y"
 license       = "MIT"
 
+installExt = @["nim"]
 bin = @["y"]
 
 # Dependencies

tests/passNimFlags/passNimFlags.nim

@@ -0,0 +1 @@
+when not defined(passNimIsWorking): {.error: "-d:passNimIsWorking wasn't passed to the compiler"}

tests/passNimFlags/passNimFlags.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "SolitudeSF"
+description   = "Test nimble install flag forwarding"
+license       = "BSD"
+bin           = @["passNimFlags"]
+
+# Dependencies
+
+requires "nim >= 0.13.0"

tests/recursive/recursive.nimble

@@ -0,0 +1,25 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Test package"
+license       = "BSD"
+
+# Dependencies
+
+requires "nim >= 0.12.1"
+
+let
+  callNimble = getEnv("NIMBLE_TEST_BINARY_PATH")
+doAssert callNimble.len != 0, "NIMBLE_TEST_BINARY_PATH not set"
+
+task recurse, "Level 1":
+  echo 1
+  exec callNimble & " recurse2"
+
+task recurse2, "Level 2":
+  echo 2
+  exec callNimble & " recurse3"
+
+task recurse3, "Level 3":
+  echo 3

tests/revdep/mydep/mydep.nimble

@@ -0,0 +1,10 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Random dep"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0"

tests/revdep/pkgNoDep/pkgA.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package A"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0"
+

tests/revdep/pkgWithDep/pkgA.nimble

@@ -0,0 +1,11 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "Correctly structured package A"
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.0", "mydep"
+

tests/run/run.nimble

@@ -0,0 +1,14 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+bin           = @["run"]
+
+
+
+# Dependencies
+
+requires "nim >= 0.19.0"

tests/run/src/run.nim

@@ -0,0 +1,4 @@
+import os
+
+when isMainModule:
+  echo("Testing `nimble run`: ", commandLineParams())

tests/testCommand/testOverride/myTester.nim

@@ -0,0 +1 @@
+echo("overriden")

tests/testCommand/testOverride/pkga.nimble

@@ -0,0 +1,9 @@
+version       = "0.1.0"
+author        = "John Doe"
+description   = "Nimble Test"
+license       = "BSD"
+
+skipFiles = @["myTester.nim"]
+
+task test, "Custom tester":
+  exec "nim c -r myTester.nim"

tests/testCommand/testsCWD/testing123.nimble

@@ -0,0 +1,4 @@
+version       = "0.1.0"
+author        = "John Doe"
+description   = "Nimble Test"
+license       = "BSD"

tests/testCommand/testsCWD/tests/tcwd.nim

@@ -0,0 +1,2 @@
+import os
+echo(getCurrentDir())

tests/testCommand/testsFail/testing123.nim

@@ -0,0 +1,4 @@
+
+proc myFunc*() =
+    echo "Executing my func"
+

tests/testCommand/testsFail/testing123.nimble

@@ -0,0 +1,4 @@
+version       = "0.1.0"
+author        = "John Doe"
+description   = "Nimble Test"
+license       = "BSD"

tests/testCommand/testsFail/tests/t1.nim

@@ -0,0 +1,6 @@
+import testing123, unittest
+
+test "can compile nimble":
+  echo "First test"
+  myFunc()
+

tests/testCommand/testsFail/tests/t2.nim

@@ -0,0 +1,6 @@
+import testing123, unittest
+
+test "can compile nimble":
+  echo "Failing Second test"
+  assert(false)
+

tests/testCommand/testsFail/tests/t3.nim

@@ -0,0 +1,7 @@
+import testing123, unittest
+
+test "can compile nimble":
+  echo "Third test"
+  myFunc()
+
+

tests/testCommand/testsIgnore/testing123.nim

@@ -0,0 +1,4 @@
+
+proc myFunc*() =
+    echo "Executing my func"
+

tests/testCommand/testsIgnore/testing123.nimble

@@ -0,0 +1,4 @@
+version       = "0.1.0"
+author        = "John Doe"
+description   = "Nimble Test"
+license       = "BSD"

tests/testCommand/testsIgnore/tests/foobar/tignored.nim

@@ -0,0 +1 @@
+echo "Should be ignored"

tests/testCommand/testsIgnore/tests/ignore.nim

@@ -0,0 +1 @@
+echo "Should be ignored"

tests/testCommand/testsIgnore/tests/taccept.nim

@@ -0,0 +1,7 @@
+import testing123, unittest
+
+test "can accept":
+  echo "First test"
+  myFunc()
+
+

tests/testCommand/testsPass/testing123.nim

@@ -0,0 +1,4 @@
+
+proc myFunc*() =
+    echo "Executing my func"
+

tests/testCommand/testsPass/testing123.nimble

@@ -0,0 +1,4 @@
+version       = "0.1.0"
+author        = "John Doe"
+description   = "Nimble Test"
+license       = "BSD"

tests/testCommand/testsPass/tests/t1.nim

@@ -0,0 +1,6 @@
+import testing123, unittest
+
+test "can compile nimble":
+  echo "First test"
+  myFunc()
+

tests/testCommand/testsPass/tests/t2.nim

@@ -0,0 +1,7 @@
+import testing123, unittest
+
+test "can compile nimble":
+  echo "Second test"
+  myFunc()
+
+

tests/testCommand/testsPass/tests/t3.nim

@@ -0,0 +1,7 @@
+import testing123, unittest
+
+test "can compile nimble":
+  echo "Third test"
+  myFunc()
+
+

tests/versionClashes/aporiaScenario/aporiaScenario.nimble

@@ -0,0 +1,13 @@
+# Package
+
+version       = "0.1.0"
+author        = "Dominik Picheta"
+description   = " "
+license       = "MIT"
+
+# Dependencies
+
+requires "nim >= 0.15.2"
+requires "https://github.com/nimble-test/packagea.git#head"
+requires "https://github.com/nimble-test/packagea.git >= 0.5"
+