Allow locally stored package list files to be specified in config (#368)

readme.markdown

@@ -297,6 +297,10 @@ nimbleDir = r"C:\Nimble\"
 [PackageList]
 name = "CustomPackages"
 url = "http://mydomain.org/packages.json"
+
+[PackageList]
+name = "Local project packages"
+path = r"C:\Projects\Nim\packages.json"
 ```
 
 You can currently configure the following in this file:
@@ -307,12 +311,13 @@ You can currently configure the following in this file:
   application packages. If ``true`` this will add ``chcp 65001`` to the
   .cmd stubs generated in ``~/.nimble/bin/``.
   **Default:** ``true``
-* ``[PackageList]`` + ``name`` + ``url`` - You can use this section to specify
+* ``[PackageList]`` + ``name`` + (``url``|``path``) - You can use this section to specify
   a new custom package list. Multiple package lists can be specified. Nimble
   defaults to the "Official" package list, you can override it by specifying
   a ``[PackageList]`` section named "official". Multiple URLs can be specified
   under each section, Nimble will try each in succession if
-  downloading from the first fails.
+  downloading from the first fails. Alternately, ``path`` can specify a
+  local file path to copy a package list .json file from.
 * ``cloneUsingHttps`` - Whether to replace any ``git://`` inside URLs with
   ``https://``.
   **Default: true**

src/nimble.nim

@@ -48,17 +48,17 @@ proc refresh(options: Options) =
   if parameter.len > 0:
     if parameter.isUrl:
       let cmdLine = PackageList(name: "commandline", urls: @[parameter])
-      downloadList(cmdLine, options)
+      fetchList(cmdLine, options)
     else:
       if parameter notin options.config.packageLists:
         let msg = "Package list with the specified name not found."
         raise newException(NimbleError, msg)
 
-      downloadList(options.config.packageLists[parameter], options)
+      fetchList(options.config.packageLists[parameter], options)
   else:
     # Try each package list in config
     for name, list in options.config.packageLists:
-      downloadList(list, options)
+      fetchList(list, options)
 
 proc checkInstallFile(pkgInfo: PackageInfo,
                       origDir, file: string): bool =

src/nimblepkg/config.nim

@@ -15,6 +15,7 @@ type
   PackageList* = object
     name*: string
     urls*: seq[string]
+    path*: string
 
 proc initConfig(): Config =
   result.nimbleDir = getHomeDir() / ".nimble"
@@ -35,6 +36,7 @@ proc initConfig(): Config =
 proc initPackageList(): PackageList =
   result.name = ""
   result.urls = @[]
+  result.path = ""
 
 proc addCurrentPkgList(config: var Config, currentPackageList: PackageList) =
   if currentPackageList.name.len > 0:
@@ -53,7 +55,6 @@ proc parseConfig*(): Config =
     if f != nil:
       display("Warning", "Using deprecated config file at " & confFile,
               Warning, HighPriority)
-
   if f != nil:
     display("Reading", "config file at " & confFile, priority = LowPriority)
     var p: CfgParser
@@ -64,6 +65,10 @@ proc parseConfig*(): Config =
       var e = next(p)
       case e.kind
       of cfgEof:
+        if currentPackageList.urls.len == 0 and currentPackageList.path == "":
+          raise newException(NimbleError, "Package list '$1' requires either url or path" % currentPackageList.name)
+        if currentPackageList.urls.len > 0 and currentPackageList.path != "":
+          raise newException(NimbleError, "Attempted to specify `url` and `path` for the same package list '$1'" % currentPackageList.name)
         addCurrentPkgList(result, currentPackageList)
         break
       of cfgSectionStart:
@@ -97,6 +102,14 @@ proc parseConfig*(): Config =
           of "packagelist":
             currentPackageList.urls.add(e.value)
           else: assert false
+        of "path":
+          case currentSection.normalize
+          of "packagelist":
+            if currentPackageList.path.len > 0:
+              raise newException(NimbleError, "Attempted to specify more than one `path` for the same package list.")
+            else:
+              currentPackageList.path = e.value.normalize
+          else: assert false
         else:
           raise newException(NimbleError, "Unable to parse config file:" &
                                      " Unknown key: " & e.key)

src/nimblepkg/packageinfo.nim

@@ -160,53 +160,69 @@ proc validatePackagesList(path: string): bool =
   except ValueError, JsonParsingError:
     return false
 
-proc downloadList*(list: PackageList, options: Options) =
+proc fetchList*(list: PackageList, options: Options) =
-  ## Downloads the specified package list and saves it in $nimbleDir.
+  ## Downloads or copies the specified package list and saves it in $nimbleDir.
-  display("Downloading", list.name & " package list", priority = HighPriority)
+  let verb = if list.urls.len > 0: "Downloading" else: "Copying"
+  display(verb, list.name & " package list", priority = HighPriority)
 
-  var lastError = ""
+  var
-  for i in 0 .. <list.urls.len:
-    let url = list.urls[i]
-    display("Trying", url)
-    let tempPath = options.getNimbleDir() / "packages_temp.json"
-
-    # Grab the proxy
-    let proxy = getProxy(options)
-    if not proxy.isNil:
-      var maskedUrl = proxy.url
-      if maskedUrl.password.len > 0: maskedUrl.password = "***"
-      display("Connecting", "to proxy at " & $maskedUrl,
-              priority = LowPriority)
-
-    try:
-      downloadFile(url, tempPath, proxy = getProxy(options))
-    except:
-      let message = "Could not download: " & getCurrentExceptionMsg()
-      display("Warning:", message, Warning)
-      lastError = message
-      continue
-
-    if not validatePackagesList(tempPath):
-      lastError = "Downloaded packages.json file is invalid"
-      display("Warning:", lastError & ", discarding.", Warning)
-      continue
-
-    copyFile(tempPath,
-        options.getNimbleDir() / "packages_$1.json" % list.name.toLowerAscii())
-    display("Success", "Package list downloaded.", Success, HighPriority)
     lastError = ""
-    break
+    copyFromPath = ""
+  if list.urls.len > 0:
+    let httpclient = newHttpClient(proxy = getProxy(options))
+    for i in 0 .. <list.urls.len:
+      let url = list.urls[i]
+      display("Trying", url)
+      let tempPath = options.getNimbleDir() / "packages_temp.json"
+
+      # Grab the proxy
+      let proxy = getProxy(options)
+      if not proxy.isNil:
+        var maskedUrl = proxy.url
+        if maskedUrl.password.len > 0: maskedUrl.password = "***"
+        display("Connecting", "to proxy at " & $maskedUrl,
+                priority = LowPriority)
+
+      try:
+        httpclient.downloadFile(url, tempPath)
+      except:
+        let message = "Could not download: " & getCurrentExceptionMsg()
+        display("Warning:", message, Warning)
+        lastError = message
+        continue
+
+      if not validatePackagesList(tempPath):
+        lastError = "Downloaded packages.json file is invalid"
+        display("Warning:", lastError & ", discarding.", Warning)
+        continue
+
+      copyFromPath = tempPath
+      display("Success", "Package list downloaded.", Success, HighPriority)
+      lastError = ""
+      break
+
+  elif list.path != "":
+    if not validatePackagesList(list.path):
+      lastError = "Copied packages.json file is invalid"
+      display("Warning:", lastError & ", discarding.", Warning)
+    else:
+      copyFromPath = list.path
+      display("Success", "Package list copied.", Success, HighPriority)
 
   if lastError.len != 0:
     raise newException(NimbleError, "Refresh failed\n" & lastError)
 
+  if copyFromPath.len > 0:
+    copyFile(copyFromPath,
+        options.getNimbleDir() / "packages_$1.json" % list.name.toLowerAscii())
+
 proc readPackageList(name: string, options: Options): JsonNode =
   # If packages.json is not present ask the user if they want to download it.
   if needsRefresh(options):
     if options.prompt("No local packages.json found, download it from " &
             "internet?"):
       for name, list in options.config.packageLists:
-        downloadList(list, options)
+        fetchList(list, options)
     else:
       raise newException(NimbleError, "Please run nimble refresh.")
   return parseFile(options.getNimbleDir() / "packages_" &

tests/issue368/packages.json

@@ -0,0 +1,14 @@
+[
+  {
+    "name": "discordnim",
+    "url": "https://github.com/Krognol/discordnim",
+    "method": "git",
+    "tags": [
+      "library",
+      "discord"
+    ],
+    "description": "Discord library for Nim",
+    "license": "MIT",
+    "web": "https://github.com/Krognol/discordnim"
+  }
+]

tests/tester.nim

@@ -33,7 +33,6 @@ proc execNimble(args: varargs[string]): tuple[output: string, exitCode: int] =
   quotedArgs = quoted_args.map((x: string) => ("\"" & x & "\""))
 
   result = execCmdEx(quotedArgs.join(" "))
-  #echo(result.output)
 
 proc processOutput(output: string): seq[string] =
   output.strip.splitLines().filter((x: string) => (x.len > 0))
@@ -138,9 +137,9 @@ proc safeMoveFile(src, dest: string) =
     copyFile(src, dest)
     removeFile(src)
 
-test "can refresh with custom urls":
+template testRefresh(body: untyped) =
   # Backup current config
-  let configFile = getConfigDir() / "nimble" / "nimble.ini"
+  let configFile {.inject.} = getConfigDir() / "nimble" / "nimble.ini"
   let configBakFile = getConfigDir() / "nimble" / "nimble.ini.bak"
   if fileExists(configFile):
     safeMoveFile(configFile, configBakFile)
@@ -148,29 +147,73 @@ test "can refresh with custom urls":
   # Ensure config dir exists
   createDir(getConfigDir() / "nimble")
 
-  writeFile(configFile, """
+  body
-    [PackageList]
-    name = "official"
-    url = "http://google.com"
-    url = "http://google.com/404"
-    url = "http://irclogs.nim-lang.org/packages.json"
-    url = "http://nim-lang.org/nimble/packages.json"
-  """.unindent)
-
-  let (output, exitCode) = execNimble(["refresh", "--verbose"])
-  let lines = output.strip.splitLines()
-  check exitCode == QuitSuccess
-  check inLines(lines, "config file at")
-  check inLines(lines, "official package list")
-  check inLines(lines, "http://google.com")
-  check inLines(lines, "packages.json file is invalid")
-  check inLines(lines, "404 not found")
-  check inLines(lines, "Package list downloaded.")
 
   # Restore config
   if fileExists(configBakFile):
     safeMoveFile(configBakFile, configFile)
 
+test "can refresh with custom urls":
+  testRefresh():
+    writeFile(configFile, """
+      [PackageList]
+      name = "official"
+      url = "http://google.com"
+      url = "http://google.com/404"
+      url = "http://irclogs.nim-lang.org/packages.json"
+      url = "http://nim-lang.org/nimble/packages.json"
+    """.unindent)
+
+    let (output, exitCode) = execNimble(["refresh", "--verbose"])
+    let lines = output.strip.splitLines()
+    check exitCode == QuitSuccess
+    check inLines(lines, "config file at")
+    check inLines(lines, "official package list")
+    check inLines(lines, "http://google.com")
+    check inLines(lines, "packages.json file is invalid")
+    check inLines(lines, "404 not found")
+    check inLines(lines, "Package list downloaded.")
+
+test "can refresh with local package list":
+  testRefresh():
+    writeFile(configFile, """
+      [PackageList]
+      name = "local"
+      path = "$1"
+    """.unindent % (getCurrentDir() / "issue368" / "packages.json"))
+    let (output, exitCode) = execNimble(["refresh", "--verbose"])
+    let lines = output.strip.splitLines()
+    check inLines(lines, "config file at")
+    check inLines(lines, "Copying")
+    check inLines(lines, "Package list copied.")
+    check exitCode == QuitSuccess
+
+test "package list source required":
+  testRefresh():
+    writeFile(configFile, """
+      [PackageList]
+      name = "local"
+    """)
+    let (output, exitCode) = execNimble(["refresh", "--verbose"])
+    let lines = output.strip.splitLines()
+    check inLines(lines, "config file at")
+    check inLines(lines, "Package list 'local' requires either url or path")
+    check exitCode == QuitFailure
+
+test "package list can only have one source":
+  testRefresh():
+    writeFile(configFile, """
+      [PackageList]
+      name = "local"
+      path = "$1"
+      url = "http://nim-lang.org/nimble/packages.json"
+    """)
+    let (output, exitCode) = execNimble(["refresh", "--verbose"])
+    let lines = output.strip.splitLines()
+    check inLines(lines, "config file at")
+    check inLines(lines, "Attempted to specify `url` and `path` for the same package list 'local'")
+    check exitCode == QuitFailure
+
 test "can install nimscript package":
   cd "nimscript":
     check execNimble(["install", "-y"]).exitCode == QuitSuccess