Implements configurable package list URLs. Implements #75.

src/nimble.nim

@@ -32,8 +32,6 @@ else:
 
 const
   nimbleVersion = "0.7.0"
-  defaultPackageURL =
-      "https://github.com/nim-lang/packages/raw/master/packages.json"
 
 proc writeVersion() =
   echo("nimble v$# compiled at $# $#" %
@@ -55,16 +53,40 @@ proc promptCustom(question, default: string): string =
 proc update(options: Options) =
   ## Downloads the package list from the specified URL.
   ##
-  ## If the download is successful, the global didUpdatePackages is set to
+  ## If the download is not successful, an exception is raised.
-  ## true. Otherwise an exception is raised on error.
+  let parameter =
-  let url =
+    if options.action.typ == actionUpdate:
-    if options.action.typ == actionUpdate and options.action.optionalURL != "":
       options.action.optionalURL
     else:
-      defaultPackageURL
+      ""
-  echo("Downloading package list from " & url)
+
-  downloadFile(url, options.getNimbleDir() / "packages.json")
+  proc downloadList(list: PackageList, options: Options) =
-  echo("Done.")
+    echo("Downloading \"", list.name, "\" package list")
+    for i in 0 .. <list.urls.len:
+      let url = list.urls[i]
+      echo("Trying ", url, "...")
+      let tempPath = options.getNimbleDir() / "packages_temp.json"
+      try:
+        downloadFile(url, tempPath)
+      except:
+        if i == <list.urls.len:
+          raise
+        echo("Could not download: ", getCurrentExceptionMsg())
+        continue
+      if not validatePackagesList(tempPath):
+        echo("Downloaded packages.json file is invalid, discarding.")
+        continue
+      copyFile(tempPath,
+          options.getNimbleDir() / "packages_$1.json" % list.name.toLower())
+      echo("Done.")
+      break
+
+  if parameter.isUrl:
+    downloadList(PackageList(name: "commandline", urls: @[parameter]), options)
+  else:
+    # Try each package list in config
+    for name, list in options.config.packageLists:
+      downloadList(list, options)
 
 proc checkInstallFile(pkgInfo: PackageInfo,
                       origDir, file: string): bool =
@@ -461,13 +483,13 @@ proc getDownloadInfo*(pv: PkgTuple, options: Options,
     return (checkUrlType(pv.name), pv.name)
   else:
     var pkg: Package
-    if getPackage(pv.name, options.getNimbleDir() / "packages.json", pkg):
+    if getPackage(pv.name, options, pkg):
       return (pkg.downloadMethod.getDownloadMethod(), pkg.url)
     else:
       # If package is not found give the user a chance to update
       # package.json
       if doPrompt and
-          options.prompt(pv.name & " not found in local packages.json, " &
+          options.prompt(pv.name & " not found in any local packages.json, " &
                          "check internet for updated packages?"):
         update(options)
         return getDownloadInfo(pv, options, doPrompt)
@@ -481,13 +503,13 @@ proc install(packages: seq[PkgTuple],
     result = installFromDir(getCurrentDir(), false, options, "")
   else:
     # If packages.json is not present ask the user if they want to download it.
-    if not existsFile(options.getNimbleDir / "packages.json"):
+    if needsRefresh(options):
       if doPrompt and
-          options.prompt("Local packages.json not found, download it from " &
+          options.prompt("No local packages.json found, download it from " &
               "internet?"):
         update(options)
       else:
-        quit("Please run nimble update.", QuitFailure)
+        quit("Please run nimble refresh.", QuitFailure)
 
     # Install each package.
     for pv in packages:
@@ -555,9 +577,9 @@ proc search(options: Options) =
   assert options.action.typ == actionSearch
   if options.action.search == @[]:
     raise newException(NimbleError, "Please specify a search string.")
-  if not existsFile(options.getNimbleDir() / "packages.json"):
+  if needsRefresh(options):
-    raise newException(NimbleError, "Please run nimble update.")
+    raise newException(NimbleError, "Please run nimble refresh.")
-  let pkgList = getPackageList(options.getNimbleDir() / "packages.json")
+  let pkgList = getPackageList(options)
   var found = false
   template onFound: stmt =
     echoPackage(pkg)
@@ -581,9 +603,9 @@ proc search(options: Options) =
     echo("No package found.")
 
 proc list(options: Options) =
-  if not existsFile(options.getNimbleDir() / "packages.json"):
+  if needsRefresh(options):
-    raise newException(NimbleError, "Please run nimble update.")
+    raise newException(NimbleError, "Please run nimble refresh.")
-  let pkgList = getPackageList(options.getNimbleDir() / "packages.json")
+  let pkgList = getPackageList(options)
   for pkg in pkgList:
     echoPackage(pkg)
     if options.queryVersions:

src/nimblepkg/config.nim

@@ -1,6 +1,6 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
-import parsecfg, streams, strutils, os
+import parsecfg, streams, strutils, os, tables
 
 import tools, version, nimbletypes
 
@@ -8,7 +8,11 @@ type
   Config* = object
     nimbleDir*: string
     chcp*: bool # Whether to change the code page in .cmd files on Win.
+    packageLists*: Table[string, PackageList] ## URLs to packages.json files
 
+  PackageList* = object
+    name*: string
+    urls*: seq[string]
 
 proc initConfig(): Config =
   if getNimrodVersion() > newVersion("0.9.6"):
@@ -18,6 +22,22 @@ proc initConfig(): Config =
 
   result.chcp = true
 
+  result.packageLists = initTable[string, PackageList]()
+  let defaultPkgList = PackageList(name: "Official", urls: @[
+    "http://irclogs.nim-lang.org/packages.json",
+    "http://nim-lang.org/nimble/packages.json",
+    "https://github.com/nim-lang/packages/raw/master/packages.json"
+  ])
+  result.packageLists["official"] = defaultPkgList
+
+proc initPackageList(): PackageList =
+  result.name = ""
+  result.urls = @[]
+
+proc addCurrentPkgList(config: var Config, currentPackageList: PackageList) =
+  if currentPackageList.name.len > 0:
+    config.packageLists[currentPackageList.name.normalize] = currentPackageList
+
 proc parseConfig*(): Config =
   result = initConfig()
   var confFile = getConfigDir() / "nimble" / "nimble.ini"
@@ -34,12 +54,23 @@ proc parseConfig*(): Config =
     echo("Reading from config file at ", confFile)
     var p: CfgParser
     open(p, f, confFile)
+    var currentSection = ""
+    var currentPackageList = initPackageList()
     while true:
       var e = next(p)
       case e.kind
       of cfgEof:
+        addCurrentPkgList(result, currentPackageList)
         break
-      of cfgSectionStart: discard
+      of cfgSectionStart:
+        addCurrentPkgList(result, currentPackageList)
+        currentSection = e.section
+        case currentSection.normalize
+        of "packagelist":
+          currentPackageList = initPackageList()
+        else:
+          raise newException(NimbleError, "Unable to parse config file:" &
+                             " Unknown section: " & e.key)
       of cfgKeyValuePair, cfgOption:
         case e.key.normalize
         of "nimbledir":
@@ -48,6 +79,16 @@ proc parseConfig*(): Config =
             result.nimbleDir = e.value
         of "chcp":
           result.chcp = parseBool(e.value)
+        of "name":
+          case currentSection.normalize
+          of "packagelist":
+            currentPackageList.name = e.value
+          else: assert false
+        of "url":
+          case currentSection.normalize
+          of "packagelist":
+            currentPackageList.urls.add(e.value)
+          else: assert false
         else:
           raise newException(NimbleError, "Unable to parse config file:" &
                                      " Unknown key: " & e.key)

src/nimblepkg/packageinfo.nim

@@ -1,6 +1,6 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
-import parsecfg, json, streams, strutils, parseutils, os
+import parsecfg, json, streams, strutils, parseutils, os, sets, tables
 import version, tools, nimbletypes, options
 
 type
@@ -127,25 +127,35 @@ proc readMetaData*(path: string): MetaData =
   let jsonmeta = parseJson(cont)
   result.url = jsonmeta["url"].str
 
-proc getPackage*(pkg: string, packagesPath: string, resPkg: var Package): bool =
+proc getPackage*(pkg: string, options: Options,
-  ## Searches ``packagesPath`` file saving into ``resPkg`` the found package.
+    resPkg: var Package): bool =
+  ## Searches any packages.json files defined in ``options.config.packageLists``
+  ## Saves the found package into ``resPkg``.
   ##
   ## Pass in ``pkg`` the name of the package you are searching for. As
   ## convenience the proc returns a boolean specifying if the ``resPkg`` was
   ## successfully filled with good data.
-  let packages = parseFile(packagesPath)
+  for name, list in options.config.packageLists:
-  for p in packages:
+    echo("Searching in \"", name, "\" package list...")
-    if p["name"].str == pkg:
+    let packages = parseFile(options.getNimbleDir() /
-      resPkg = p.fromJson()
+        "packages_" & name.toLower() & ".json")
-      return true
+    for p in packages:
+      if p["name"].str == pkg:
+        resPkg = p.fromJson()
+        return true
 
-proc getPackageList*(packagesPath: string): seq[Package] =
+proc getPackageList*(options: Options): seq[Package] =
-  ## Returns the list of packages found at the specified path.
+  ## Returns the list of packages found in the downloaded packages.json files.
   result = @[]
-  let packages = parseFile(packagesPath)
+  var namesAdded = initSet[string]()
-  for p in packages:
+  for name, list in options.config.packageLists:
-    let pkg: Package = p.fromJson()
+    let packages = parseFile(options.getNimbleDir() /
-    result.add(pkg)
+        "packages_" & name.toLower() & ".json")
+    for p in packages:
+      let pkg: Package = p.fromJson()
+      if pkg.name notin namesAdded:
+        result.add(pkg)
+        namesAdded.incl(pkg.name)
 
 proc findNimbleFile*(dir: string; error: bool): string =
   result = ""
@@ -250,6 +260,27 @@ proc getDownloadDirName*(pkg: Package, verRange: VersionRange): string =
     result.add "_"
     result.add verSimple
 
+proc needsRefresh*(options: Options): bool =
+  ## Determines whether a ``nimble refresh`` is needed.
+  ##
+  ## In the future this will check a stored time stamp to determine how long
+  ## ago the package list was refreshed.
+  result = true
+  for name, list in options.config.packageLists:
+    if fileExists(options.getNimbleDir() / "packages_" & name & ".json"):
+      result = false
+
+proc validatePackagesList*(path: string): bool =
+  ## Determines whether package list at ``path`` is valid.
+  try:
+    let pkgList = parseFile(path)
+    if pkgList.kind == JArray:
+      if pkgList.len == 0:
+        echo("WARNING: ", path, " contains no packages.")
+      return true
+  except ValueError, JsonParsingError:
+    return false
+
 when isMainModule:
   doAssert getNameVersion("/home/user/.nimble/libs/packagea-0.1") ==
       ("packagea", "0.1")

tests/tester.nim

@@ -18,6 +18,38 @@ template cd*(dir: string, body: stmt) =
 proc processOutput(output: string): seq[string] =
   output.strip.splitLines().filter((x: string) => (x.len > 0))
 
+test "can refresh with default urls":
+  check execCmdEx(path & " refresh").exitCode == QuitSuccess
+
+test "can refresh with custom urls":
+  # Backup current config
+  let configFile = getConfigDir() / "nimble" / "nimble.ini"
+  let configBakFile = getConfigDir() / "nimble" / "nimble.ini.bak"
+  if fileExists(configFile):
+    moveFile(configFile, configBakFile)
+  writeFile(configFile, """
+    [PackageList]
+    name = "official"
+    url = "http://google.com"
+    url = "http://google.com/404"
+    url = "http://irclogs.nim-lang.org/packages.json"
+    url = "http://nim-lang.org/nimble/packages.json"
+  """.unindent)
+
+  let (output, exitCode) = execCmdEx(path & " refresh")
+  let lines = output.strip.splitLines()
+  check exitCode == QuitSuccess
+  check "reading from config file" in lines[0].normalize
+  check "downloading \"official\" package list" in lines[1].normalize
+  check "trying http://google.com" in lines[2].normalize
+  check "packages.json file is invalid" in lines[3].normalize
+  check "404 not found" in lines[5].normalize
+  check "done" in lines[^1].normalize
+
+  # Restore config
+  if fileExists(configBakFile):
+    moveFile(configBakFile, configFile)
+
 test "can install nimscript package":
   cd "nimscript":
     check execCmdEx("../" & path & " install -y").exitCode == QuitSuccess