Optional version and uri, no Std copy, reuse deps, jbb libc check

.travis.yml

@@ -13,7 +13,7 @@ language: c
 env:
   - BRANCH=0.20.2
   - BRANCH=1.0.6
-  - BRANCH=1.2.0
+  - BRANCH=1.2.2
   - BRANCH=devel
 
 cache:

appveyor.yml

@@ -11,6 +11,7 @@ environment:
   matrix:
     - NIM_VERSION: 0.20.2
     - NIM_VERSION: 1.0.6
+    - NIM_VERSION: 1.2.2
 
 for:
 -

nimterop/build.nim

@@ -2,6 +2,8 @@ import hashes, macros, osproc, sets, strformat, strutils, tables
 
 import os except findExe, sleep
 
+export extractFilename, `/`
+
 type
   BuildType* = enum
     btAutoconf, btCmake
@@ -850,6 +852,31 @@ template fixOutDir() {.dirty.} =
   let
     outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir
 
+proc compareVersions*(ver1, ver2: string): int =
+  ## Compare two version strings x.y.z and return -1, 0, 1
+  ##
+  ## ver1 < ver2 = -1
+  ## ver1 = ver2 = 0
+  ## ver1 > ver2 = 1
+  let
+    ver1seq = ver1.replace("-", "").split('.')
+    ver2seq = ver2.replace("-", "").split('.')
+  for i in 0 ..< ver1seq.len:
+    let
+      p1 = ver1seq[i]
+      p2 = if i < ver2seq.len: ver2seq[i] else: "0"
+
+    try:
+      let
+        h1 = p1.parseHexInt()
+        h2 = p2.parseHexInt()
+
+      if h1 < h2: return -1
+      elif h1 > h2: return 1
+    except ValueError:
+      if p1 < p2: return -1
+      elif p1 > p2: return 1
+
 # Conan support
 include conan
 
@@ -913,10 +940,10 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string =
     uri = uri
 
   if "$#" in uri or "$1" in uri:
-    doAssert version.len != 0, "Need version for Conan uri"
+    doAssert version.len != 0, "Need version for Conan.io uri: " & uri
     uri = uri % version
-  else:
+  elif version.len != 0:
-    doAssert version.len == 0, "Conan uri does not contain version"
+    uri = uri & "/" & version
 
   let
     pkg = newConanPackageFromUri(uri, shared)
@@ -934,6 +961,7 @@ proc getJBBPath(header, uri, outdir, version: string): string =
   let
     spl = uri.split('/', 1)
     name = spl[0]
+    hasVersion = version.len != 0
 
   var
     ver =
@@ -942,11 +970,15 @@ proc getJBBPath(header, uri, outdir, version: string): string =
       else:
         ""
 
-  if "$#" in ver or "$1" in ver:
+  if ver.len != 0:
-    doAssert version.len != 0, "Need version for BinaryBuilder.org uri"
+    if "$#" in ver or "$1" in ver:
-    ver = ver % version
+      doAssert hasVersion, "Need version for BinaryBuilder.org uri: " & uri
-  else:
+      ver = ver % version
-    doAssert version.len == 0, "BinaryBuilder.org uri does not contain version"
+    elif hasVersion:
+      doAssert false, "Version in both uri `" & uri & "` and `-d:xxxSetVer=\"" &
+        version & "\"` for BinaryBuilder.org"
+  elif hasVersion:
+    ver = version
 
   let
     pkg = newJBBPackage(name, ver)
@@ -1129,10 +1161,10 @@ macro getHeader*(
   ## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux
   ## `-d:xxxGit` - clone source from a git repo specified in `giturl`
   ## `-d:xxxDL` - download source from `dlurl` and extract if required
-  ## `-d:xxxConan` - download headers and binary from conan.io using `conanuri`
+  ## `-d:xxxConan` - download headers and binary from Conan.io using `conanuri` with
-  ##   typically formatted as `name/version[@user/channel][:bhash]`
+  ##   format `pkgname[/version[@user/channel][:bhash]]`
-  ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi`
+  ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` with
-  ##   typically formatted as `name/version`
+  ##   format `pkgname[/version]`
   ##
   ## This allows a single wrapper to be used in different ways depending on the user's needs.
   ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is.
@@ -1145,8 +1177,15 @@ macro getHeader*(
   ## one of the other methods.
   ##
   ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag
-  ## name for Git whereas for DL, Conan and BinaryBuilder.org, it replaces `$1` in the URL
+  ## name for `Git` whereas for `DL`, `Conan` and `JBB`, it replaces `$1` in the URL
-  ## defined.
+  ## if specified. Specifying `-d:xxxSetVer` without a `$1` will download that version for
+  ## `Conan` and `JBB` if available. If no version is specified, the latest release of the
+  ## package is downloaded. For `Conan`, `-d:xxxSetVer` can also be used to set additional
+  ## URI information:
+  ##   `-d:xxxSetVer=1.9.0@bincrafters/stable:bhash`
+  ##
+  ## If `conanuri` or `jbburi` are not defined and `Conan` or `JBB` is selected, the `header`
+  ## filename is used instead.
   ##
   ## All defines can also be set in code using `setDefines()` and checked for using
   ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`.
@@ -1165,7 +1204,7 @@ macro getHeader*(
   ## dependencies to that directory. This prevents any runtime failures if `outdir` gets
   ## removed or its contents changed. By default, `libdir` is set to the output directory
   ## where the program binary will be created. The values of `xxxLPath` and `xxxLDeps` will
-  ## reflect this new location.
+  ## reflect this new location. `libdir` is ignored for `Std` mode.
   ##
   ## `-d:xxxStatic` can be specified to statically link with the library instead. This
   ## will automatically add a `{.passL.}` call to the static library for convenience. Note
@@ -1210,6 +1249,10 @@ macro getHeader*(
     origname = header.extractFilename().split(".")[0]
     name = origname.split(seps = AllChars-Letters-Digits).join()
 
+    # Default to origname if not specified
+    conanuri = if conanuri.len != 0: conanuri else: origname
+    jbburi = if jbburi.len != 0: jbburi else: origname
+
     # -d:xxx for this header
     stdStr = name & "Std"
     gitStr = name & "Git"
@@ -1316,7 +1359,7 @@ macro getHeader*(
 
     let
       # Library binary path - build if not standard / conan / jbb
-      lpath {.compiletime.} =
+      lpath {.compileTime.} =
         when useStd:
           stdLPath
         elif `nameConan` or `nameJBB`:
@@ -1325,11 +1368,14 @@ macro getHeader*(
           buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`)
 
       # Library dependecy paths
-      ldeps {.compiletime.}: seq[string] =
+      ldeps {.compileTime.}: seq[string] =
-        when `nameConan`:
+        when not useStd:
-          getConanLDeps(`outdir`)
+          when `nameConan`:
-        elif `nameJBB`:
+            getConanLDeps(`outdir`)
-          getJBBLDeps(`outdir`, not `nameStatic`)
+          elif `nameJBB`:
+            getJBBLDeps(`outdir`, not `nameStatic`)
+          else:
+            @[]
         else:
           @[]
 
@@ -1346,20 +1392,6 @@ macro getHeader*(
         "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name`
       doAssert lpath.len != 0, "\nLibrary " & `lname` & " not found"
 
-    proc extractFilenameStatic(str: string): string {.compiletime.} =
-      var
-        pos = -1
-      for i in countdown(str.len - 1, 0):
-        if str[i] == '/' or str[i] == '\\':
-          pos = i + 1
-          break
-      result = str[pos .. ^1]
-
-    proc joinPathStatic(str1, str2: string): string {.compiletime.} =
-      let
-        sep = when defined(Windows): "\\" else: "/"
-      result = str1 & sep & str2
-
     when `nameStatic`:
       const
         `lpath`* = lpath
@@ -1376,28 +1408,34 @@ macro getHeader*(
           echo "# Including dependencies " & `ldeps`.join(" ")
     else:
       const
-        `lpath`* = joinPathStatic(`libdir`, lpath.extractFilenameStatic())
+        `lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath
-        `ldeps`* = block:
+        `ldeps`* =
-          var
+          when not useStd:
-            ldeps = ldeps
+            block:
-            copied: seq[string]
+              var
-          for i in 0 ..< ldeps.len:
+                ldeps = ldeps
-            let
+                copied: seq[string]
-              lname = ldeps[i].extractFilenameStatic()
+              for i in 0 ..< ldeps.len:
-              ldeptgt = joinPathStatic(`libdir`, lname)
+                let
-            if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt):
+                  lname = ldeps[i].extractFilename()
-              cpFile(ldeps[i], ldeptgt, psymlink = true)
+                  ldeptgt = `libdir` / lname
-              copied.add lname
+                if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt):
-            ldeps[i] = ldeptgt
+                  cpFile(ldeps[i], ldeptgt, psymlink = true)
-          if copied.len != 0:
+                  copied.add lname
-            echo "# Copying dependencies: " & copied.join(" ") & "\n#   to " & `libdir`
+                ldeps[i] = ldeptgt
-          ldeps
+              # Copy downloaded dependencies to `libdir`
+              if copied.len != 0:
+                echo "# Copying dependencies: " & copied.join(" ") & "\n#   to " & `libdir`
+              ldeps
+          else:
+            ldeps
 
       static:
-        # Copy shared libraries and dependencies to `libdir`
+          when not useStd:
-        if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`):
+            # Copy downloaded shared libraries to `libdir`
-          echo "# Copying " & `lpath`.extractFilenameStatic() & " to " & `libdir`
+            if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`):
-          cpFile(lpath, `lpath`)
+              echo "# Copying " & `lpath`.extractFilename() & " to " & `libdir`
+              cpFile(lpath, `lpath`)
 
-        echo "# Including library " & `lpath`
+          echo "# Including library " & `lpath`
   )

nimterop/conan.nim

@@ -42,11 +42,14 @@ const
 
 var
   # Bintray download URL for explicit `user/channel`
-  conanBaseAltUrl {.compiletime.} = {
+  conanBaseAltUrl {.compileTime.} = {
     "bincrafters": "https://bintray.com/bincrafters/public-conan",
     "conan": "https://bintray.com/conan-community/conan"
   }.toTable()
 
+  # Reuse dependencies already downloaded
+  gConanRequires {.compileTime.}: Table[string, ConanPackage]
+
 proc addAltConanBaseUrl*(name, url: string) =
   # Add an alternate base URL for a custom conan repo on bintray
   conanBaseAltUrl[name] = url
@@ -144,13 +147,28 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac
       if channel.len != 0:
         query &= "/" & channel
 
+  echo &"# Searching Conan.io for latest version of {name}"
+
   let
     j1 = jsonGet(conanSearchUrl % ["query", query])
     res = j1.getOrDefault("results").getElems()
 
-  if res.len != 0:
+  # Return latest comparing versions - prefer @_/_
-    # Return last entry - latest
+  var
-    result = newConanPackageFromUri(res[^1].getStr())
+    latest = ""
+    latestv = ""
+  for i in 0 ..< res.len:
+    let
+      str = res[i].getStr()
+    if "@_/_" in str:
+      let
+        ver = str.split('/')[1].split('@')[0]
+      if latestv.len == 0 or compareVersions(ver, latestv) > 0:
+        latestv = ver
+        latest = str
+
+  if latest.len != 0:
+    result = newConanPackageFromUri(latest)
 
 proc searchConan*(pkg: ConanPackage): ConanPackage =
   ## Search for latest package based on incomplete package info
@@ -288,6 +306,9 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision
   ##
   ## If omitted, the latest revision (first) is downloaded
   fixOutDir()
+
+  doAssert bld.revisions.len != 0, "No build revisions found for Conan.io package " & pkg.getUriFromConanPackage()
+
   let
     revision =
       if revision.len != 0:
@@ -326,7 +347,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision
   rmFile(outdir / conanManifest)
 
 proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string)
-proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) =
+proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) =
   ## Download latest recipe/build/revision of `pkg` to `outdir`
   ##
   ## High-level API that handles the end to end Conan process flow to find
@@ -339,11 +360,13 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) =
       else:
         pkg
 
-    cpkg = loadConanInfo(outdir)
+  if main:
+    let
+      cpkg = loadConanInfo(outdir)
+
+    if cpkg == pkg:
+      return
 
-  if cpkg == pkg:
-    return
-  elif clean:
     cleanDir(outdir)
 
   echo &"# Downloading {pkg.name} v{pkg.version} from Conan"
@@ -352,7 +375,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) =
 
   doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center"
 
-  echo &"# Downloading {pkg.name} v{pkg.version} from Conan"
+  echo &"# Downloading {pkg.name} v{pkg.version} from Conan.io"
   for recipe, builds in pkg.recipes:
     for build in builds:
       if pkg.bhash.len == 0 or pkg.bhash == build.bhash:
@@ -362,7 +385,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) =
         break
     break
 
-  if clean:
+  if main:
     pkg.saveConanInfo(outdir)
 
 proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) =
@@ -374,10 +397,17 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) =
   if bld.options["shared"] == "False":
     for req in bld.requires:
       let
-        rpkg = newConanPackageFromUri(req, shared = false)
+        name = req.split('/')[0]
+      if gConanRequires.hasKey(name):
+        # Reuse dep already downloaded
+        pkg.requires.add gConanRequires[name]
+      else:
+        let
+          rpkg = newConanPackageFromUri(req, shared = false)
 
-      downloadConan(rpkg, outdir, clean = false)
+        downloadConan(rpkg, outdir, main = false)
-      pkg.requires.add rpkg
+        pkg.requires.add rpkg
+        gConanRequires[name] = rpkg
 
 proc getConanLDeps*(pkg: ConanPackage, outdir: string, main = true): seq[string] =
   ## Get all Conan libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps

nimterop/globals.nim

@@ -146,7 +146,7 @@ when defined(TOAST):
       gecho join(args, "").getCommented()
 else:
   var
-    gStateCT* {.compiletime, used.} = new(State)
+    gStateCT* {.compileTime, used.} = new(State)
 
 template nBl*(s: typed): untyped {.used.} =
   (s.len != 0)

nimterop/jbb.nim

@@ -20,6 +20,10 @@ const
   jbbProject = "Project.toml"
   jbbArtifacts = "Artifacts.toml"
 
+var
+  # Reuse dependencies already downloaded
+  gJBBRequires {.compileTime.}: Table[string, JBBPackage]
+
 proc `==`*(pkg1, pkg2: JBBPackage): bool =
   ## Check if two JBBPackage objects are equal
   (not pkg1.isNil and not pkg2.isNil and
@@ -79,19 +83,26 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) =
       let
         line = line.strip()
       if line.len != 0:
-        if line.startsWith("arch = ") and not found:
+        let
-          let
+          spl = line.split(" = ", 1)
-            barch = line.split(" = ")[1].strip(chars = {'"'})
+          name = spl[0]
-          if barch == arch:
+          val = if spl.len == 2: spl[1].strip(chars = {'"', ' '}) else: ""
-            found = true
+
-        elif line.startsWith("os = ") and found:
+        # Match arch, os and glibc on Linux to find download URL
-          let
+        case name
-            bos = line.split(" = ")[1].strip(chars = {'"'})
+        of "arch":
-          if bos != os:
+          if val == arch and not found: found = true
-            found = false
+        of "os":
-        elif line.startsWith("url = ") and found:
+          if val != os and found: found = false
-          pkg.url = line.split(" = ")[1].strip(chars = {'"'})
+        of "libc":
-          break
+          when defined(Linux):
+            if val != "glibc" and found: found = false
+        of "url":
+          if found:
+            pkg.url = val
+            break
+        else:
+          discard
 
 proc findJBBLibs(pkg: JBBPackage, outdir: string) =
   pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dylib)[0-9.]*", outdir)
@@ -143,18 +154,19 @@ proc saveJBBInfo*(pkg: JBBPackage, outdir: string) =
   writeFile(file, $$pkg)
 
 proc dlJBBRequires*(pkg: JBBPackage, outdir: string)
-proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) =
+proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) =
   ## Download `pkg` from BinaryBuilder.org to `outdir`
   ##
   ## High-level API that handles the end to end JBB process flow to find
   ## latest package binary and downloads and extracts it to `outdir`.
   fixOutDir()
-  let
+  if main:
-    cpkg = loadJBBInfo(outdir)
+    let
+      cpkg = loadJBBInfo(outdir)
+
+    if cpkg == pkg:
+      return
 
-  if cpkg == pkg:
-    return
-  elif clean:
     cleanDir(outdir)
 
   pkg.getJBBRepo(outdir)
@@ -174,14 +186,21 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) =
 
   pkg.dlJBBRequires(outdir)
 
-  if clean:
+  if main:
     pkg.saveJBBInfo(outdir)
 
 proc dlJBBRequires*(pkg: JBBPackage, outdir: string) =
   ## Download all required dependencies of this `pkg`
   fixOutDir()
-  for rpkg in pkg.requires:
+  for i in 0 ..< pkg.requires.len:
-    downloadJBB(rpkg, outdir, clean = false)
+    let
+      rpkg = pkg.requires[i]
+    if gJBBRequires.hasKey(rpkg.name):
+      # Reuse dep already downloaded
+      pkg.requires[i] = gJBBRequires[rpkg.name]
+    else:
+      downloadJBB(rpkg, outdir, main = false)
+      gJBBRequires[rpkg.name] = rpkg
 
 proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): seq[string] =
   ## Get all BinaryBuilder.org libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps

tests/libssh2.nim

@@ -6,7 +6,7 @@ const
 getHeader(
   header = "libssh2.h",
   conanuri = "libssh2/$1",
-  jbburi = "libssh2/$1",
+  jbburi = "libssh2/1.9.0",
   outdir = outdir
 )
 
@@ -45,6 +45,3 @@ echo "zlib version = " & (block:
   else:
     ""
 )
-
-static:
-  echo libssh2LDeps

tests/lzma.nim

@@ -24,6 +24,8 @@ getHeader(
   "lzma.h",
   giturl = "https://github.com/xz-mirror/xz",
   dlurl = "https://tukaani.org/xz/xz-$1.tar.gz",
+  conanuri = "xz_utils",
+  jbburi = "xz",
   outdir = baseDir,
   conFlags = "--disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo"
 )

tests/zlib.nim

@@ -27,8 +27,6 @@ getHeader(
   "zlib.h",
   giturl = "https://github.com/madler/zlib",
   dlurl = "http://zlib.net/zlib-$1.tar.gz",
-  conanuri = "zlib/$1",
-  jbburi = "zlib/$1",
   outdir = baseDir,
   altNames = "z,zlib"
 )