Add binarybuilder.org support, conan improvements, headerLDeps

README.md

@@ -54,6 +54,7 @@ getHeader(
   giturl = "https://github.com/username/repo",            # Git repo URL
   dlurl = "https://website.org/download/repo-$1.tar.gz",  # Download URL for archive or raw file
   conanuri = "repo/$1",                                   # Conan.io URI
+  jbburi = "repo/$1",                                     # BinaryBuilder.org URI
   outdir = baseDir,                                       # Where to download/build/search
   conFlags = "--disable-comp --enable-feature",           # Flags to pass configure script
   cmakeFlags = "-DENABLE_STATIC_LIB=ON"                   # Flags to pass to Cmake
@@ -75,8 +76,8 @@ Module documentation for the build API can be found [here](https://nimterop.gith
 The above wrapper is generic and allows the end user to control how it works. Note that `headerPath` is derived from `header.h` so if you have `SDL.h` as the argument to `getHeader()`, it generates `SDLPath` and `SDLLPath` and is controlled by `-d:SDLStatic`, `-d:SDLGit` and so forth.
 
 - If the library is already installed in `/usr/include` then the `-d:headerStd` define to Nim can be used to instruct `getHeader()` to search for `header.h` in the standard system path.
-- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL, `-d:headerDL` to get the source from download URL or `-d:headerConan` to download from https://conan.io/center.
+- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL, `-d:headerDL` to get the source from download URL, `-d:headerConan` to download from https://conan.io/center or `-d:headerJBB` to download from https://binarybuilder.org.
-  - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git and for DL and Conan, it replaces `$1` in the URL if specified.
+  - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git and for DL, Conan and JBB, it replaces `$1` in the URL if specified.
 - If no flag is provided, `getHeader()` simply looks for the library in `outdir`. The user could use Git submodules or manually download or check-in the library to that directory and `getHeader()` will use it directly.
 
 #### Pre build
@@ -93,16 +94,21 @@ Flags can be specified to these tools via `getHeader()` or directly via the unde
 
 #### Linking
 
-- If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`.
+If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`.
-- `getHeader()` searches for libraries based on the header name by default:
+
-  - `libheader.so` or `libheader.a` on Linux
+All dependency libraries (supported by Conan and JBB) will be returned in `headerLDeps`. Static libraries and dependencies are automatically linked using `{.passL.}`. Conan shared libs include all dependencies whereas JBB shared libs expect the required dependencies to be in the same location or in `LD_LIBRARY_PATH`.
-  - `libheader.dylib` on OSX
+
-  - `header.dll` or `header.a` on Windows
+`getHeader()` searches for libraries based on the header name by default:
-- If a library has a different header and library binary name, `altNames` can be used to configure an alternate name of library binary.
+- `libheader.so` or `libheader.a` on Linux
-  - For example, Bzip2 has `bzlib.h` but the library is `libbz2.so` so `altNames = "bz2"`.
+- `libheader.dylib` on OSX
-  - In the example above, `altNames = "hdr"` so `getHeader()` will look for `libhdr.so`, `hdr.dll`, etc.
+- `header.dll`, `header.a` or `header.lib` on Windows
-  - See [bzlib.nim](https://github.com/genotrance/nimarchive/blob/master/nimarchive/bzlib.nim) for an example.
+
-- [lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library that allows both static and dynamic linking.
+If a library has a different header and library binary name, `altNames` can be used to configure an alternate name of library binary.
+- For example, Bzip2 has `bzlib.h` but the library is `libbz2.so` so `altNames = "bz2"`.
+- In the example above, `altNames = "hdr"` so `getHeader()` will look for `libhdr.so`, `hdr.dll`, etc.
+- See [bzlib.nim](https://github.com/genotrance/nimarchive/blob/master/nimarchive/bzlib.nim) for an example.
+
+[lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library that allows both static and dynamic linking.
 
 #### User control
 

nimterop/build.nim

@@ -158,7 +158,7 @@ proc mkDir*(dir: string) =
       flag = when not defined(Windows): "-p" else: ""
     discard execAction(&"mkdir {flag} {dir.sanitizePath}", retry = 2)
 
-proc cpFile*(source, dest: string, move=false) =
+proc cpFile*(source, dest: string, move = false) =
   ## Copy a file from `source` to `dest` at compile time
   let
     source = source.replace("/", $DirSep)
@@ -214,6 +214,25 @@ proc cleanDir*(dir: string) =
     else:
       rmFile(path)
 
+proc cpTree*(source, dest: string, move = false) =
+  ## Copy contents of source dir to the destination, not the directory itself
+  for kind, path in walkDir(source, relative = true):
+    if kind == pcDir:
+      cpTree(source / path, dest / path, move)
+      if move:
+        rmDir(source / path)
+    else:
+      if not dirExists(dest):
+        mkDir(dest)
+      if move:
+        mvFile(source / path, dest / path)
+      else:
+        cpFile(source / path, dest / path)
+
+proc mvTree*(source, dest: string) =
+  ## Move contents of source dir to the destination, not the directory itself
+  cpTree(source, dest, move = true)
+
 proc getProjectCacheDir*(name: string, forceClean = true): string =
   ## Get a cache directory where all nimterop artifacts can be stored
   ##
@@ -242,7 +261,7 @@ proc getProjectCacheDir*(name: string, forceClean = true): string =
     echo "# Removing " & result
     rmDir(result)
 
-proc extractZip*(zipfile, outdir: string) =
+proc extractZip*(zipfile, outdir: string, quiet = false) =
   ## Extract a zip file using `powershell` on Windows and `unzip` on other
   ## systems to the specified output directory
   var cmd = "unzip -o $#"
@@ -251,10 +270,11 @@ proc extractZip*(zipfile, outdir: string) =
           "'System.IO.Compression.FileSystem'; " &
           "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\""
 
-  echo "# Extracting " & zipfile
+  if not quiet:
+    echo "# Extracting " & zipfile
   discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}")
 
-proc extractTar*(tarfile, outdir: string) =
+proc extractTar*(tarfile, outdir: string, quiet = false) =
   ## Extract a tar file using `tar`, `7z` or `7za` to the specified output directory
   var
     cmd = ""
@@ -284,7 +304,8 @@ proc extractTar*(tarfile, outdir: string) =
 
   doAssert cmd.len != 0, "No extraction tool - tar, 7z, 7za - available for " & tarfile.sanitizePath
 
-  echo "# Extracting " & tarfile
+  if not quiet:
+    echo "# Extracting " & tarfile
   discard execAction(&"cd {outdir.sanitizePath} && {cmd}")
   if name.len != 0:
     rmFile(outdir / name)
@@ -316,9 +337,9 @@ proc downloadUrl*(url, outdir: string, quiet = false) =
     discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 1)
 
     if ext == ".zip":
-      extractZip(file, outdir)
+      extractZip(file, outdir, quiet)
     elif ext in archives:
-      extractTar(file, outdir)
+      extractTar(file, outdir, quiet)
 
 proc gitReset*(outdir: string) =
   ## Hard reset the git repository at the specified directory
@@ -342,7 +363,7 @@ proc gitCheckout*(file, outdir: string) =
     sleep(500)
     echo "#   Retrying ..."
 
-proc gitPull*(url: string, outdir = "", plist = "", checkout = "") =
+proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) =
   ## Pull the specified git repository to the output directory
   ##
   ## `plist` is the list of specific files and directories or wildcards
@@ -362,7 +383,8 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") =
 
   mkDir(outdir)
 
-  echo "# Setting up Git repo: " & url
+  if not quiet:
+    echo "# Setting up Git repo: " & url
   discard execAction(&"cd {outdirQ} && git init .")
   discard execAction(&"cd {outdirQ} && git remote add origin {url}")
 
@@ -378,20 +400,32 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") =
   discard execAction(&"cd {outdirQ} && git clean -fxd")
 
   if checkout.len != 0:
-    echo "# Checking out " & checkout
+    if not quiet:
+      echo "# Checking out " & checkout
     discard execAction(&"cd {outdirQ} && git fetch", retry = 1)
     discard execAction(&"cd {outdirQ} && git checkout {checkout}")
   else:
-    echo "# Pulling repository"
+    if not quiet:
+      echo "# Pulling repository"
     discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 1)
 
-proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string =
+proc gitTags*(outdir: string): seq[string] =
-  ## Find the file in the specified directory
+  ## Get all the git tags in the specified directory
+  let
+    cmd = &"cd {outdir.sanitizePath} && git tag"
+    tags = execAction(cmd).output.splitLines()
+  for tag in tags:
+    let
+      tag = tag.strip()
+    if tag.len != 0:
+      result.add tag
+
+proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[string] =
+  ## Find all matching files in the specified directory
   ##
   ## `file` is a regular expression if `regex` is true
   ##
-  ## Turn off recursive search with `recurse` and stop on first match with
+  ## Turn off recursive search with `recurse`
-  ## `first`. Without it, the shortest match is returned.
   var
     cmd =
       when defined(Windows):
@@ -435,10 +469,22 @@ proc findFile*(file: string, dir: string, recurse = true, first = false, regex =
             ""
         else:
           line
+      if f.len != 0:
+        result.add f
 
-      if (f.len != 0 and (result.len == 0 or result.len > f.len)):
+proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string =
-        result = f
+  ## Find the file in the specified directory
-        if first: break
+  ##
+  ## `file` is a regular expression if `regex` is true
+  ##
+  ## Turn off recursive search with `recurse` and stop on first match with
+  ## `first`. Without it, the shortest match is returned.
+  let
+    matches = findFiles(file, dir, recurse, regex)
+  for match in matches:
+    if (result.len == 0 or result.len > match.len):
+      result = match
+      if first: break
 
 proc flagBuild*(base: string, flags: openArray[string]): string =
   ## Simple helper proc to generate flags for `configure`, `cmake`, etc.
@@ -533,7 +579,7 @@ proc configure*(path, check: string, flags = "") =
 
     echoDebug execAction(cmd).output
 
-  doAssert (path / check).fileExists(), "# Configure failed"
+  doAssert (path / check).fileExists(), "Configure failed"
 
 proc getCmakePropertyStr(name, property, value: string): string =
   &"\nset_target_properties({name} PROPERTIES {property} \"{value}\")\n"
@@ -630,7 +676,7 @@ proc cmake*(path, check, flags: string) =
 
   echoDebug execAction(cmd).output
 
-  doAssert (path / check).fileExists(), "# cmake failed"
+  doAssert (path / check).fileExists(), "cmake failed"
 
 proc make*(path, check: string, flags = "", regex = false) =
   ## Run the `make` command to build all binaries in the specified path
@@ -666,7 +712,7 @@ proc make*(path, check: string, flags = "", regex = false) =
 
   echoDebug execAction(cmd).output
 
-  doAssert findFile(check, path, regex = regex).len != 0, "# make failed"
+  doAssert findFile(check, path, regex = regex).len != 0, "make failed"
 
 proc getCompilerMode*(path: string): string =
   ## Determines a target language mode from an input filename, if one is not already specified.
@@ -768,9 +814,16 @@ proc getGccInfo*(): tuple[arch, os, compiler, version: string] =
   else:
     result.compiler = "gcc"
 
+template fixOutDir() {.dirty.} =
+  let
+    outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir
+
 # Conan support
 include conan
 
+# Julia Binary Builder support
+include jbb
+
 proc getStdPath(header, mode: string): string =
   for inc in getGccPaths(mode):
     result = findFile(header, inc, recurse = false, first = true)
@@ -839,11 +892,41 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string =
 
   result = findFile(header, outdir)
 
-proc getConanLPath(outdir: string): string =
+proc getConanLDeps(outdir: string): seq[string] =
   let
     pkg = loadConanInfo(outdir)
 
-  result = pkg.getConanLibs(outdir).join(" ")
+  result = pkg.getConanLDeps(outdir)
+
+proc getJBBPath(header, uri, outdir, version: string): string =
+  let
+    spl = uri.split('/', 1)
+    name = spl[0]
+
+  var
+    ver =
+      if spl.len == 2:
+        spl[1]
+      else:
+        ""
+
+  if "$#" in ver or "$1" in ver:
+    doAssert version.len != 0, "Need version for BinaryBuilder.org uri"
+    ver = ver % version
+  else:
+    doAssert version.len == 0, "BinaryBuilder.org uri does not contain version"
+
+  let
+    pkg = newJBBPackage(name, ver)
+  downloadJBB(pkg, outdir)
+
+  result = findFile(header, outdir)
+
+proc getJBBLDeps(outdir: string, shared: bool): seq[string] =
+  let
+    pkg = loadJBBInfo(outdir)
+
+  result = pkg.getJBBLDeps(outdir, shared)
 
 proc getLocalPath(header, outdir: string): string =
   if outdir.len != 0:
@@ -932,7 +1015,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT
       buildStatus.built = true
 
   let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir
-  doAssert buildStatus.built, &"\n# Build configuration failed - {error}\n"
+  doAssert buildStatus.built, &"\nBuild configuration failed - {error}\n"
 
   result = findFile(lname, outdir, regex = true)
 
@@ -1000,7 +1083,7 @@ macro isDefined*(def: untyped): untyped =
 
 macro getHeader*(
   header: static[string], giturl: static[string] = "", dlurl: static[string] = "",
-  conanuri: static[string] = "", outdir: static[string] = "",
+  conanuri: static[string] = "", jbburi: static[string] = "", outdir: static[string] = "",
   conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "",
   altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped =
   ## Get the path to a header file for wrapping with
@@ -1015,19 +1098,22 @@ macro getHeader*(
   ## `-d:xxxDL` - download source from `dlurl` and extract if required
   ## `-d:xxxConan` - download headers and binary from conan.io using `conanuri`
   ##   typically formatted as `name/version[@user/channel][:bhash]`
+  ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi`
+  ##   typically formatted as `name/version`
   ##
   ## This allows a single wrapper to be used in different ways depending on the user's needs.
   ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is.
   ## The user can opt to download the sources to `outdir` using any other method such as
   ## git sub-modules, vendoring or pointing to a repository that was already cloned.
   ##
-  ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL` or
+  ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL`,
-  ## `Conan`. This allows using a system installed library if available before falling back
+  ## `Conan` or `JBB`. This allows using a system installed library if available before
-  ## to manual building. The user would need to specify both `-d:xxxStd` and one of the other
+  ## falling back to manual building. The user would need to specify both `-d:xxxStd` and
-  ## methods.
+  ## one of the other methods.
   ##
   ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag
-  ## name for Git whereas for DL and Conan, it replaces `$1` in the URL defined.
+  ## name for Git whereas for DL, Conan and BinaryBuilder.org, it replaces `$1` in the URL
+  ## defined.
   ##
   ## All defines can also be set in code using `setDefines()` and checked for using
   ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`.
@@ -1035,7 +1121,7 @@ macro getHeader*(
   ## The library is then configured (with `cmake` or `autotools` if possible) and built
   ## using `make`, unless using `-d:xxxStd` which presumes that the system package
   ## manager was used to install prebuilt headers and binaries, or using `-d:xxxConan`
-  ## which downloads pre-built binaries.
+  ## or `-d:xxxJBB` which download pre-built binaries.
   ##
   ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call
   ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can
@@ -1043,8 +1129,9 @@ macro getHeader*(
   ##
   ## `-d:xxxStatic` can be specified to statically link with the library instead. This
   ## will automatically add a `{.passL.}` call to the static library for convenience. Note
-  ## that `-d:xxxConan` downloads all dependency libs as well and the `xxxLPath` will
+  ## that `-d:xxxConan` and `-d:xxxJBB` download all dependency libs as well and the
-  ## include paths to all of them separated by space in the right order for linking.
+  ## `xxxLPath` will include paths to all of them separated by space in the right order for
+  ## linking.
   ##
   ## Note also that Conan currently builds all OSX binaries on 10.14 so older versions of
   ## OSX will complain if statically linking to these binaries. Further, all Conan binaries
@@ -1088,6 +1175,7 @@ macro getHeader*(
     gitStr = name & "Git"
     dlStr = name & "DL"
     conanStr = name & "Conan"
+    jbbStr = name & "JBB"
 
     staticStr = name & "Static"
     verStr = name & "SetVer"
@@ -1097,12 +1185,14 @@ macro getHeader*(
     nameGit = newIdentNode(gitStr)
     nameDL = newIdentNode(dlStr)
     nameConan = newIdentNode(conanStr)
+    nameJBB = newIdentNode(jbbStr)
 
     nameStatic = newIdentNode(staticStr)
 
     # Consts to generate
     path = newIdentNode(name & "Path")
     lpath = newIdentNode(name & "LPath")
+    ldeps = newIdentNode(name & "LDeps")
     version = newIdentNode(verStr)
     lname = newIdentNode(name & "LName")
     preBuild = newIdentNode(name & "PreBuild")
@@ -1115,6 +1205,7 @@ macro getHeader*(
     gitVal = gDefines.hasKey(gitStr)
     dlVal = gDefines.hasKey(dlStr)
     conanVal = gDefines.hasKey(conanStr)
+    jbbVal = gDefines.hasKey(jbbStr)
     staticVal = gDefines.hasKey(staticStr)
     verVal =
       if gDefines.hasKey(verStr):
@@ -1137,16 +1228,19 @@ macro getHeader*(
       `nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1
       `nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1
       `nameConan`* = when defined(`nameConan`): true else: `conanVal` == 1
+      `nameJBB`* = when defined(`nameJBB`): true else: `jbbVal` == 1
       `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1
 
     # Search for header in outdir (after retrieving code) depending on -d:xxx mode
-    proc getPath(header, giturl, dlurl, conanuri, outdir, version: string, shared: bool): string =
+    proc getPath(header, giturl, dlurl, conanuri, jbburi, outdir, version: string, shared: bool): string =
       when `nameGit`:
         getGitPath(header, giturl, outdir, version)
       elif `nameDL`:
         getDlPath(header, dlurl, outdir, version)
       elif `nameConan`:
         getConanPath(header, conanuri, outdir, version, shared)
+      elif `nameJBB`:
+        getJBBPath(header, jbburi, outdir, version)
       else:
         getLocalPath(header, outdir)
 
@@ -1171,10 +1265,10 @@ macro getHeader*(
         when useStd:
           stdPath
         else:
-          getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`)
+          getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`)
 
-    # Run preBuild hook before building library if not Std or Conan
+    # Run preBuild hook before building library if not Std, Conan or JBB
-    when not (useStd or `nameConan`) and declared(`preBuild`):
+    when not (useStd or `nameConan` or `nameJBB`) and declared(`preBuild`):
       static:
         `preBuild`(`outdir`, prePath)
 
@@ -1183,28 +1277,36 @@ macro getHeader*(
       `lpath`* =
         when useStd:
           stdLPath
-        elif `nameConan`:
+        elif `nameConan` or `nameJBB`:
-          when `nameStatic`:
+          findFile(`lname`, `outdir`, regex = true)
-            getConanLPath(`outdir`)
-          else:
-            findFile(`lname`, `outdir`, regex = true)
         else:
           buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`)
 
+      # Library dependecy paths
+      `ldeps`*: seq[string] =
+        when `nameConan`:
+          getConanLDeps(`outdir`)
+        elif `nameJBB`:
+          getJBBLDeps(`outdir`, not `nameStatic`)
+        else:
+          @[]
+
       # Header path - search again in case header is generated in build
       `path`* =
         if prePath.len != 0:
           prePath
         else:
-          getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`)
+          getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`)
 
     static:
       doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " &
-        "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL or -d:$1Conan not specified" % `name`
+        "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name`
       doAssert `lpath`.len != 0, "\nLibrary " & `lname` & " not found"
       echo "# Including library " & `lpath`
 
-    # Automatically link with static libary
+    # Automatically link with static library and dependencies
     when `nameStatic`:
       {.passL: `lpath`.}
+      if `ldeps`.len != 0:
+        {.passL: `ldeps`.join(" ").}
   )

nimterop/conan.nim

@@ -11,8 +11,8 @@ type
 
     bhash*: string
     shared*: bool
-    headers*: seq[string]
+    sharedLibs*: seq[string]
-    libs*: seq[string]
+    staticLibs*: seq[string]
     requires*: seq[ConanPackage]
 
   ConanBuild* = ref object
@@ -26,14 +26,14 @@ type
 
 const
   # Conan API urls
-  baseUrl = "https://conan.bintray.com/v2/conans"
+  conanBaseUrl = "https://conan.bintray.com/v2/conans"
-  searchUrl = baseUrl & "/search?q=$query"
+  conanSearchUrl = conanBaseUrl & "/search?q=$query"
-  pkgUrl = baseUrl & "/$name/$version/$user/$channel/search$query"
+  conanPkgUrl = conanBaseUrl & "/$name/$version/$user/$channel/search$query"
-  cfgUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions"
+  conanCfgUrl = conanBaseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions"
-  dlUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions/$revision/files/$file"
+  conanDlUrl = conanBaseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions/$revision/files/$file"
 
   # Bintray download sub-URL for explicit `user/channel` (not _/_)
-  dlAltUrl = "/download_file?file_path=$user%2F$name%2F$version%2F$channel%2F0%2Fpackage%2F$build%2F0%2F$file"
+  conanDlAltUrl = "/download_file?file_path=$user%2F$name%2F$version%2F$channel%2F0%2Fpackage%2F$build%2F0%2F$file"
 
   # Strings
   conanInfo = "conaninfo.json"
@@ -42,18 +42,14 @@ const
 
 var
   # Bintray download URL for explicit `user/channel`
-  baseAltUrl {.compiletime.} = {
+  conanBaseAltUrl {.compiletime.} = {
     "bincrafters": "https://bintray.com/bincrafters/public-conan",
     "conan": "https://bintray.com/conan-community/conan"
   }.toTable()
 
-template fixOutDir() {.dirty.} =
+proc addAltConanBaseUrl*(name, url: string) =
-  let
-    outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir
-
-proc addAltBaseUrl*(name, url: string) =
   # Add an alternate base URL for a custom conan repo on bintray
-  baseAltUrl[name] = url
+  conanBaseAltUrl[name] = url
 
 proc jsonGet(url: string): JsonNode =
   # Make HTTP call and return content as JSON
@@ -149,7 +145,7 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac
         query &= "/" & channel
 
   let
-    j1 = jsonGet(searchUrl % ["query", query])
+    j1 = jsonGet(conanSearchUrl % ["query", query])
     res = j1.getOrDefault("results").getElems()
 
   if res.len != 0:
@@ -193,7 +189,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") =
           query.replace("&", "%20and%20")
       else: ""
 
-    url = pkgUrl % [
+    url = conanPkgUrl % [
       "name", pkg.name,
       "version", pkg.version,
       "user", pkg.user,
@@ -234,7 +230,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") =
 proc getConanRevisions*(pkg: ConanPackage, bld: ConanBuild) =
   ## Get all revisions of a build
   let
-    url = cfgUrl % [
+    url = conanCfgUrl % [
       "name", pkg.name,
       "version", pkg.version,
       "user", pkg.user,
@@ -269,7 +265,7 @@ proc saveConanInfo*(pkg: ConanPackage, outdir: string) =
   writeFile(file, $$pkg)
 
 proc parseConanManifest(pkg: ConanPackage, outdir: string) =
-  # Get all header and library info from downloaded conan package
+  # Get all library info from downloaded conan package
   let
     file = outdir / conanManifest
 
@@ -279,12 +275,13 @@ proc parseConanManifest(pkg: ConanPackage, outdir: string) =
     for line in data.splitLines():
       let
         line = line.split(':')[0]
-      if line.startsWith("include/"):
+      if line.startsWith("lib/"):
-        pkg.headers.add line
+        if line.endsWith(".a") or line.endsWith(".lib"):
-      elif line.startsWith("lib/"):
+          pkg.staticLibs.add line
-        pkg.libs.add line
+        elif line.endsWith(".so"):
+          pkg.sharedLibs.add line
       elif line.startsWith("bin/") and line.endsWith("dll"):
-        pkg.libs.add line
+        pkg.sharedLibs.add line
 
 proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision = "") =
   ## Download specific `revision` of `bld` to `outdir`
@@ -300,7 +297,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision
 
     url =
       if pkg.user == "_":
-        dlUrl % [
+        conanDlUrl % [
           "name", pkg.name,
           "version", pkg.version,
           "user", pkg.user,
@@ -311,7 +308,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision
           "file", conanPackage
         ]
       else:
-        baseAltUrl[pkg.user] & dlAltUrl % [
+        conanBaseAltUrl[pkg.user] & conanDlAltUrl % [
           "name", pkg.name,
           "version", pkg.version,
           "user", pkg.user,
@@ -351,7 +348,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) =
 
   pkg.getConanBuilds()
 
-  doAssert pkg.recipes.len != 0, &"# Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center"
+  doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center"
 
   echo &"# Downloading {pkg.name} v{pkg.version} from Conan"
   for recipe, builds in pkg.recipes:
@@ -375,18 +372,25 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) =
   if bld.options["shared"] == "False":
     for req in bld.requires:
       let
-        rpkg = newConanPackageFromUri(req)
+        rpkg = newConanPackageFromUri(req, shared = false)
 
       downloadConan(rpkg, outdir, clean = false)
       pkg.requires.add rpkg
 
-proc getConanLibs*(pkg: ConanPackage, outdir: string): seq[string] =
+proc getConanLDeps*(pkg: ConanPackage, outdir: string, main = true): seq[string] =
-  ## Get all Conan libs (.so|.a|.lib|.dll) in pkg, including deps
+  ## Get all Conan libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps
   ## in descending order
   ##
   ## `outdir` is prefixed to each entry
-  for lib in pkg.libs:
+  let
-    result.add outdir / lib
+    libs = if pkg.shared: pkg.sharedLibs else: pkg.staticLibs
+    str = if pkg.shared: "shared" else: "static"
+
+  doAssert libs.len != 0, &"No {str} libs found for {pkg.name} in {outdir}"
+
+  if not main:
+    for lib in libs:
+      result.add outdir / lib
 
   for cpkg in pkg.requires:
-    result.add cpkg.getConanLibs(outdir)
+    result.add cpkg.getConanLDeps(outdir, main = false)

nimterop/jbb.nim

@@ -0,0 +1,210 @@
+import marshal, os, strutils
+
+type
+  JBBPackage* = ref object
+    ## JBBPackage type that stores package information
+    name*: string
+    version*: string
+
+    url*: string
+
+    sharedLibs*: seq[string]
+    staticLibs*: seq[string]
+    requires*: seq[JBBPackage]
+
+const
+  # JBB URLs
+  jbbBaseUrl = "https://github.com/JuliaBinaryWrappers/$1_jll.jl"
+
+  jbbInfo = "jbbinfo.json"
+  jbbProject = "Project.toml"
+  jbbArtifacts = "Artifacts.toml"
+
+proc `==`*(pkg1, pkg2: JBBPackage): bool =
+  ## Check if two JBBPackage objects are equal
+  (not pkg1.isNil and not pkg2.isNil and
+    pkg1.name == pkg2.name and
+    pkg1.version == pkg2.version)
+
+proc newJBBPackage*(name, version: string): JBBPackage =
+  ## Create a new JBBPackage with specified name and version
+  result = new(JBBPackage)
+  result.name = name
+  result.version = version
+
+proc parseJBBProject(pkg: JBBPackage, outdir: string) =
+  # Get all dependencies from Project.toml
+  let
+    file = outdir / jbbProject
+
+  if fileExists(file):
+    let
+      data = readFile(file)
+    var
+      deps = false
+
+    doAssert pkg.version in data, &"{pkg.name} v{pkg.version} not found"
+
+    for line in data.splitLines():
+      let
+        line = line.strip()
+      if line.len != 0:
+        if line.startsWith('['):
+          if line == "[deps]":
+            deps = true
+          else:
+            deps = false
+        elif deps:
+          let
+            name = line.split()[0]
+          if name.endsWith("_jll"):
+            pkg.requires.add newJBBPackage(name[0 .. ^5], "")
+
+proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) =
+  # Get build information from Artifacts.toml
+  let
+    file = outdir / jbbArtifacts
+
+    (arch, os, _, _) = getGccInfo()
+
+  if fileExists(file):
+    let
+      data = readFile(file)
+
+    doAssert pkg.version in data, &"{pkg.name} v{pkg.version} not found"
+
+    var
+      found = false
+    for line in data.splitLines():
+      let
+        line = line.strip()
+      if line.len != 0:
+        if line.startsWith("arch = ") and not found:
+          let
+            barch = line.split(" = ")[1].strip(chars = {'"'})
+          if barch == arch:
+            found = true
+        elif line.startsWith("os = ") and found:
+          let
+            bos = line.split(" = ")[1].strip(chars = {'"'})
+          if bos != os:
+            found = false
+        elif line.startsWith("url = ") and found:
+          pkg.url = line.split(" = ")[1].strip(chars = {'"'})
+          break
+
+proc findJBBLibs(pkg: JBBPackage, outdir: string) =
+  pkg.sharedLibs = findFiles("lib[\\\\/].*\\.(so|dylib)", outdir)
+  pkg.sharedLibs.add findFiles("bin[\\\\/].*\\.(dll)", outdir)
+  for i in 0 ..< pkg.sharedLibs.len:
+    if pkg.sharedLibs[i].isAbsolute:
+      pkg.sharedLibs[i] = pkg.sharedLibs[i][outdir.len+1 .. ^1]
+
+  for lib in findFiles("lib[\\\\/].*\\.(a|lib)$", outdir):
+    if not lib.endsWith(".dll.a"):
+      if lib.isAbsolute:
+        pkg.staticLibs.add lib[outdir.len+1 .. ^1]
+      else:
+        pkg.staticLibs.add lib
+
+proc getJBBRepo*(pkg: JBBPackage, outdir: string) =
+  ## Clone JBB package repo and checkout version tag if version is
+  ## specified in package
+  let
+    path = outdir / "repos" / pkg.name
+
+  gitPull(
+    jbbBaseUrl % pkg.name,
+    outdir = path,
+    plist = "*.toml",
+    "master",
+    quiet = true
+  )
+
+  if pkg.version.len != 0:
+    # Checkout correct tag
+    let
+      tags = gitTags(path)
+    for i in tags.len - 1 .. 0:
+      if pkg.version in tags[i] and i != tags.len - 1:
+        gitCheckout(path, tags[i-1])
+
+  pkg.parseJBBProject(path)
+  pkg.parseJBBArtifacts(path)
+
+proc loadJBBInfo*(outdir: string): JBBPackage =
+  ## Load cached package info from `outdir/jbbinfo.json`
+  fixOutDir()
+  let
+    file = outdir / jbbInfo
+
+  if fileExists(file):
+    result = to[JBBPackage](readFile(file))
+
+proc saveJBBInfo*(pkg: JBBPackage, outdir: string) =
+  ## Save downloaded package info to `outdir/jbbinfo.json`
+  fixOutDir()
+  let
+    file = outdir / jbbInfo
+
+  writeFile(file, $$pkg)
+
+proc dlJBBRequires*(pkg: JBBPackage, outdir: string)
+proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) =
+  ## Download `pkg` from BinaryBuilder.org to `outdir`
+  ##
+  ## High-level API that handles the end to end JBB process flow to find
+  ## latest package binary and downloads and extracts it to `outdir`.
+  fixOutDir()
+  let
+    cpkg = loadJBBInfo(outdir)
+
+  if cpkg == pkg:
+    return
+  elif clean:
+    cleanDir(outdir)
+
+  pkg.getJBBRepo(outdir)
+
+  doAssert pkg.url.len != 0, &"Failed to download {pkg.name} info from BinaryBuilder.org"
+
+  let
+    vstr =
+      if pkg.version.len != 0:
+        &" v{pkg.version}"
+      else:
+        ""
+    path = outdir / "downloads" / pkg.name
+  echo &"# Downloading {pkg.name}{vstr} from BinaryBuilder.org"
+  downloadUrl(pkg.url, path, quiet = true)
+  pkg.findJBBLibs(path)
+  mvTree(path, outdir)
+
+  pkg.dlJBBRequires(outdir)
+
+  if clean:
+    pkg.saveJBBInfo(outdir)
+
+proc dlJBBRequires*(pkg: JBBPackage, outdir: string) =
+  ## Download all required dependencies of this `pkg`
+  fixOutDir()
+  for rpkg in pkg.requires:
+    downloadJBB(rpkg, outdir, clean = false)
+
+proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): seq[string] =
+  ## Get all BinaryBuilder.org libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps
+  ## in descending order
+  ##
+  ## `outdir` is prefixed to each entry
+  let
+    libs = if shared: pkg.sharedLibs else: pkg.staticLibs
+    str = if shared: "shared" else: "static"
+
+  doAssert libs.len != 0, &"No {str} libs found for {pkg.name} in {outdir}"
+
+  if not main:
+    for lib in libs:
+      result.add outdir / lib
+
+  for cpkg in pkg.requires:
+    result.add cpkg.getJBBLDeps(outdir, shared, main = false)

tests/getheader.nims

@@ -23,6 +23,7 @@ var
 
 testCall(cmd & lrcmd, "No build files found", 1)
 testCall(cmd & " -d:libssh2Conan" & sshcmd, "Need version for Conan uri", 1)
+testCall(cmd & " -d:libssh2JBB" & sshcmd, "Need version for BinaryBuilder.org uri", 1)
 
 when defined(posix):
   # stdlib
@@ -43,6 +44,11 @@ else:
   # conan static for Windows
   testCall(cmd & " -d:zlibConan -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0)
 
+# JBB
+testCall(cmd & " -d:libssh2JBB -d:libssh2SetVer=1.9.0" & sshcmd, zexp, 0)
+testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11" & zrcmd, zexp, 0)
+testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0)
+
 # git
 testCall(cmd & " -d:envTest" & zrcmd, zexp, 0)
 testCall(cmd & " -d:envTestStatic" & zrcmd, zexp, 0, delete = false)

tests/libssh2.nim

@@ -6,6 +6,7 @@ const
 getHeader(
   header = "libssh2.h",
   conanuri = "libssh2/$1",
+  jbburi = "libssh2/$1",
   outdir = outdir
 )
 
@@ -18,12 +19,12 @@ cOverride:
 when not libssh2Static:
   cImport(libssh2Path, recurse = true, dynlib = "libssh2LPath", flags = "-f:ast2 -c -E_ -F_")
 
-  when not defined(Windows):
+  when not defined(Windows) and not isDefined(libssh2JBB):
     proc zlibVersion(): cstring {.importc, dynlib: libssh2LPath.}
 else:
   cImport(libssh2Path, recurse = true, flags = "-f:ast2 -c -E_ -F_")
 
-  when not defined(Windows):
+  when not defined(Windows) and not isDefined(libssh2JBB):
     proc zlibVersion(): cstring {.importc.}
 
   {.passL: "-lpthread".}
@@ -39,8 +40,11 @@ if session == nil:
 libssh2_session_set_blocking(session, 0.cint)
 
 echo "zlib version = " & (block:
-  when not defined(Windows):
+  when not defined(Windows) and not isDefined(libssh2JBB):
     $zlibVersion()
   else:
     ""
 )
+
+static:
+  echo libssh2LDeps

tests/zlib.nim

@@ -28,6 +28,7 @@ getHeader(
   giturl = "https://github.com/madler/zlib",
   dlurl = "http://zlib.net/zlib-$1.tar.gz",
   conanuri = "zlib/$1",
+  jbburi = "zlib/$1",
   outdir = baseDir,
   altNames = "z,zlib"
 )
@@ -72,3 +73,6 @@ else:
   cImport(zlibPath, recurse = true, flags = FLAGS)
 
 echo "zlib version = " & $zlibVersion()
+
+when isDefined(zlibJBB) and isDefined(zlibStatic):
+  {.passL: "-no-pie".}