From 2b3b5e2bec8f7c39540f0da5b4331f5e90c46843 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 15 Jun 2020 21:37:35 -0500 Subject: [PATCH] Add binarybuilder.org support, conan improvements, headerLDeps --- README.md | 30 ++++--- nimterop/build.nim | 196 ++++++++++++++++++++++++++++++---------- nimterop/conan.nim | 78 ++++++++-------- nimterop/jbb.nim | 210 +++++++++++++++++++++++++++++++++++++++++++ tests/getheader.nims | 6 ++ tests/libssh2.nim | 14 +-- tests/zlib.nim | 4 + 7 files changed, 437 insertions(+), 101 deletions(-) create mode 100644 nimterop/jbb.nim diff --git a/README.md b/README.md index 3fd48e2..ca17b90 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ getHeader( giturl = "https://github.com/username/repo", # Git repo URL dlurl = "https://website.org/download/repo-$1.tar.gz", # Download URL for archive or raw file conanuri = "repo/$1", # Conan.io URI + jbburi = "repo/$1", # BinaryBuilder.org URI outdir = baseDir, # Where to download/build/search conFlags = "--disable-comp --enable-feature", # Flags to pass configure script cmakeFlags = "-DENABLE_STATIC_LIB=ON" # Flags to pass to Cmake @@ -75,8 +76,8 @@ Module documentation for the build API can be found [here](https://nimterop.gith The above wrapper is generic and allows the end user to control how it works. Note that `headerPath` is derived from `header.h` so if you have `SDL.h` as the argument to `getHeader()`, it generates `SDLPath` and `SDLLPath` and is controlled by `-d:SDLStatic`, `-d:SDLGit` and so forth. - If the library is already installed in `/usr/include` then the `-d:headerStd` define to Nim can be used to instruct `getHeader()` to search for `header.h` in the standard system path. -- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL, `-d:headerDL` to get the source from download URL or `-d:headerConan` to download from https://conan.io/center. - - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git and for DL and Conan, it replaces `$1` in the URL if specified. +- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL, `-d:headerDL` to get the source from download URL, `-d:headerConan` to download from https://conan.io/center or `-d:headerJBB` to download from https://binarybuilder.org. + - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git and for DL, Conan and JBB, it replaces `$1` in the URL if specified. - If no flag is provided, `getHeader()` simply looks for the library in `outdir`. The user could use Git submodules or manually download or check-in the library to that directory and `getHeader()` will use it directly. #### Pre build @@ -93,16 +94,21 @@ Flags can be specified to these tools via `getHeader()` or directly via the unde #### Linking -- If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`. -- `getHeader()` searches for libraries based on the header name by default: - - `libheader.so` or `libheader.a` on Linux - - `libheader.dylib` on OSX - - `header.dll` or `header.a` on Windows -- If a library has a different header and library binary name, `altNames` can be used to configure an alternate name of library binary. - - For example, Bzip2 has `bzlib.h` but the library is `libbz2.so` so `altNames = "bz2"`. - - In the example above, `altNames = "hdr"` so `getHeader()` will look for `libhdr.so`, `hdr.dll`, etc. - - See [bzlib.nim](https://github.com/genotrance/nimarchive/blob/master/nimarchive/bzlib.nim) for an example. -- [lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library that allows both static and dynamic linking. +If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`. + +All dependency libraries (supported by Conan and JBB) will be returned in `headerLDeps`. Static libraries and dependencies are automatically linked using `{.passL.}`. Conan shared libs include all dependencies whereas JBB shared libs expect the required dependencies to be in the same location or in `LD_LIBRARY_PATH`. + +`getHeader()` searches for libraries based on the header name by default: +- `libheader.so` or `libheader.a` on Linux +- `libheader.dylib` on OSX +- `header.dll`, `header.a` or `header.lib` on Windows + +If a library has a different header and library binary name, `altNames` can be used to configure an alternate name of library binary. +- For example, Bzip2 has `bzlib.h` but the library is `libbz2.so` so `altNames = "bz2"`. +- In the example above, `altNames = "hdr"` so `getHeader()` will look for `libhdr.so`, `hdr.dll`, etc. +- See [bzlib.nim](https://github.com/genotrance/nimarchive/blob/master/nimarchive/bzlib.nim) for an example. + +[lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library that allows both static and dynamic linking. #### User control diff --git a/nimterop/build.nim b/nimterop/build.nim index 74e6a61..eda27e7 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -158,7 +158,7 @@ proc mkDir*(dir: string) = flag = when not defined(Windows): "-p" else: "" discard execAction(&"mkdir {flag} {dir.sanitizePath}", retry = 2) -proc cpFile*(source, dest: string, move=false) = +proc cpFile*(source, dest: string, move = false) = ## Copy a file from `source` to `dest` at compile time let source = source.replace("/", $DirSep) @@ -214,6 +214,25 @@ proc cleanDir*(dir: string) = else: rmFile(path) +proc cpTree*(source, dest: string, move = false) = + ## Copy contents of source dir to the destination, not the directory itself + for kind, path in walkDir(source, relative = true): + if kind == pcDir: + cpTree(source / path, dest / path, move) + if move: + rmDir(source / path) + else: + if not dirExists(dest): + mkDir(dest) + if move: + mvFile(source / path, dest / path) + else: + cpFile(source / path, dest / path) + +proc mvTree*(source, dest: string) = + ## Move contents of source dir to the destination, not the directory itself + cpTree(source, dest, move = true) + proc getProjectCacheDir*(name: string, forceClean = true): string = ## Get a cache directory where all nimterop artifacts can be stored ## @@ -242,7 +261,7 @@ proc getProjectCacheDir*(name: string, forceClean = true): string = echo "# Removing " & result rmDir(result) -proc extractZip*(zipfile, outdir: string) = +proc extractZip*(zipfile, outdir: string, quiet = false) = ## Extract a zip file using `powershell` on Windows and `unzip` on other ## systems to the specified output directory var cmd = "unzip -o $#" @@ -251,10 +270,11 @@ proc extractZip*(zipfile, outdir: string) = "'System.IO.Compression.FileSystem'; " & "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" - echo "# Extracting " & zipfile + if not quiet: + echo "# Extracting " & zipfile discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}") -proc extractTar*(tarfile, outdir: string) = +proc extractTar*(tarfile, outdir: string, quiet = false) = ## Extract a tar file using `tar`, `7z` or `7za` to the specified output directory var cmd = "" @@ -284,7 +304,8 @@ proc extractTar*(tarfile, outdir: string) = doAssert cmd.len != 0, "No extraction tool - tar, 7z, 7za - available for " & tarfile.sanitizePath - echo "# Extracting " & tarfile + if not quiet: + echo "# Extracting " & tarfile discard execAction(&"cd {outdir.sanitizePath} && {cmd}") if name.len != 0: rmFile(outdir / name) @@ -316,9 +337,9 @@ proc downloadUrl*(url, outdir: string, quiet = false) = discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 1) if ext == ".zip": - extractZip(file, outdir) + extractZip(file, outdir, quiet) elif ext in archives: - extractTar(file, outdir) + extractTar(file, outdir, quiet) proc gitReset*(outdir: string) = ## Hard reset the git repository at the specified directory @@ -342,7 +363,7 @@ proc gitCheckout*(file, outdir: string) = sleep(500) echo "# Retrying ..." -proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = +proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = ## Pull the specified git repository to the output directory ## ## `plist` is the list of specific files and directories or wildcards @@ -362,7 +383,8 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = mkDir(outdir) - echo "# Setting up Git repo: " & url + if not quiet: + echo "# Setting up Git repo: " & url discard execAction(&"cd {outdirQ} && git init .") discard execAction(&"cd {outdirQ} && git remote add origin {url}") @@ -378,20 +400,32 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = discard execAction(&"cd {outdirQ} && git clean -fxd") if checkout.len != 0: - echo "# Checking out " & checkout + if not quiet: + echo "# Checking out " & checkout discard execAction(&"cd {outdirQ} && git fetch", retry = 1) discard execAction(&"cd {outdirQ} && git checkout {checkout}") else: - echo "# Pulling repository" + if not quiet: + echo "# Pulling repository" discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 1) -proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string = - ## Find the file in the specified directory +proc gitTags*(outdir: string): seq[string] = + ## Get all the git tags in the specified directory + let + cmd = &"cd {outdir.sanitizePath} && git tag" + tags = execAction(cmd).output.splitLines() + for tag in tags: + let + tag = tag.strip() + if tag.len != 0: + result.add tag + +proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[string] = + ## Find all matching files in the specified directory ## ## `file` is a regular expression if `regex` is true ## - ## Turn off recursive search with `recurse` and stop on first match with - ## `first`. Without it, the shortest match is returned. + ## Turn off recursive search with `recurse` var cmd = when defined(Windows): @@ -435,10 +469,22 @@ proc findFile*(file: string, dir: string, recurse = true, first = false, regex = "" else: line + if f.len != 0: + result.add f - if (f.len != 0 and (result.len == 0 or result.len > f.len)): - result = f - if first: break +proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string = + ## Find the file in the specified directory + ## + ## `file` is a regular expression if `regex` is true + ## + ## Turn off recursive search with `recurse` and stop on first match with + ## `first`. Without it, the shortest match is returned. + let + matches = findFiles(file, dir, recurse, regex) + for match in matches: + if (result.len == 0 or result.len > match.len): + result = match + if first: break proc flagBuild*(base: string, flags: openArray[string]): string = ## Simple helper proc to generate flags for `configure`, `cmake`, etc. @@ -533,7 +579,7 @@ proc configure*(path, check: string, flags = "") = echoDebug execAction(cmd).output - doAssert (path / check).fileExists(), "# Configure failed" + doAssert (path / check).fileExists(), "Configure failed" proc getCmakePropertyStr(name, property, value: string): string = &"\nset_target_properties({name} PROPERTIES {property} \"{value}\")\n" @@ -630,7 +676,7 @@ proc cmake*(path, check, flags: string) = echoDebug execAction(cmd).output - doAssert (path / check).fileExists(), "# cmake failed" + doAssert (path / check).fileExists(), "cmake failed" proc make*(path, check: string, flags = "", regex = false) = ## Run the `make` command to build all binaries in the specified path @@ -666,7 +712,7 @@ proc make*(path, check: string, flags = "", regex = false) = echoDebug execAction(cmd).output - doAssert findFile(check, path, regex = regex).len != 0, "# make failed" + doAssert findFile(check, path, regex = regex).len != 0, "make failed" proc getCompilerMode*(path: string): string = ## Determines a target language mode from an input filename, if one is not already specified. @@ -768,9 +814,16 @@ proc getGccInfo*(): tuple[arch, os, compiler, version: string] = else: result.compiler = "gcc" +template fixOutDir() {.dirty.} = + let + outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir + # Conan support include conan +# Julia Binary Builder support +include jbb + proc getStdPath(header, mode: string): string = for inc in getGccPaths(mode): result = findFile(header, inc, recurse = false, first = true) @@ -839,11 +892,41 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string = result = findFile(header, outdir) -proc getConanLPath(outdir: string): string = +proc getConanLDeps(outdir: string): seq[string] = let pkg = loadConanInfo(outdir) - - result = pkg.getConanLibs(outdir).join(" ") + + result = pkg.getConanLDeps(outdir) + +proc getJBBPath(header, uri, outdir, version: string): string = + let + spl = uri.split('/', 1) + name = spl[0] + + var + ver = + if spl.len == 2: + spl[1] + else: + "" + + if "$#" in ver or "$1" in ver: + doAssert version.len != 0, "Need version for BinaryBuilder.org uri" + ver = ver % version + else: + doAssert version.len == 0, "BinaryBuilder.org uri does not contain version" + + let + pkg = newJBBPackage(name, ver) + downloadJBB(pkg, outdir) + + result = findFile(header, outdir) + +proc getJBBLDeps(outdir: string, shared: bool): seq[string] = + let + pkg = loadJBBInfo(outdir) + + result = pkg.getJBBLDeps(outdir, shared) proc getLocalPath(header, outdir: string): string = if outdir.len != 0: @@ -932,7 +1015,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT buildStatus.built = true let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir - doAssert buildStatus.built, &"\n# Build configuration failed - {error}\n" + doAssert buildStatus.built, &"\nBuild configuration failed - {error}\n" result = findFile(lname, outdir, regex = true) @@ -1000,7 +1083,7 @@ macro isDefined*(def: untyped): untyped = macro getHeader*( header: static[string], giturl: static[string] = "", dlurl: static[string] = "", - conanuri: static[string] = "", outdir: static[string] = "", + conanuri: static[string] = "", jbburi: static[string] = "", outdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = ## Get the path to a header file for wrapping with @@ -1015,19 +1098,22 @@ macro getHeader*( ## `-d:xxxDL` - download source from `dlurl` and extract if required ## `-d:xxxConan` - download headers and binary from conan.io using `conanuri` ## typically formatted as `name/version[@user/channel][:bhash]` + ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` + ## typically formatted as `name/version` ## ## This allows a single wrapper to be used in different ways depending on the user's needs. ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. ## The user can opt to download the sources to `outdir` using any other method such as ## git sub-modules, vendoring or pointing to a repository that was already cloned. ## - ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL` or - ## `Conan`. This allows using a system installed library if available before falling back - ## to manual building. The user would need to specify both `-d:xxxStd` and one of the other - ## methods. + ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL`, + ## `Conan` or `JBB`. This allows using a system installed library if available before + ## falling back to manual building. The user would need to specify both `-d:xxxStd` and + ## one of the other methods. ## ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag - ## name for Git whereas for DL and Conan, it replaces `$1` in the URL defined. + ## name for Git whereas for DL, Conan and BinaryBuilder.org, it replaces `$1` in the URL + ## defined. ## ## All defines can also be set in code using `setDefines()` and checked for using ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`. @@ -1035,7 +1121,7 @@ macro getHeader*( ## The library is then configured (with `cmake` or `autotools` if possible) and built ## using `make`, unless using `-d:xxxStd` which presumes that the system package ## manager was used to install prebuilt headers and binaries, or using `-d:xxxConan` - ## which downloads pre-built binaries. + ## or `-d:xxxJBB` which download pre-built binaries. ## ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can @@ -1043,9 +1129,10 @@ macro getHeader*( ## ## `-d:xxxStatic` can be specified to statically link with the library instead. This ## will automatically add a `{.passL.}` call to the static library for convenience. Note - ## that `-d:xxxConan` downloads all dependency libs as well and the `xxxLPath` will - ## include paths to all of them separated by space in the right order for linking. - ## + ## that `-d:xxxConan` and `-d:xxxJBB` download all dependency libs as well and the + ## `xxxLPath` will include paths to all of them separated by space in the right order for + ## linking. + ## ## Note also that Conan currently builds all OSX binaries on 10.14 so older versions of ## OSX will complain if statically linking to these binaries. Further, all Conan binaries ## for Windows are built with Visual Studio so static linking the `.lib` files with gcc @@ -1088,6 +1175,7 @@ macro getHeader*( gitStr = name & "Git" dlStr = name & "DL" conanStr = name & "Conan" + jbbStr = name & "JBB" staticStr = name & "Static" verStr = name & "SetVer" @@ -1097,12 +1185,14 @@ macro getHeader*( nameGit = newIdentNode(gitStr) nameDL = newIdentNode(dlStr) nameConan = newIdentNode(conanStr) + nameJBB = newIdentNode(jbbStr) nameStatic = newIdentNode(staticStr) # Consts to generate path = newIdentNode(name & "Path") lpath = newIdentNode(name & "LPath") + ldeps = newIdentNode(name & "LDeps") version = newIdentNode(verStr) lname = newIdentNode(name & "LName") preBuild = newIdentNode(name & "PreBuild") @@ -1115,6 +1205,7 @@ macro getHeader*( gitVal = gDefines.hasKey(gitStr) dlVal = gDefines.hasKey(dlStr) conanVal = gDefines.hasKey(conanStr) + jbbVal = gDefines.hasKey(jbbStr) staticVal = gDefines.hasKey(staticStr) verVal = if gDefines.hasKey(verStr): @@ -1137,16 +1228,19 @@ macro getHeader*( `nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1 `nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1 `nameConan`* = when defined(`nameConan`): true else: `conanVal` == 1 + `nameJBB`* = when defined(`nameJBB`): true else: `jbbVal` == 1 `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, conanuri, outdir, version: string, shared: bool): string = + proc getPath(header, giturl, dlurl, conanuri, jbburi, outdir, version: string, shared: bool): string = when `nameGit`: getGitPath(header, giturl, outdir, version) elif `nameDL`: getDlPath(header, dlurl, outdir, version) elif `nameConan`: getConanPath(header, conanuri, outdir, version, shared) + elif `nameJBB`: + getJBBPath(header, jbburi, outdir, version) else: getLocalPath(header, outdir) @@ -1171,10 +1265,10 @@ macro getHeader*( when useStd: stdPath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) - # Run preBuild hook before building library if not Std or Conan - when not (useStd or `nameConan`) and declared(`preBuild`): + # Run preBuild hook before building library if not Std, Conan or JBB + when not (useStd or `nameConan` or `nameJBB`) and declared(`preBuild`): static: `preBuild`(`outdir`, prePath) @@ -1183,28 +1277,36 @@ macro getHeader*( `lpath`* = when useStd: stdLPath - elif `nameConan`: - when `nameStatic`: - getConanLPath(`outdir`) - else: - findFile(`lname`, `outdir`, regex = true) + elif `nameConan` or `nameJBB`: + findFile(`lname`, `outdir`, regex = true) else: buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) + # Library dependecy paths + `ldeps`*: seq[string] = + when `nameConan`: + getConanLDeps(`outdir`) + elif `nameJBB`: + getJBBLDeps(`outdir`, not `nameStatic`) + else: + @[] + # Header path - search again in case header is generated in build `path`* = if prePath.len != 0: prePath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) static: doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & - "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL or -d:$1Conan not specified" % `name` + "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name` doAssert `lpath`.len != 0, "\nLibrary " & `lname` & " not found" echo "# Including library " & `lpath` - # Automatically link with static libary + # Automatically link with static library and dependencies when `nameStatic`: {.passL: `lpath`.} + if `ldeps`.len != 0: + {.passL: `ldeps`.join(" ").} ) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index 95ab24f..bfe2e94 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -11,8 +11,8 @@ type bhash*: string shared*: bool - headers*: seq[string] - libs*: seq[string] + sharedLibs*: seq[string] + staticLibs*: seq[string] requires*: seq[ConanPackage] ConanBuild* = ref object @@ -26,14 +26,14 @@ type const # Conan API urls - baseUrl = "https://conan.bintray.com/v2/conans" - searchUrl = baseUrl & "/search?q=$query" - pkgUrl = baseUrl & "/$name/$version/$user/$channel/search$query" - cfgUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions" - dlUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions/$revision/files/$file" + conanBaseUrl = "https://conan.bintray.com/v2/conans" + conanSearchUrl = conanBaseUrl & "/search?q=$query" + conanPkgUrl = conanBaseUrl & "/$name/$version/$user/$channel/search$query" + conanCfgUrl = conanBaseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions" + conanDlUrl = conanBaseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions/$revision/files/$file" # Bintray download sub-URL for explicit `user/channel` (not _/_) - dlAltUrl = "/download_file?file_path=$user%2F$name%2F$version%2F$channel%2F0%2Fpackage%2F$build%2F0%2F$file" + conanDlAltUrl = "/download_file?file_path=$user%2F$name%2F$version%2F$channel%2F0%2Fpackage%2F$build%2F0%2F$file" # Strings conanInfo = "conaninfo.json" @@ -42,18 +42,14 @@ const var # Bintray download URL for explicit `user/channel` - baseAltUrl {.compiletime.} = { + conanBaseAltUrl {.compiletime.} = { "bincrafters": "https://bintray.com/bincrafters/public-conan", "conan": "https://bintray.com/conan-community/conan" }.toTable() -template fixOutDir() {.dirty.} = - let - outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir - -proc addAltBaseUrl*(name, url: string) = +proc addAltConanBaseUrl*(name, url: string) = # Add an alternate base URL for a custom conan repo on bintray - baseAltUrl[name] = url + conanBaseAltUrl[name] = url proc jsonGet(url: string): JsonNode = # Make HTTP call and return content as JSON @@ -137,7 +133,7 @@ proc getUriFromConanPackage*(pkg: ConanPackage): string = proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPackage = ## Search for package by `name` and optional `version`, `user` and `channel` - ## + ## ## Search is quite slow so it is preferable to specify a version and use `getConanBuilds()` var query = name @@ -149,7 +145,7 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac query &= "/" & channel let - j1 = jsonGet(searchUrl % ["query", query]) + j1 = jsonGet(conanSearchUrl % ["query", query]) res = j1.getOrDefault("results").getElems() if res.len != 0: @@ -193,7 +189,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = query.replace("&", "%20and%20") else: "" - url = pkgUrl % [ + url = conanPkgUrl % [ "name", pkg.name, "version", pkg.version, "user", pkg.user, @@ -234,7 +230,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = proc getConanRevisions*(pkg: ConanPackage, bld: ConanBuild) = ## Get all revisions of a build let - url = cfgUrl % [ + url = conanCfgUrl % [ "name", pkg.name, "version", pkg.version, "user", pkg.user, @@ -269,22 +265,23 @@ proc saveConanInfo*(pkg: ConanPackage, outdir: string) = writeFile(file, $$pkg) proc parseConanManifest(pkg: ConanPackage, outdir: string) = - # Get all header and library info from downloaded conan package + # Get all library info from downloaded conan package let file = outdir / conanManifest - + if fileExists(file): let data = readFile(file) for line in data.splitLines(): let line = line.split(':')[0] - if line.startsWith("include/"): - pkg.headers.add line - elif line.startsWith("lib/"): - pkg.libs.add line + if line.startsWith("lib/"): + if line.endsWith(".a") or line.endsWith(".lib"): + pkg.staticLibs.add line + elif line.endsWith(".so"): + pkg.sharedLibs.add line elif line.startsWith("bin/") and line.endsWith("dll"): - pkg.libs.add line + pkg.sharedLibs.add line proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision = "") = ## Download specific `revision` of `bld` to `outdir` @@ -300,7 +297,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision url = if pkg.user == "_": - dlUrl % [ + conanDlUrl % [ "name", pkg.name, "version", pkg.version, "user", pkg.user, @@ -311,7 +308,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision "file", conanPackage ] else: - baseAltUrl[pkg.user] & dlAltUrl % [ + conanBaseAltUrl[pkg.user] & conanDlAltUrl % [ "name", pkg.name, "version", pkg.version, "user", pkg.user, @@ -331,7 +328,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = ## Download latest recipe/build/revision of `pkg` to `outdir` - ## + ## ## High-level API that handles the end to end Conan process flow to find ## latest package binary and downloads and extracts it to `outdir`. fixOutDir() @@ -351,7 +348,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = pkg.getConanBuilds() - doAssert pkg.recipes.len != 0, &"# Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" + doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" echo &"# Downloading {pkg.name} v{pkg.version} from Conan" for recipe, builds in pkg.recipes: @@ -375,18 +372,25 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = if bld.options["shared"] == "False": for req in bld.requires: let - rpkg = newConanPackageFromUri(req) + rpkg = newConanPackageFromUri(req, shared = false) downloadConan(rpkg, outdir, clean = false) pkg.requires.add rpkg -proc getConanLibs*(pkg: ConanPackage, outdir: string): seq[string] = - ## Get all Conan libs (.so|.a|.lib|.dll) in pkg, including deps +proc getConanLDeps*(pkg: ConanPackage, outdir: string, main = true): seq[string] = + ## Get all Conan libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps ## in descending order - ## + ## ## `outdir` is prefixed to each entry - for lib in pkg.libs: - result.add outdir / lib + let + libs = if pkg.shared: pkg.sharedLibs else: pkg.staticLibs + str = if pkg.shared: "shared" else: "static" + + doAssert libs.len != 0, &"No {str} libs found for {pkg.name} in {outdir}" + + if not main: + for lib in libs: + result.add outdir / lib for cpkg in pkg.requires: - result.add cpkg.getConanLibs(outdir) + result.add cpkg.getConanLDeps(outdir, main = false) diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim new file mode 100644 index 0000000..9f0d173 --- /dev/null +++ b/nimterop/jbb.nim @@ -0,0 +1,210 @@ +import marshal, os, strutils + +type + JBBPackage* = ref object + ## JBBPackage type that stores package information + name*: string + version*: string + + url*: string + + sharedLibs*: seq[string] + staticLibs*: seq[string] + requires*: seq[JBBPackage] + +const + # JBB URLs + jbbBaseUrl = "https://github.com/JuliaBinaryWrappers/$1_jll.jl" + + jbbInfo = "jbbinfo.json" + jbbProject = "Project.toml" + jbbArtifacts = "Artifacts.toml" + +proc `==`*(pkg1, pkg2: JBBPackage): bool = + ## Check if two JBBPackage objects are equal + (not pkg1.isNil and not pkg2.isNil and + pkg1.name == pkg2.name and + pkg1.version == pkg2.version) + +proc newJBBPackage*(name, version: string): JBBPackage = + ## Create a new JBBPackage with specified name and version + result = new(JBBPackage) + result.name = name + result.version = version + +proc parseJBBProject(pkg: JBBPackage, outdir: string) = + # Get all dependencies from Project.toml + let + file = outdir / jbbProject + + if fileExists(file): + let + data = readFile(file) + var + deps = false + + doAssert pkg.version in data, &"{pkg.name} v{pkg.version} not found" + + for line in data.splitLines(): + let + line = line.strip() + if line.len != 0: + if line.startsWith('['): + if line == "[deps]": + deps = true + else: + deps = false + elif deps: + let + name = line.split()[0] + if name.endsWith("_jll"): + pkg.requires.add newJBBPackage(name[0 .. ^5], "") + +proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = + # Get build information from Artifacts.toml + let + file = outdir / jbbArtifacts + + (arch, os, _, _) = getGccInfo() + + if fileExists(file): + let + data = readFile(file) + + doAssert pkg.version in data, &"{pkg.name} v{pkg.version} not found" + + var + found = false + for line in data.splitLines(): + let + line = line.strip() + if line.len != 0: + if line.startsWith("arch = ") and not found: + let + barch = line.split(" = ")[1].strip(chars = {'"'}) + if barch == arch: + found = true + elif line.startsWith("os = ") and found: + let + bos = line.split(" = ")[1].strip(chars = {'"'}) + if bos != os: + found = false + elif line.startsWith("url = ") and found: + pkg.url = line.split(" = ")[1].strip(chars = {'"'}) + break + +proc findJBBLibs(pkg: JBBPackage, outdir: string) = + pkg.sharedLibs = findFiles("lib[\\\\/].*\\.(so|dylib)", outdir) + pkg.sharedLibs.add findFiles("bin[\\\\/].*\\.(dll)", outdir) + for i in 0 ..< pkg.sharedLibs.len: + if pkg.sharedLibs[i].isAbsolute: + pkg.sharedLibs[i] = pkg.sharedLibs[i][outdir.len+1 .. ^1] + + for lib in findFiles("lib[\\\\/].*\\.(a|lib)$", outdir): + if not lib.endsWith(".dll.a"): + if lib.isAbsolute: + pkg.staticLibs.add lib[outdir.len+1 .. ^1] + else: + pkg.staticLibs.add lib + +proc getJBBRepo*(pkg: JBBPackage, outdir: string) = + ## Clone JBB package repo and checkout version tag if version is + ## specified in package + let + path = outdir / "repos" / pkg.name + + gitPull( + jbbBaseUrl % pkg.name, + outdir = path, + plist = "*.toml", + "master", + quiet = true + ) + + if pkg.version.len != 0: + # Checkout correct tag + let + tags = gitTags(path) + for i in tags.len - 1 .. 0: + if pkg.version in tags[i] and i != tags.len - 1: + gitCheckout(path, tags[i-1]) + + pkg.parseJBBProject(path) + pkg.parseJBBArtifacts(path) + +proc loadJBBInfo*(outdir: string): JBBPackage = + ## Load cached package info from `outdir/jbbinfo.json` + fixOutDir() + let + file = outdir / jbbInfo + + if fileExists(file): + result = to[JBBPackage](readFile(file)) + +proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = + ## Save downloaded package info to `outdir/jbbinfo.json` + fixOutDir() + let + file = outdir / jbbInfo + + writeFile(file, $$pkg) + +proc dlJBBRequires*(pkg: JBBPackage, outdir: string) +proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) = + ## Download `pkg` from BinaryBuilder.org to `outdir` + ## + ## High-level API that handles the end to end JBB process flow to find + ## latest package binary and downloads and extracts it to `outdir`. + fixOutDir() + let + cpkg = loadJBBInfo(outdir) + + if cpkg == pkg: + return + elif clean: + cleanDir(outdir) + + pkg.getJBBRepo(outdir) + + doAssert pkg.url.len != 0, &"Failed to download {pkg.name} info from BinaryBuilder.org" + + let + vstr = + if pkg.version.len != 0: + &" v{pkg.version}" + else: + "" + path = outdir / "downloads" / pkg.name + echo &"# Downloading {pkg.name}{vstr} from BinaryBuilder.org" + downloadUrl(pkg.url, path, quiet = true) + pkg.findJBBLibs(path) + mvTree(path, outdir) + + pkg.dlJBBRequires(outdir) + + if clean: + pkg.saveJBBInfo(outdir) + +proc dlJBBRequires*(pkg: JBBPackage, outdir: string) = + ## Download all required dependencies of this `pkg` + fixOutDir() + for rpkg in pkg.requires: + downloadJBB(rpkg, outdir, clean = false) + +proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): seq[string] = + ## Get all BinaryBuilder.org libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps + ## in descending order + ## + ## `outdir` is prefixed to each entry + let + libs = if shared: pkg.sharedLibs else: pkg.staticLibs + str = if shared: "shared" else: "static" + + doAssert libs.len != 0, &"No {str} libs found for {pkg.name} in {outdir}" + + if not main: + for lib in libs: + result.add outdir / lib + + for cpkg in pkg.requires: + result.add cpkg.getJBBLDeps(outdir, shared, main = false) diff --git a/tests/getheader.nims b/tests/getheader.nims index 349b9bb..1d0ba1d 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -23,6 +23,7 @@ var testCall(cmd & lrcmd, "No build files found", 1) testCall(cmd & " -d:libssh2Conan" & sshcmd, "Need version for Conan uri", 1) +testCall(cmd & " -d:libssh2JBB" & sshcmd, "Need version for BinaryBuilder.org uri", 1) when defined(posix): # stdlib @@ -43,6 +44,11 @@ else: # conan static for Windows testCall(cmd & " -d:zlibConan -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) +# JBB +testCall(cmd & " -d:libssh2JBB -d:libssh2SetVer=1.9.0" & sshcmd, zexp, 0) +testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11" & zrcmd, zexp, 0) +testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) + # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) testCall(cmd & " -d:envTestStatic" & zrcmd, zexp, 0, delete = false) diff --git a/tests/libssh2.nim b/tests/libssh2.nim index b7e591c..48b8ba9 100644 --- a/tests/libssh2.nim +++ b/tests/libssh2.nim @@ -6,6 +6,7 @@ const getHeader( header = "libssh2.h", conanuri = "libssh2/$1", + jbburi = "libssh2/$1", outdir = outdir ) @@ -14,16 +15,16 @@ cOverride: stat = object stat64 = object SOCKET = object - + when not libssh2Static: cImport(libssh2Path, recurse = true, dynlib = "libssh2LPath", flags = "-f:ast2 -c -E_ -F_") - when not defined(Windows): + when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc, dynlib: libssh2LPath.} else: cImport(libssh2Path, recurse = true, flags = "-f:ast2 -c -E_ -F_") - when not defined(Windows): + when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc.} {.passL: "-lpthread".} @@ -39,8 +40,11 @@ if session == nil: libssh2_session_set_blocking(session, 0.cint) echo "zlib version = " & (block: - when not defined(Windows): + when not defined(Windows) and not isDefined(libssh2JBB): $zlibVersion() else: "" -) \ No newline at end of file +) + +static: + echo libssh2LDeps \ No newline at end of file diff --git a/tests/zlib.nim b/tests/zlib.nim index 15400d4..2bd9395 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -28,6 +28,7 @@ getHeader( giturl = "https://github.com/madler/zlib", dlurl = "http://zlib.net/zlib-$1.tar.gz", conanuri = "zlib/$1", + jbburi = "zlib/$1", outdir = baseDir, altNames = "z,zlib" ) @@ -72,3 +73,6 @@ else: cImport(zlibPath, recurse = true, flags = FLAGS) echo "zlib version = " & $zlibVersion() + +when isDefined(zlibJBB) and isDefined(zlibStatic): + {.passL: "-no-pie".}