From 1820fdffcf8af4b1c5ecca8bdc548c206e24b439 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 18 Jun 2020 10:22:22 -0500 Subject: [PATCH] Optional version and uri, no Std copy, reuse deps, jbb libc check --- .travis.yml | 2 +- appveyor.yml | 1 + nimterop/build.nim | 150 +++++++++++++++++++++++++++---------------- nimterop/conan.nim | 58 +++++++++++++---- nimterop/globals.nim | 2 +- nimterop/jbb.nim | 63 +++++++++++------- tests/libssh2.nim | 5 +- tests/lzma.nim | 2 + tests/zlib.nim | 2 - 9 files changed, 185 insertions(+), 100 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4e9bde..173da68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ language: c env: - BRANCH=0.20.2 - BRANCH=1.0.6 - - BRANCH=1.2.0 + - BRANCH=1.2.2 - BRANCH=devel cache: diff --git a/appveyor.yml b/appveyor.yml index d67b01d..b572ea1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,6 +11,7 @@ environment: matrix: - NIM_VERSION: 0.20.2 - NIM_VERSION: 1.0.6 + - NIM_VERSION: 1.2.2 for: - diff --git a/nimterop/build.nim b/nimterop/build.nim index dbdcd4c..b391f7a 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -2,6 +2,8 @@ import hashes, macros, osproc, sets, strformat, strutils, tables import os except findExe, sleep +export extractFilename, `/` + type BuildType* = enum btAutoconf, btCmake @@ -850,6 +852,31 @@ template fixOutDir() {.dirty.} = let outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir +proc compareVersions*(ver1, ver2: string): int = + ## Compare two version strings x.y.z and return -1, 0, 1 + ## + ## ver1 < ver2 = -1 + ## ver1 = ver2 = 0 + ## ver1 > ver2 = 1 + let + ver1seq = ver1.replace("-", "").split('.') + ver2seq = ver2.replace("-", "").split('.') + for i in 0 ..< ver1seq.len: + let + p1 = ver1seq[i] + p2 = if i < ver2seq.len: ver2seq[i] else: "0" + + try: + let + h1 = p1.parseHexInt() + h2 = p2.parseHexInt() + + if h1 < h2: return -1 + elif h1 > h2: return 1 + except ValueError: + if p1 < p2: return -1 + elif p1 > p2: return 1 + # Conan support include conan @@ -913,10 +940,10 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string = uri = uri if "$#" in uri or "$1" in uri: - doAssert version.len != 0, "Need version for Conan uri" + doAssert version.len != 0, "Need version for Conan.io uri: " & uri uri = uri % version - else: - doAssert version.len == 0, "Conan uri does not contain version" + elif version.len != 0: + uri = uri & "/" & version let pkg = newConanPackageFromUri(uri, shared) @@ -934,6 +961,7 @@ proc getJBBPath(header, uri, outdir, version: string): string = let spl = uri.split('/', 1) name = spl[0] + hasVersion = version.len != 0 var ver = @@ -942,11 +970,15 @@ proc getJBBPath(header, uri, outdir, version: string): string = else: "" - if "$#" in ver or "$1" in ver: - doAssert version.len != 0, "Need version for BinaryBuilder.org uri" - ver = ver % version - else: - doAssert version.len == 0, "BinaryBuilder.org uri does not contain version" + if ver.len != 0: + if "$#" in ver or "$1" in ver: + doAssert hasVersion, "Need version for BinaryBuilder.org uri: " & uri + ver = ver % version + elif hasVersion: + doAssert false, "Version in both uri `" & uri & "` and `-d:xxxSetVer=\"" & + version & "\"` for BinaryBuilder.org" + elif hasVersion: + ver = version let pkg = newJBBPackage(name, ver) @@ -1129,10 +1161,10 @@ macro getHeader*( ## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux ## `-d:xxxGit` - clone source from a git repo specified in `giturl` ## `-d:xxxDL` - download source from `dlurl` and extract if required - ## `-d:xxxConan` - download headers and binary from conan.io using `conanuri` - ## typically formatted as `name/version[@user/channel][:bhash]` - ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` - ## typically formatted as `name/version` + ## `-d:xxxConan` - download headers and binary from Conan.io using `conanuri` with + ## format `pkgname[/version[@user/channel][:bhash]]` + ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` with + ## format `pkgname[/version]` ## ## This allows a single wrapper to be used in different ways depending on the user's needs. ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. @@ -1145,8 +1177,15 @@ macro getHeader*( ## one of the other methods. ## ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag - ## name for Git whereas for DL, Conan and BinaryBuilder.org, it replaces `$1` in the URL - ## defined. + ## name for `Git` whereas for `DL`, `Conan` and `JBB`, it replaces `$1` in the URL + ## if specified. Specifying `-d:xxxSetVer` without a `$1` will download that version for + ## `Conan` and `JBB` if available. If no version is specified, the latest release of the + ## package is downloaded. For `Conan`, `-d:xxxSetVer` can also be used to set additional + ## URI information: + ## `-d:xxxSetVer=1.9.0@bincrafters/stable:bhash` + ## + ## If `conanuri` or `jbburi` are not defined and `Conan` or `JBB` is selected, the `header` + ## filename is used instead. ## ## All defines can also be set in code using `setDefines()` and checked for using ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`. @@ -1165,7 +1204,7 @@ macro getHeader*( ## dependencies to that directory. This prevents any runtime failures if `outdir` gets ## removed or its contents changed. By default, `libdir` is set to the output directory ## where the program binary will be created. The values of `xxxLPath` and `xxxLDeps` will - ## reflect this new location. + ## reflect this new location. `libdir` is ignored for `Std` mode. ## ## `-d:xxxStatic` can be specified to statically link with the library instead. This ## will automatically add a `{.passL.}` call to the static library for convenience. Note @@ -1210,6 +1249,10 @@ macro getHeader*( origname = header.extractFilename().split(".")[0] name = origname.split(seps = AllChars-Letters-Digits).join() + # Default to origname if not specified + conanuri = if conanuri.len != 0: conanuri else: origname + jbburi = if jbburi.len != 0: jbburi else: origname + # -d:xxx for this header stdStr = name & "Std" gitStr = name & "Git" @@ -1316,7 +1359,7 @@ macro getHeader*( let # Library binary path - build if not standard / conan / jbb - lpath {.compiletime.} = + lpath {.compileTime.} = when useStd: stdLPath elif `nameConan` or `nameJBB`: @@ -1325,11 +1368,14 @@ macro getHeader*( buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) # Library dependecy paths - ldeps {.compiletime.}: seq[string] = - when `nameConan`: - getConanLDeps(`outdir`) - elif `nameJBB`: - getJBBLDeps(`outdir`, not `nameStatic`) + ldeps {.compileTime.}: seq[string] = + when not useStd: + when `nameConan`: + getConanLDeps(`outdir`) + elif `nameJBB`: + getJBBLDeps(`outdir`, not `nameStatic`) + else: + @[] else: @[] @@ -1346,20 +1392,6 @@ macro getHeader*( "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name` doAssert lpath.len != 0, "\nLibrary " & `lname` & " not found" - proc extractFilenameStatic(str: string): string {.compiletime.} = - var - pos = -1 - for i in countdown(str.len - 1, 0): - if str[i] == '/' or str[i] == '\\': - pos = i + 1 - break - result = str[pos .. ^1] - - proc joinPathStatic(str1, str2: string): string {.compiletime.} = - let - sep = when defined(Windows): "\\" else: "/" - result = str1 & sep & str2 - when `nameStatic`: const `lpath`* = lpath @@ -1376,28 +1408,34 @@ macro getHeader*( echo "# Including dependencies " & `ldeps`.join(" ") else: const - `lpath`* = joinPathStatic(`libdir`, lpath.extractFilenameStatic()) - `ldeps`* = block: - var - ldeps = ldeps - copied: seq[string] - for i in 0 ..< ldeps.len: - let - lname = ldeps[i].extractFilenameStatic() - ldeptgt = joinPathStatic(`libdir`, lname) - if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt): - cpFile(ldeps[i], ldeptgt, psymlink = true) - copied.add lname - ldeps[i] = ldeptgt - if copied.len != 0: - echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` - ldeps + `lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath + `ldeps`* = + when not useStd: + block: + var + ldeps = ldeps + copied: seq[string] + for i in 0 ..< ldeps.len: + let + lname = ldeps[i].extractFilename() + ldeptgt = `libdir` / lname + if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt): + cpFile(ldeps[i], ldeptgt, psymlink = true) + copied.add lname + ldeps[i] = ldeptgt + # Copy downloaded dependencies to `libdir` + if copied.len != 0: + echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` + ldeps + else: + ldeps static: - # Copy shared libraries and dependencies to `libdir` - if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): - echo "# Copying " & `lpath`.extractFilenameStatic() & " to " & `libdir` - cpFile(lpath, `lpath`) + when not useStd: + # Copy downloaded shared libraries to `libdir` + if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): + echo "# Copying " & `lpath`.extractFilename() & " to " & `libdir` + cpFile(lpath, `lpath`) - echo "# Including library " & `lpath` + echo "# Including library " & `lpath` ) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index fcb0495..6e761df 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -42,11 +42,14 @@ const var # Bintray download URL for explicit `user/channel` - conanBaseAltUrl {.compiletime.} = { + conanBaseAltUrl {.compileTime.} = { "bincrafters": "https://bintray.com/bincrafters/public-conan", "conan": "https://bintray.com/conan-community/conan" }.toTable() + # Reuse dependencies already downloaded + gConanRequires {.compileTime.}: Table[string, ConanPackage] + proc addAltConanBaseUrl*(name, url: string) = # Add an alternate base URL for a custom conan repo on bintray conanBaseAltUrl[name] = url @@ -144,13 +147,28 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac if channel.len != 0: query &= "/" & channel + echo &"# Searching Conan.io for latest version of {name}" + let j1 = jsonGet(conanSearchUrl % ["query", query]) res = j1.getOrDefault("results").getElems() - if res.len != 0: - # Return last entry - latest - result = newConanPackageFromUri(res[^1].getStr()) + # Return latest comparing versions - prefer @_/_ + var + latest = "" + latestv = "" + for i in 0 ..< res.len: + let + str = res[i].getStr() + if "@_/_" in str: + let + ver = str.split('/')[1].split('@')[0] + if latestv.len == 0 or compareVersions(ver, latestv) > 0: + latestv = ver + latest = str + + if latest.len != 0: + result = newConanPackageFromUri(latest) proc searchConan*(pkg: ConanPackage): ConanPackage = ## Search for latest package based on incomplete package info @@ -288,6 +306,9 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision ## ## If omitted, the latest revision (first) is downloaded fixOutDir() + + doAssert bld.revisions.len != 0, "No build revisions found for Conan.io package " & pkg.getUriFromConanPackage() + let revision = if revision.len != 0: @@ -326,7 +347,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision rmFile(outdir / conanManifest) proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) -proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = +proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = ## Download latest recipe/build/revision of `pkg` to `outdir` ## ## High-level API that handles the end to end Conan process flow to find @@ -339,11 +360,13 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = else: pkg - cpkg = loadConanInfo(outdir) + if main: + let + cpkg = loadConanInfo(outdir) + + if cpkg == pkg: + return - if cpkg == pkg: - return - elif clean: cleanDir(outdir) echo &"# Downloading {pkg.name} v{pkg.version} from Conan" @@ -352,7 +375,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" - echo &"# Downloading {pkg.name} v{pkg.version} from Conan" + echo &"# Downloading {pkg.name} v{pkg.version} from Conan.io" for recipe, builds in pkg.recipes: for build in builds: if pkg.bhash.len == 0 or pkg.bhash == build.bhash: @@ -362,7 +385,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = break break - if clean: + if main: pkg.saveConanInfo(outdir) proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = @@ -374,10 +397,17 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = if bld.options["shared"] == "False": for req in bld.requires: let - rpkg = newConanPackageFromUri(req, shared = false) + name = req.split('/')[0] + if gConanRequires.hasKey(name): + # Reuse dep already downloaded + pkg.requires.add gConanRequires[name] + else: + let + rpkg = newConanPackageFromUri(req, shared = false) - downloadConan(rpkg, outdir, clean = false) - pkg.requires.add rpkg + downloadConan(rpkg, outdir, main = false) + pkg.requires.add rpkg + gConanRequires[name] = rpkg proc getConanLDeps*(pkg: ConanPackage, outdir: string, main = true): seq[string] = ## Get all Conan libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 8062ac8..c79f4a5 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -146,7 +146,7 @@ when defined(TOAST): gecho join(args, "").getCommented() else: var - gStateCT* {.compiletime, used.} = new(State) + gStateCT* {.compileTime, used.} = new(State) template nBl*(s: typed): untyped {.used.} = (s.len != 0) diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim index de59a87..5d421f0 100644 --- a/nimterop/jbb.nim +++ b/nimterop/jbb.nim @@ -20,6 +20,10 @@ const jbbProject = "Project.toml" jbbArtifacts = "Artifacts.toml" +var + # Reuse dependencies already downloaded + gJBBRequires {.compileTime.}: Table[string, JBBPackage] + proc `==`*(pkg1, pkg2: JBBPackage): bool = ## Check if two JBBPackage objects are equal (not pkg1.isNil and not pkg2.isNil and @@ -79,19 +83,26 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = let line = line.strip() if line.len != 0: - if line.startsWith("arch = ") and not found: - let - barch = line.split(" = ")[1].strip(chars = {'"'}) - if barch == arch: - found = true - elif line.startsWith("os = ") and found: - let - bos = line.split(" = ")[1].strip(chars = {'"'}) - if bos != os: - found = false - elif line.startsWith("url = ") and found: - pkg.url = line.split(" = ")[1].strip(chars = {'"'}) - break + let + spl = line.split(" = ", 1) + name = spl[0] + val = if spl.len == 2: spl[1].strip(chars = {'"', ' '}) else: "" + + # Match arch, os and glibc on Linux to find download URL + case name + of "arch": + if val == arch and not found: found = true + of "os": + if val != os and found: found = false + of "libc": + when defined(Linux): + if val != "glibc" and found: found = false + of "url": + if found: + pkg.url = val + break + else: + discard proc findJBBLibs(pkg: JBBPackage, outdir: string) = pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dylib)[0-9.]*", outdir) @@ -143,18 +154,19 @@ proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = writeFile(file, $$pkg) proc dlJBBRequires*(pkg: JBBPackage, outdir: string) -proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) = +proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = ## Download `pkg` from BinaryBuilder.org to `outdir` ## ## High-level API that handles the end to end JBB process flow to find ## latest package binary and downloads and extracts it to `outdir`. fixOutDir() - let - cpkg = loadJBBInfo(outdir) + if main: + let + cpkg = loadJBBInfo(outdir) + + if cpkg == pkg: + return - if cpkg == pkg: - return - elif clean: cleanDir(outdir) pkg.getJBBRepo(outdir) @@ -174,14 +186,21 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) = pkg.dlJBBRequires(outdir) - if clean: + if main: pkg.saveJBBInfo(outdir) proc dlJBBRequires*(pkg: JBBPackage, outdir: string) = ## Download all required dependencies of this `pkg` fixOutDir() - for rpkg in pkg.requires: - downloadJBB(rpkg, outdir, clean = false) + for i in 0 ..< pkg.requires.len: + let + rpkg = pkg.requires[i] + if gJBBRequires.hasKey(rpkg.name): + # Reuse dep already downloaded + pkg.requires[i] = gJBBRequires[rpkg.name] + else: + downloadJBB(rpkg, outdir, main = false) + gJBBRequires[rpkg.name] = rpkg proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): seq[string] = ## Get all BinaryBuilder.org libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps diff --git a/tests/libssh2.nim b/tests/libssh2.nim index 48b8ba9..172f6ae 100644 --- a/tests/libssh2.nim +++ b/tests/libssh2.nim @@ -6,7 +6,7 @@ const getHeader( header = "libssh2.h", conanuri = "libssh2/$1", - jbburi = "libssh2/$1", + jbburi = "libssh2/1.9.0", outdir = outdir ) @@ -45,6 +45,3 @@ echo "zlib version = " & (block: else: "" ) - -static: - echo libssh2LDeps \ No newline at end of file diff --git a/tests/lzma.nim b/tests/lzma.nim index cff39de..9bda18e 100644 --- a/tests/lzma.nim +++ b/tests/lzma.nim @@ -24,6 +24,8 @@ getHeader( "lzma.h", giturl = "https://github.com/xz-mirror/xz", dlurl = "https://tukaani.org/xz/xz-$1.tar.gz", + conanuri = "xz_utils", + jbburi = "xz", outdir = baseDir, conFlags = "--disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo" ) diff --git a/tests/zlib.nim b/tests/zlib.nim index 2bd9395..53384c3 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -27,8 +27,6 @@ getHeader( "zlib.h", giturl = "https://github.com/madler/zlib", dlurl = "http://zlib.net/zlib-$1.tar.gz", - conanuri = "zlib/$1", - jbburi = "zlib/$1", outdir = baseDir, altNames = "z,zlib" )