jyapayne/nimterop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Optional version and uri, no Std copy, reuse deps, jbb libc check

Browse source
This commit is contained in:
Ganesh Viswanathan 2020-06-18 10:22:22 -05:00
commit 1820fdffcf
9 changed files with 185 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.travis.yml
View file

@ -13,7 +13,7 @@ language: c
env:
- BRANCH=0.20.2
- BRANCH=1.0.6
- BRANCH=1.2.0
- BRANCH=1.2.2
- BRANCH=devel
cache:

1
appveyor.yml
View file

@ -11,6 +11,7 @@ environment:
matrix:
- NIM_VERSION: 0.20.2
- NIM_VERSION: 1.0.6
- NIM_VERSION: 1.2.2
for:
-

150
nimterop/build.nim
View file

@ -2,6 +2,8 @@ import hashes, macros, osproc, sets, strformat, strutils, tables
import os except findExe, sleep
export extractFilename, `/`
type
BuildType* = enum
btAutoconf, btCmake
@ -850,6 +852,31 @@ template fixOutDir() {.dirty.} =
let
outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir
proc compareVersions*(ver1, ver2: string): int =
## Compare two version strings x.y.z and return -1, 0, 1
##
## ver1 < ver2 = -1
## ver1 = ver2 = 0
## ver1 > ver2 = 1
let
ver1seq = ver1.replace("-", "").split('.')
ver2seq = ver2.replace("-", "").split('.')
for i in 0 ..< ver1seq.len:
let
p1 = ver1seq[i]
p2 = if i < ver2seq.len: ver2seq[i] else: "0"
try:
let
h1 = p1.parseHexInt()
h2 = p2.parseHexInt()
if h1 < h2: return -1
elif h1 > h2: return 1
except ValueError:
if p1 < p2: return -1
elif p1 > p2: return 1
# Conan support
include conan
@ -913,10 +940,10 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string =
uri = uri
if "$#" in uri or "$1" in uri:
doAssert version.len != 0, "Need version for Conan uri"
doAssert version.len != 0, "Need version for Conan.io uri: " & uri
uri = uri % version
else:
doAssert version.len == 0, "Conan uri does not contain version"
elif version.len != 0:
uri = uri & "/" & version
let
pkg = newConanPackageFromUri(uri, shared)
@ -934,6 +961,7 @@ proc getJBBPath(header, uri, outdir, version: string): string =
let
spl = uri.split('/', 1)
name = spl[0]
hasVersion = version.len != 0
var
ver =
@ -942,11 +970,15 @@ proc getJBBPath(header, uri, outdir, version: string): string =
else:
""
if "$#" in ver or "$1" in ver:
doAssert version.len != 0, "Need version for BinaryBuilder.org uri"
ver = ver % version
else:
doAssert version.len == 0, "BinaryBuilder.org uri does not contain version"
if ver.len != 0:
if "$#" in ver or "$1" in ver:
doAssert hasVersion, "Need version for BinaryBuilder.org uri: " & uri
ver = ver % version
elif hasVersion:
doAssert false, "Version in both uri `" & uri & "` and `-d:xxxSetVer=\"" &
version & "\"` for BinaryBuilder.org"
elif hasVersion:
ver = version
let
pkg = newJBBPackage(name, ver)
@ -1129,10 +1161,10 @@ macro getHeader*(
## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux
## `-d:xxxGit` - clone source from a git repo specified in `giturl`
## `-d:xxxDL` - download source from `dlurl` and extract if required
## `-d:xxxConan` - download headers and binary from conan.io using `conanuri`
## typically formatted as `name/version[@user/channel][:bhash]`
## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi`
## typically formatted as `name/version`
## `-d:xxxConan` - download headers and binary from Conan.io using `conanuri` with
## format `pkgname[/version[@user/channel][:bhash]]`
## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` with
## format `pkgname[/version]`
##
## This allows a single wrapper to be used in different ways depending on the user's needs.
## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is.
@ -1145,8 +1177,15 @@ macro getHeader*(
## one of the other methods.
##
## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag
## name for Git whereas for DL, Conan and BinaryBuilder.org, it replaces `$1` in the URL
## defined.
## name for `Git` whereas for `DL`, `Conan` and `JBB`, it replaces `$1` in the URL
## if specified. Specifying `-d:xxxSetVer` without a `$1` will download that version for
## `Conan` and `JBB` if available. If no version is specified, the latest release of the
## package is downloaded. For `Conan`, `-d:xxxSetVer` can also be used to set additional
## URI information:
## `-d:xxxSetVer=1.9.0@bincrafters/stable:bhash`
##
## If `conanuri` or `jbburi` are not defined and `Conan` or `JBB` is selected, the `header`
## filename is used instead.
##
## All defines can also be set in code using `setDefines()` and checked for using
## `isDefined()` which checks for defines set from both `-d` and `setDefines()`.
@ -1165,7 +1204,7 @@ macro getHeader*(
## dependencies to that directory. This prevents any runtime failures if `outdir` gets
## removed or its contents changed. By default, `libdir` is set to the output directory
## where the program binary will be created. The values of `xxxLPath` and `xxxLDeps` will
## reflect this new location.
## reflect this new location. `libdir` is ignored for `Std` mode.
##
## `-d:xxxStatic` can be specified to statically link with the library instead. This
## will automatically add a `{.passL.}` call to the static library for convenience. Note
@ -1210,6 +1249,10 @@ macro getHeader*(
origname = header.extractFilename().split(".")[0]
name = origname.split(seps = AllChars-Letters-Digits).join()
# Default to origname if not specified
conanuri = if conanuri.len != 0: conanuri else: origname
jbburi = if jbburi.len != 0: jbburi else: origname
# -d:xxx for this header
stdStr = name & "Std"
gitStr = name & "Git"
@ -1316,7 +1359,7 @@ macro getHeader*(
let
# Library binary path - build if not standard / conan / jbb
lpath {.compiletime.} =
lpath {.compileTime.} =
when useStd:
stdLPath
elif `nameConan` or `nameJBB`:
@ -1325,11 +1368,14 @@ macro getHeader*(
buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`)
# Library dependecy paths
ldeps {.compiletime.}: seq[string] =
when `nameConan`:
getConanLDeps(`outdir`)
elif `nameJBB`:
getJBBLDeps(`outdir`, not `nameStatic`)
ldeps {.compileTime.}: seq[string] =
when not useStd:
when `nameConan`:
getConanLDeps(`outdir`)
elif `nameJBB`:
getJBBLDeps(`outdir`, not `nameStatic`)
else:
@[]
else:
@[]
@ -1346,20 +1392,6 @@ macro getHeader*(
"missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name`
doAssert lpath.len != 0, "\nLibrary " & `lname` & " not found"
proc extractFilenameStatic(str: string): string {.compiletime.} =
var
pos = -1
for i in countdown(str.len - 1, 0):
if str[i] == '/' or str[i] == '\\':
pos = i + 1
break
result = str[pos .. ^1]
proc joinPathStatic(str1, str2: string): string {.compiletime.} =
let
sep = when defined(Windows): "\\" else: "/"
result = str1 & sep & str2
when `nameStatic`:
const
`lpath`* = lpath
@ -1376,28 +1408,34 @@ macro getHeader*(
echo "# Including dependencies " & `ldeps`.join(" ")
else:
const
`lpath`* = joinPathStatic(`libdir`, lpath.extractFilenameStatic())
`ldeps`* = block:
var
ldeps = ldeps
copied: seq[string]
for i in 0 ..< ldeps.len:
let
lname = ldeps[i].extractFilenameStatic()
ldeptgt = joinPathStatic(`libdir`, lname)
if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt):
cpFile(ldeps[i], ldeptgt, psymlink = true)
copied.add lname
ldeps[i] = ldeptgt
if copied.len != 0:
echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir`
ldeps
`lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath
`ldeps`* =
when not useStd:
block:
var
ldeps = ldeps
copied: seq[string]
for i in 0 ..< ldeps.len:
let
lname = ldeps[i].extractFilename()
ldeptgt = `libdir` / lname
if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt):
cpFile(ldeps[i], ldeptgt, psymlink = true)
copied.add lname
ldeps[i] = ldeptgt
# Copy downloaded dependencies to `libdir`
if copied.len != 0:
echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir`
ldeps
else:
ldeps
static:
# Copy shared libraries and dependencies to `libdir`
if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`):
echo "# Copying " & `lpath`.extractFilenameStatic() & " to " & `libdir`
cpFile(lpath, `lpath`)
when not useStd:
# Copy downloaded shared libraries to `libdir`
if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`):
echo "# Copying " & `lpath`.extractFilename() & " to " & `libdir`
cpFile(lpath, `lpath`)
echo "# Including library " & `lpath`
echo "# Including library " & `lpath`
)

58
nimterop/conan.nim
View file

@ -42,11 +42,14 @@ const
var
# Bintray download URL for explicit `user/channel`
conanBaseAltUrl {.compiletime.} = {
conanBaseAltUrl {.compileTime.} = {
"bincrafters": "https://bintray.com/bincrafters/public-conan",
"conan": "https://bintray.com/conan-community/conan"
}.toTable()
# Reuse dependencies already downloaded
gConanRequires {.compileTime.}: Table[string, ConanPackage]
proc addAltConanBaseUrl*(name, url: string) =
# Add an alternate base URL for a custom conan repo on bintray
conanBaseAltUrl[name] = url
@ -144,13 +147,28 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac
if channel.len != 0:
query &= "/" & channel
echo &"# Searching Conan.io for latest version of {name}"
let
j1 = jsonGet(conanSearchUrl % ["query", query])
res = j1.getOrDefault("results").getElems()
if res.len != 0:
# Return last entry - latest
result = newConanPackageFromUri(res[^1].getStr())
# Return latest comparing versions - prefer @_/_
var
latest = ""
latestv = ""
for i in 0 ..< res.len:
let
str = res[i].getStr()
if "@_/_" in str:
let
ver = str.split('/')[1].split('@')[0]
if latestv.len == 0 or compareVersions(ver, latestv) > 0:
latestv = ver
latest = str
if latest.len != 0:
result = newConanPackageFromUri(latest)
proc searchConan*(pkg: ConanPackage): ConanPackage =
## Search for latest package based on incomplete package info
@ -288,6 +306,9 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision
##
## If omitted, the latest revision (first) is downloaded
fixOutDir()
doAssert bld.revisions.len != 0, "No build revisions found for Conan.io package " & pkg.getUriFromConanPackage()
let
revision =
if revision.len != 0:
@ -326,7 +347,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision
rmFile(outdir / conanManifest)
proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string)
proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) =
proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) =
## Download latest recipe/build/revision of `pkg` to `outdir`
##
## High-level API that handles the end to end Conan process flow to find
@ -339,11 +360,13 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) =
else:
pkg
cpkg = loadConanInfo(outdir)
if main:
let
cpkg = loadConanInfo(outdir)
if cpkg == pkg:
return
if cpkg == pkg:
return
elif clean:
cleanDir(outdir)
echo &"# Downloading {pkg.name} v{pkg.version} from Conan"
@ -352,7 +375,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) =
doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center"
echo &"# Downloading {pkg.name} v{pkg.version} from Conan"
echo &"# Downloading {pkg.name} v{pkg.version} from Conan.io"
for recipe, builds in pkg.recipes:
for build in builds:
if pkg.bhash.len == 0 or pkg.bhash == build.bhash:
@ -362,7 +385,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) =
break
break
if clean:
if main:
pkg.saveConanInfo(outdir)
proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) =
@ -374,10 +397,17 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) =
if bld.options["shared"] == "False":
for req in bld.requires:
let
rpkg = newConanPackageFromUri(req, shared = false)
name = req.split('/')[0]
if gConanRequires.hasKey(name):
# Reuse dep already downloaded
pkg.requires.add gConanRequires[name]
else:
let
rpkg = newConanPackageFromUri(req, shared = false)
downloadConan(rpkg, outdir, clean = false)
pkg.requires.add rpkg
downloadConan(rpkg, outdir, main = false)
pkg.requires.add rpkg
gConanRequires[name] = rpkg
proc getConanLDeps*(pkg: ConanPackage, outdir: string, main = true): seq[string] =
## Get all Conan libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps

2
nimterop/globals.nim
View file

@ -146,7 +146,7 @@ when defined(TOAST):
gecho join(args, "").getCommented()
else:
var
gStateCT* {.compiletime, used.} = new(State)
gStateCT* {.compileTime, used.} = new(State)
template nBl*(s: typed): untyped {.used.} =
(s.len != 0)

63
nimterop/jbb.nim
View file

@ -20,6 +20,10 @@ const
jbbProject = "Project.toml"
jbbArtifacts = "Artifacts.toml"
var
# Reuse dependencies already downloaded
gJBBRequires {.compileTime.}: Table[string, JBBPackage]
proc `==`*(pkg1, pkg2: JBBPackage): bool =
## Check if two JBBPackage objects are equal
(not pkg1.isNil and not pkg2.isNil and
@ -79,19 +83,26 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) =
let
line = line.strip()
if line.len != 0:
if line.startsWith("arch = ") and not found:
let
barch = line.split(" = ")[1].strip(chars = {'"'})
if barch == arch:
found = true
elif line.startsWith("os = ") and found:
let
bos = line.split(" = ")[1].strip(chars = {'"'})
if bos != os:
found = false
elif line.startsWith("url = ") and found:
pkg.url = line.split(" = ")[1].strip(chars = {'"'})
break
let
spl = line.split(" = ", 1)
name = spl[0]
val = if spl.len == 2: spl[1].strip(chars = {'"', ' '}) else: ""
# Match arch, os and glibc on Linux to find download URL
case name
of "arch":
if val == arch and not found: found = true
of "os":
if val != os and found: found = false
of "libc":
when defined(Linux):
if val != "glibc" and found: found = false
of "url":
if found:
pkg.url = val
break
else:
discard
proc findJBBLibs(pkg: JBBPackage, outdir: string) =
pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dylib)[0-9.]*", outdir)
@ -143,18 +154,19 @@ proc saveJBBInfo*(pkg: JBBPackage, outdir: string) =
writeFile(file, $$pkg)
proc dlJBBRequires*(pkg: JBBPackage, outdir: string)
proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) =
proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) =
## Download `pkg` from BinaryBuilder.org to `outdir`
##
## High-level API that handles the end to end JBB process flow to find
## latest package binary and downloads and extracts it to `outdir`.
fixOutDir()
let
cpkg = loadJBBInfo(outdir)
if main:
let
cpkg = loadJBBInfo(outdir)
if cpkg == pkg:
return
if cpkg == pkg:
return
elif clean:
cleanDir(outdir)
pkg.getJBBRepo(outdir)
@ -174,14 +186,21 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) =
pkg.dlJBBRequires(outdir)
if clean:
if main:
pkg.saveJBBInfo(outdir)
proc dlJBBRequires*(pkg: JBBPackage, outdir: string) =
## Download all required dependencies of this `pkg`
fixOutDir()
for rpkg in pkg.requires:
downloadJBB(rpkg, outdir, clean = false)
for i in 0 ..< pkg.requires.len:
let
rpkg = pkg.requires[i]
if gJBBRequires.hasKey(rpkg.name):
# Reuse dep already downloaded
pkg.requires[i] = gJBBRequires[rpkg.name]
else:
downloadJBB(rpkg, outdir, main = false)
gJBBRequires[rpkg.name] = rpkg
proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): seq[string] =
## Get all BinaryBuilder.org libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps

5
tests/libssh2.nim
View file

@ -6,7 +6,7 @@ const
getHeader(
header = "libssh2.h",
conanuri = "libssh2/$1",
jbburi = "libssh2/$1",
jbburi = "libssh2/1.9.0",
outdir = outdir
)
@ -45,6 +45,3 @@ echo "zlib version = " & (block:
else:
""
)
static:
echo libssh2LDeps

2
tests/lzma.nim
View file

@ -24,6 +24,8 @@ getHeader(
"lzma.h",
giturl = "https://github.com/xz-mirror/xz",
dlurl = "https://tukaani.org/xz/xz-$1.tar.gz",
conanuri = "xz_utils",
jbburi = "xz",
outdir = baseDir,
conFlags = "--disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo"
)

2
tests/zlib.nim
View file

@ -27,8 +27,6 @@ getHeader(
"zlib.h",
giturl = "https://github.com/madler/zlib",
dlurl = "http://zlib.net/zlib-$1.tar.gz",
conanuri = "zlib/$1",
jbburi = "zlib/$1",
outdir = baseDir,
altNames = "z,zlib"
)