Initial conan.io support

This commit is contained in:
Ganesh Viswanathan 2020-06-09 23:21:11 -05:00
commit 290ad2b967
2 changed files with 468 additions and 16 deletions

View file

@ -206,6 +206,14 @@ proc rmDir*(dir: string) =
## Remove a directory or pattern at compile time
rmFile(dir, dir = true)
proc cleanDir*(dir: string) =
## Remove all contents of a directory at compile time
for kind, path in walkDir(dir):
if kind == pcDir:
rmDir(path)
else:
rmFile(path)
proc getProjectCacheDir*(name: string, forceClean = true): string =
## Get a cache directory where all nimterop artifacts can be stored
##
@ -281,7 +289,7 @@ proc extractTar*(tarfile, outdir: string) =
if name.len != 0:
rmFile(outdir / name)
proc downloadUrl*(url, outdir: string) =
proc downloadUrl*(url, outdir: string, quiet = false) =
## Download a file using `curl` or `wget` (or `powershell` on Windows) to the specified directory
##
## If an archive file, it is automatically extracted after download.
@ -291,7 +299,8 @@ proc downloadUrl*(url, outdir: string) =
archives = @[".zip", ".xz", ".gz", ".bz2", ".tgz", ".tar"]
if not (ext in archives and fileExists(outdir/file)):
echo "# Downloading " & file
if not quiet:
echo "# Downloading " & file
mkDir(outdir)
var cmd = findExe("curl")
if cmd.len != 0:
@ -304,7 +313,7 @@ proc downloadUrl*(url, outdir: string) =
cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#"
else:
doAssert false, "No download tool available - curl, wget"
discard execAction(cmd % [url, (outdir/file).sanitizePath], retry = 1)
discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 1)
if ext == ".zip":
extractZip(file, outdir)
@ -732,6 +741,36 @@ proc getGccLibPaths*(mode: string): seq[string] =
when defined(osx):
result.add "/usr/lib"
proc getGccInfo*(): tuple[arch, os, compiler, version: string] =
let
(outp, _) = execAction(&"{getCompiler()} -v")
for line in outp.splitLines():
if line.startsWith("Target: "):
result.arch = line.split(' ')[1].split('-')[0]
result.os =
if "linux" in line:
"linux"
elif "android" in line:
"android"
elif "darwin" in line:
"macos"
elif "w64" in line or "mingw" in line:
"windows"
else:
"unknown"
elif " version " in line:
result.version = line.split(" version ")[1].split(' ')[0]
if "clang" in outp:
if result.os == "macos":
result.compiler = "apple-clang"
else:
result.compiler = "clang"
else:
result.compiler = "gcc"
# Conan support
include conan
proc getStdPath(header, mode: string): string =
for inc in getGccPaths(mode):
result = findFile(header, inc, recurse = false, first = true)
@ -784,6 +823,28 @@ proc getDlPath(header, url, outdir, version: string): string =
result = findFile(header, outdir)
proc getConanPath(header, uri, outdir, version: string, shared: bool): string =
var
uri = uri
if "$#" in uri or "$1" in uri:
doAssert version.len != 0, "Need version for Conan uri"
uri = uri % version
else:
doAssert version.len == 0, "Conan uri does not contain version"
let
pkg = newConanPackageFromUri(uri, shared)
downloadConan(pkg, outdir)
result = findFile(header, outdir)
proc getConanLPath(outdir: string): string =
let
pkg = loadConanInfo(outdir)
result = pkg.getConanLibs(outdir).join(" ")
proc getLocalPath(header, outdir: string): string =
if outdir.len != 0:
result = findFile(header, outdir)
@ -937,7 +998,9 @@ macro isDefined*(def: untyped): untyped =
false
)
macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: static[string] = "", outdir: static[string] = "",
macro getHeader*(
header: static[string], giturl: static[string] = "", dlurl: static[string] = "",
conanuri: static[string] = "", outdir: static[string] = "",
conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "",
altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped =
## Get the path to a header file for wrapping with
@ -950,16 +1013,18 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux
## `-d:xxxGit` - clone source from a git repo specified in `giturl`
## `-d:xxxDL` - download source from `dlurl` and extract if required
## `-d:xxxConan` - download headers and binary from conan.io using `conanuri`
## typically formatted as `name/version[@user/channel][:bhash]`
##
## This allows a single wrapper to be used in different ways depending on the user's needs.
## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is.
##
## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git` or `DL`.
## This allows using a system installed library if available before falling back to manual
## building.
## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL` or
## `Conan`. This allows using a system installed library if available before falling back
## to manual building.
##
## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag
## name for Git whereas for DL, it replaces `$1` in the URL defined.
## name for Git whereas for DL and Conan, it replaces `$1` in the URL defined.
##
## All defines can also be set in code using `setDefines()`.
##
@ -1006,6 +1071,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
stdStr = name & "Std"
gitStr = name & "Git"
dlStr = name & "DL"
conanStr = name & "Conan"
staticStr = name & "Static"
verStr = name & "SetVer"
@ -1014,6 +1080,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
nameStd = newIdentNode(stdStr)
nameGit = newIdentNode(gitStr)
nameDL = newIdentNode(dlStr)
nameConan = newIdentNode(conanStr)
nameStatic = newIdentNode(staticStr)
@ -1031,6 +1098,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
stdVal = gDefines.hasKey(stdStr)
gitVal = gDefines.hasKey(gitStr)
dlVal = gDefines.hasKey(dlStr)
conanVal = gDefines.hasKey(conanStr)
staticVal = gDefines.hasKey(staticStr)
verVal =
if gDefines.hasKey(verStr):
@ -1052,14 +1120,17 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
`nameStd`* = when defined(`nameStd`): true else: `stdVal` == 1
`nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1
`nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1
`nameConan`* = when defined(`nameConan`): true else: `conanVal` == 1
`nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1
# Search for header in outdir (after retrieving code) depending on -d:xxx mode
proc getPath(header, giturl, dlurl, outdir, version: string): string =
proc getPath(header, giturl, dlurl, conanuri, outdir, version: string, shared: bool): string =
when `nameGit`:
getGitPath(header, giturl, outdir, version)
elif `nameDL`:
getDlPath(header, dlurl, outdir, version)
elif `nameConan`:
getConanPath(header, conanuri, outdir, version, shared)
else:
getLocalPath(header, outdir)
@ -1077,23 +1148,30 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
stdLPath =
when `nameStd`: getStdLibPath(`lname`, `mode`) else: ""
useStd = stdPath.len != 0 and stdLPath.len != 0
# Look elsewhere if requested while prioritizing standard paths
prePath =
when stdPath.len != 0 and stdLPath.len != 0:
when useStd:
stdPath
else:
getPath(`header`, `giturl`, `dlurl`, `outdir`, `version`)
getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`)
# Run preBuild hook before building library if not standard
when (prePath != stdPath or prePath.len == 0) and declared(`preBuild`):
# Run preBuild hook before building library if not Std or Conan
when not (useStd or `nameConan`) and declared(`preBuild`):
static:
`preBuild`(`outdir`, prePath)
const
# Library binary path - build if not standard
`lpath`* =
when stdPath.len != 0 and stdLPath.len != 0:
when useStd:
stdLPath
elif `nameConan`:
when `nameStatic`:
getConanLPath(`outdir`)
else:
findFile(`lname`, `outdir`, regex = true)
else:
buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`)
@ -1102,10 +1180,11 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta
if prePath.len != 0:
prePath
else:
getPath(`header`, `giturl`, `dlurl`, `outdir`, `version`)
getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`)
static:
doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & "missing/empty outdir or -d:$1Std -d:$1Git or -d:$1DL not specified" % `name`
doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " &
"missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL or -d:$1Conan not specified" % `name`
doAssert `lpath`.len != 0, "\nLibrary " & `lname` & " not found"
echo "# Including library " & `lpath`

373
nimterop/conan.nim Normal file
View file

@ -0,0 +1,373 @@
import json, marshal, os, strformat, strutils, tables
type
ConanPackage* = ref object
## ConanPackage type that stores conan uri and recipes/builds/revisions
name*: string
version*: string
user*: string
channel*: string
recipes*: OrderedTableRef[string, seq[ConanBuild]]
bhash*: string
shared*: bool
headers*: seq[string]
libs*: seq[string]
requires*: seq[ConanPackage]
ConanBuild* = ref object
## Build type that stores build specific info and revisions
bhash*: string
settings*: TableRef[string, string]
options*: TableRef[string, string]
requires*: seq[string]
recipe_hash*: string
revisions*: seq[string]
const
# Conan API urls
baseUrl = "https://conan.bintray.com/v2/conans"
searchUrl = baseUrl & "/search?q=$query"
pkgUrl = baseUrl & "/$name/$version/$user/$channel/search$query"
cfgUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions"
dlUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions/$revision/files/$file"
# Bintray download sub-URL for explicit `user/channel` (not _/_)
dlAltUrl = "/download_file?file_path=$user%2F$name%2F$version%2F$channel%2F0%2Fpackage%2F$build%2F0%2F$file"
# Strings
conanInfo = "conaninfo.json"
conanPackage = "conan_package.tgz"
conanManifest = "conanmanifest.txt"
# https://github.com/kdheepak/binary-builder-downloader/blob/master/src/bbd.nim
# https://bintray.com/conan/conan-center
# https://bintray.com/bincrafters/public-conan/
# https://docs.conan.io/en/latest/howtos/manage_shared_libraries.html#manage-shared
var
# Bintray download URL for explicit `user/channel`
baseAltUrl {.compiletime.} = {
"bincrafters": "https://bintray.com/bincrafters/public-conan",
"conan": "https://bintray.com/conan-community/conan"
}.toTable()
template fixOutDir() {.dirty.} =
let
outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir
proc addAltBaseUrl*(name, url: string) =
# Add an alternate base URL for a custom conan repo on bintray
baseAltUrl[name] = url
proc jsonGet(url: string): JsonNode =
# Make HTTP call and return content as JSON
let
temp = getTempDir()
file = temp / url.extractFilename()
downloadUrl(url, temp, quiet = true)
result = readFile(file).parseJson()
rmFile(file)
proc `==`(pkg1, pkg2: ConanPackage): bool =
(not pkg1.isNil and not pkg2.isNil and
pkg1.name == pkg2.name and
pkg1.version == pkg2.version and
pkg1.user == pkg2.user and
pkg1.channel == pkg2.channel and
pkg1.bhash == pkg2.bhash and
pkg1.shared == pkg2.shared)
proc newConanPackage*(name, version, user = "_", channel = "_", bhash = "", shared = true): ConanPackage =
## Create a new ConanPackage with specified name and version
result = new(ConanPackage)
result.name = name
result.version = version
result.user = user
result.channel = channel
result.recipes = newOrderedTable[string, seq[ConanBuild]](2)
result.bhash = bhash
result.shared = shared
proc newConanPackageFromUri*(uri: string, shared = true): ConanPackage =
## Create a new ConanPackage from a conan uri
##
## name/version[@user/channel][:bhash]
var
name, version, user, channel, bhash: string
spl = uri.split(":")
if spl.len > 1:
bhash = spl[1]
spl = spl[0].split('/')
name = spl[0]
user = "_"
channel = "_"
if spl.len > 2:
channel = spl[2]
if spl.len > 1:
spl = spl[1].split('@')
version = spl[0]
if spl.len > 1:
user = spl[1]
result = newConanPackage(name, version, user, channel, bhash, shared)
proc getUriFromConanPackage*(pkg: ConanPackage): string =
## Convert a ConanPackage to a conan uri
result = pkg.name
if pkg.version.len != 0:
result &= "/" & pkg.version
if pkg.user.len != 0:
result &= "@" & pkg.user
if pkg.channel.len != 0:
result &= "/" & pkg.channel
if pkg.bhash.len != 0:
result &= ":" & pkg.bhash
proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPackage =
## Search for package by `name` and optional `version`, `user` and `channel`
var
query = name
if version.len != 0:
query &= "/" & version
if user.len != 0:
query &= "@" & user
if channel.len != 0:
query &= "/" & channel
let
j1 = jsonGet(searchUrl % ["query", query])
res = j1.getOrDefault("results").getElems()
if res.len != 0:
# Return last entry - latest
result = newConanPackageFromUri(res[^1].getStr())
proc searchConan*(pkg: ConanPackage): ConanPackage =
## Search for latest package based on incomplete package info
result = searchConan(pkg.name, pkg.version, pkg.user, pkg.channel)
proc getConanBuilds*(pkg: ConanPackage, filter = "") =
## Get all builds for a package based on the C compiler's target OS/arch info
##
## `filter` can be used to tweak search terms
## e.g. build_type=Debug&compiler=clang
let
(arch, os, compiler, _) = getGccInfo()
query =
if pkg.bhash.len == 0:
block:
var
query = &"?q=arch={arch}&os={os.capitalizeAscii()}"
if "build_type" notin filter:
query &= "&build_type=Release"
if "shared=" notin filter:
query &= &"&options.shared={($pkg.shared).capitalizeAscii()}"
if filter.len != 0:
query &= &"&{filter}"
if "compiler=" notin filter and os != "windows":
query &= &"&compiler={compiler}"
query.replace("&", "%20and%20")
else: ""
url = pkgUrl % [
"name", pkg.name,
"version", pkg.version,
"user", pkg.user,
"channel", pkg.channel,
"query", query
]
j1 = jsonGet(url)
if not j1.isNil:
for bhash, bdata in j1.getFields():
if pkg.bhash.len == 0 or pkg.bhash == bhash:
let
bld = new(ConanBuild)
settings = bdata.getOrDefault("settings")
options = bdata.getOrDefault("options")
requires = bdata.getOrDefault("requires")
bld.bhash = bhash
if not settings.isNil:
bld.settings = newTable[string, string](8)
for key, value in settings.getFields():
bld.settings[key] = value.getStr()
if not options.isNil:
bld.options = newTable[string, string](8)
for key, value in options.getFields():
bld.options[key] = value.getStr()
bld.requires = requires.to(seq[string])
bld.recipe_hash = bdata.getOrDefault("recipe_hash").getStr()
if pkg.recipes.hasKey(bld.recipe_hash):
pkg.recipes[bld.recipe_hash].add bld
else:
pkg.recipes[bld.recipe_hash] = @[bld]
# Only need first or matching build
break
proc getConanRevisions*(pkg: ConanPackage, bld: ConanBuild) =
## Get all revisions of a build
let
url = cfgUrl % [
"name", pkg.name,
"version", pkg.version,
"user", pkg.user,
"channel", pkg.channel,
"recipe", bld.recipe_hash,
"build", bld.bhash
]
j1 = jsonGet(url)
if not j1.isNil:
let
revs = j1.getOrDefault("revisions")
for i in revs:
bld.revisions.add i.getOrDefault("revision").getStr()
proc loadConanInfo*(outdir: string): ConanPackage =
## Load cached package info from `outdir/conaninfo.txt`
fixOutDir()
let
file = outdir / conanInfo
if fileExists(file):
result = to[ConanPackage](readFile(file))
proc saveConanInfo*(pkg: ConanPackage, outdir: string) =
## Save downloaded package info to `outdir/conaninfo.txt`
fixOutDir()
let
file = outdir / conanInfo
writeFile(file, $$pkg)
proc parseConanManifest(pkg: ConanPackage, outdir: string) =
let
file = outdir / conanManifest
if fileExists(file):
let
data = readFile(file)
for line in data.splitLines():
let
line = line.split(':')[0]
if line.startsWith("include/"):
pkg.headers.add line
elif line.startsWith("lib/"):
pkg.libs.add line
elif line.startsWith("bin/") and line.endsWith("dll"):
pkg.libs.add line
proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision = "") =
## Download specific `revision` of `bld` to `outdir`
##
## If omitted, the latest revision (first) is downloaded
fixOutDir()
let
revision =
if revision.len != 0:
revision
else:
bld.revisions[0]
url =
if pkg.user == "_":
dlUrl % [
"name", pkg.name,
"version", pkg.version,
"user", pkg.user,
"channel", pkg.channel,
"recipe", bld.recipe_hash,
"build", bld.bhash,
"revision", revision,
"file", conanPackage
]
else:
baseAltUrl[pkg.user] & dlAltUrl % [
"name", pkg.name,
"version", pkg.version,
"user", pkg.user,
"channel", pkg.channel,
"build", bld.bhash,
"file", conanPackage
]
downloadUrl(url, outdir, quiet = true)
downloadUrl(url.replace(conanPackage, conanManifest), outdir, quiet = true)
pkg.parseConanManifest(outdir)
rmFile(outdir / url.extractFilename())
rmFile(outdir / conanManifest)
proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string)
proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) =
## Download latest recipe/build/revision of `pkg` to `outdir`
fixOutDir()
let
pkg =
if pkg.version.len == 0:
searchConan(pkg)
else:
pkg
cpkg = loadConanInfo(outdir)
if cpkg == pkg:
return
elif clean:
cleanDir(outdir)
pkg.getConanBuilds()
for recipe, builds in pkg.recipes:
for build in builds:
if pkg.bhash.len == 0 or pkg.bhash == build.bhash:
pkg.getConanRevisions(build)
pkg.dlConanBuild(build, outdir)
pkg.dlConanRequires(build, outdir)
break
break
if clean:
pkg.saveConanInfo(outdir)
proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) =
## Download all required dependencies of this `bld`
##
## This is not required for shared libs since conan builds them
## with all dependencies statically linked in
fixOutDir()
if bld.options["shared"] == "False":
for req in bld.requires:
let
rpkg = newConanPackageFromUri(req)
downloadConan(rpkg, outdir, clean = false)
pkg.requires.add rpkg
proc getConanLibs*(pkg: ConanPackage, outdir: string): seq[string] =
## Get all Conan libs (.so|.a|.lib|.dll) in pkg, including deps
## in descending order
##
## `outdir` is prefixed to each entry
for lib in pkg.libs:
result.add outdir / lib
for cpkg in pkg.requires:
result.add cpkg.getConanLibs(outdir)