From 5b0a5ab14611f0f562ae96c75ee5d7ca634ae2ea Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 22 Aug 2019 15:59:13 -0700 Subject: [PATCH 01/21] Rename git to build --- nimterop/all.nim | 2 +- nimterop/build.nim | 294 +++++++++++++++++++++++++++++++++++++++++ nimterop/cimport.nim | 2 +- nimterop/getters.nim | 2 +- nimterop/git.nim | 295 +----------------------------------------- nimterop/setup.nim | 2 +- nimterop/template.nim | 2 +- nimterop/templite.nim | 2 +- tests/tpcre.nim | 2 +- tests/tsoloud.nim | 2 +- 10 files changed, 303 insertions(+), 302 deletions(-) create mode 100644 nimterop/build.nim diff --git a/nimterop/all.nim b/nimterop/all.nim index 78249a1..c024d01 100644 --- a/nimterop/all.nim +++ b/nimterop/all.nim @@ -4,4 +4,4 @@ Module that should import everything so that `nim doc --project nimtero/all` run # TODO: make sure it does import everything. -import "."/[cimport, git, types, plugin, compat] +import "."/[cimport, build, types, plugin, compat] diff --git a/nimterop/build.nim b/nimterop/build.nim new file mode 100644 index 0000000..72315a6 --- /dev/null +++ b/nimterop/build.nim @@ -0,0 +1,294 @@ +import os, osproc, strformat, strutils + +import "."/[compat] + +proc execAction*(cmd: string, nostderr=false): string = + ## Execute an external command - supported at compile time + ## + ## Checks if command exits successfully before returning. If not, an + ## error is raised. + var + ccmd = "" + ret = 0 + when defined(Windows): + ccmd = "cmd /c " & cmd + elif defined(posix): + ccmd = cmd + else: + doAssert false + + when nimvm: + (result, ret) = gorgeEx(ccmd) + else: + let opt = if nostderr: {poUsePath} else: {poStdErrToStdOut, poUsePath} + (result, ret) = execCmdEx(ccmd, opt) + doAssert ret == 0, "Command failed: " & $(ret, nostderr) & "\nccmd: " & ccmd & "\nresult:\n" & result + +proc findExe*(exe: string): string = + ## Find the specified executable using the which/where command - supported + ## at compile time + var + cmd = + when defined(windows): + "where " & exe + else: + "which " & exe + + (oup, code) = gorgeEx(cmd) + + if code == 0: + return oup.strip() + +proc mkDir*(dir: string) = + ## Create a directory at cmopile time + ## + ## The `os` module is not available at compile time so a few + ## crucial helper functions are included with nimterop. + if not dirExists(dir): + let + flag = when not defined(Windows): "-p" else: "" + discard execAction(&"mkdir {flag} {dir.quoteShell}") + +proc cpFile*(source, dest: string, move=false) = + ## Copy a file from source to destination at compile time + let + source = source.replace("/", $DirSep) + dest = dest.replace("/", $DirSep) + cmd = + when defined(Windows): + if move: + "move /y" + else: + "copy /y" + else: + if move: + "mv -f" + else: + "cp -f" + + discard execAction(&"{cmd} {source.quoteShell} {dest.quoteShell}") + +proc mvFile*(source, dest: string) = + ## Move a file from source to destination at compile time + cpFile(source, dest, move=true) + +proc rmFile*(source: string, dir = false) = + ## Remove a file or pattern at compile time + let + source = source.replace("/", $DirSep) + cmd = + when defined(Windows): + if dir: + "rd /s/q" + else: + "del /s/q/f" + else: + "rm -rf" + + discard execAction(&"{cmd} {source.quoteShell}") + +proc rmDir*(source: string) = + ## Remove a directory or pattern at compile time + rmFile(source, dir = true) + +proc extractZip*(zipfile, outdir: string) = + ## Extract a zip file using powershell on Windows and unzip on other + ## systems to the specified output directory + var cmd = "unzip -o $#" + if defined(Windows): + cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A " & + "'System.IO.Compression.FileSystem'; " & + "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" + + echo "# Extracting " & zipfile + discard execAction(&"cd {outdir.quoteShell} && {cmd % zipfile}") + +proc downloadUrl*(url, outdir: string) = + ## Download a file using curl or wget (or powershell on Windows) to the specified directory + ## + ## If a zip file, it is automatically extracted after download. + let + file = url.extractFilename() + ext = file.splitFile().ext.toLowerAscii() + + if not (ext == ".zip" and fileExists(outdir/file)): + echo "# Downloading " & file + mkDir(outdir) + var cmd = findExe("curl") + if cmd.len != 0: + cmd &= " -L $# -o $#" + else: + cmd = findExe("wget") + if cmd.len != 0: + cmd &= " $# -o $#" + elif defined(Windows): + cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" + else: + doAssert false, "No download tool available - curl, wget" + discard execAction(cmd % [url, (outdir/file).quoteShell]) + + if ext == ".zip": + extractZip(file, outdir) + +proc gitReset*(outdir: string) = + ## Hard reset the git repository at the specified directory + echo "# Resetting " & outdir + + let cmd = &"cd {outdir.quoteShell} && git reset --hard" + while execAction(cmd).contains("Permission denied"): + sleep(1000) + echo "# Retrying ..." + +proc gitCheckout*(file, outdir: string) = + ## Checkout the specified file in the git repository specified + ## + ## This effectively resets all changes in the file and can be + ## used to undo any changes that were made to source files to enable + ## successful wrapping with `cImport()` or `c2nImport()`. + echo "# Resetting " & file + let file2 = file.relativePath outdir + let cmd = &"cd {outdir.quoteShell} && git checkout {file2.quoteShell}" + while execAction(cmd).contains("Permission denied"): + sleep(500) + echo "# Retrying ..." + +proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = + ## Pull the specified git repository to the output directory + ## + ## `plist` is the list of specific files and directories or wildcards + ## to sparsely checkout. Multiple values can be specified one entry per + ## line. It is optional and if omitted, the entire repository will be + ## checked out. + ## + ## `checkout` is the git tag, branch or commit hash to checkout once + ## the repository is downloaded. This allows for pinning to a specific + ## version of the code. + if dirExists(outdir/".git"): + gitReset(outdir) + return + + let + outdirQ = outdir.quoteShell + + mkDir(outdir) + + echo "# Setting up Git repo: " & url + discard execAction(&"cd {outdirQ} && git init .") + discard execAction(&"cd {outdirQ} && git remote add origin {url}") + + if plist.len != 0: + # If a specific list of files is required, create a sparse checkout + # file for git in its config directory + let sparsefile = outdir / ".git/info/sparse-checkout" + + discard execAction(&"cd {outdirQ} && git config core.sparsecheckout true") + writeFile(sparsefile, plist) + + if checkout.len != 0: + echo "# Checking out " & checkout + discard execAction(&"cd {outdirQ} && git pull --tags origin master") + discard execAction(&"cd {outdirQ} && git checkout {checkout}") + else: + echo "# Pulling repository" + discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master") + +proc configure*(path, check: string, flags = "") = + ## Run the GNU `configure` command to generate all Makefiles or other + ## build scripts in the specified path + ## + ## If a `configure` script is not present and an `autogen.sh` script + ## is present, it will be run before attempting `configure`. + ## + ## `check` is a file that will be generated by the `configure` command. + ## This is required to prevent configure from running on every build. It + ## is relative to the `path` and should not be an absolute path. + ## + ## `flags` are any flags that should be passed to the `configure` command. + if (path / check).fileExists(): + return + + echo "# Configuring " & path + + if not fileExists(path / "configure"): + for i in @[path / "autogen.sh", path / "build" / "autogen.sh"]: + if fileExists(i): + echo "# Running autogen.sh" + + discard execAction(&"cd {i.parentDir().quoteShell} && bash autogen.sh") + + break + + if fileExists(path / "configure"): + echo "# Running configure " & flags + + var + cmd = &"cd {path.quoteShell} && bash configure" + if flags.len != 0: + cmd &= &" {flags}" + + echo execAction(cmd) + + doAssert (path / check).fileExists(), "# Configure failed" + +proc cmake*(path, check, flags: string) = + ## Run the `cmake` command to generate all Makefiles or other + ## build scripts in the specified path + ## + ## `path` will be created since typically `cmake` is run in an + ## empty directory. + ## + ## `check` is a file that will be generated by the `cmake` command. + ## This is required to prevent `cmake` from running on every build. It + ## is relative to the `path` and should not be an absolute path. + ## + ## `flags` are any flags that should be passed to the `cmake` command. + ## Unlike `configure`, it is required since typically it will be the + ## path to the repository, typically `..` when `path` is a subdir. + if (path / check).fileExists(): + return + + echo "# Running cmake " & flags + echo "# Path: " & path + + mkDir(path) + + var + cmd = &"cd {path.quoteShell} && cmake {flags}" + + echo execAction(cmd) + + doAssert (path / check).fileExists(), "# cmake failed" + +proc make*(path, check: string, flags = "") = + ## Run the `make` command to build all binaries in the specified path + ## + ## `check` is a file that will be generated by the `make` command. + ## This is required to prevent `make` from running on every build. It + ## is relative to the `path` and should not be an absolute path. + ## + ## `flags` are any flags that should be passed to the `make` command. + ## + ## If make.exe is missing and mingw32-make.exe is available, it will + ## be copied over to make.exe in the same location. + if (path / check).fileExists(): + return + + echo "# Running make " & flags + echo "# Path: " & path + + var + cmd = findExe("make") + + if cmd.len == 0: + cmd = findExe("mingw32-make") + if cmd.len != 0: + cpFile(cmd, cmd.replace("mingw32-make", "make")) + doAssert cmd.len != 0, "Make not found" + + cmd = &"cd {path.quoteShell} && make" + if flags.len != 0: + cmd &= &" {flags}" + + echo execAction(cmd) + + doAssert (path / check).fileExists(), "# make failed" diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 506f18f..0e875ed 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -21,7 +21,7 @@ const CIMPORT {.used.} = 1 include "."/globals -import "."/[git, paths, types] +import "."/[build, paths, types] export types proc interpPath(dir: string): string= diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 51d63a7..04ed5cf 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -2,7 +2,7 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import regex -import "."/[git, globals, plugin, treesitter/api] +import "."/[build, globals, plugin, treesitter/api] const gReserved = """ addr and as asm diff --git a/nimterop/git.nim b/nimterop/git.nim index 72315a6..7a0cf2a 100644 --- a/nimterop/git.nim +++ b/nimterop/git.nim @@ -1,294 +1 @@ -import os, osproc, strformat, strutils - -import "."/[compat] - -proc execAction*(cmd: string, nostderr=false): string = - ## Execute an external command - supported at compile time - ## - ## Checks if command exits successfully before returning. If not, an - ## error is raised. - var - ccmd = "" - ret = 0 - when defined(Windows): - ccmd = "cmd /c " & cmd - elif defined(posix): - ccmd = cmd - else: - doAssert false - - when nimvm: - (result, ret) = gorgeEx(ccmd) - else: - let opt = if nostderr: {poUsePath} else: {poStdErrToStdOut, poUsePath} - (result, ret) = execCmdEx(ccmd, opt) - doAssert ret == 0, "Command failed: " & $(ret, nostderr) & "\nccmd: " & ccmd & "\nresult:\n" & result - -proc findExe*(exe: string): string = - ## Find the specified executable using the which/where command - supported - ## at compile time - var - cmd = - when defined(windows): - "where " & exe - else: - "which " & exe - - (oup, code) = gorgeEx(cmd) - - if code == 0: - return oup.strip() - -proc mkDir*(dir: string) = - ## Create a directory at cmopile time - ## - ## The `os` module is not available at compile time so a few - ## crucial helper functions are included with nimterop. - if not dirExists(dir): - let - flag = when not defined(Windows): "-p" else: "" - discard execAction(&"mkdir {flag} {dir.quoteShell}") - -proc cpFile*(source, dest: string, move=false) = - ## Copy a file from source to destination at compile time - let - source = source.replace("/", $DirSep) - dest = dest.replace("/", $DirSep) - cmd = - when defined(Windows): - if move: - "move /y" - else: - "copy /y" - else: - if move: - "mv -f" - else: - "cp -f" - - discard execAction(&"{cmd} {source.quoteShell} {dest.quoteShell}") - -proc mvFile*(source, dest: string) = - ## Move a file from source to destination at compile time - cpFile(source, dest, move=true) - -proc rmFile*(source: string, dir = false) = - ## Remove a file or pattern at compile time - let - source = source.replace("/", $DirSep) - cmd = - when defined(Windows): - if dir: - "rd /s/q" - else: - "del /s/q/f" - else: - "rm -rf" - - discard execAction(&"{cmd} {source.quoteShell}") - -proc rmDir*(source: string) = - ## Remove a directory or pattern at compile time - rmFile(source, dir = true) - -proc extractZip*(zipfile, outdir: string) = - ## Extract a zip file using powershell on Windows and unzip on other - ## systems to the specified output directory - var cmd = "unzip -o $#" - if defined(Windows): - cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A " & - "'System.IO.Compression.FileSystem'; " & - "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" - - echo "# Extracting " & zipfile - discard execAction(&"cd {outdir.quoteShell} && {cmd % zipfile}") - -proc downloadUrl*(url, outdir: string) = - ## Download a file using curl or wget (or powershell on Windows) to the specified directory - ## - ## If a zip file, it is automatically extracted after download. - let - file = url.extractFilename() - ext = file.splitFile().ext.toLowerAscii() - - if not (ext == ".zip" and fileExists(outdir/file)): - echo "# Downloading " & file - mkDir(outdir) - var cmd = findExe("curl") - if cmd.len != 0: - cmd &= " -L $# -o $#" - else: - cmd = findExe("wget") - if cmd.len != 0: - cmd &= " $# -o $#" - elif defined(Windows): - cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" - else: - doAssert false, "No download tool available - curl, wget" - discard execAction(cmd % [url, (outdir/file).quoteShell]) - - if ext == ".zip": - extractZip(file, outdir) - -proc gitReset*(outdir: string) = - ## Hard reset the git repository at the specified directory - echo "# Resetting " & outdir - - let cmd = &"cd {outdir.quoteShell} && git reset --hard" - while execAction(cmd).contains("Permission denied"): - sleep(1000) - echo "# Retrying ..." - -proc gitCheckout*(file, outdir: string) = - ## Checkout the specified file in the git repository specified - ## - ## This effectively resets all changes in the file and can be - ## used to undo any changes that were made to source files to enable - ## successful wrapping with `cImport()` or `c2nImport()`. - echo "# Resetting " & file - let file2 = file.relativePath outdir - let cmd = &"cd {outdir.quoteShell} && git checkout {file2.quoteShell}" - while execAction(cmd).contains("Permission denied"): - sleep(500) - echo "# Retrying ..." - -proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = - ## Pull the specified git repository to the output directory - ## - ## `plist` is the list of specific files and directories or wildcards - ## to sparsely checkout. Multiple values can be specified one entry per - ## line. It is optional and if omitted, the entire repository will be - ## checked out. - ## - ## `checkout` is the git tag, branch or commit hash to checkout once - ## the repository is downloaded. This allows for pinning to a specific - ## version of the code. - if dirExists(outdir/".git"): - gitReset(outdir) - return - - let - outdirQ = outdir.quoteShell - - mkDir(outdir) - - echo "# Setting up Git repo: " & url - discard execAction(&"cd {outdirQ} && git init .") - discard execAction(&"cd {outdirQ} && git remote add origin {url}") - - if plist.len != 0: - # If a specific list of files is required, create a sparse checkout - # file for git in its config directory - let sparsefile = outdir / ".git/info/sparse-checkout" - - discard execAction(&"cd {outdirQ} && git config core.sparsecheckout true") - writeFile(sparsefile, plist) - - if checkout.len != 0: - echo "# Checking out " & checkout - discard execAction(&"cd {outdirQ} && git pull --tags origin master") - discard execAction(&"cd {outdirQ} && git checkout {checkout}") - else: - echo "# Pulling repository" - discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master") - -proc configure*(path, check: string, flags = "") = - ## Run the GNU `configure` command to generate all Makefiles or other - ## build scripts in the specified path - ## - ## If a `configure` script is not present and an `autogen.sh` script - ## is present, it will be run before attempting `configure`. - ## - ## `check` is a file that will be generated by the `configure` command. - ## This is required to prevent configure from running on every build. It - ## is relative to the `path` and should not be an absolute path. - ## - ## `flags` are any flags that should be passed to the `configure` command. - if (path / check).fileExists(): - return - - echo "# Configuring " & path - - if not fileExists(path / "configure"): - for i in @[path / "autogen.sh", path / "build" / "autogen.sh"]: - if fileExists(i): - echo "# Running autogen.sh" - - discard execAction(&"cd {i.parentDir().quoteShell} && bash autogen.sh") - - break - - if fileExists(path / "configure"): - echo "# Running configure " & flags - - var - cmd = &"cd {path.quoteShell} && bash configure" - if flags.len != 0: - cmd &= &" {flags}" - - echo execAction(cmd) - - doAssert (path / check).fileExists(), "# Configure failed" - -proc cmake*(path, check, flags: string) = - ## Run the `cmake` command to generate all Makefiles or other - ## build scripts in the specified path - ## - ## `path` will be created since typically `cmake` is run in an - ## empty directory. - ## - ## `check` is a file that will be generated by the `cmake` command. - ## This is required to prevent `cmake` from running on every build. It - ## is relative to the `path` and should not be an absolute path. - ## - ## `flags` are any flags that should be passed to the `cmake` command. - ## Unlike `configure`, it is required since typically it will be the - ## path to the repository, typically `..` when `path` is a subdir. - if (path / check).fileExists(): - return - - echo "# Running cmake " & flags - echo "# Path: " & path - - mkDir(path) - - var - cmd = &"cd {path.quoteShell} && cmake {flags}" - - echo execAction(cmd) - - doAssert (path / check).fileExists(), "# cmake failed" - -proc make*(path, check: string, flags = "") = - ## Run the `make` command to build all binaries in the specified path - ## - ## `check` is a file that will be generated by the `make` command. - ## This is required to prevent `make` from running on every build. It - ## is relative to the `path` and should not be an absolute path. - ## - ## `flags` are any flags that should be passed to the `make` command. - ## - ## If make.exe is missing and mingw32-make.exe is available, it will - ## be copied over to make.exe in the same location. - if (path / check).fileExists(): - return - - echo "# Running make " & flags - echo "# Path: " & path - - var - cmd = findExe("make") - - if cmd.len == 0: - cmd = findExe("mingw32-make") - if cmd.len != 0: - cpFile(cmd, cmd.replace("mingw32-make", "make")) - doAssert cmd.len != 0, "Make not found" - - cmd = &"cd {path.quoteShell} && make" - if flags.len != 0: - cmd &= &" {flags}" - - echo execAction(cmd) - - doAssert (path / check).fileExists(), "# make failed" +include build \ No newline at end of file diff --git a/nimterop/setup.nim b/nimterop/setup.nim index 8e70e64..4ee375b 100644 --- a/nimterop/setup.nim +++ b/nimterop/setup.nim @@ -1,6 +1,6 @@ import os, strutils -import "."/[git, paths] +import "."/[build, paths] proc treesitterSetup*() = gitPull("https://github.com/tree-sitter/tree-sitter", incDir() / "treesitter", """ diff --git a/nimterop/template.nim b/nimterop/template.nim index 1c9fc7c..b8b894a 100644 --- a/nimterop/template.nim +++ b/nimterop/template.nim @@ -1,6 +1,6 @@ import os, strutils -import nimterop/[cimport, git, paths] +import nimterop/[cimport, build, paths] # Documentation: # https://github.com/nimterop/nimterop diff --git a/nimterop/templite.nim b/nimterop/templite.nim index 71262df..7d8eb5b 100644 --- a/nimterop/templite.nim +++ b/nimterop/templite.nim @@ -1,6 +1,6 @@ import os, strutils -import nimterop/[cimport, git, paths] +import nimterop/[cimport, build, paths] const baseDir = currentSourcePath.parentDir()/"build" diff --git a/tests/tpcre.nim b/tests/tpcre.nim index 705525f..5588850 100644 --- a/tests/tpcre.nim +++ b/tests/tpcre.nim @@ -1,6 +1,6 @@ import os -import nimterop/[cimport, git, paths] +import nimterop/[cimport, build, paths] const baseDir = nimteropBuildDir()/"pcre" diff --git a/tests/tsoloud.nim b/tests/tsoloud.nim index f46dd10..12966a8 100644 --- a/tests/tsoloud.nim +++ b/tests/tsoloud.nim @@ -1,4 +1,4 @@ -import os, nimterop/[cimport, git, paths] +import os, nimterop/[cimport, build, paths] const baseDir = nimteropBuildDir()/"soloud" From 7ef73147a69f224f3df36081afcc7a235cd70b33 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 23 Aug 2019 07:10:54 -0700 Subject: [PATCH 02/21] getHeader and supporting procs --- nimterop/build.nim | 197 ++++++++++++++++++++++++++++++++++++++++++- nimterop/cimport.nim | 42 +++------ 2 files changed, 210 insertions(+), 29 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 72315a6..342e6bc 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1,4 +1,6 @@ -import os, osproc, strformat, strutils +import macros, osproc, sequtils, strformat, strutils + +import os except findExe import "."/[compat] @@ -292,3 +294,196 @@ proc make*(path, check: string, flags = "") = echo execAction(cmd) doAssert (path / check).fileExists(), "# make failed" + +proc findFile*(file, dir: string): string = + ## Find the file in the specified directory + for f in walkDirRec(dir): + if f.extractFilename() == file: + if result.len == 0 or result.len > f.len: + result = f + +proc getGccPaths*(mode = "c"): seq[string] = + var + nul = when defined(Windows): "nul" else: "/dev/null" + mmode = if mode == "cpp": "c++" else: mode + inc = false + + (outp, ret) = gorgeEx(&"""{getEnv("CC", "gcc")} -Wp,-v -x{mmode} {nul}""") + + for line in outp.splitLines(): + if "#include <...> search starts here" in line: + inc = true + continue + elif "End of search list" in line: + break + if inc: + result.add line.strip() + +proc getStdPath(header: string): string = + for inc in getGccPaths(): + result = findFile(header, inc) + if result.len != 0: + break + +proc getGitPath(header, url, outdir, version: string): string = + doAssert url.len != 0, "No git url setup for " & header + doAssert findExe("git").len != 0, "git executable missing" + + gitPull(url, outdir, checkout = version) + + result = findFile(header, outdir) + +proc getDlPath(header, url, outdir, version: string): string = + doAssert url.len != 0, "No download url setup for " & header + + var + dlurl = url + if "$#" in url or "$1" in url: + doAssert version.len != 0, "Need version for download url" + dlurl = url % version + else: + doAssert version.len == 0, "Download url does not contain version" + + downloadUrl(dlurl, outdir) + + var + dirname = "" + for kind, path in walkDir(outdir, relative = true): + if kind == pcFile and path != dlurl.extractFilename(): + dirname = "" + break + elif kind == pcDir: + if dirname.len == 0: + dirname = path + else: + dirname = "" + break + + if dirname.len != 0: + for kind, path in walkDir(outdir / dirname, relative = true): + mvFile(outdir / dirname / path, outdir / path) + + result = findFile(header, outdir) + +proc getLocalPath(header, outdir: string): string = + if outdir.len != 0: + result = findFile(header, outdir) + +proc buildLibrary(outdir, conFlags, conStaticLib, conDynLib, cmakeFlags, cmakeStaticLib, cmakeDynLib, makeFlags: string) = + var + conDeps = false + conDepStr = "" + cmakeDeps = false + cmakeDepStr = "" + + if fileExists(outdir / "CMakeLists.txt"): + if findExe("cmake").len != 0: + if cmakeStaticLib.len != 0 or cmakeDynLib.len != 0: + var + gen = "" + when defined(windows): + if findExe("sh").len != 0: + gen = "MSYS Makefiles" + else: + gen = "MinGW Makefiles" + else: + gen = "Unix Makefiles" + cmake(outdir / "build", "Makefile", &".. -G {gen.quoteShell} {cmakeFlags}") + cmakeDeps = true + let + check = if cmakeStaticLib.len != 0: cmakeStaticLib else: cmakeDynLib + make(outdir / "build", check, makeFlags) + else: + cmakeDepStr &= "cmakeStatibLib / cmakeDynLib not specified" + else: + cmakeDepStr &= "cmake executable missing" + + template cfgCommon() {.dirty.} = + if (conStaticLib.len != 0 or conDynLib.len != 0): + configure(outdir, "Makefile", conFlags) + conDeps = true + let + check = if conStaticLib.len != 0: conStaticLib else: conDynLib + make(outdir, check, makeFlags) + else: + conDepStr &= "conStaticLib / conDynLib not specified" + + if not cmakeDeps: + if not fileExists(outdir / "configure"): + if fileExists(outdir / "autogen.sh") or fileExists(outdir / "build" / "autogen.sh"): + if findExe("aclocal").len != 0: + if findExe("autoconf").len != 0: + if findExe("libtoolize").len != 0: + cfgCommon() + else: + conDepStr &= "libtoolize executable missing" + else: + conDepStr &= "autoconf executable missing" + else: + conDepStr &= "aclocal executable missing" + else: + if findExe("bash").len != 0: + cfgCommon() + else: + conDepStr &= "bash executable missing" + + var + error = "" + if not cmakeDeps and cmakeDepStr.len != 0: + error &= &"cmake capable but {cmakeDepStr}\n" + if not conDeps and conDepStr.len != 0: + error &= &"configure capable but {conDepStr}\n" + if error.len == 0: + error = "No build files found in " & outdir + doAssert cmakeDeps or conDeps, &"\n# Build configuration failed - {error}\n" + +macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: static[string] = "", outdir: static[string] = "", + conFlags: static[string] = "", conStaticLib: static[string] = "", conDynLib: static[string] = "", + cmakeFlags: static[string] = "", cmakeStaticLib: static[string] = "", cmakeDynLib: static[string] = "", + makeFlags: static[string] = ""): untyped = + ## Get the path to a header file for wrapping with + ## `cImport() `_ or + ## `c2nImport() `_. + ## + ## Checks defines based on the header name (e.g. lzma from lzma.h), to use different + ## ways to obtain the source. + ## + ## ``-d:xxxStd`` - search standard system paths. E.g. ``/usr/include`` and ``/usr/lib`` on Linux + ## ``-d:xxxGit`` - clone source from a git repo specified in ``giturl`` + ## ``-d:xxxDL`` - download source from ``dlurl`` and extract if required + ## + ## This allows a single wrapper to be used in different ways depending on the user's needs. + ## If no defines are specified, ``outdir`` will be searched for the header. + ## + ## The library is then configured (either with cmake or autotools if possible) and then built + ## using make. + var + name = header.split(".")[0] + + stdName = newIdentNode(name & "Std") + gitName = newIdentNode(name & "Git") + dlName = newIdentNode(name & "DL") + + path = newIdentNode(name & "Path") + version = newIdentNode(name & "Version") + + result = newNimNode(nnkStmtList) + result.add(quote do: + const `version`* {.strdefine.} = "" + + when defined(`stdName`): + const `path`* = getStdPath(`header`) + else: + const `path`* = + when defined(`gitName`): + getGitPath(`header`, `giturl`, `outdir`, `version`) + elif defined(`dlName`): + getDlPath(`header`, `dlurl`, `outdir`, `version`) + else: + getLocalPath(`header`, `outdir`) + + static: + doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & "missing/empty outdir or -d:$1Std -d:$1Git or -d:$1DL not specified" % `name` + + buildLibrary(`outdir`, `conFlags`, `conStaticLib`, `conDynLib`, `cmakeFlags`, `cmakeStaticLib`, `cmakeDynLib`, `makeFlags`) + ) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 0e875ed..fc03bc5 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -168,14 +168,6 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "", (result, ret) = gorgeEx(cmd, cache=getCacheValue(fullpath)) doAssert ret == 0, getToastError(result) -proc getGccPaths(mode = "c"): string = - var - ret = 0 - nul = when defined(Windows): "nul" else: "/dev/null" - mmode = if mode == "cpp": "c++" else: mode - - (result, ret) = gorgeEx(&"""{getEnv("CC", "gcc")} -Wp,-v -x{mmode} {nul}""") - macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not ## accurate, it may be required to hand wrap them. Define them in a @@ -236,7 +228,8 @@ proc cSkipSymbol*(skips: seq[string]) {.compileTime.} = gStateCT.symOverride.add skips macro cPlugin*(body): untyped = - ## When `cOverride() `_ and `cSkipSymbol() `_ + ## When `cOverride() `_ and + ## `cSkipSymbol() `_ ## are not adequate, the `cPlugin() `_ macro can be used ## to customize the generated Nim output. The following callbacks are available at ## this time. @@ -322,7 +315,8 @@ proc cDebug*() {.compileTime.} = proc cDisableCaching*() {.compileTime.} = ## Disable caching of generated Nim code - useful during wrapper development ## - ## If files included by header being processed by `cImport() `_ + ## If files included by header being processed by + ## `cImport() `_ ## change and affect the generated content, they will be ignored and the cached ## value will continue to be used . Use `cDisableCaching() `_ ## to avoid this scenario during development. @@ -334,8 +328,8 @@ proc cDisableCaching*() {.compileTime.} = macro cDefine*(name: static string, val: static string = ""): untyped = ## ``#define`` an identifer that is forwarded to the C/C++ preprocessor if ## called within `cImport() `_ - ## or `c2nImport() `_ as well as to the - ## C/C++ compiler during Nim compilation using ``{.passC: "-DXXX".}`` + ## or `c2nImport() `_ + ## as well as to the C/C++ compiler during Nim compilation using ``{.passC: "-DXXX".}`` result = newNimNode(nnkStmtList) @@ -370,8 +364,8 @@ proc cAddSearchDir*(dir: string) {.compileTime.} = macro cIncludeDir*(dir: static string): untyped = ## Add an include directory that is forwarded to the C/C++ preprocessor if ## called within `cImport() `_ - ## or `c2nImport() `_ as well as to the - ## C/C++ compiler during Nim compilation using ``{.passC: "-IXXX".}``. + ## or `c2nImport() `_ + ## as well as to the C/C++ compiler during Nim compilation using ``{.passC: "-IXXX".}``. var dir = interpPath(dir) result = newNimNode(nnkStmtList) @@ -392,16 +386,8 @@ proc cAddStdDir*(mode = "c") {.compileTime.} = static: cAddStdDir() import os doAssert cSearchPath("math.h").existsFile - var - inc = false - for line in getGccPaths(mode).splitLines(): - if "#include <...> search starts here" in line: - inc = true - continue - elif "End of search list" in line: - break - if inc: - cAddSearchDir line.strip() + for inc in getGccPaths(mode): + cAddSearchDir inc macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = ## Compile and link C/C++ implementation into resulting binary using ``{.compile.}`` @@ -528,8 +514,8 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st ## cImport("pcre.h", dynlib="dynpcre") ## ## If ``dynlib`` is not specified, the C/C++ implementation files can be compiled in - ## with `cCompile() `_, or the ``{.passL.}`` pragma - ## can be used to specify the static lib to link. + ## with `cCompile() `_, or the + ## ``{.passL.}`` pragma can be used to specify the static lib to link. ## ## ``mode`` is purely for forward compatibility when toast adds C++ support. It can ## be ignored for the foreseeable future. @@ -562,8 +548,8 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: mode: static string = "c", flags: static string = ""): untyped = ## Import all supported definitions from specified header file using ``c2nim`` ## - ## Similar to `cImport() `_ but uses ``c2nim`` to generate - ## the Nim wrapper instead of ``toast``. Note that neither + ## Similar to `cImport() `_ + ## but uses ``c2nim`` to generate the Nim wrapper instead of ``toast``. Note that neither ## `cOverride() `_, `cSkipSymbol() `_ ## nor `cPlugin() `_ have any impact on ``c2nim``. ## From 182d473973294585a5348d6fc1d01dd209136a4c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 23 Aug 2019 16:00:43 -0500 Subject: [PATCH 03/21] findFile w/regex, make -j, lib support for getHeader, noexcept / throw --- nimterop.nimble | 6 +- nimterop/build.nim | 210 ++++++++++++++++++++++++++++++------------- nimterop/grammar.nim | 1 + 3 files changed, 153 insertions(+), 64 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 9910444..1014e66 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -36,12 +36,16 @@ proc testAll() = execCmd "nim cpp -r tests/tnimterop_cpp.nim" execTest "tests/tpcre.nim" - ## platform specific tests + # platform specific tests when defined(Windows): execTest "tests/tmath.nim" if defined(OSX) or defined(Windows) or not existsEnv("TRAVIS"): tsoloud() # requires some libraries on linux, need them installed in TRAVIS + # getHeader tests + withDir("tests"): + execCmd("nim e getheader.nims") + const htmldocsDir = "build/htmldocs" when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9): diff --git a/nimterop/build.nim b/nimterop/build.nim index 342e6bc..53237c0 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1,4 +1,4 @@ -import macros, osproc, sequtils, strformat, strutils +import macros, osproc, regex, sequtils, strformat, strutils import os except findExe @@ -194,6 +194,29 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = echo "# Pulling repository" discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master") +proc findFile*(file: string|Regex, dir: string, recurse = true, first = false): string = + ## Find the file in the specified directory + ## + ## ``file`` can be a string or a regex object + ## + ## Turn off recursive search with ``recurse`` and stop on first match with + ## ``first``. Without it, the shortest match is returned. + when file is Regex: + var + rm: RegexMatch + + for f in walkDirRec(dir, followFilter = if recurse: {pcDir} else: {}): + let + fn = f.extractFilename() + when file is string: + if (result.len == 0 or result.len > f.len) and fn == file: + result = f + if first: break + else: + if (result.len == 0 or result.len > f.len) and fn.match(file, rm): + result = f + if first: break + proc configure*(path, check: string, flags = "") = ## Run the GNU `configure` command to generate all Makefiles or other ## build scripts in the specified path @@ -261,7 +284,7 @@ proc cmake*(path, check, flags: string) = doAssert (path / check).fileExists(), "# cmake failed" -proc make*(path, check: string, flags = "") = +proc make*(path, check: string|Regex, flags = "") = ## Run the `make` command to build all binaries in the specified path ## ## `check` is a file that will be generated by the `make` command. @@ -272,7 +295,7 @@ proc make*(path, check: string, flags = "") = ## ## If make.exe is missing and mingw32-make.exe is available, it will ## be copied over to make.exe in the same location. - if (path / check).fileExists(): + if findFile(check, path).len != 0: return echo "# Running make " & flags @@ -293,14 +316,7 @@ proc make*(path, check: string, flags = "") = echo execAction(cmd) - doAssert (path / check).fileExists(), "# make failed" - -proc findFile*(file, dir: string): string = - ## Find the file in the specified directory - for f in walkDirRec(dir): - if f.extractFilename() == file: - if result.len == 0 or result.len > f.len: - result = f + doAssert findFile(check, path).len != 0, "# make failed" proc getGccPaths*(mode = "c"): seq[string] = var @@ -308,7 +324,7 @@ proc getGccPaths*(mode = "c"): seq[string] = mmode = if mode == "cpp": "c++" else: mode inc = false - (outp, ret) = gorgeEx(&"""{getEnv("CC", "gcc")} -Wp,-v -x{mmode} {nul}""") + (outp, _) = gorgeEx(&"""{getEnv("CC", "gcc")} -Wp,-v -x{mmode} {nul}""") for line in outp.splitLines(): if "#include <...> search starts here" in line: @@ -317,11 +333,38 @@ proc getGccPaths*(mode = "c"): seq[string] = elif "End of search list" in line: break if inc: - result.add line.strip() + var + path = line.strip() + path.normalizePath() + if path notin result: + result.add path + +proc getGccLibPaths*(mode = "c"): seq[string] = + var + nul = when defined(Windows): "nul" else: "/dev/null" + mmode = if mode == "cpp": "c++" else: mode + + (outp, _) = gorgeEx(&"""{getEnv("CC", "gcc")} -v -x{mmode} {nul}""") + + for line in outp.splitLines(): + if "LIBRARY_PATH=" in line: + for path in line[13 .. ^1].split(PathSep): + var + path = path.strip() + path.normalizePath() + if path notin result: + result.add path + break proc getStdPath(header: string): string = for inc in getGccPaths(): - result = findFile(header, inc) + result = findFile(header, inc, recurse = false, first = true) + if result.len != 0: + break + +proc getStdLibPath(lname: string): string = + for lib in getGccLibPaths(): + result = findFile(re(lname), lib, recurse = false, first = true) if result.len != 0: break @@ -369,44 +412,49 @@ proc getLocalPath(header, outdir: string): string = if outdir.len != 0: result = findFile(header, outdir) -proc buildLibrary(outdir, conFlags, conStaticLib, conDynLib, cmakeFlags, cmakeStaticLib, cmakeDynLib, makeFlags: string) = +proc getNumProcs(): string = + when defined(windows): + getEnv("NUMBER_OF_PROCESSORS").strip() + elif defined(linux): + execAction("nproc").strip() + elif defined(macosx): + execAction("sysctl -n hw.ncpu").strip() + else: + "1" + +proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): string = var conDeps = false conDepStr = "" cmakeDeps = false cmakeDepStr = "" + lpath = findFile(re(lname), outdir) + makeFlagsProc = &"-j {getNumProcs()} {makeFlags}" + + if lpath.len != 0: + return lpath if fileExists(outdir / "CMakeLists.txt"): if findExe("cmake").len != 0: - if cmakeStaticLib.len != 0 or cmakeDynLib.len != 0: - var - gen = "" - when defined(windows): - if findExe("sh").len != 0: - gen = "MSYS Makefiles" - else: - gen = "MinGW Makefiles" + var + gen = "" + when defined(windows): + if findExe("sh").len != 0: + gen = "MSYS Makefiles" else: - gen = "Unix Makefiles" - cmake(outdir / "build", "Makefile", &".. -G {gen.quoteShell} {cmakeFlags}") - cmakeDeps = true - let - check = if cmakeStaticLib.len != 0: cmakeStaticLib else: cmakeDynLib - make(outdir / "build", check, makeFlags) + gen = "MinGW Makefiles" else: - cmakeDepStr &= "cmakeStatibLib / cmakeDynLib not specified" + gen = "Unix Makefiles" + cmake(outdir / "build", "Makefile", &".. -G {gen.quoteShell} {cmakeFlags}") + cmakeDeps = true + make(outdir / "build", re(lname), makeFlagsProc) else: cmakeDepStr &= "cmake executable missing" template cfgCommon() {.dirty.} = - if (conStaticLib.len != 0 or conDynLib.len != 0): - configure(outdir, "Makefile", conFlags) - conDeps = true - let - check = if conStaticLib.len != 0: conStaticLib else: conDynLib - make(outdir, check, makeFlags) - else: - conDepStr &= "conStaticLib / conDynLib not specified" + configure(outdir, "Makefile", conFlags) + conDeps = true + make(outdir, re(lname), makeFlagsProc) if not cmakeDeps: if not fileExists(outdir / "configure"): @@ -437,53 +485,89 @@ proc buildLibrary(outdir, conFlags, conStaticLib, conDynLib, cmakeFlags, cmakeSt error = "No build files found in " & outdir doAssert cmakeDeps or conDeps, &"\n# Build configuration failed - {error}\n" + result = findFile(re(lname), outdir) + +proc getDynlibExt(): string = + when defined(windows): + result = ".dll" + elif defined(linux): + result = ".so[0-9.]*" + elif defined(macosx): + result = ".dylib[0-9.]*" + macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: static[string] = "", outdir: static[string] = "", - conFlags: static[string] = "", conStaticLib: static[string] = "", conDynLib: static[string] = "", - cmakeFlags: static[string] = "", cmakeStaticLib: static[string] = "", cmakeDynLib: static[string] = "", - makeFlags: static[string] = ""): untyped = + conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = ""): untyped = ## Get the path to a header file for wrapping with ## `cImport() `_ or ## `c2nImport() `_. ## - ## Checks defines based on the header name (e.g. lzma from lzma.h), to use different - ## ways to obtain the source. + ## This proc checks -d:xxx defines based on the header name (e.g. lzma from lzma.h), + ## and accordingly employs different ways to obtain the source. ## ## ``-d:xxxStd`` - search standard system paths. E.g. ``/usr/include`` and ``/usr/lib`` on Linux ## ``-d:xxxGit`` - clone source from a git repo specified in ``giturl`` ## ``-d:xxxDL`` - download source from ``dlurl`` and extract if required ## ## This allows a single wrapper to be used in different ways depending on the user's needs. - ## If no defines are specified, ``outdir`` will be searched for the header. + ## If no -d:xxx defines are specified, ``outdir`` will be searched for the header. ## - ## The library is then configured (either with cmake or autotools if possible) and then built - ## using make. + ## The library is then configured (with cmake or autotools if possible) and built + ## using make, unless using ``-d:xxxStd`` which presumes that the system package + ## manager was used to install prebuilt headers and binaries. + ## + ## The header path is stored in ``const xxxPath`` and can be used in a ``cImport()`` call + ## in the calling wrapper. The dynamic library path is stored in ``const xxxLPath`` and can + ## be used for the ``dynlib`` parameter (within quotes). + ## + ## ``-d:xxxStatic`` can be specified to statically link with the library instead. This + ## will automatically add a ``{.passL.}`` call to the static library for convenience. var name = header.split(".")[0] - stdName = newIdentNode(name & "Std") - gitName = newIdentNode(name & "Git") - dlName = newIdentNode(name & "DL") + nameStd = newIdentNode(name & "Std") + nameGit = newIdentNode(name & "Git") + nameDL = newIdentNode(name & "DL") + + nameStatic = newIdentNode(name & "Static") path = newIdentNode(name & "Path") + lpath = newIdentNode(name & "LPath") version = newIdentNode(name & "Version") + lname = newIdentNode(name & "LName") + + lre = "(lib)?$1[0-9.\\-]*\\" % name result = newNimNode(nnkStmtList) result.add(quote do: - const `version`* {.strdefine.} = "" - - when defined(`stdName`): - const `path`* = getStdPath(`header`) - else: - const `path`* = - when defined(`gitName`): - getGitPath(`header`, `giturl`, `outdir`, `version`) - elif defined(`dlName`): - getDlPath(`header`, `dlurl`, `outdir`, `version`) + const + `version`* {.strdefine.} = "" + `lname` = + when defined(`nameStatic`): + `lre` & ".a" else: - getLocalPath(`header`, `outdir`) + `lre` & getDynlibExt() - static: - doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & "missing/empty outdir or -d:$1Std -d:$1Git or -d:$1DL not specified" % `name` + when defined(`nameStd`): + const + `path`* = getStdPath(`header`) + `lpath`* = getStdLibPath(`lname`) + else: + const + `path`* = + when defined(`nameGit`): + getGitPath(`header`, `giturl`, `outdir`, `version`) + elif defined(`nameDL`): + getDlPath(`header`, `dlurl`, `outdir`, `version`) + else: + getLocalPath(`header`, `outdir`) - buildLibrary(`outdir`, `conFlags`, `conStaticLib`, `conDynLib`, `cmakeFlags`, `cmakeStaticLib`, `cmakeDynLib`, `makeFlags`) + `lpath`* = buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`) + + static: + doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & "missing/empty outdir or -d:$1Std -d:$1Git or -d:$1DL not specified" % `name` + doAssert `lpath`.len != 0, "\nLibrary " & `lname` & " not found" + echo "# Including library " & `lpath` + + when defined(`nameStatic`): + {.passL: `lpath`.} ) diff --git a/nimterop/grammar.nim b/nimterop/grammar.nim index a66442f..6e2db16 100644 --- a/nimterop/grammar.nim +++ b/nimterop/grammar.nim @@ -73,6 +73,7 @@ proc initGrammar(): Grammar = (type_identifier) ) {paramListGrammar} + (noexcept|throw_specifier?) ) """ From ba2bd6e40a1a4b954fd0d508f5cc98d0f06d62bc Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 23 Aug 2019 16:16:00 -0500 Subject: [PATCH 04/21] Add getheader test --- .travis.yml | 4 ++-- nimterop/types.nim | 24 ++++++++++++++---------- tests/getheader.nims | 42 ++++++++++++++++++++++++++++++++++++++++++ tests/lzma.nim | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 12 deletions(-) create mode 100644 tests/getheader.nims create mode 100644 tests/lzma.nim diff --git a/.travis.yml b/.travis.yml index f323941..e56a392 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,13 +6,13 @@ language: c env: - BRANCH=0.19.6 - - BRANCH=0.20.0 + - BRANCH=0.20.2 - BRANCH=devel cache: directories: - "$HOME/.choosenim/toolchains/nim-0.19.6" - - "$HOME/.choosenim/toolchains/nim-0.20.0" + - "$HOME/.choosenim/toolchains/nim-0.20.2" install: # `set -u` failed for ubuntu: /home/travis/.travis/job_stages: line 107: secure: unbound variable diff --git a/nimterop/types.nim b/nimterop/types.nim index 5b4ad5f..709aa79 100644 --- a/nimterop/types.nim +++ b/nimterop/types.nim @@ -13,25 +13,29 @@ when (NimMajor, NimMinor, NimPatch) < (0, 19, 9): type Time {.importc: "time_t", header: "".} = distinct int64 elif defined(posix): import posix - type time_t* = Time + type + time_t* = Time + wchar_t* {.importc.} = object else: import std/time_t as time_t_temp type time_t* = time_t_temp.Time + when defined(c): + # http://www.cplusplus.com/reference/cwchar/wchar_t/ + # In C++, wchar_t is a distinct fundamental type (and thus it is + # not defined in nor any other header). + type + wchar_t* {.importc, header:"".} = object + elif defined(cpp): + type + wchar_t* {.importc.} = object + type ptrdiff_t* = ByteAddress type va_list* {.importc, header:"".} = object -when defined(c): - # http://www.cplusplus.com/reference/cwchar/wchar_t/ In C++, wchar_t is a distinct fundamental type (and thus it is not defined in nor any other header). - type - wchar_t* {.importc, header:"".} = object -elif defined(cpp): - type - wchar_t* {.importc.} = object - template enumOp*(op, typ, typout) = proc op*(x: typ, y: int): typout {.borrow.} proc op*(x: int, y: typ): typout {.borrow.} @@ -75,4 +79,4 @@ template defineEnum*(typ) = proc `/`*(x: typ, y: int): typ = `/`(x, y.typ) proc `/`*(x: int, y: typ): typ = `/`(x.typ, y) - proc `$` *(x: typ): string {.borrow.} \ No newline at end of file + proc `$` *(x: typ): string {.borrow.} diff --git a/tests/getheader.nims b/tests/getheader.nims new file mode 100644 index 0000000..57584a6 --- /dev/null +++ b/tests/getheader.nims @@ -0,0 +1,42 @@ +import strutils + +proc testCall(cmd, output: string, exitCode: int, delete = true) = + if delete: + rmDir("build/liblzma") + echo cmd + var + ccmd = + when defined(windows): + "cmd /c " & cmd + else: + cmd + (outp, exitC) = gorgeEx(ccmd) + echo outp + doAssert exitC == exitCode, $exitC + doAssert outp.contains(output), outp + +var + cmd = "nim c -f" + rcmd = " -r lzma.nim" + exp = "liblzma version = " + +when defined(linux): + testCall(cmd & rcmd, "No build files found", 1) + + # stdlib + testCall(cmd & " -d:lzmaStd" & rcmd, exp, 0) + testCall(cmd & " -d:lzmaStd -d:lzmaStatic" & rcmd, exp, 0) + + # git + testCall(cmd & " -d:lzmaGit" & rcmd, exp, 0) + testCall(cmd & " -d:lzmaGit -d:lzmaStatic" & rcmd, exp, 0, delete = false) + + # git tag + testCall(cmd & " -d:lzmaGit -d:lzmaVersion=v5.2.0" & rcmd, exp & "5.2.0", 0) + testCall(cmd & " -d:lzmaGit -d:lzmaStatic -d:lzmaVersion=v5.2.0" & rcmd, exp & "5.2.0", 0, delete = false) + testCall("cd build/liblzma && git branch", "v5.2.0", 0, delete = false) + + # dl + testCall(cmd & " -d:lzmaDL" & rcmd, "Need version", 1) + testCall(cmd & " -d:lzmaDL -d:lzmaVersion=v5.2.4" & rcmd, exp & "5.2.4", 0) + testCall(cmd & " -d:lzmaDL -d:lzmaStatic -d:lzmaVersion=v5.2.4" & rcmd, exp & "5.2.4", 0, delete = false) diff --git a/tests/lzma.nim b/tests/lzma.nim new file mode 100644 index 0000000..0cb92f5 --- /dev/null +++ b/tests/lzma.nim @@ -0,0 +1,41 @@ +import os, strutils + +import nimterop/[build, cimport] + +const + baseDir = currentSourcePath.parentDir()/"build/liblzma" + +static: + cDebug() + +getHeader( + "lzma.h", + giturl = "https://github.com/xz-mirror/xz", + dlurl = "https://github.com/xz-mirror/xz/archive/$1.zip", + outdir = baseDir, + conFlags = "--disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo" +) + +cPlugin: + import strutils + + proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = + sym.name = sym.name.strip(chars = {'_'}) + +cOverride: + type + lzma_internal = object + lzma_index = object + lzma_index_hash = object + + lzma_options_lzma = object + lzma_stream_flags = object + lzma_block = object + lzma_index_iter = object + +when not defined(lzmaStatic): + cImport(lzmaPath, recurse = true, dynlib = "lzmaLPath") +else: + cImport(lzmaPath, recurse = true) + +echo "liblzma version = " & $lzma_version_string() \ No newline at end of file From 9c51c824182a1b3180cecdd86b5af6c9dbc958ab Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 30 Aug 2019 19:08:35 -0500 Subject: [PATCH 05/21] Add autopoint detection --- .travis.yml | 6 +++++- appveyor.yml | 4 +++- nimterop/build.nim | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e56a392..564c34d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,11 @@ os: - linux - osx +addons: + apt: + packages: + - autopoint + language: c env: @@ -15,7 +20,6 @@ cache: - "$HOME/.choosenim/toolchains/nim-0.20.2" install: - # `set -u` failed for ubuntu: /home/travis/.travis/job_stages: line 107: secure: unbound variable - set -e - export CHOOSENIM_CHOOSE_VERSION=$BRANCH - | diff --git a/appveyor.yml b/appveyor.yml index 9525673..5386bbc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,7 +10,7 @@ matrix: environment: matrix: - NIM_VERSION: 0.19.6 - - NIM_VERSION: 0.20.0 + - NIM_VERSION: 0.20.2 for: - @@ -54,6 +54,8 @@ for: - image: Ubuntu install: + - sudo apt-get update + - sudo apt-get --yes --force-yes install liblzma-dev liblzma5 autopoint - if [ ! -e /home/appveyor/binaries ]; then echo $NIM_VERSION && mkdir /home/appveyor/binaries && diff --git a/nimterop/build.nim b/nimterop/build.nim index 53237c0..dfeb2d4 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -462,7 +462,10 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): strin if findExe("aclocal").len != 0: if findExe("autoconf").len != 0: if findExe("libtoolize").len != 0: - cfgCommon() + if findExe("autopoint").len != 0: + cfgCommon() + else: + conDepStr &= "autopoint executable missing" else: conDepStr &= "libtoolize executable missing" else: From 6bc0c4aa3b82a0ccd3144530b81a5cd922dc1bb3 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 4 Sep 2019 00:35:33 -0500 Subject: [PATCH 06/21] OSX support for getHeader, handle symlinks --- .travis.yml | 2 +- nimterop/build.nim | 14 +++++++++++--- nimterop/getters.nim | 6 ++++++ nimterop/toast.nim | 2 +- tests/getheader.nims | 2 +- 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 564c34d..d866260 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ install: - | curl https://nim-lang.org/choosenim/init.sh -sSf > init.sh sh init.sh -y - - export PATH=$HOME/.nimble/bin:$PATH + - export PATH="$HOME/.nimble/bin:/usr/local/opt/gettext/bin:$PATH" script: - set -e diff --git a/nimterop/build.nim b/nimterop/build.nim index dfeb2d4..0585d36 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -205,7 +205,8 @@ proc findFile*(file: string|Regex, dir: string, recurse = true, first = false): var rm: RegexMatch - for f in walkDirRec(dir, followFilter = if recurse: {pcDir} else: {}): + for f in walkDirRec(dir, yieldFilter = {pcFile, pcLinkToFile}, + followFilter = if recurse: {pcDir} else: {}): let fn = f.extractFilename() when file is string: @@ -343,8 +344,9 @@ proc getGccLibPaths*(mode = "c"): seq[string] = var nul = when defined(Windows): "nul" else: "/dev/null" mmode = if mode == "cpp": "c++" else: mode + linker = when defined(OSX): "-Xlinker" else: "" - (outp, _) = gorgeEx(&"""{getEnv("CC", "gcc")} -v -x{mmode} {nul}""") + (outp, _) = gorgeEx(&"""{getEnv("CC", "gcc")} {linker} -v -x{mmode} {nul}""") for line in outp.splitLines(): if "LIBRARY_PATH=" in line: @@ -355,6 +357,12 @@ proc getGccLibPaths*(mode = "c"): seq[string] = if path notin result: result.add path break + elif '\t' in line: + var + path = line.strip() + path.normalizePath() + if path notin result: + result.add path proc getStdPath(header: string): string = for inc in getGccPaths(): @@ -461,7 +469,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): strin if fileExists(outdir / "autogen.sh") or fileExists(outdir / "build" / "autogen.sh"): if findExe("aclocal").len != 0: if findExe("autoconf").len != 0: - if findExe("libtoolize").len != 0: + if findExe("libtoolize").len != 0 or findExe("glibtoolize").len != 0: if findExe("autopoint").len != 0: cfgCommon() else: diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 04ed5cf..f1559db 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -403,3 +403,9 @@ proc loadPlugin*(gState: State, sourcePath: string) = gState.onSymbol = cast[OnSymbol](lib.symAddr("onSymbol")) doAssert gState.onSymbol != nil, "onSymbol() load failed from " & pdll + +proc expandSymlinkAbs*(path: string): string = + try: + result = path.expandSymlink().absolutePath(path.parentDir()) + except: + result = path diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 6e38670..b66a3c6 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -145,7 +145,7 @@ proc main( if gState.pnim: printNimHeader() for src in source: - gState.process(src, astTable) + gState.process(src.expandSymlinkAbs(), astTable) when isMainModule: import cligen diff --git a/tests/getheader.nims b/tests/getheader.nims index 57584a6..6d0f73a 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -20,7 +20,7 @@ var rcmd = " -r lzma.nim" exp = "liblzma version = " -when defined(linux): +when defined(posix): testCall(cmd & rcmd, "No build files found", 1) # stdlib From 5bf5c7445da4849dc7bf01086ed2b7117a79ed41 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 5 Sep 2019 16:04:37 -0500 Subject: [PATCH 07/21] autoreconf, don't find tools --- nimterop/build.nim | 87 ++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 0585d36..15581eb 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -225,6 +225,9 @@ proc configure*(path, check: string, flags = "") = ## If a `configure` script is not present and an `autogen.sh` script ## is present, it will be run before attempting `configure`. ## + ## Next, if `configure.ac` or `configure.in` exist, `autoreconf` will + ## be executed. + ## ## `check` is a file that will be generated by the `configure` command. ## This is required to prevent configure from running on every build. It ## is relative to the `path` and should not be an absolute path. @@ -236,11 +239,20 @@ proc configure*(path, check: string, flags = "") = echo "# Configuring " & path if not fileExists(path / "configure"): - for i in @[path / "autogen.sh", path / "build" / "autogen.sh"]: - if fileExists(i): + for i in @["autogen.sh", "build" / "autogen.sh"]: + if fileExists(path / i): echo "# Running autogen.sh" - discard execAction(&"cd {i.parentDir().quoteShell} && bash autogen.sh") + echo execAction(&"cd {(path / i).parentDir().quoteShell} && bash autogen.sh") + + break + + if not fileExists(path / "configure"): + for i in @["configure.ac", "configure.in"]: + if fileExists(path / i): + echo "# Running autoreconf" + + echo execAction(&"cd {path.quoteShell} && autoreconf -fi") break @@ -438,54 +450,45 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): strin cmakeDepStr = "" lpath = findFile(re(lname), outdir) makeFlagsProc = &"-j {getNumProcs()} {makeFlags}" + made = false + makePath = outdir if lpath.len != 0: return lpath - if fileExists(outdir / "CMakeLists.txt"): - if findExe("cmake").len != 0: - var - gen = "" - when defined(windows): - if findExe("sh").len != 0: - gen = "MSYS Makefiles" - else: - gen = "MinGW Makefiles" - else: - gen = "Unix Makefiles" - cmake(outdir / "build", "Makefile", &".. -G {gen.quoteShell} {cmakeFlags}") - cmakeDeps = true - make(outdir / "build", re(lname), makeFlagsProc) - else: - cmakeDepStr &= "cmake executable missing" - - template cfgCommon() {.dirty.} = - configure(outdir, "Makefile", conFlags) - conDeps = true - make(outdir, re(lname), makeFlagsProc) - - if not cmakeDeps: - if not fileExists(outdir / "configure"): - if fileExists(outdir / "autogen.sh") or fileExists(outdir / "build" / "autogen.sh"): - if findExe("aclocal").len != 0: - if findExe("autoconf").len != 0: - if findExe("libtoolize").len != 0 or findExe("glibtoolize").len != 0: - if findExe("autopoint").len != 0: - cfgCommon() - else: - conDepStr &= "autopoint executable missing" - else: - conDepStr &= "libtoolize executable missing" + if not fileExists(outdir / "Makefile"): + if fileExists(outdir / "CMakeLists.txt"): + if findExe("cmake").len != 0: + var + gen = "" + when defined(windows): + if findExe("sh").len != 0: + gen = "MSYS Makefiles" else: - conDepStr &= "autoconf executable missing" + gen = "MinGW Makefiles" else: - conDepStr &= "aclocal executable missing" - else: + gen = "Unix Makefiles" + cmake(outdir / "build", "Makefile", &".. -G {gen.quoteShell} {cmakeFlags}") + cmakeDeps = true + makePath = outdir / "build" + else: + cmakeDepStr &= "cmake executable missing" + + if not cmakeDeps: if findExe("bash").len != 0: - cfgCommon() + for file in @["configure", "configure.ac", "configure.in", "autogen.sh", "build/autogen.sh"]: + if fileExists(outdir / file): + configure(outdir, "Makefile", conFlags) + conDeps = true + + break else: conDepStr &= "bash executable missing" + if fileExists(makePath / "Makefile"): + make(makePath, re(lname), makeFlagsProc) + made = true + var error = "" if not cmakeDeps and cmakeDepStr.len != 0: @@ -494,7 +497,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): strin error &= &"configure capable but {conDepStr}\n" if error.len == 0: error = "No build files found in " & outdir - doAssert cmakeDeps or conDeps, &"\n# Build configuration failed - {error}\n" + doAssert cmakeDeps or conDeps or made, &"\n# Build configuration failed - {error}\n" result = findFile(re(lname), outdir) From 2049787f435440a51c7989ca9939a89d9b2aeff6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 6 Sep 2019 11:57:14 -0700 Subject: [PATCH 08/21] More archive support, Windows lzma test --- nimterop/build.nim | 40 ++++++++++++++++++++++++++++++++++++++-- tests/getheader.nims | 8 ++++---- tests/lzma.nim | 2 +- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 15581eb..ee7c4ee 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1,4 +1,4 @@ -import macros, osproc, regex, sequtils, strformat, strutils +import macros, osproc, regex, strformat, strutils import os except findExe @@ -105,6 +105,39 @@ proc extractZip*(zipfile, outdir: string) = echo "# Extracting " & zipfile discard execAction(&"cd {outdir.quoteShell} && {cmd % zipfile}") +proc extractTar*(tarfile, outdir: string) = + ## Extract a tar file using tar, 7z or 7za to the specified output directory + var + cmd = "" + name = "" + + if findExe("tar").len != 0: + let + ext = tarfile.splitFile().ext.toLowerAscii() + typ = + case ext + of ".gz", ".tgz": "z" + of ".xz": "J" + of ".bz2": "j" + else: "" + + cmd = "tar xvf" & typ & " " & tarfile.quoteShell + else: + for i in ["7z", "7za"]: + if findExe(i).len != 0: + cmd = i & " x $#" % tarfile.quoteShell + + name = tarfile.splitFile().name + if ".tar" in name.toLowerAscii(): + cmd &= " && " & i & " x $#" % name.quoteShell + + break + + echo "# Extracting " & tarfile + discard execAction(&"cd {outdir.quoteShell} && {cmd}") + if name.len != 0: + rmFile(outdir / name) + proc downloadUrl*(url, outdir: string) = ## Download a file using curl or wget (or powershell on Windows) to the specified directory ## @@ -112,8 +145,9 @@ proc downloadUrl*(url, outdir: string) = let file = url.extractFilename() ext = file.splitFile().ext.toLowerAscii() + archives = @[".zip", ".xz", ".gz", ".bz2", ".tgz", ".tar"] - if not (ext == ".zip" and fileExists(outdir/file)): + if not (ext in archives and fileExists(outdir/file)): echo "# Downloading " & file mkDir(outdir) var cmd = findExe("curl") @@ -131,6 +165,8 @@ proc downloadUrl*(url, outdir: string) = if ext == ".zip": extractZip(file, outdir) + elif ext in archives: + extractTar(file, outdir) proc gitReset*(outdir: string) = ## Hard reset the git repository at the specified directory diff --git a/tests/getheader.nims b/tests/getheader.nims index 6d0f73a..840a114 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -36,7 +36,7 @@ when defined(posix): testCall(cmd & " -d:lzmaGit -d:lzmaStatic -d:lzmaVersion=v5.2.0" & rcmd, exp & "5.2.0", 0, delete = false) testCall("cd build/liblzma && git branch", "v5.2.0", 0, delete = false) - # dl - testCall(cmd & " -d:lzmaDL" & rcmd, "Need version", 1) - testCall(cmd & " -d:lzmaDL -d:lzmaVersion=v5.2.4" & rcmd, exp & "5.2.4", 0) - testCall(cmd & " -d:lzmaDL -d:lzmaStatic -d:lzmaVersion=v5.2.4" & rcmd, exp & "5.2.4", 0, delete = false) +# dl +testCall(cmd & " -d:lzmaDL" & rcmd, "Need version", 1) +testCall(cmd & " -d:lzmaDL -d:lzmaVersion=5.2.4" & rcmd, exp & "5.2.4", 0) +testCall(cmd & " -d:lzmaDL -d:lzmaStatic -d:lzmaVersion=5.2.4" & rcmd, exp & "5.2.4", 0, delete = false) diff --git a/tests/lzma.nim b/tests/lzma.nim index 0cb92f5..422f94d 100644 --- a/tests/lzma.nim +++ b/tests/lzma.nim @@ -11,7 +11,7 @@ static: getHeader( "lzma.h", giturl = "https://github.com/xz-mirror/xz", - dlurl = "https://github.com/xz-mirror/xz/archive/$1.zip", + dlurl = "https://tukaani.org/xz/xz-$1.tar.gz", outdir = baseDir, conFlags = "--disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo" ) From 9f196e3f77be2728ec7cd5e8caa51b7b882637cd Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 6 Sep 2019 23:10:28 -0500 Subject: [PATCH 09/21] Fix paths, git for Windows build --- appveyor.yml | 42 +++++++++++++++------------- nimterop/build.nim | 56 ++++++++++++++++++++----------------- nimterop/cimport.nim | 20 ++++++------- nimterop/compat.nim | 12 ++++++-- nimterop/getters.nim | 17 +++++------ nimterop/treesitter/api.nim | 12 ++++---- nimterop/treesitter/c.nim | 2 +- nimterop/treesitter/cpp.nim | 2 +- 8 files changed, 87 insertions(+), 76 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 5386bbc..50d9813 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,8 @@ version: '{build}' image: + - Visual Studio 2015 - Ubuntu - - Visual Studio 2017 matrix: fast_finish: true @@ -16,13 +16,12 @@ for: - matrix: only: - - image: Visual Studio 2017 + - image: Visual Studio 2015 environment: - ARCH: 32 - MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf - MINGW_ARCHIVE: i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - SFNET_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686 + ARCH: 64 + GIT_URL: https://github.com/git-for-windows/git/releases/download/v2.23.0.windows.1/ + GIT_ARCHIVE: PortableGit-2.23.0-64-bit.7z.exe install: - CD c:\ @@ -30,20 +29,23 @@ for: echo %NIM_VERSION% && MKDIR binaries && CD binaries && - appveyor DownloadFile "%MINGW_URL%/%MINGW_ARCHIVE%/download" -FileName "%MINGW_ARCHIVE%" && - 7z x -y "%MINGW_ARCHIVE%"> nul && - del "%MINGW_ARCHIVE%" && + MKDIR git && + CD git && + appveyor DownloadFile "%GIT_URL%/%GIT_ARCHIVE%" -FileName "%GIT_ARCHIVE%" && + 7z x -y -bd "%GIT_ARCHIVE%"> nul && + del "%GIT_ARCHIVE%" && + CD .. && appveyor DownloadFile "https://nim-lang.org/download/nim-%NIM_VERSION%_x%ARCH%.zip" -FileName "nim-%NIM_VERSION%_x%ARCH%.zip" && 7z x -y "nim-%NIM_VERSION%_x%ARCH%.zip"> nul && del "nim-%NIM_VERSION%_x%ARCH%.zip") - - SET PATH=c:\binaries\mingw%ARCH%\bin;c:\binaries\nim-%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% + - SET PATH=c:\binaries\git\bin;C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;c:\binaries\nim-%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - CD %APPVEYOR_BUILD_FOLDER% - on_finish: - - 7z a -r buildlogs-win-pkgs.zip %USERPROFILE%\.nimble\pkgs - - appveyor PushArtifact buildlogs-win-pkgs.zip - - 7z a -r buildlogs-win-projects.zip c:\projects\* - - appveyor PushArtifact buildlogs-win-projects.zip +# on_finish: +# - 7z a -r buildlogs-win-pkgs.zip %USERPROFILE%\.nimble\pkgs +# - appveyor PushArtifact buildlogs-win-pkgs.zip +# - 7z a -r buildlogs-win-projects.zip c:\projects\* +# - appveyor PushArtifact buildlogs-win-projects.zip cache: - c:\binaries @@ -71,11 +73,11 @@ for: - export PATH=/home/appveyor/binaries/nim-$NIM_VERSION/bin:~/.nimble/bin:$PATH - cd $APPVEYOR_BUILD_FOLDER - on_finish: - - zip -r -q buildlogs-lin-pkgs.zip ~/.nimble/pkgs - - appveyor PushArtifact buildlogs-lin-pkgs.zip - - zip -r -q buildlogs-lin-projects.zip /home/appveyor/projects - - appveyor PushArtifact buildlogs-lin-projects.zip +# on_finish: +# - zip -r -q buildlogs-lin-pkgs.zip ~/.nimble/pkgs +# - appveyor PushArtifact buildlogs-lin-pkgs.zip +# - zip -r -q buildlogs-lin-projects.zip /home/appveyor/projects +# - appveyor PushArtifact buildlogs-lin-projects.zip cache: - /home/appveyor/binaries diff --git a/nimterop/build.nim b/nimterop/build.nim index ee7c4ee..7a1aaf9 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -4,6 +4,11 @@ import os except findExe import "."/[compat] +proc sanitizePath*(path: string, noQuote = false): string = + result = path.multiReplace([("\\\\", $DirSep), ("\\", $DirSep), ("/", $DirSep)]) + if not noQuote: + result = result.quoteShell + proc execAction*(cmd: string, nostderr=false): string = ## Execute an external command - supported at compile time ## @@ -39,7 +44,7 @@ proc findExe*(exe: string): string = (oup, code) = gorgeEx(cmd) if code == 0: - return oup.strip() + return oup.splitLines()[0].strip() proc mkDir*(dir: string) = ## Create a directory at cmopile time @@ -49,7 +54,7 @@ proc mkDir*(dir: string) = if not dirExists(dir): let flag = when not defined(Windows): "-p" else: "" - discard execAction(&"mkdir {flag} {dir.quoteShell}") + discard execAction(&"mkdir {flag} {dir.sanitizePath}") proc cpFile*(source, dest: string, move=false) = ## Copy a file from source to destination at compile time @@ -68,7 +73,7 @@ proc cpFile*(source, dest: string, move=false) = else: "cp -f" - discard execAction(&"{cmd} {source.quoteShell} {dest.quoteShell}") + discard execAction(&"{cmd} {source.sanitizePath} {dest.sanitizePath}") proc mvFile*(source, dest: string) = ## Move a file from source to destination at compile time @@ -87,7 +92,7 @@ proc rmFile*(source: string, dir = false) = else: "rm -rf" - discard execAction(&"{cmd} {source.quoteShell}") + discard execAction(&"{cmd} {source.sanitizePath}") proc rmDir*(source: string) = ## Remove a directory or pattern at compile time @@ -103,7 +108,7 @@ proc extractZip*(zipfile, outdir: string) = "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" echo "# Extracting " & zipfile - discard execAction(&"cd {outdir.quoteShell} && {cmd % zipfile}") + discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}") proc extractTar*(tarfile, outdir: string) = ## Extract a tar file using tar, 7z or 7za to the specified output directory @@ -121,20 +126,22 @@ proc extractTar*(tarfile, outdir: string) = of ".bz2": "j" else: "" - cmd = "tar xvf" & typ & " " & tarfile.quoteShell + cmd = "tar xvf" & typ & " " & tarfile.sanitizePath else: for i in ["7z", "7za"]: if findExe(i).len != 0: - cmd = i & " x $#" % tarfile.quoteShell + cmd = i & " x $#" % tarfile.sanitizePath name = tarfile.splitFile().name if ".tar" in name.toLowerAscii(): - cmd &= " && " & i & " x $#" % name.quoteShell + cmd &= " && " & i & " x $#" % name.sanitizePath break + doAssert cmd.len != 0, "No extraction tool - tar, 7z, 7za - available for " & tarfile.sanitizePath + echo "# Extracting " & tarfile - discard execAction(&"cd {outdir.quoteShell} && {cmd}") + discard execAction(&"cd {outdir.sanitizePath} && {cmd}") if name.len != 0: rmFile(outdir / name) @@ -152,7 +159,7 @@ proc downloadUrl*(url, outdir: string) = mkDir(outdir) var cmd = findExe("curl") if cmd.len != 0: - cmd &= " -L $# -o $#" + cmd &= " -Lk $# -o $#" else: cmd = findExe("wget") if cmd.len != 0: @@ -161,7 +168,7 @@ proc downloadUrl*(url, outdir: string) = cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" else: doAssert false, "No download tool available - curl, wget" - discard execAction(cmd % [url, (outdir/file).quoteShell]) + discard execAction(cmd % [url, (outdir/file).sanitizePath]) if ext == ".zip": extractZip(file, outdir) @@ -172,7 +179,7 @@ proc gitReset*(outdir: string) = ## Hard reset the git repository at the specified directory echo "# Resetting " & outdir - let cmd = &"cd {outdir.quoteShell} && git reset --hard" + let cmd = &"cd {outdir.sanitizePath} && git reset --hard" while execAction(cmd).contains("Permission denied"): sleep(1000) echo "# Retrying ..." @@ -185,7 +192,7 @@ proc gitCheckout*(file, outdir: string) = ## successful wrapping with `cImport()` or `c2nImport()`. echo "# Resetting " & file let file2 = file.relativePath outdir - let cmd = &"cd {outdir.quoteShell} && git checkout {file2.quoteShell}" + let cmd = &"cd {outdir.sanitizePath} && git checkout {file2.sanitizePath}" while execAction(cmd).contains("Permission denied"): sleep(500) echo "# Retrying ..." @@ -206,7 +213,7 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = return let - outdirQ = outdir.quoteShell + outdirQ = outdir.sanitizePath mkDir(outdir) @@ -279,7 +286,7 @@ proc configure*(path, check: string, flags = "") = if fileExists(path / i): echo "# Running autogen.sh" - echo execAction(&"cd {(path / i).parentDir().quoteShell} && bash autogen.sh") + echo execAction(&"cd {(path / i).parentDir().sanitizePath} && bash autogen.sh") break @@ -288,7 +295,7 @@ proc configure*(path, check: string, flags = "") = if fileExists(path / i): echo "# Running autoreconf" - echo execAction(&"cd {path.quoteShell} && autoreconf -fi") + echo execAction(&"cd {path.sanitizePath} && autoreconf -fi") break @@ -296,7 +303,7 @@ proc configure*(path, check: string, flags = "") = echo "# Running configure " & flags var - cmd = &"cd {path.quoteShell} && bash configure" + cmd = &"cd {path.sanitizePath} && bash configure" if flags.len != 0: cmd &= &" {flags}" @@ -327,7 +334,7 @@ proc cmake*(path, check, flags: string) = mkDir(path) var - cmd = &"cd {path.quoteShell} && cmake {flags}" + cmd = &"cd {path.sanitizePath} && cmake {flags}" echo execAction(cmd) @@ -359,7 +366,7 @@ proc make*(path, check: string|Regex, flags = "") = cpFile(cmd, cmd.replace("mingw32-make", "make")) doAssert cmd.len != 0, "Make not found" - cmd = &"cd {path.quoteShell} && make" + cmd = &"cd {path.sanitizePath} && make" if flags.len != 0: cmd &= &" {flags}" @@ -383,8 +390,7 @@ proc getGccPaths*(mode = "c"): seq[string] = break if inc: var - path = line.strip() - path.normalizePath() + path = line.strip().myNormalizedPath() if path notin result: result.add path @@ -400,15 +406,13 @@ proc getGccLibPaths*(mode = "c"): seq[string] = if "LIBRARY_PATH=" in line: for path in line[13 .. ^1].split(PathSep): var - path = path.strip() - path.normalizePath() + path = path.strip().myNormalizedPath() if path notin result: result.add path break elif '\t' in line: var - path = line.strip() - path.normalizePath() + path = line.strip().myNormalizedPath() if path notin result: result.add path @@ -504,7 +508,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): strin gen = "MinGW Makefiles" else: gen = "Unix Makefiles" - cmake(outdir / "build", "Makefile", &".. -G {gen.quoteShell} {cmakeFlags}") + cmake(outdir / "build", "Makefile", &".. -G {gen.sanitizePath} {cmakeFlags}") cmakeDeps = true makePath = outdir / "build" else: diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index fc03bc5..3c99c81 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -79,11 +79,11 @@ proc getFileDate(fullpath: string): string = ret = 0 cmd = when defined(Windows): - &"cmd /c for %a in ({fullpath.quoteShell}) do echo %~ta" + &"cmd /c for %a in ({fullpath.sanitizePath}) do echo %~ta" elif defined(Linux): - &"stat -c %y {fullpath.quoteShell}" + &"stat -c %y {fullpath.sanitizePath}" elif defined(OSX): - &"stat -f %m {fullpath.quoteShell}" + &"stat -f %m {fullpath.sanitizePath}" (result, ret) = gorgeEx(cmd) @@ -120,7 +120,7 @@ proc getNimCheckError(output: string): tuple[tmpFile, errors: string] = getCurrentCompilerExe() else: "nim" - (check, _) = gorgeEx(&"{nim} check {result.tmpFile.quoteShell}") + (check, _) = gorgeEx(&"{nim} check {result.tmpFile.sanitizePath}") result.errors = "\n\n" & check @@ -131,7 +131,7 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "", cmd = when defined(Windows): "cmd /c " else: "" let toastExe = toastExePath() - doAssert fileExists(toastExe), "toast not compiled: " & toastExe.quoteShell & + doAssert fileExists(toastExe), "toast not compiled: " & toastExe.sanitizePath & " make sure 'nimble build' or 'nimble install' built it" cmd &= &"{toastExe} --preprocess" @@ -145,7 +145,7 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "", cmd.add &" --defines+={i.quoteShell}" for i in gStateCT.includeDirs: - cmd.add &" --includeDirs+={i.quoteShell}" + cmd.add &" --includeDirs+={i.sanitizePath}" if not noNimout: cmd.add &" --pnim" @@ -157,12 +157,12 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "", cmd.add &" --symOverride={gStateCT.symOverride.join(\",\")}" when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9): - cmd.add &" --nim:{getCurrentCompilerExe().quoteShell}" + cmd.add &" --nim:{getCurrentCompilerExe().sanitizePath}" if gStateCT.pluginSourcePath.nBl: - cmd.add &" --pluginSourcePath={gStateCT.pluginSourcePath.quoteShell}" + cmd.add &" --pluginSourcePath={gStateCT.pluginSourcePath.sanitizePath}" - cmd.add &" {fullpath.quoteShell}" + cmd.add &" {fullpath.sanitizePath}" # see https://github.com/nimterop/nimterop/issues/69 (result, ret) = gorgeEx(cmd, cache=getCacheValue(fullpath)) @@ -527,7 +527,7 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st let fullpath = findPath(filename) - echo "# Importing " & fullpath + echo "# Importing " & fullpath.sanitizePath let output = getToast(fullpath, recurse, dynlib, mode, flags) diff --git a/nimterop/compat.nim b/nimterop/compat.nim index 7f89358..1115252 100644 --- a/nimterop/compat.nim +++ b/nimterop/compat.nim @@ -6,10 +6,18 @@ put everything that requires `when (NimMajor, NimMinor, NimPatch)` here import os when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9): + proc myNormalizedPath*(path: string): string = path.normalizedPath() + export relativePath + else: import std/[ospaths,strutils] + proc myNormalizedPath*(path: string): string = + result = path.normalizedPath() + when defined(windows): + result = result.strip(trailing = false, chars = {'\\'}) + proc relativePath*(file, base: string): string = ## naive version of `os.relativePath` ; remove after nim >= 0.19.9 runnableExamples: @@ -17,8 +25,8 @@ else: check: "/foo/bar/baz/log.txt".unixToNativePath.relativePath("/foo/bar".unixToNativePath) == "baz/log.txt".unixToNativePath "foo/bar/baz/log.txt".unixToNativePath.relativePath("foo/bar".unixToNativePath) == "baz/log.txt".unixToNativePath - var base = base.normalizedPath - var file = file.normalizedPath + var base = base.myNormalizedPath + var file = file.myNormalizedPath if not base.endsWith DirSep: base.add DirSep doAssert file.startsWith base result = file[base.len .. ^1] diff --git a/nimterop/getters.nim b/nimterop/getters.nim index f1559db..c5c839d 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -73,9 +73,6 @@ const gTypeMap = { "long double": "clongdouble" }.toTable() -proc sanitizePath*(path: string): string = - path.multiReplace([("\\\\", $DirSep), ("\\", $DirSep), ("/", $DirSep)]) - proc getType*(str: string): string = if str == "void": return "object" @@ -205,15 +202,15 @@ proc getPreprocessor*(gState: State, fullpath: string, mode = "cpp"): string = rdata: seq[string] = @[] start = false - sfile = fullpath.sanitizePath + sfile = fullpath.sanitizePath(noQuote = true) for inc in gState.includeDirs: - cmd &= &"-I{inc.quoteShell} " + cmd &= &"-I{inc.sanitizePath} " for def in gState.defines: cmd &= &"-D{def} " - cmd &= &"{fullpath.quoteShell}" + cmd &= &"{fullpath.sanitizePath}" # Include content only from file for line in execAction(cmd).splitLines(): @@ -221,19 +218,19 @@ proc getPreprocessor*(gState: State, fullpath: string, mode = "cpp"): string = if line.len > 1 and line[0 .. 1] == "# ": start = false let - saniLine = line.sanitizePath + saniLine = line.sanitizePath(noQuote = true) if sfile in saniLine: start = true elif not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: start = true elif gState.recurse: let - pDir = sfile.expandFilename().parentDir().sanitizePath() + pDir = sfile.expandFilename().parentDir().sanitizePath(noQuote = true) if pDir.len == 0 or pDir in saniLine: start = true else: for inc in gState.includeDirs: - if inc.absolutePath().sanitizePath in saniLine: + if inc.absolutePath().sanitizePath(noQuote = true) in saniLine: start = true break else: @@ -395,7 +392,7 @@ proc loadPlugin*(gState: State, sourcePath: string) = pdll = sourcePath.dll if not fileExists(pdll) or sourcePath.getLastModificationTime() > pdll.getLastModificationTime(): - discard execAction(&"{gState.nim.quoteShell} c --app:lib {sourcePath.quoteShell}") + discard execAction(&"{gState.nim.sanitizePath} c --app:lib {sourcePath.sanitizePath}") doAssert fileExists(pdll), "No plugin binary generated for " & sourcePath let lib = loadLib(pdll) diff --git a/nimterop/treesitter/api.nim b/nimterop/treesitter/api.nim index 62ebe3b..2c11a60 100644 --- a/nimterop/treesitter/api.nim +++ b/nimterop/treesitter/api.nim @@ -7,17 +7,17 @@ import ".."/[setup, paths, types] static: treesitterSetup() -const sourcePath = incDir() / "treesitter/lib" +const sourcePath = incDir() / "treesitter" / "lib" when defined(Linux): {.passC: "-std=c11".} {.passC: "-DUTF8PROC_STATIC".} -{.passC: "-I$1/include" % sourcePath.} -{.passC: "-I$1/src" % sourcePath.} -{.passC: "-I$1/../../utf8proc" % sourcePath.} +{.passC: "-I$1" % (sourcePath / "include").} +{.passC: "-I$1" % (sourcePath / "src").} +{.passC: "-I$1" % (sourcePath / ".." / ".." / "utf8proc").} -{.compile: sourcePath / "src/lib.c".} +{.compile: sourcePath / "src" / "lib.c".} ### Generated below @@ -27,7 +27,7 @@ defineEnum(TSInputEncoding) defineEnum(TSSymbolType) defineEnum(TSLogType) const - headerapi {.used.} = sourcePath / "include/tree_sitter/api.h" + headerapi {.used.} = sourcePath / "include" / "tree_sitter" / "api.h" TREE_SITTER_LANGUAGE_VERSION* = 9 TSInputEncodingUTF8* = 0.TSInputEncoding TSInputEncodingUTF16* = 1.TSInputEncoding diff --git a/nimterop/treesitter/c.nim b/nimterop/treesitter/c.nim index a39ee65..38fa003 100644 --- a/nimterop/treesitter/c.nim +++ b/nimterop/treesitter/c.nim @@ -5,7 +5,7 @@ import ".."/[setup, paths] static: treesitterCSetup() -const srcDir = incDir() / "treesitter_c/src" +const srcDir = incDir() / "treesitter_c" / "src" import "."/api diff --git a/nimterop/treesitter/cpp.nim b/nimterop/treesitter/cpp.nim index 1b47206..131e57f 100644 --- a/nimterop/treesitter/cpp.nim +++ b/nimterop/treesitter/cpp.nim @@ -5,7 +5,7 @@ import ".."/[setup, paths] static: treesitterCppSetup() -const srcDir = incDir() / "treesitter_cpp/src" +const srcDir = incDir() / "treesitter_cpp" / "src" {.passC: "-I$1" % srcDir.} From 1ccf4c8e41c9aeb60f0a611d8a0d2ed06e4f96eb Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Sep 2019 18:52:24 -0500 Subject: [PATCH 10/21] Readme update --- README.md | 39 +++++++++++++++++++++++++++++++++++---- nimterop/template.nim | 5 ++++- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6dfcb04..982a2e7 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,15 @@ Nim has one of the best FFI you can find - importing C/C++ is supported out of t The goal of nimterop is to leverage the [tree-sitter](http://tree-sitter.github.io/tree-sitter/) engine to parse C/C++ code and then convert relevant portions of the AST into Nim definitions. [tree-sitter](https://github.com/tree-sitter) is a Github sponsored project that can parse a variety of languages into an AST which is then leveraged by the [Atom](https://atom.io/) editor for syntax highlighting and code folding. The advantages of this approach are multifold: - Benefit from the tree-sitter community's investment into language parsing - Wrap what is recognized in the AST rather than completely failing due to parsing errors -- Avoid depending on Nim compiler API which is evolving constantly and makes backwards compatibility a bit challenging -Most of the functionality is contained within the `toast` binary that is built when nimterop is installed and can be used standalone similar to how c2nim can be used today. In addition, nimterop also offers an API to pull in the generated Nim content directly into an application. +Most of the wrapping functionality is contained within the `toast` binary that is built when nimterop is installed and can be used standalone similar to how c2nim can be used today. In addition, nimterop also offers an API to pull in the generated Nim content directly into an application and other nimgen functionality that helps in automating the wrapping process. There is also support to statically or dynamically link to system installed libraries or downloading and building them with `autoconf` or `cmake` from a Git repo or source archive. -The nimterop feature set is still limited to C but is expanding rapidly. C++ support will be added once most popular C libraries can be wrapped seamlessly. +The nimterop wrapping functionality is still limited to C but is constantly expanding. C++ support will be added once most popular C libraries can be wrapped seamlessly. Meanwhile, `c2nim` can also be used in place of `toast` with the `c2nImport()` API call. Nimterop has seen some adoption within the community and the simplicity and success of this approach justifies additional investment of time and effort. Regardless, the goal is to make interop seamless so nimterop will focus on wrapping headers and not the outright conversion of C/C++ implementation. +Also, given tree-sitter can parse a variety of other languages, there might also be value in investigating how to wrap Rust and Go libraries. + __Installation__ Nimterop can be installed via [Nimble](https://github.com/nim-lang/nimble): @@ -37,8 +38,37 @@ This will download and install nimterop in the standard Nimble package location, __Usage__ +Detailed documentation can be found [here](https://nimterop.github.io/nimterop/theindex.html). + Check out the [wiki](https://github.com/nimterop/nimterop/wiki/Wrappers) for a list of all known wrappers that have been created using nimterop. Please do add your project once you are done so that others can benefit from your work. +Using the high-level `getHeader` API to perform all building and linking automatically: +```nim +import nimterop/[build, cimport] + +static: + cDebug() + +getHeader( + "header.h", + giturl = "https://github.com/username/repo", + dlurl = "https://website.org/download/repo-$1.tar.gz", + outdir = "build", + conFlags = "--disable-comp --enable-feature" +) + +when not defined(headerStatic): + cImport(headerPath, recurse = true, dynlib = "headerLPath") +else: + cImport(headerPath, recurse = true) +``` +This allows the user to control how the wrapper works - either pass `-d:headerStd` to search for `header.h` in the standard system path, `-d:headerGit` to clone the source from the specified git URL or `-d:headerDL` to get the source from download URL. Further, the `-d:headerVersion=X.Y.Z` flag can be used to specify which version to use. It is used as the tag name for Git whereas for DL, it replaces `$1` in the URL defined. + +The `-d:headerStatic` attempts to statically link the library. If it is omitted, the library is dynamically linked instead. + +[lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library using this high-level API. + +The traditional approach is to manually compile in the code: ```nim import nimterop/cimport @@ -52,10 +82,11 @@ cImport("clib.h") cCompile("clib/src/*.c") ``` -Check out [template.nim](https://github.com/nimterop/nimterop/blob/master/nimterop/template.nim) as a starting point for wrapping a new library. The template can be copied and trimmed down and modified as required. [templite.nim](https://github.com/nimterop/nimterop/blob/master/nimterop/templite.nim) is a shorter version for more experienced users. +Check out [template.nim](https://github.com/nimterop/nimterop/blob/master/nimterop/template.nim) as a starting point for wrapping a library using the traditional approach. The template can be copied and trimmed down and modified as required. [templite.nim](https://github.com/nimterop/nimterop/blob/master/nimterop/templite.nim) is a shorter version for more experienced users. Refer to the ```tests``` directory for examples on how the library can be used. + The `toast` binary can also be used directly on the CLI: ``` diff --git a/nimterop/template.nim b/nimterop/template.nim index b8b894a..f625280 100644 --- a/nimterop/template.nim +++ b/nimterop/template.nim @@ -33,7 +33,10 @@ src/*.c # Run GNU configure on the source when defined(posix): - configure(srcDir, fileThatShouldGetGenerated) + configure(srcDir, fileThatShouldGetGenerated, flagsToConfigure) + + # Run cmake on the source + cmake(srcDir/"build", fileThatShouldGetGenerated, flagsToCmake) # Run standard file/directory operations with mkDir(), cpFile(), mvFile() From a867c726a937c3f44219a5264980f6e3ccf789f4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 7 Sep 2019 23:36:16 -0500 Subject: [PATCH 11/21] Normalize symlinks --- nimterop/getters.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index c5c839d..1954623 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -403,6 +403,6 @@ proc loadPlugin*(gState: State, sourcePath: string) = proc expandSymlinkAbs*(path: string): string = try: - result = path.expandSymlink().absolutePath(path.parentDir()) + result = path.expandSymlink().absolutePath(path.parentDir()).normalizedPath() except: result = path From 3a2395360712d2c6f27221e0887b7e3cad0be7a1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Sep 2019 18:54:48 -0500 Subject: [PATCH 12/21] Update to v0.2.0 --- nimterop.nimble | 2 +- nimterop/getters.nim | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 1014e66..565c3e4 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.1.0" +version = "0.2.0" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 1954623..54040c0 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -2,7 +2,7 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import regex -import "."/[build, globals, plugin, treesitter/api] +import "."/[build, compat, globals, plugin, treesitter/api] const gReserved = """ addr and as asm @@ -403,6 +403,6 @@ proc loadPlugin*(gState: State, sourcePath: string) = proc expandSymlinkAbs*(path: string): string = try: - result = path.expandSymlink().absolutePath(path.parentDir()).normalizedPath() + result = path.expandSymlink().absolutePath(path.parentDir()).myNormalizedPath() except: result = path From b0447677f1e02ec7d6b070e374e5e92b3ad0ce51 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Sep 2019 15:33:44 -0700 Subject: [PATCH 13/21] Build pre-hook, altNames, uInt, getType() calls, add zlib test Fix subdir header, static lib name, [u]int[\d]+ --- .travis.yml | 2 - README.md | 2 +- nimterop/build.nim | 163 ++++++++++++++++++++++++++++++++++--------- nimterop/cimport.nim | 68 +++++++++--------- nimterop/getters.nim | 12 +++- nimterop/grammar.nim | 8 +-- tests/getheader.nims | 46 ++++++++---- tests/zlib.nim | 68 ++++++++++++++++++ 8 files changed, 279 insertions(+), 90 deletions(-) create mode 100644 tests/zlib.nim diff --git a/.travis.yml b/.travis.yml index d866260..d48afd0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,6 @@ cache: - "$HOME/.choosenim/toolchains/nim-0.20.2" install: - - set -e - export CHOOSENIM_CHOOSE_VERSION=$BRANCH - | curl https://nim-lang.org/choosenim/init.sh -sSf > init.sh @@ -28,7 +27,6 @@ install: - export PATH="$HOME/.nimble/bin:/usr/local/opt/gettext/bin:$PATH" script: - - set -e - nimble --verbose install -y - nimble --verbose test - nimble --verbose --nimbleDir:`pwd`/build/fakenimble install nimterop -y diff --git a/README.md b/README.md index 982a2e7..fcab924 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ when not defined(headerStatic): else: cImport(headerPath, recurse = true) ``` -This allows the user to control how the wrapper works - either pass `-d:headerStd` to search for `header.h` in the standard system path, `-d:headerGit` to clone the source from the specified git URL or `-d:headerDL` to get the source from download URL. Further, the `-d:headerVersion=X.Y.Z` flag can be used to specify which version to use. It is used as the tag name for Git whereas for DL, it replaces `$1` in the URL defined. +This allows the user to control how the wrapper works - either pass `-d:headerStd` to search for `header.h` in the standard system path, `-d:headerGit` to clone the source from the specified git URL or `-d:headerDL` to get the source from download URL. Further, the `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to use. It is used as the tag name for Git whereas for DL, it replaces `$1` in the URL defined. The `-d:headerStatic` attempts to statically link the library. If it is omitted, the library is dynamically linked instead. diff --git a/nimterop/build.nim b/nimterop/build.nim index 7a1aaf9..2563bb8 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -32,7 +32,7 @@ proc execAction*(cmd: string, nostderr=false): string = doAssert ret == 0, "Command failed: " & $(ret, nostderr) & "\nccmd: " & ccmd & "\nresult:\n" & result proc findExe*(exe: string): string = - ## Find the specified executable using the which/where command - supported + ## Find the specified executable using the `which`/`where` command - supported ## at compile time var cmd = @@ -47,7 +47,7 @@ proc findExe*(exe: string): string = return oup.splitLines()[0].strip() proc mkDir*(dir: string) = - ## Create a directory at cmopile time + ## Create a directory at compile time ## ## The `os` module is not available at compile time so a few ## crucial helper functions are included with nimterop. @@ -57,7 +57,7 @@ proc mkDir*(dir: string) = discard execAction(&"mkdir {flag} {dir.sanitizePath}") proc cpFile*(source, dest: string, move=false) = - ## Copy a file from source to destination at compile time + ## Copy a file from `source` to `dest` at compile time let source = source.replace("/", $DirSep) dest = dest.replace("/", $DirSep) @@ -76,7 +76,7 @@ proc cpFile*(source, dest: string, move=false) = discard execAction(&"{cmd} {source.sanitizePath} {dest.sanitizePath}") proc mvFile*(source, dest: string) = - ## Move a file from source to destination at compile time + ## Move a file from `source` to `dest` at compile time cpFile(source, dest, move=true) proc rmFile*(source: string, dir = false) = @@ -99,7 +99,7 @@ proc rmDir*(source: string) = rmFile(source, dir = true) proc extractZip*(zipfile, outdir: string) = - ## Extract a zip file using powershell on Windows and unzip on other + ## Extract a zip file using `powershell` on Windows and `unzip` on other ## systems to the specified output directory var cmd = "unzip -o $#" if defined(Windows): @@ -111,7 +111,7 @@ proc extractZip*(zipfile, outdir: string) = discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}") proc extractTar*(tarfile, outdir: string) = - ## Extract a tar file using tar, 7z or 7za to the specified output directory + ## Extract a tar file using `tar`, `7z` or `7za` to the specified output directory var cmd = "" name = "" @@ -146,9 +146,9 @@ proc extractTar*(tarfile, outdir: string) = rmFile(outdir / name) proc downloadUrl*(url, outdir: string) = - ## Download a file using curl or wget (or powershell on Windows) to the specified directory + ## Download a file using `curl` or `wget` (or `powershell` on Windows) to the specified directory ## - ## If a zip file, it is automatically extracted after download. + ## If an archive file, it is automatically extracted after download. let file = url.extractFilename() ext = file.splitFile().ext.toLowerAscii() @@ -185,7 +185,7 @@ proc gitReset*(outdir: string) = echo "# Retrying ..." proc gitCheckout*(file, outdir: string) = - ## Checkout the specified file in the git repository specified + ## Checkout the specified `file` in the git repository at `outdir` ## ## This effectively resets all changes in the file and can be ## used to undo any changes that were made to source files to enable @@ -240,13 +240,17 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = proc findFile*(file: string|Regex, dir: string, recurse = true, first = false): string = ## Find the file in the specified directory ## - ## ``file`` can be a string or a regex object + ## `file` can be a string or a regex object ## - ## Turn off recursive search with ``recurse`` and stop on first match with - ## ``first``. Without it, the shortest match is returned. + ## Turn off recursive search with `recurse` and stop on first match with + ## `first`. Without it, the shortest match is returned. when file is Regex: var rm: RegexMatch + else: + let + dir = dir / file.parentDir() + file = file.extractFilename for f in walkDirRec(dir, yieldFilter = {pcFile, pcLinkToFile}, followFilter = if recurse: {pcDir} else: {}): @@ -311,6 +315,52 @@ proc configure*(path, check: string, flags = "") = doAssert (path / check).fileExists(), "# Configure failed" +proc getCmakePropertyStr(name, property, value: string): string = + &"\nset_target_properties({name} PROPERTIES {property} \"{value}\")\n" + +proc setCmakeProperty*(outdir, name, property, value: string) = + ## Set a `cmake` property in `outdir / CMakeLists.txt` - usable in the `xxxPreBuild` hook + ## for `getHeader()` + ## + ## `set_target_properties(name PROPERTIES property "value")` + let + cm = outdir / "CMakeLists.txt" + if cm.fileExists(): + cm.writeFile( + cm.readFile() & getCmakePropertyStr(name, property, value) + ) + +proc setCmakeLibName*(outdir, name, prefix = "", oname = "", suffix = "") = + ## Set a `cmake` property in `outdir / CMakeLists.txt` to specify a custom library output + ## name - usable in the `xxxPreBuild` hook for `getHeader()` + ## + ## `prefix` is typically `lib` + ## `oname` is the library name + ## `suffix` is typically `.a` + ## + ## Sometimes, `cmake` generates non-standard library names - e.g. zlib compiles to + ## `libzlibstatic.a` on Windows. This proc can help rename it to `libzlib.a` so that `getHeader()` + ## can find it after the library is compiled. + ## + ## ``` + ## set_target_properties(name PROPERTIES PREFIX "prefix") + ## set_target_properties(name PROPERTIES OUTPUT_NAME "oname") + ## set_target_properties(name PROPERTIES SUFFIX "suffix") + ## ``` + let + cm = outdir / "CMakeLists.txt" + if cm.fileExists(): + var + str = "" + if prefix.len != 0: + str &= getCmakePropertyStr(name, "PREFIX", prefix) + if oname.len != 0: + str &= getCmakePropertyStr(name, "OUTPUT_NAME", oname) + if suffix.len != 0: + str &= getCmakePropertyStr(name, "SUFFIX", suffix) + if str.len != 0: + cm.writeFile(cm.readFile() & str) + proc cmake*(path, check, flags: string) = ## Run the `cmake` command to generate all Makefiles or other ## build scripts in the specified path @@ -349,7 +399,7 @@ proc make*(path, check: string|Regex, flags = "") = ## ## `flags` are any flags that should be passed to the `make` command. ## - ## If make.exe is missing and mingw32-make.exe is available, it will + ## If `make.exe` is missing and `mingw32-make.exe` is available, it will ## be copied over to make.exe in the same location. if findFile(check, path).len != 0: return @@ -394,6 +444,9 @@ proc getGccPaths*(mode = "c"): seq[string] = if path notin result: result.add path + when defined(osx): + result.add execAction("xcrun --show-sdk-path").strip() & "/usr/include" + proc getGccLibPaths*(mode = "c"): seq[string] = var nul = when defined(Windows): "nul" else: "/dev/null" @@ -416,6 +469,9 @@ proc getGccLibPaths*(mode = "c"): seq[string] = if path notin result: result.add path + when defined(osx): + result.add "/usr/lib" + proc getStdPath(header: string): string = for inc in getGccPaths(): result = findFile(header, inc, recurse = false, first = true) @@ -503,12 +559,19 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): strin gen = "" when defined(windows): if findExe("sh").len != 0: - gen = "MSYS Makefiles" + let + uname = execAction("sh -c uname -a").toLowerAscii() + if uname.contains("msys"): + gen = "MSYS Makefiles".quoteShell + elif uname.contains("mingw"): + gen = "MinGW Makefiles".quoteShell & " -DCMAKE_SH=\"CMAKE_SH-NOTFOUND\"" + else: + echo "Unsupported system: " & uname else: - gen = "MinGW Makefiles" + gen = "MinGW Makefiles".quoteShell else: - gen = "Unix Makefiles" - cmake(outdir / "build", "Makefile", &".. -G {gen.sanitizePath} {cmakeFlags}") + gen = "Unix Makefiles".quoteShell + cmake(outdir / "build", "Makefile", &".. -G {gen} {cmakeFlags}") cmakeDeps = true makePath = outdir / "build" else: @@ -550,33 +613,54 @@ proc getDynlibExt(): string = result = ".dylib[0-9.]*" macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: static[string] = "", outdir: static[string] = "", - conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = ""): untyped = + conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", + altNames: static[string] = ""): untyped = ## Get the path to a header file for wrapping with ## `cImport() `_ or ## `c2nImport() `_. ## - ## This proc checks -d:xxx defines based on the header name (e.g. lzma from lzma.h), + ## This proc checks `-d:xxx` defines based on the header name (e.g. lzma from lzma.h), ## and accordingly employs different ways to obtain the source. ## - ## ``-d:xxxStd`` - search standard system paths. E.g. ``/usr/include`` and ``/usr/lib`` on Linux - ## ``-d:xxxGit`` - clone source from a git repo specified in ``giturl`` - ## ``-d:xxxDL`` - download source from ``dlurl`` and extract if required + ## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux + ## `-d:xxxGit` - clone source from a git repo specified in `giturl` + ## `-d:xxxDL` - download source from `dlurl` and extract if required ## ## This allows a single wrapper to be used in different ways depending on the user's needs. - ## If no -d:xxx defines are specified, ``outdir`` will be searched for the header. + ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. ## - ## The library is then configured (with cmake or autotools if possible) and built - ## using make, unless using ``-d:xxxStd`` which presumes that the system package + ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag + ## name for Git whereas for DL, it replaces `$1` in the URL defined. + ## + ## The library is then configured (with `cmake` or `autotools` if possible) and built + ## using `make`, unless using `-d:xxxStd` which presumes that the system package ## manager was used to install prebuilt headers and binaries. ## - ## The header path is stored in ``const xxxPath`` and can be used in a ``cImport()`` call - ## in the calling wrapper. The dynamic library path is stored in ``const xxxLPath`` and can - ## be used for the ``dynlib`` parameter (within quotes). + ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call + ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can + ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. ## - ## ``-d:xxxStatic`` can be specified to statically link with the library instead. This - ## will automatically add a ``{.passL.}`` call to the static library for convenience. + ## `-d:xxxStatic` can be specified to statically link with the library instead. This + ## will automatically add a `{.passL.}` call to the static library for convenience. + ## + ## `conFlags`, `cmakeFlags` and `makeFlags` allow sending custom parameters to `configure`, + ## `cmake` and `make` in case additional configuration is required as part of the build process. + ## + ## `altNames` is a list of alternate names for the library - e.g. zlib uses `zlib.h` for the header but + ## the typical lib name is `libz.so` and not `libzlib.so`. In this case, `altNames = "z"`. Comma + ## separate for multiple alternate names. + ## + ## `xxxPreBuild` is a hook that is called after the source code is pulled from Git or downloaded but + ## before the library is built. This might be needed if some initial prep needs to be done before + ## compilation. A few values are provided to the hook to help provide context: + ## + ## `outdir` is the same `outdir` passed in and `header` is the discovered header path in the + ## downloaded source code. + ## + ## Simply define `proc xxxPreBuild(outdir, header: string)` in the wrapper and it will get called + ## prior to the build process. var - name = header.split(".")[0] + name = header.extractFilename().split(".")[0] nameStd = newIdentNode(name & "Std") nameGit = newIdentNode(name & "Git") @@ -586,10 +670,18 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta path = newIdentNode(name & "Path") lpath = newIdentNode(name & "LPath") - version = newIdentNode(name & "Version") + version = newIdentNode(name & "SetVer") lname = newIdentNode(name & "LName") + preBuild = newIdentNode(name & "PreBuild") - lre = "(lib)?$1[0-9.\\-]*\\" % name + lre = "(lib)?$1[_]?(static)?[0-9.\\-]*\\" + + if altNames.len != 0: + let + names = "(" & name & "|" & altNames.replace(",", "|") & ")" + lre = lre % names + else: + lre = lre % name result = newNimNode(nnkStmtList) result.add(quote do: @@ -615,6 +707,11 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta else: getLocalPath(`header`, `outdir`) + when declared(`preBuild`): + static: + `preBuild`(`outdir`, `path`) + + const `lpath`* = buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`) static: diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 3c99c81..b295344 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -6,7 +6,7 @@ as a starting point for wrapping a new library. The template can be copied and trimmed down and modified as required. `templite.nim `_ is a shorter version for more experienced users. -All ``{.compileTime.}`` procs must be used in a compile time context, e.g. using: +All `{.compileTime.}` procs must be used in a compile time context, e.g. using: .. code-block:: c @@ -194,7 +194,7 @@ macro cOverride*(body): untyped = ## proc svGetCallerInfo(fileName: var cstring; lineNumber: var cint) ## ## Using the `cOverride() `_ block, nimterop - ## can be instructed to skip over ``svGetCallerInfo()``. This works for procs, + ## can be instructed to skip over `svGetCallerInfo()`. This works for procs, ## consts and types. ## ## `cOverride() `_ only affects calls to @@ -254,7 +254,7 @@ macro cPlugin*(body): untyped = ## - `nskEnumField` for enum (field) names, though they are in the global namespace as `nskConst` ## - `nskProc` - for proc names ## - ## ``nimterop/plugins`` is implicitly imported to provide access to standard plugin facilities. + ## `nimterop/plugins` is implicitly imported to provide access to standard plugin facilities. ## ## `cPlugin() `_ only affects calls to ## `cImport() `_ that follow it. @@ -288,7 +288,7 @@ macro cPlugin*(body): untyped = gStateCT.pluginSourcePath = path proc cSearchPath*(path: string): string {.compileTime.}= - ## Get full path to file or directory ``path`` in search path configured + ## Get full path to file or directory `path` in search path configured ## using `cAddSearchDir() `_ and ## `cAddStdDir() `_. ## @@ -321,15 +321,15 @@ proc cDisableCaching*() {.compileTime.} = ## value will continue to be used . Use `cDisableCaching() `_ ## to avoid this scenario during development. ## - ## ``nim -f`` was broken prior to 0.19.4 but can also be used to flush the cached content. + ## `nim -f` was broken prior to 0.19.4 but can also be used to flush the cached content. gStateCT.nocache = true macro cDefine*(name: static string, val: static string = ""): untyped = - ## ``#define`` an identifer that is forwarded to the C/C++ preprocessor if + ## `#define` an identifer that is forwarded to the C/C++ preprocessor if ## called within `cImport() `_ ## or `c2nImport() `_ - ## as well as to the C/C++ compiler during Nim compilation using ``{.passC: "-DXXX".}`` + ## as well as to the C/C++ compiler during Nim compilation using `{.passC: "-DXXX".}` result = newNimNode(nnkStmtList) @@ -350,7 +350,7 @@ macro cDefine*(name: static string, val: static string = ""): untyped = echo result.repr & "\n" proc cAddSearchDir*(dir: string) {.compileTime.} = - ## Add directory ``dir`` to the search path used in calls to + ## Add directory `dir` to the search path used in calls to ## `cSearchPath() `_. runnableExamples: import paths, os @@ -365,7 +365,7 @@ macro cIncludeDir*(dir: static string): untyped = ## Add an include directory that is forwarded to the C/C++ preprocessor if ## called within `cImport() `_ ## or `c2nImport() `_ - ## as well as to the C/C++ compiler during Nim compilation using ``{.passC: "-IXXX".}``. + ## as well as to the C/C++ compiler during Nim compilation using `{.passC: "-IXXX".}`. var dir = interpPath(dir) result = newNimNode(nnkStmtList) @@ -380,7 +380,7 @@ macro cIncludeDir*(dir: static string): untyped = echo result.repr proc cAddStdDir*(mode = "c") {.compileTime.} = - ## Add the standard ``c`` [default] or ``cpp`` include paths to search + ## Add the standard `c` [default] or `cpp` include paths to search ## path used in calls to `cSearchPath() `_ runnableExamples: static: cAddStdDir() @@ -390,24 +390,24 @@ proc cAddStdDir*(mode = "c") {.compileTime.} = cAddSearchDir inc macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = - ## Compile and link C/C++ implementation into resulting binary using ``{.compile.}`` + ## Compile and link C/C++ implementation into resulting binary using `{.compile.}` ## - ## ``path`` can be a specific file or contain wildcards: + ## `path` can be a specific file or contain wildcards: ## ## .. code-block:: nim ## ## cCompile("file.c") ## cCompile("path/to/*.c") ## - ## ``mode`` recursively searches for code files in ``path``. + ## `mode` recursively searches for code files in `path`. ## - ## ``c`` searches for ``*.c`` whereas ``cpp`` searches for ``*.C *.cpp *.c++ *.cc *.cxx`` + ## `c` searches for `*.c` whereas `cpp` searches for `*.C *.cpp *.c++ *.cc *.cxx` ## ## .. code-block:: nim ## ## cCompile("path/to/dir", "cpp") ## - ## ``exclude`` can be used to exclude files by partial string match. Comma separated to + ## `exclude` can be used to exclude files by partial string match. Comma separated to ## specify multiple exclude strings ## ## .. code-block:: nim @@ -485,16 +485,16 @@ macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = macro cImport*(filename: static string, recurse: static bool = false, dynlib: static string = "", mode: static string = "c", flags: static string = ""): untyped = ## Import all supported definitions from specified header file. Generated - ## content is cached in ``nimcache`` until ``filename`` changes unless - ## `cDisableCaching() `_ is set. ``nim -f`` + ## content is cached in `nimcache` until `filename` changes unless + ## `cDisableCaching() `_ is set. `nim -f` ## can also be used after Nim v0.19.4 to flush the cache. ## - ## ``recurse`` can be used to generate Nim wrappers from ``#include`` files - ## referenced in ``filename``. This is only done for files in the same - ## directory as ``filename`` or in a directory added using + ## `recurse` can be used to generate Nim wrappers from `#include` files + ## referenced in `filename`. This is only done for files in the same + ## directory as `filename` or in a directory added using ## `cIncludeDir() `_ ## - ## ``dynlib`` can be used to specify the Nim string to use to specify the dynamic + ## `dynlib` can be used to specify the Nim string to use to specify the dynamic ## library to load the imported symbols from. For example: ## ## .. code-block:: nim @@ -513,14 +513,14 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st ## ## cImport("pcre.h", dynlib="dynpcre") ## - ## If ``dynlib`` is not specified, the C/C++ implementation files can be compiled in + ## If `dynlib` is not specified, the C/C++ implementation files can be compiled in ## with `cCompile() `_, or the - ## ``{.passL.}`` pragma can be used to specify the static lib to link. + ## `{.passL.}` pragma can be used to specify the static lib to link. ## - ## ``mode`` is purely for forward compatibility when toast adds C++ support. It can + ## `mode` is purely for forward compatibility when toast adds C++ support. It can ## be ignored for the foreseeable future. ## - ## ``flags`` can be used to pass any other command line arguments to ``toast``. + ## `flags` can be used to pass any other command line arguments to `toast`. result = newNimNode(nnkStmtList) @@ -546,24 +546,24 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: static string = "", mode: static string = "c", flags: static string = ""): untyped = - ## Import all supported definitions from specified header file using ``c2nim`` + ## Import all supported definitions from specified header file using `c2nim` ## ## Similar to `cImport() `_ - ## but uses ``c2nim`` to generate the Nim wrapper instead of ``toast``. Note that neither + ## but uses `c2nim` to generate the Nim wrapper instead of `toast`. Note that neither ## `cOverride() `_, `cSkipSymbol() `_ - ## nor `cPlugin() `_ have any impact on ``c2nim``. + ## nor `cPlugin() `_ have any impact on `c2nim`. ## - ## ``toast`` is only used to preprocess the header file and recurse + ## `toast` is only used to preprocess the header file and recurse ## if specified. ## - ## ``mode`` should be set to ``cpp`` for c2nim to wrap C++ headers. + ## `mode` should be set to `cpp` for c2nim to wrap C++ headers. ## - ## ``flags`` can be used to pass other command line arguments to ``c2nim``. + ## `flags` can be used to pass other command line arguments to `c2nim`. ## - ## ``nimterop`` does not depend on ``c2nim`` as a ``nimble`` dependency so it + ## `nimterop` does not depend on `c2nim` as a `nimble` dependency so it ## does not get installed automatically. Any wrapper or library that requires this proc - ## needs to install ``c2nim`` with ``nimble install c2nim`` or add it as a dependency in - ## its own ``.nimble`` file. + ## needs to install `c2nim` with `nimble install c2nim` or add it as a dependency in + ## its own `.nimble` file. result = newNimNode(nnkStmtList) diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 54040c0..20fd8b3 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -26,7 +26,7 @@ when while xor yield""".split(Whitespace).toSet() -const gTypeMap = { +const gTypeMap* = { # char "char": "cchar", "signed char": "cschar", @@ -39,6 +39,8 @@ const gTypeMap = { "signed short int": "cshort", "unsigned short": "cushort", "unsigned short int": "cushort", + "uShort": "cushort", + "u_short": "cushort", # int "int": "cint", @@ -47,6 +49,8 @@ const gTypeMap = { "ssize_t": "cint", "unsigned": "cuint", "unsigned int": "cuint", + "uInt": "cuint", + "u_int": "cuint", "size_t": "cuint", # long @@ -57,6 +61,8 @@ const gTypeMap = { "off_t": "clong", "unsigned long": "culong", "unsigned long int": "culong", + "uLong": "culong", + "u_long": "culong", # long long "long long": "clonglong", @@ -79,8 +85,8 @@ proc getType*(str: string): string = result = str.strip(chars={'_'}). replace(re"\s+", " "). - replace(re"([u]?int[\d]+)_t", "$1"). - replace(re"([u]?int)ptr_t", "ptr $1") + replace(re"^([u]?int[\d]+)_t$", "$1"). + replace(re"^([u]?int)ptr_t$", "ptr $1") if gTypeMap.hasKey(result): result = gTypeMap[result] diff --git a/nimterop/grammar.nim b/nimterop/grammar.nim index 6e2db16..bfc5def 100644 --- a/nimterop/grammar.nim +++ b/nimterop/grammar.nim @@ -91,7 +91,7 @@ proc initGrammar(): Grammar = """ template funcParamCommon(fname, pname, ptyp, pptr, pout, count, i: untyped): untyped = - ptyp = nimState.getIdentifier(nimState.data[i].val, nskType, fname) + ptyp = nimState.getIdentifier(nimState.data[i].val, nskType, fname).getType() pptr = "" while i+1 < nimState.data.len and nimState.data[i+1].name == "pointer_declarator": @@ -137,7 +137,7 @@ proc initGrammar(): Grammar = var i = 0 - typ = nimState.getIdentifier(nimState.data[i].val, nskType) + typ = nimState.getIdentifier(nimState.data[i].val, nskType).getType() name = "" nname = "" tptr = "" @@ -161,7 +161,7 @@ proc initGrammar(): Grammar = let pragma = nimState.getPragma(nimState.getImportC(name, nname)) - if typ.nBl and nname.nBl and nimState.addNewIdentifer(nname): + if nname notin gTypeMap and typ.nBl and nname.nBl and nimState.addNewIdentifer(nname): if i < nimState.data.len and nimState.data[^1].name == "function_declarator": var fname = nname @@ -575,7 +575,7 @@ proc initGrammar(): Grammar = if fnname.nBl and nimState.addNewIdentifer(fnname): let - ftyp = nimState.getIdentifier(nimState.data[0].val, nskType, fnname) + ftyp = nimState.getIdentifier(nimState.data[0].val, nskType, fnname).getType() pragma = nimState.getPragma(nimState.getImportC(fname, fnname), "cdecl") if fptr.len != 0 or ftyp != "object": diff --git a/tests/getheader.nims b/tests/getheader.nims index 840a114..e489380 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -3,6 +3,7 @@ import strutils proc testCall(cmd, output: string, exitCode: int, delete = true) = if delete: rmDir("build/liblzma") + rmDir("build/zlib") echo cmd var ccmd = @@ -17,26 +18,45 @@ proc testCall(cmd, output: string, exitCode: int, delete = true) = var cmd = "nim c -f" - rcmd = " -r lzma.nim" - exp = "liblzma version = " + lrcmd = " -r lzma.nim" + zrcmd = " -r zlib.nim" + lexp = "liblzma version = " + zexp = "zlib version = " + +testCall(cmd & lrcmd, "No build files found", 1) when defined(posix): - testCall(cmd & rcmd, "No build files found", 1) - # stdlib - testCall(cmd & " -d:lzmaStd" & rcmd, exp, 0) - testCall(cmd & " -d:lzmaStd -d:lzmaStatic" & rcmd, exp, 0) + testCall(cmd & " -d:lzmaStd" & lrcmd, lexp, 0) + testCall(cmd & " -d:lzmaStd -d:lzmaStatic" & lrcmd, lexp, 0) + + when not defined(osx): + testCall(cmd & " -d:zlibStd" & zrcmd, zexp, 0) + testCall(cmd & " -d:zlibStd -d:zlibStatic" & zrcmd, zexp, 0) # git - testCall(cmd & " -d:lzmaGit" & rcmd, exp, 0) - testCall(cmd & " -d:lzmaGit -d:lzmaStatic" & rcmd, exp, 0, delete = false) + testCall(cmd & " -d:lzmaGit" & lrcmd, lexp, 0) + testCall(cmd & " -d:lzmaGit -d:lzmaStatic" & lrcmd, lexp, 0, delete = false) # git tag - testCall(cmd & " -d:lzmaGit -d:lzmaVersion=v5.2.0" & rcmd, exp & "5.2.0", 0) - testCall(cmd & " -d:lzmaGit -d:lzmaStatic -d:lzmaVersion=v5.2.0" & rcmd, exp & "5.2.0", 0, delete = false) + testCall(cmd & " -d:lzmaGit -d:lzmaSetVer=v5.2.0" & lrcmd, lexp & "5.2.0", 0) + testCall(cmd & " -d:lzmaGit -d:lzmaStatic -d:lzmaSetVer=v5.2.0" & lrcmd, lexp & "5.2.0", 0, delete = false) testCall("cd build/liblzma && git branch", "v5.2.0", 0, delete = false) +# git +testCall(cmd & " -d:zlibGit" & zrcmd, zexp, 0) +testCall(cmd & " -d:zlibGit -d:zlibStatic" & zrcmd, zexp, 0, delete = false) + +# git tag +testCall(cmd & " -d:zlibGit -d:zlibSetVer=v1.2.10" & zrcmd, zexp & "1.2.10", 0) +testCall(cmd & " -d:zlibGit -d:zlibStatic -d:zlibSetVer=v1.2.10" & zrcmd, zexp & "1.2.10", 0, delete = false) +testCall("cd build/zlib && git branch", "v1.2.10", 0, delete = false) + # dl -testCall(cmd & " -d:lzmaDL" & rcmd, "Need version", 1) -testCall(cmd & " -d:lzmaDL -d:lzmaVersion=5.2.4" & rcmd, exp & "5.2.4", 0) -testCall(cmd & " -d:lzmaDL -d:lzmaStatic -d:lzmaVersion=5.2.4" & rcmd, exp & "5.2.4", 0, delete = false) +testCall(cmd & " -d:lzmaDL" & lrcmd, "Need version", 1) +testCall(cmd & " -d:lzmaDL -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0) +testCall(cmd & " -d:lzmaDL -d:lzmaStatic -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0, delete = false) + +# dl +testCall(cmd & " -d:zlibDL -d:zlibSetVer=1.2.11" & zrcmd, zexp & "1.2.11", 0) +testCall(cmd & " -d:zlibDL -d:zlibStatic -d:zlibSetVer=1.2.11" & zrcmd, zexp & "1.2.11", 0, delete = false) diff --git a/tests/zlib.nim b/tests/zlib.nim new file mode 100644 index 0000000..f2cc16a --- /dev/null +++ b/tests/zlib.nim @@ -0,0 +1,68 @@ +import os, strutils + +import nimterop/[build, cimport] + +const + baseDir = currentSourcePath.parentDir()/"build"/"zlib" + +static: + cDebug() + +proc zlibPreBuild(outdir, path: string) = + let + mf = outdir / "Makefile" + if mf.fileExists(): + # Delete default Makefile + if mf.readFile().contains("configure first"): + mf.rmFile() + when defined(windows): + # Fix static lib name on Windows + setCmakeLibName(outdir, "zlibstatic", prefix = "lib", oname = "zlib", suffix = ".a") + +getHeader( + "zlib.h", + giturl = "https://github.com/madler/zlib", + dlurl = "http://zlib.net/zlib-$1.tar.gz", + outdir = baseDir, + altNames = "z" +) + +cPlugin: + import regex, strutils + + proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = + sym.name = sym.name.replace(re"_[_]+", "_").strip(chars = {'_'}) + +cOverride: + type + voidpf = ptr object + voidpc = ptr object + voidp = ptr object + uLongf = culong + z_size_t = culong + z_crc_t = culong + alloc_func* {.importc.} = proc(opaque: voidpf, items, size: uint) {.cdecl.} + Bytef {.importc.} = object + + when defined(posix): + type + pthread_mutex_s = object + pthread_cond_s = object + pthread_rwlock_arch_t = object + extension = object + fd_set = object + +when defined(posix): + static: + cSkipSymbol(@["u_int8_t", "u_int16_t", "u_int32_t", "u_int64_t"]) + +when defined(zlibGit) or defined(zlibDL): + when dirExists(baseDir / "build"): + cIncludeDir(baseDir / "build") + +when not defined(zlibStatic): + cImport(zlibPath, recurse = true, dynlib = "zlibLPath") +else: + cImport(zlibPath, recurse = true) + +echo "zlib version = " & $zlibVersion() From 4ed751ad5a64c868cf5cc087114827b426a9c07e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Sep 2019 21:56:29 -0700 Subject: [PATCH 14/21] Add ccache support --- nimterop/build.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 2563bb8..f2d4692 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -571,9 +571,11 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): strin gen = "MinGW Makefiles".quoteShell else: gen = "Unix Makefiles".quoteShell - cmake(outdir / "build", "Makefile", &".. -G {gen} {cmakeFlags}") + if findExe("ccache").len != 0: + gen &= " -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + makePath = outdir / "buildcache" + cmake(makePath, "Makefile", &".. -G {gen} {cmakeFlags}") cmakeDeps = true - makePath = outdir / "build" else: cmakeDepStr &= "cmake executable missing" From f21315ff1747ce0814f3a1e480f7b2e01ad9a278 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 23 Sep 2019 14:52:34 -0500 Subject: [PATCH 15/21] More build helpers --- nimterop/build.nim | 40 ++++++++++++++++++++++++++++++++++++++-- tests/zlib.nim | 4 ++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index f2d4692..494bc48 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -4,8 +4,8 @@ import os except findExe import "."/[compat] -proc sanitizePath*(path: string, noQuote = false): string = - result = path.multiReplace([("\\\\", $DirSep), ("\\", $DirSep), ("/", $DirSep)]) +proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = + result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) if not noQuote: result = result.quoteShell @@ -265,6 +265,20 @@ proc findFile*(file: string|Regex, dir: string, recurse = true, first = false): result = f if first: break +proc flagBuild*(base: string, flags: openArray[string]): string = + ## Simple helper proc to generate flags for `configure`, `cmake`, etc. + ## + ## Every entry in `flags` is replaced into the `base` string and + ## concatenated to the result. + ## + ## E.g. + ## `base = "--disable-$#"` + ## `flags = @["one", "two"]` + ## + ## `flagBuild(base, flags) => " --disable-one --disable-two"` + for i in flags: + result &= " " & base % i + proc configure*(path, check: string, flags = "") = ## Run the GNU `configure` command to generate all Makefiles or other ## build scripts in the specified path @@ -318,6 +332,15 @@ proc configure*(path, check: string, flags = "") = proc getCmakePropertyStr(name, property, value: string): string = &"\nset_target_properties({name} PROPERTIES {property} \"{value}\")\n" +proc getCmakeIncludePath*(paths: openArray[string]): string = + ## Create a `cmake` flag to specify custom include paths + ## + ## Result can be included in the `flag` parameter for `cmake()` or + ## the `cmakeFlags` parameter for `getHeader()`. + for path in paths: + result &= path & ";" + result = " -DCMAKE_INCLUDE_PATH=" & result[0 .. ^2].sanitizePath(sep = "/") + proc setCmakeProperty*(outdir, name, property, value: string) = ## Set a `cmake` property in `outdir / CMakeLists.txt` - usable in the `xxxPreBuild` hook ## for `getHeader()` @@ -361,6 +384,19 @@ proc setCmakeLibName*(outdir, name, prefix = "", oname = "", suffix = "") = if str.len != 0: cm.writeFile(cm.readFile() & str) +proc setCmakePositionIndependentCode*(outdir: string) = + ## Set a `cmake` directive to create libraries with -fPIC enabled + let + cm = outdir / "CMakeLists.txt" + if cm.fileExists(): + let + pic = "set(CMAKE_POSITION_INDEPENDENT_CODE ON)" + cmd = cm.readFile() + if not cmd.contains(pic): + cm.writeFile( + pic & "\n" & cmd + ) + proc cmake*(path, check, flags: string) = ## Run the `cmake` command to generate all Makefiles or other ## build scripts in the specified path diff --git a/tests/zlib.nim b/tests/zlib.nim index f2cc16a..6da7cdc 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -57,8 +57,8 @@ when defined(posix): cSkipSymbol(@["u_int8_t", "u_int16_t", "u_int32_t", "u_int64_t"]) when defined(zlibGit) or defined(zlibDL): - when dirExists(baseDir / "build"): - cIncludeDir(baseDir / "build") + when dirExists(baseDir / "buildcache"): + cIncludeDir(baseDir / "buildcache") when not defined(zlibStatic): cImport(zlibPath, recurse = true, dynlib = "zlibLPath") From fad9fd78f30eda750e615f69dd88f28158effbce Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 25 Sep 2019 21:15:14 -0500 Subject: [PATCH 16/21] Add env var support for defines --- .travis.yml | 2 + appveyor.yml | 1 + nimterop/build.nim | 90 ++++++++++++++++++++++++++++++++++++++------ tests/getheader.nims | 8 ++-- tests/lzma.nim | 9 ++++- tests/zlib.nim | 9 ++++- 6 files changed, 99 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index d48afd0..b347f73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,14 @@ language: c env: - BRANCH=0.19.6 - BRANCH=0.20.2 + - BRANCH=1.0.0 - BRANCH=devel cache: directories: - "$HOME/.choosenim/toolchains/nim-0.19.6" - "$HOME/.choosenim/toolchains/nim-0.20.2" + - "$HOME/.choosenim/toolchains/nim-1.0.0" install: - export CHOOSENIM_CHOOSE_VERSION=$BRANCH diff --git a/appveyor.yml b/appveyor.yml index 50d9813..0d9ba77 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,6 +11,7 @@ environment: matrix: - NIM_VERSION: 0.19.6 - NIM_VERSION: 0.20.2 + - NIM_VERSION: 1.0.0 for: - diff --git a/nimterop/build.nim b/nimterop/build.nim index 494bc48..24ae87c 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1,4 +1,4 @@ -import macros, osproc, regex, strformat, strutils +import macros, osproc, regex, strformat, strutils, tables import os except findExe @@ -650,6 +650,48 @@ proc getDynlibExt(): string = elif defined(macosx): result = ".dylib[0-9.]*" +var + gDefines {.compileTime.} = initTable[string, string]() + +macro setDefines*(defs: static openArray[string]): untyped = + ## Specify `-d:xxx` values in code instead of having to rely on the command + ## line or `cfg` or `nims` files. + ## + ## At this time, Nim does not allow creation of `-d:xxx` defines in code. In + ## addition, Nim only loads config files for the module being compiled but not + ## for imported packages. This becomes a challenge when wanting to ship a wrapper + ## library that wants to control `getHeader()` for an underlying package. + ## + ## E.g. nimarchive wanting to set `-d:lzmaStatic` + ## + ## The consumer of nimarchive would need to set such defines as part of their + ## project, making it inconvenient. + ## + ## By calling this proc with the defines preferred before importing such a module, + ## the caller can set the behavior in code instead. + ## + ## .. code-block:: nim + ## + ## setDefines(@["lzmaStatic", "lzmaDL", "lzmaSetVer=5.2.4"]) + ## + ## import lzma + for def in defs: + let + nv = def.strip().split("=", maxsplit = 1) + if nv.len != 0: + let + n = nv[0] + v = + if nv.len == 2: + nv[1] + else: + "" + gDefines[n] = v + +macro clearDefines*(): untyped = + ## Clear all defines set using `setDefines()`. + gDefines.clear() + macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: static[string] = "", outdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", altNames: static[string] = ""): untyped = @@ -670,6 +712,8 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag ## name for Git whereas for DL, it replaces `$1` in the URL defined. ## + ## All defines can also be set in code using `setDefines()`. + ## ## The library is then configured (with `cmake` or `autotools` if possible) and built ## using `make`, unless using `-d:xxxStd` which presumes that the system package ## manager was used to install prebuilt headers and binaries. @@ -700,20 +744,37 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta var name = header.extractFilename().split(".")[0] - nameStd = newIdentNode(name & "Std") - nameGit = newIdentNode(name & "Git") - nameDL = newIdentNode(name & "DL") + stdStr = name & "Std" + gitStr = name & "Git" + dlStr = name & "DL" - nameStatic = newIdentNode(name & "Static") + staticStr = name & "Static" + verStr = name & "SetVer" + + nameStd = newIdentNode(stdStr) + nameGit = newIdentNode(gitStr) + nameDL = newIdentNode(dlStr) + + nameStatic = newIdentNode(staticStr) path = newIdentNode(name & "Path") lpath = newIdentNode(name & "LPath") - version = newIdentNode(name & "SetVer") + version = newIdentNode(verStr) lname = newIdentNode(name & "LName") preBuild = newIdentNode(name & "PreBuild") lre = "(lib)?$1[_]?(static)?[0-9.\\-]*\\" + stdVal = gDefines.hasKey(stdStr) + gitVal = gDefines.hasKey(gitStr) + dlVal = gDefines.hasKey(dlStr) + staticVal = gDefines.hasKey(staticStr) + verVal = + if gDefines.hasKey(verStr): + gDefines[verStr] + else: + "" + if altNames.len != 0: let names = "(" & name & "|" & altNames.replace(",", "|") & ")" @@ -724,23 +785,28 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta result = newNimNode(nnkStmtList) result.add(quote do: const - `version`* {.strdefine.} = "" + `nameStd`* = when defined(`nameStd`): true else: `stdVal` == 1 + `nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1 + `nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1 + `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 + + `version`* {.strdefine.} = `verVal` `lname` = - when defined(`nameStatic`): + when `nameStatic`: `lre` & ".a" else: `lre` & getDynlibExt() - when defined(`nameStd`): + when `nameStd`: const `path`* = getStdPath(`header`) `lpath`* = getStdLibPath(`lname`) else: const `path`* = - when defined(`nameGit`): + when `nameGit`: getGitPath(`header`, `giturl`, `outdir`, `version`) - elif defined(`nameDL`): + elif `nameDL`: getDlPath(`header`, `dlurl`, `outdir`, `version`) else: getLocalPath(`header`, `outdir`) @@ -757,6 +823,6 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta doAssert `lpath`.len != 0, "\nLibrary " & `lname` & " not found" echo "# Including library " & `lpath` - when defined(`nameStatic`): + when `nameStatic`: {.passL: `lpath`.} ) diff --git a/tests/getheader.nims b/tests/getheader.nims index e489380..c122484 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -27,8 +27,8 @@ testCall(cmd & lrcmd, "No build files found", 1) when defined(posix): # stdlib - testCall(cmd & " -d:lzmaStd" & lrcmd, lexp, 0) - testCall(cmd & " -d:lzmaStd -d:lzmaStatic" & lrcmd, lexp, 0) + testCall(cmd & " -d:envTest" & lrcmd, lexp, 0) + testCall(cmd & " -d:envTestStatic" & lrcmd, lexp, 0) when not defined(osx): testCall(cmd & " -d:zlibStd" & zrcmd, zexp, 0) @@ -44,8 +44,8 @@ when defined(posix): testCall("cd build/liblzma && git branch", "v5.2.0", 0, delete = false) # git -testCall(cmd & " -d:zlibGit" & zrcmd, zexp, 0) -testCall(cmd & " -d:zlibGit -d:zlibStatic" & zrcmd, zexp, 0, delete = false) +testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) +testCall(cmd & " -d:envTestStatic" & zrcmd, zexp, 0, delete = false) # git tag testCall(cmd & " -d:zlibGit -d:zlibSetVer=v1.2.10" & zrcmd, zexp & "1.2.10", 0) diff --git a/tests/lzma.nim b/tests/lzma.nim index 422f94d..32f9fa0 100644 --- a/tests/lzma.nim +++ b/tests/lzma.nim @@ -8,6 +8,11 @@ const static: cDebug() +when defined(envTest): + setDefines(@["lzmaStd"]) +elif defined(envTestStatic): + setDefines(@["lzmaStd", "lzmaStatic"]) + getHeader( "lzma.h", giturl = "https://github.com/xz-mirror/xz", @@ -33,9 +38,9 @@ cOverride: lzma_block = object lzma_index_iter = object -when not defined(lzmaStatic): +when not lzmaStatic: cImport(lzmaPath, recurse = true, dynlib = "lzmaLPath") else: cImport(lzmaPath, recurse = true) -echo "liblzma version = " & $lzma_version_string() \ No newline at end of file +echo "liblzma version = " & $lzma_version_string() diff --git a/tests/zlib.nim b/tests/zlib.nim index 6da7cdc..371d41f 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -19,6 +19,11 @@ proc zlibPreBuild(outdir, path: string) = # Fix static lib name on Windows setCmakeLibName(outdir, "zlibstatic", prefix = "lib", oname = "zlib", suffix = ".a") +when defined(envTest): + setDefines(@["zlibGit"]) +elif defined(envTestStatic): + setDefines(@["zlibGit", "zlibStatic"]) + getHeader( "zlib.h", giturl = "https://github.com/madler/zlib", @@ -56,11 +61,11 @@ when defined(posix): static: cSkipSymbol(@["u_int8_t", "u_int16_t", "u_int32_t", "u_int64_t"]) -when defined(zlibGit) or defined(zlibDL): +when zlibGit or zlibDL: when dirExists(baseDir / "buildcache"): cIncludeDir(baseDir / "buildcache") -when not defined(zlibStatic): +when not zlibStatic: cImport(zlibPath, recurse = true, dynlib = "zlibLPath") else: cImport(zlibPath, recurse = true) From 86aea481ac3433ddeae7832fc9749f99e12d5f7b Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 1 Oct 2019 11:33:24 -0500 Subject: [PATCH 17/21] Add retries for file ops --- nimterop/build.nim | 15 ++++++++++----- tests/lzma.nim | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 24ae87c..e36324a 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -9,7 +9,7 @@ proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = if not noQuote: result = result.quoteShell -proc execAction*(cmd: string, nostderr=false): string = +proc execAction*(cmd: string, retry = 0, nostderr = false): string = ## Execute an external command - supported at compile time ## ## Checks if command exits successfully before returning. If not, an @@ -29,7 +29,12 @@ proc execAction*(cmd: string, nostderr=false): string = else: let opt = if nostderr: {poUsePath} else: {poStdErrToStdOut, poUsePath} (result, ret) = execCmdEx(ccmd, opt) - doAssert ret == 0, "Command failed: " & $(ret, nostderr) & "\nccmd: " & ccmd & "\nresult:\n" & result + if ret != 0: + if retry > 0: + sleep(500) + result = execAction(cmd, retry = retry - 1) + else: + doAssert true, "Command failed: " & $(ret, nostderr) & "\nccmd: " & ccmd & "\nresult:\n" & result proc findExe*(exe: string): string = ## Find the specified executable using the `which`/`where` command - supported @@ -54,7 +59,7 @@ proc mkDir*(dir: string) = if not dirExists(dir): let flag = when not defined(Windows): "-p" else: "" - discard execAction(&"mkdir {flag} {dir.sanitizePath}") + discard execAction(&"mkdir {flag} {dir.sanitizePath}", retry = 2) proc cpFile*(source, dest: string, move=false) = ## Copy a file from `source` to `dest` at compile time @@ -73,7 +78,7 @@ proc cpFile*(source, dest: string, move=false) = else: "cp -f" - discard execAction(&"{cmd} {source.sanitizePath} {dest.sanitizePath}") + discard execAction(&"{cmd} {source.sanitizePath} {dest.sanitizePath}", retry = 2) proc mvFile*(source, dest: string) = ## Move a file from `source` to `dest` at compile time @@ -92,7 +97,7 @@ proc rmFile*(source: string, dir = false) = else: "rm -rf" - discard execAction(&"{cmd} {source.sanitizePath}") + discard execAction(&"{cmd} {source.sanitizePath}", retry = 2) proc rmDir*(source: string) = ## Remove a directory or pattern at compile time diff --git a/tests/lzma.nim b/tests/lzma.nim index 32f9fa0..2368b55 100644 --- a/tests/lzma.nim +++ b/tests/lzma.nim @@ -3,7 +3,7 @@ import os, strutils import nimterop/[build, cimport] const - baseDir = currentSourcePath.parentDir()/"build/liblzma" + baseDir = currentSourcePath.parentDir()/"build"/"liblzma" static: cDebug() From 4994e56a3f625995db43f3c31502311192a4c125 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 1 Oct 2019 12:38:22 -0500 Subject: [PATCH 18/21] Fix sleep --- nimterop/build.nim | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index e36324a..a3dd95f 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1,6 +1,6 @@ import macros, osproc, regex, strformat, strutils, tables -import os except findExe +import os except findExe, sleep import "."/[compat] @@ -9,6 +9,17 @@ proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = if not noQuote: result = result.quoteShell +proc sleep*(milsecs: int) = + ## Sleep at compile time + let + cmd = + when defined(windows): + "cmd /c timeout " + else: + "sleep " + + (oup, ret) = gorgeEx(cmd & $(milsecs / 1000)) + proc execAction*(cmd: string, retry = 0, nostderr = false): string = ## Execute an external command - supported at compile time ## From 23d2b869553f41b4676607cfd32178b24288259c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 3 Oct 2019 10:54:51 -0500 Subject: [PATCH 19/21] Fix wchar_t for nimdoc --- nimterop/types.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/types.nim b/nimterop/types.nim index 709aa79..8492f00 100644 --- a/nimterop/types.nim +++ b/nimterop/types.nim @@ -20,7 +20,7 @@ else: import std/time_t as time_t_temp type time_t* = time_t_temp.Time - when defined(c): + when defined(c) or defined(nimdoc): # http://www.cplusplus.com/reference/cwchar/wchar_t/ # In C++, wchar_t is a distinct fundamental type (and thus it is # not defined in nor any other header). From dee898dc51c4fddebd16f27af88443edaa46a95a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 3 Oct 2019 13:45:56 -0500 Subject: [PATCH 20/21] Move docs generation to module --- nimterop.nimble | 56 ++++++++++------------------------------------- nimterop/docs.nim | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 45 deletions(-) create mode 100644 nimterop/docs.nim diff --git a/nimterop.nimble b/nimterop.nimble index 565c3e4..5825f5a 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -14,7 +14,7 @@ installFiles = @["config.nims"] # Dependencies requires "nim >= 0.19.2", "regex >= 0.10.0", "cligen >= 0.9.17" -import strformat +import nimterop/docs proc execCmd(cmd: string) = echo "execCmd:" & cmd @@ -24,61 +24,27 @@ proc execTest(test: string) = execCmd "nim c -r " & test execCmd "nim cpp -r " & test -proc tsoloud() = - execTest "tests/tsoloud.nim" - task buildToast, "build toast": - # If need to manually rebuild (automatically built on 1st need) - execCmd(&"nim c -d:release nimterop/toast.nim") + execCmd("nim c -d:danger nimterop/toast.nim") + +task docs, "Generate docs": + buildDocs(@["nimterop/all.nim"], "build/htmldocs") + +task test, "Test": + buildToastTask() -proc testAll() = execTest "tests/tnimterop_c.nim" execCmd "nim cpp -r tests/tnimterop_cpp.nim" execTest "tests/tpcre.nim" - # platform specific tests + # Platform specific tests when defined(Windows): execTest "tests/tmath.nim" if defined(OSX) or defined(Windows) or not existsEnv("TRAVIS"): - tsoloud() # requires some libraries on linux, need them installed in TRAVIS + execTest "tests/tsoloud.nim" # getHeader tests withDir("tests"): execCmd("nim e getheader.nims") -const htmldocsDir = "build/htmldocs" - -when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9): - import os - proc getNimRootDir(): string = - #[ - hack, but works - alternatively (but more complex), use (from a nim file, not nims otherwise - you get Error: ambiguous call; both system.fileExists): - import "$nim/testament/lib/stdtest/specialpaths.nim" - nimRootDir - ]# - fmt"{currentSourcePath}".parentDir.parentDir.parentDir - -proc runNimDoc() = - execCmd &"nim doc -o:{htmldocsDir} --project --index:on nimterop/all.nim" - execCmd &"nim buildIndex -o:{htmldocsDir}/theindex.html {htmldocsDir}" - when declared(getNimRootDir): - #[ - this enables doc search, works at least locally with: - cd {htmldocsDir} && python -m SimpleHTTPServer 9009 - ]# - execCmd &"nim js -o:{htmldocsDir}/dochack.js {getNimRootDir()}/tools/dochack/dochack.nim" - -task test, "Test": - buildToastTask() - testAll() - runNimDoc() - -task docs, "Generate docs": - runNimDoc() - -task docsPublish, "Generate and publish docs": - # Uses: pip install ghp-import - runNimDoc() - execCmd &"ghp-import --no-jekyll -fp {htmldocsDir}" + docsTask() diff --git a/nimterop/docs.nim b/nimterop/docs.nim new file mode 100644 index 0000000..d7cb1a7 --- /dev/null +++ b/nimterop/docs.nim @@ -0,0 +1,42 @@ +import macros, strformat + +when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9): + from os import parentDir + proc getNimRootDir(): string = + #[ + hack, but works + alternatively (but more complex), use (from a nim file, not nims otherwise + you get Error: ambiguous call; both system.fileExists): + import "$nim/testament/lib/stdtest/specialpaths.nim" + nimRootDir + ]# + fmt"{currentSourcePath}".parentDir.parentDir.parentDir + +proc buildDocs*(files: seq[string], path: string, baseDir = getProjectPath() & "/") = + ## Generate docs for all specified nim `files` to the specified `path` + ## + ## `baseDir` is the project path by default and `files` and `path` are relative + ## to that directory. Set to "" if using absolute paths. + ## + ## Use the `--publish` flag with nimble to publish docs contained in + ## `path` to Github in the `gh-pages` branch. This requires the ghp-import + ## package for Python: `pip install ghp-import` + ## + ## WARNING: `--publish` will destroy any existing content in this branch. + let + path = baseDir & path + for file in files: + echo gorge(&"nim doc -o:{path} --project --index:on {baseDir & file}") + + echo gorge(&"nim buildIndex -o:{path}/theindex.html {path}") + when declared(getNimRootDir): + #[ + this enables doc search, works at least locally with: + cd {path} && python -m SimpleHTTPServer 9009 + ]# + echo gorge(&"nim js -o:{path}/dochack.js {getNimRootDir()}/tools/dochack/dochack.nim") + + for i in 0 .. paramCount(): + if paramStr(i) == "--publish": + echo gorge(&"ghp-import --no-jekyll -fp {path}") + break From b12d77c98d6cb4f6a8ad29c47e8964831cb1990d Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 3 Oct 2019 16:29:01 -0500 Subject: [PATCH 21/21] Fix empty project path --- nimterop/docs.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nimterop/docs.nim b/nimterop/docs.nim index d7cb1a7..0b46967 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -24,6 +24,11 @@ proc buildDocs*(files: seq[string], path: string, baseDir = getProjectPath() & " ## ## WARNING: `--publish` will destroy any existing content in this branch. let + baseDir = + if baseDir == "/": + getCurrentDir() & "/" + else: + baseDir path = baseDir & path for file in files: echo gorge(&"nim doc -o:{path} --project --index:on {baseDir & file}")