From e46d3b2332d439cda073a34947cdaa803926e948 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 00:19:50 -0500 Subject: [PATCH 01/96] Implement #7 - [n.sourcefile] section, add nimclipboard --- README.md | 6 ++++++ appveyor.yml | 2 ++ src/nimgen/runcfg.nim | 13 +++++++++---- tests/nimgentest.nims | 10 ++++------ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 8b4be53..1d7b574 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ To see examples of nimgen in action check out the following wrappers:- * git sparse checkout * [nimbigwig](https://github.com/genotrance/nimbigwig) - libbigWig wrapper: [docs](http://nimgen.genotrance.com/nimbigwig) * git checkout + * [nimclipboard](https://github.com/genotrance/nimclipboard) - libclipboard wrapper: [docs](http://nimgen.genotrance.com/nimclipboard) + * git checkout * [nimfuzz](https://github.com/genotrance/nimfuzz) - fts_fuzzy_match wrapper: [docs](http://nimgen.genotrance.com/nimfuzz) * download header file * [nimkerberos](https://github.com/genotrance/nimkerberos) - WinKerberos wrapper: [docs](http://nimgen.genotrance.com/nimkerberos) @@ -134,6 +136,10 @@ File wildcards such as *.nim, ssl*.h, etc. can be used to perform tasks across a ```wildcard``` = pattern to match against. All keys following the wildcard declaration will apply to matched files +_[n.sourcefile]_ + +This section allows selection of multiple sourcefiles without requiring a detailed section for each file. Each specific file can be listed one line at a time and file wildcards can be used to include multiple source files. E.g. `$output/include/*/h`. `[n.wildcard]` definitions can be used to perform common operations on these source files if required. + _[sourcefile]_ The following keys apply to library source code and help with generating the .nim files. -win, -lin and -osx can be used for OS specific tasks. E.g. dynlib-win, pragma-win diff --git a/appveyor.yml b/appveyor.yml index a54b7ce..bcadda0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -61,6 +61,8 @@ for: on_finish: - 7z a -r buildlogs-win.zip %USERPROFILE%\.nimble\pkgs - appveyor PushArtifact buildlogs-win.zip + - 7z a -r nimgen-docs.zip %BASE_DIR%\nimgen\web + - appveyor PushArtifact nimgen-docs.zip cache: - c:\projects\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 96e067e..cc48c01 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -19,15 +19,15 @@ proc getKey(ukey: string): tuple[key: string, val: bool] = return (kv[0], false) -proc runFile*(file: string, cfgin: OrderedTableRef) = +proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, string]()) = var cfg = cfgin sfile = search(file) for pattern in gWildcards.keys(): - var match: RegexMatch + var m: RegexMatch let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") - if file.find(toPattern(pat), match): + if file.find(toPattern(pat), m): echo "Appending " & file & " " & pattern for key in gWildcards[pattern].keys(): cfg[key & "." & pattern] = gWildcards[pattern][key] @@ -224,7 +224,12 @@ proc runCfg*(cfg: string) = "n.prepare", "n.wildcard", "n.post"]: continue - runFile(file, gConfig[file]) + if file == "n.sourcefile": + for pattern in gConfig["n.sourcefile"].keys(): + for file in walkFiles(pattern.addEnv): + runFile(file) + else: + runFile(file, gConfig[file]) if gConfig.hasKey("n.post"): for post in gConfig["n.post"].keys(): diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index f5136c6..9c3e2d7 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -1,11 +1,9 @@ -import distros -import ospaths -import strutils +import distros, ospaths, strutils var - full = false - comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimfuzz", - "nimpcre", "nimrax", "nimssl", "nimssh2"] + full = true + comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", + "nimfuzz", "nimpcre", "nimrax", "nimssl", "nimssh2"] if detectOs(Windows): comps.add("nimkerberos") From 1a2ef870873f134e0df767d729ba89ecd7e1f01c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 01:37:50 -0500 Subject: [PATCH 02/96] Skip files already done --- src/nimgen/c2nim.nim | 8 ++------ src/nimgen/file.nim | 6 +++--- src/nimgen/runcfg.nim | 8 +++++++- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 07b15e0..28a3b77 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -7,14 +7,10 @@ template relativePath(path: untyped): untyped = proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = var file = search(fl) - if file == "": + if file.len() == 0: return - if file in gDoneRecursive: - return - - echo "Processing $# => $#" % [file, outfile] - gDoneRecursive.add(file) + echo " Generating " & outfile # Remove static inline function bodies removeStatic(file) diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index 1a316bf..b46f7be 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -8,7 +8,7 @@ import globals proc getNimout*(file: string, rename=true): string = result = file.splitFile().name.replace(re"[\-\.]", "_") & ".nim" if gOutput != "": - result = gOutput/result + result = gOutput & "/" & result if not rename: return @@ -35,7 +35,7 @@ proc search*(file: string): string = elif not fileExists(result) and not dirExists(result): var found = false for inc in gIncludes: - result = inc/file + result = inc & "/" & file if fileExists(result) or dirExists(result): found = true break @@ -69,7 +69,7 @@ proc rename*(file: string, renfile: string) = final = final.replace(srch, repl) newname = newname.replace(matches[0], final) - gRenames[file] = gOutput/newname + gRenames[file] = gOutput & "/" & newname # ### # Actions diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index cc48c01..a702e47 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -24,11 +24,17 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str cfg = cfgin sfile = search(file) + if sfile.len() == 0 or sfile in gDoneRecursive: + return + + echo "Processing " & sfile + gDoneRecursive.add(sfile) + for pattern in gWildcards.keys(): var m: RegexMatch let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") if file.find(toPattern(pat), m): - echo "Appending " & file & " " & pattern + echo " Appending keys for wildcard " & pattern for key in gWildcards[pattern].keys(): cfg[key & "." & pattern] = gWildcards[pattern][key] From 9e0eb256857b9dce6895b3dcb0e3f6a9dd2b5def Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 02:16:01 -0500 Subject: [PATCH 03/96] Implement #10 - per file reset --- README.md | 2 ++ src/nimgen/external.nim | 8 +++----- src/nimgen/runcfg.nim | 17 +++++++++++++---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1d7b574..df949b8 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,8 @@ The following keys apply to library source code and help with generating the .ni ```noprocess``` = do not process this source file with c2nim [default: false] - this is useful if a file only needs to be manipulated +```reset``` = reset the file back to original state after all processing [default: false] + Multiple entries for the all following keys are possible by appending any .string to the key. E.g. dynlib.win, compile.dir ```compile``` = file or dir of files of source code to {.compile.} into generated .nim diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index f642cfb..2c4a12e 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -60,15 +60,13 @@ proc gitReset*() = discard execProc("git reset --hard HEAD") -proc gitCheckout*(filename: string) {.used.} = - echo "Resetting file: $#" % [filename] +proc gitCheckout*(file: string) = + echo " Resetting " & file setCurrentDir(gOutput) defer: setCurrentDir(gProjectDir) - let adjustedFile = filename.replace(gOutput & $DirSep, "") - - discard execProc("git checkout $#" % [adjustedFile]) + discard execProc("git checkout $#" % file.replace(gOutput & "/", "")) proc gitRemotePull*(url: string, pull=true) = if dirExists(gOutput/".git"): diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index a702e47..a641150 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -24,11 +24,12 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str cfg = cfgin sfile = search(file) - if sfile.len() == 0 or sfile in gDoneRecursive: + if sfile in gDoneRecursive: return - echo "Processing " & sfile - gDoneRecursive.add(sfile) + if sfile.len() != 0: + echo "Processing " & sfile + gDoneRecursive.add(sfile) for pattern in gWildcards.keys(): var m: RegexMatch @@ -51,6 +52,7 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str let (action, val) = getKey(act) if val == true: if action == "create": + echo "Creating " & file createDir(file.splitPath().head) writeFile(file, cfg[act]) elif action in @["prepend", "append", "replace", "comment", @@ -87,7 +89,9 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str srch = act if file.splitFile().ext != ".nim": - var noprocess = false + var + noprocess = false + reset = false for act in cfg.keys(): let (action, val) = getKey(act) @@ -105,6 +109,8 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str c2nimConfig.defines = true elif action == "noprocess": noprocess = true + elif action == "reset": + reset = true elif action == "flags": c2nimConfig.flags = cfg[act] elif action == "ppflags": @@ -140,6 +146,9 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str if incout.len() != 0: prepend(outfile, incout) + if reset: + gitCheckout(sfile) + proc runCfg*(cfg: string) = if not fileExists(cfg): echo "Config doesn't exist: " & cfg From 04cd7055cf3398fd13d39bb04a549791b82b12ab Mon Sep 17 00:00:00 2001 From: data-man Date: Fri, 13 Jul 2018 12:52:11 +0300 Subject: [PATCH 04/96] Use strutils.split --- src/nimgen/external.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 2c4a12e..79426f6 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -147,7 +147,7 @@ proc runCtags*(file: string): string = fdata = "" for line in fps.splitLines(): - var spl = line.split(re"\t") + var spl = line.split('\t') if spl.len() > 4: if spl[0] != "main" and spl[3] != "member": var fn = "" From 2e332b29af2ce5e30f740aff47011744803e52f9 Mon Sep 17 00:00:00 2001 From: data-man Date: Fri, 13 Jul 2018 13:33:43 +0300 Subject: [PATCH 05/96] Use multiReplace in some places --- src/nimgen/c2nim.nim | 2 +- src/nimgen/file.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 28a3b77..4d721b4 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -48,7 +48,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = for prag in c2nimConfig.pragma: outpragma &= "{." & prag & ".}\n" - let fname = file.splitFile().name.replace(re"[\.\-]", "_") + let fname = file.splitFile().name.multiReplace([(".", "_"), ("-", "_")]) if c2nimConfig.dynlib.len() != 0: let diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index b46f7be..d78c692 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -6,7 +6,7 @@ import globals # File loction proc getNimout*(file: string, rename=true): string = - result = file.splitFile().name.replace(re"[\-\.]", "_") & ".nim" + result = file.splitFile().name.multiReplace([("-", "_"), (".", "_")]) & ".nim" if gOutput != "": result = gOutput & "/" & result From 2cfad92753025f5be7323af7e4fa58e3633e2eff Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 12:51:27 -0500 Subject: [PATCH 06/96] Fix file creation issue --- src/nimgen/runcfg.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index a641150..32146b8 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -55,6 +55,10 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str echo "Creating " & file createDir(file.splitPath().head) writeFile(file, cfg[act]) + if file in gExcludes: + gExcludes.delete(gExcludes.find(file)) + sfile = search(file) + gDoneRecursive.add(sfile) elif action in @["prepend", "append", "replace", "comment", "rename", "compile", "dynlib", "pragma", "pipe"] and sfile != "": From d1aaf70e25d00e33d6978a8c225defff25efb3b4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 13:12:46 -0500 Subject: [PATCH 07/96] Cache Linux binary --- appveyor.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index bcadda0..f0d5fac 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -65,10 +65,10 @@ for: - appveyor PushArtifact nimgen-docs.zip cache: - - c:\projects\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - - c:\projects\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz - - c:\projects\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - - c:\projects\nim-0.18.0_x32.zip + - %BASE_DIR%\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z + - %BASE_DIR%\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz + - %BASE_DIR%\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz + - %BASE_DIR%\nim-0.18.0_x32.zip - matrix: @@ -79,11 +79,12 @@ for: NIM_URL: https://nim-lang.org/download/nim-0.18.0.tar.xz NIM_ARCHIVE: nim-0.18.0.tar.xz NIM_VERSION: nim-0.18.0 + BASE_DIR: /home/appveyor/projects install: - sudo apt -qq update - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev - - cd ~/projects + - cd $BASE_DIR - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - tar xJf $NIM_ARCHIVE - cd $NIM_VERSION @@ -91,15 +92,15 @@ for: - bin/nim c -d:release koch - ./koch boot -d:release - ./koch nimble -d:release - - export PATH=~/projects/$NIM_VERSION/bin:~/.nimble/bin:$PATH - - cd ~/projects/nimgen + - export PATH=$BASE_DIR/$NIM_VERSION/bin:~/.nimble/bin:$PATH + - cd $BASE_DIR/nimgen on_finish: - zip -r -q buildlogs-lin.zip ~/.nimble/pkgs - appveyor PushArtifact buildlogs-lin.zip cache: - - ~/projects/nim-0.18.0.tar.xz + - $BASE_DIR/nim-0.18.0.tar.xz build_script: - nimble install -y From 0eaaf2e1fe94d6184d5f1473715100b9d7f40086 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 13:36:15 -0500 Subject: [PATCH 08/96] Fix base dir --- appveyor.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index f0d5fac..6b5f682 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -65,10 +65,10 @@ for: - appveyor PushArtifact nimgen-docs.zip cache: - - %BASE_DIR%\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - - %BASE_DIR%\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz - - %BASE_DIR%\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - - %BASE_DIR%\nim-0.18.0_x32.zip + - c:\projects\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z + - c:\projects\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz + - c:\projects\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz + - c:\projects\nim-0.18.0_x32.zip - matrix: @@ -100,7 +100,7 @@ for: - appveyor PushArtifact buildlogs-lin.zip cache: - - $BASE_DIR/nim-0.18.0.tar.xz + - /home/appveyor/projects/nim-0.18.0.tar.xz build_script: - nimble install -y From 29d694d630090e410a65f396adf5e91ced619d29 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 20:06:09 -0500 Subject: [PATCH 09/96] Implement #11 - comma separated for OS specific tasks --- README.md | 2 +- src/nimgen/runcfg.nim | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index df949b8..5a1945d 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ List of all directories or files to exclude from all parsing. If an entry here m _[n.prepare]_ -The following keys can be used to prepare dependencies such as downloading ZIP files, cloning Git repositories, etc. Multiple entries are possible by appending any .string to the key. E.g. download.file1. -win, -lin and -osx can be used for OS specific tasks. E.g. download-win +The following keys can be used to prepare dependencies such as downloading ZIP files, cloning Git repositories, etc. Multiple entries are possible by appending any .string to the key. E.g. download.file1. -win, -lin and -mac/osx can be used for OS specific tasks. E.g. download-win, execute-lin,mac.unique1 ```download``` = url to download to the output directory. ZIP files are automatically extracted. Files are not redownloaded if already present but re-extracted diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 32146b8..105cede 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -11,12 +11,15 @@ proc getKey(ukey: string): tuple[key: string, val: bool] = if kv.len() == 1: kv.add("") - if (kv[1] == "") or - (kv[1] == "win" and defined(Windows)) or - (kv[1] == "lin" and defined(Linux)) or - (kv[1] == "osx" and defined(MacOSX)): + if kv[1] == "": return (kv[0], true) + for ostyp in kv[1].split(","): + if (ostyp == "win" and defined(Windows)) or + (ostyp == "lin" and defined(Linux)) or + ((ostyp == "osx" or ostyp == "mac") and defined(MacOSX)): + return (kv[0], true) + return (kv[0], false) proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, string]()) = From de4cb3f7831101c91e5d450e6a44388da878a8a1 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 14 Jul 2018 10:24:53 +0900 Subject: [PATCH 10/96] Add tests for fileops --- nimgen.nimble | 1 + tests/rununittests.nim | 8 + tests/unittests/common.nim | 5 + .../data/teststaticexpectedfrontbraces.h | 675 ++++++++++++++++++ .../data/teststaticexpectednewlinebraces.h | 68 ++ tests/unittests/data/teststaticfrontbraces.h | 675 ++++++++++++++++++ .../unittests/data/teststaticnewlinebraces.h | 71 ++ .../data/teststaticnewlinebracesreadded.h | 68 ++ tests/unittests/nim.cfg | 1 + tests/unittests/testfileops.nim | 211 ++++++ 10 files changed, 1783 insertions(+) create mode 100644 tests/rununittests.nim create mode 100644 tests/unittests/common.nim create mode 100644 tests/unittests/data/teststaticexpectedfrontbraces.h create mode 100644 tests/unittests/data/teststaticexpectednewlinebraces.h create mode 100644 tests/unittests/data/teststaticfrontbraces.h create mode 100644 tests/unittests/data/teststaticnewlinebraces.h create mode 100644 tests/unittests/data/teststaticnewlinebracesreadded.h create mode 100644 tests/unittests/nim.cfg create mode 100644 tests/unittests/testfileops.nim diff --git a/nimgen.nimble b/nimgen.nimble index 59de8ed..3e65545 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -14,4 +14,5 @@ skipDirs = @["nimgen", "tests"] requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex >= 0.7.1" task test, "Test nimgen": + exec "nim c -r tests/rununittests.nim" exec "nim e tests/nimgentest.nims" diff --git a/tests/rununittests.nim b/tests/rununittests.nim new file mode 100644 index 0000000..a92ff0a --- /dev/null +++ b/tests/rununittests.nim @@ -0,0 +1,8 @@ +import os, osproc, strutils + +proc main() = + for file in walkFiles(currentSourcePath().splitPath().head / "unittests/*.nim"): + let (path, fname, ext) = file.splitFile() + if fname.startswith("test"): + discard execCmd "nim c -r " & file +main() diff --git a/tests/unittests/common.nim b/tests/unittests/common.nim new file mode 100644 index 0000000..1769b01 --- /dev/null +++ b/tests/unittests/common.nim @@ -0,0 +1,5 @@ +import unittest + +proc checkFile*(filepath, expected: string) = + let result = readFile(filepath) + check result == expected diff --git a/tests/unittests/data/teststaticexpectedfrontbraces.h b/tests/unittests/data/teststaticexpectedfrontbraces.h new file mode 100644 index 0000000..d8c6a0d --- /dev/null +++ b/tests/unittests/data/teststaticexpectedfrontbraces.h @@ -0,0 +1,675 @@ +/** + * @file ipc.h + * @brief Inter-process communication handling + * @author plutoo + * @copyright libnx Authors + */ +#pragma once +#include "../result.h" +#include "../arm/tls.h" +#include "../kernel/svc.h" + +/// IPC input header magic +#define SFCI_MAGIC 0x49434653 +/// IPC output header magic +#define SFCO_MAGIC 0x4f434653 + +/// IPC invalid object ID +#define IPC_INVALID_OBJECT_ID UINT32_MAX + +///@name IPC request building +///@{ + +/// IPC command (request) structure. +#define IPC_MAX_BUFFERS 8 +#define IPC_MAX_OBJECTS 8 + +typedef enum { + BufferType_Normal=0, ///< Regular buffer. + BufferType_Type1=1, ///< Allows ProcessMemory and shared TransferMemory. + BufferType_Invalid=2, + BufferType_Type3=3 ///< Same as Type1 except remote process is not allowed to use device-mapping. +} BufferType; + +typedef enum { + BufferDirection_Send=0, + BufferDirection_Recv=1, + BufferDirection_Exch=2, +} BufferDirection; + +typedef enum { + IpcCommandType_Invalid = 0, + IpcCommandType_LegacyRequest = 1, + IpcCommandType_Close = 2, + IpcCommandType_LegacyControl = 3, + IpcCommandType_Request = 4, + IpcCommandType_Control = 5, + IpcCommandType_RequestWithContext = 6, + IpcCommandType_ControlWithContext = 7, +} IpcCommandType; + +typedef enum { + DomainMessageType_Invalid = 0, + DomainMessageType_SendMessage = 1, + DomainMessageType_Close = 2, +} DomainMessageType; + +/// IPC domain message header. +typedef struct { + u8 Type; + u8 NumObjectIds; + u16 Length; + u32 ThisObjectId; + u32 Pad[2]; +} DomainMessageHeader; + +typedef struct { + size_t NumSend; // A + size_t NumRecv; // B + size_t NumExch; // W + const void* Buffers[IPC_MAX_BUFFERS]; + size_t BufferSizes[IPC_MAX_BUFFERS]; + BufferType BufferTypes[IPC_MAX_BUFFERS]; + + size_t NumStaticIn; // X + size_t NumStaticOut; // C + const void* Statics[IPC_MAX_BUFFERS]; + size_t StaticSizes[IPC_MAX_BUFFERS]; + u8 StaticIndices[IPC_MAX_BUFFERS]; + + bool SendPid; + size_t NumHandlesCopy; + size_t NumHandlesMove; + Handle Handles[IPC_MAX_OBJECTS]; + + size_t NumObjectIds; + u32 ObjectIds[IPC_MAX_OBJECTS]; +} IpcCommand; + +/** + * @brief Initializes an IPC command structure. + * @param cmd IPC command structure. + */ +static inline void ipcInitialize(IpcCommand* cmd);//{ +// *cmd = (IpcCommand){0}; +//} + +/// IPC buffer descriptor. +typedef struct { + u32 Size; ///< Size of the buffer. + u32 Addr; ///< Lower 32-bits of the address of the buffer + u32 Packed; ///< Packed data (including higher bits of the address) +} IpcBufferDescriptor; + +/// IPC static send-buffer descriptor. +typedef struct { + u32 Packed; ///< Packed data (including higher bits of the address) + u32 Addr; ///< Lower 32-bits of the address +} IpcStaticSendDescriptor; + +/// IPC static receive-buffer descriptor. +typedef struct { + u32 Addr; ///< Lower 32-bits of the address of the buffer + u32 Packed; ///< Packed data (including higher bits of the address) +} IpcStaticRecvDescriptor; + +/** + * @brief Adds a buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param type Buffer type. + */ +static inline void ipcAddSendBuffer(IpcCommand* cmd, const void* buffer, size_t size, BufferType type);//{ +// size_t off = cmd->NumSend; +// cmd->Buffers[off] = buffer; +// cmd->BufferSizes[off] = size; +// cmd->BufferTypes[off] = type; +// cmd->NumSend++; +//} + +/** + * @brief Adds a receive-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param type Buffer type. + */ +static inline void ipcAddRecvBuffer(IpcCommand* cmd, void* buffer, size_t size, BufferType type);//{ +// size_t off = cmd->NumSend + cmd->NumRecv; +// cmd->Buffers[off] = buffer; +// cmd->BufferSizes[off] = size; +// cmd->BufferTypes[off] = type; +// cmd->NumRecv++; +//} + +/** + * @brief Adds an exchange-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param type Buffer type. + */ +static inline void ipcAddExchBuffer(IpcCommand* cmd, void* buffer, size_t size, BufferType type);//{ +// size_t off = cmd->NumSend + cmd->NumRecv + cmd->NumExch; +// cmd->Buffers[off] = buffer; +// cmd->BufferSizes[off] = size; +// cmd->BufferTypes[off] = type; +// cmd->NumExch++; +//} + +/** + * @brief Adds a static-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddSendStatic(IpcCommand* cmd, const void* buffer, size_t size, u8 index);//{ +// size_t off = cmd->NumStaticIn; +// cmd->Statics[off] = buffer; +// cmd->StaticSizes[off] = size; +// cmd->StaticIndices[off] = index; +// cmd->NumStaticIn++; +//} + +/** + * @brief Adds a static-receive-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddRecvStatic(IpcCommand* cmd, void* buffer, size_t size, u8 index);//{ +// size_t off = cmd->NumStaticIn + cmd->NumStaticOut; +// cmd->Statics[off] = buffer; +// cmd->StaticSizes[off] = size; +// cmd->StaticIndices[off] = index; +// cmd->NumStaticOut++; +//} + +/** + * @brief Adds a smart-buffer (buffer + static-buffer pair) to an IPC command structure. + * @param cmd IPC command structure. + * @param ipc_buffer_size IPC buffer size. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddSendSmart(IpcCommand* cmd, size_t ipc_buffer_size, const void* buffer, size_t size, u8 index);//{ +// if (ipc_buffer_size != 0 && size <= ipc_buffer_size) { +// ipcAddSendBuffer(cmd, NULL, 0, BufferType_Normal); +// ipcAddSendStatic(cmd, buffer, size, index); +// } else { +// ipcAddSendBuffer(cmd, buffer, size, BufferType_Normal); +// ipcAddSendStatic(cmd, NULL, 0, index); +// } +//} + +/** + * @brief Adds a smart-receive-buffer (buffer + static-receive-buffer pair) to an IPC command structure. + * @param cmd IPC command structure. + * @param ipc_buffer_size IPC buffer size. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddRecvSmart(IpcCommand* cmd, size_t ipc_buffer_size, void* buffer, size_t size, u8 index);//{ +// if (ipc_buffer_size != 0 && size <= ipc_buffer_size) { +// ipcAddRecvBuffer(cmd, NULL, 0, BufferType_Normal); +// ipcAddRecvStatic(cmd, buffer, size, index); +// } else { +// ipcAddRecvBuffer(cmd, buffer, size, BufferType_Normal); +// ipcAddRecvStatic(cmd, NULL, 0, index); +// } +//} + +/** + * @brief Tags an IPC command structure to send the PID. + * @param cmd IPC command structure. + */ +static inline void ipcSendPid(IpcCommand* cmd);//{ +// cmd->SendPid = true; +//} + +/** + * @brief Adds a copy-handle to be sent through an IPC command structure. + * @param cmd IPC command structure. + * @param h Handle to send. + * @remark The receiving process gets a copy of the handle. + */ +static inline void ipcSendHandleCopy(IpcCommand* cmd, Handle h);//{ +// cmd->Handles[cmd->NumHandlesCopy++] = h; +//} + +/** + * @brief Adds a move-handle to be sent through an IPC command structure. + * @param cmd IPC command structure. + * @param h Handle to send. + * @remark The sending process loses ownership of the handle, which is transferred to the receiving process. + */ +static inline void ipcSendHandleMove(IpcCommand* cmd, Handle h);//{ +// cmd->Handles[cmd->NumHandlesCopy + cmd->NumHandlesMove++] = h; +//} + +/** + * @brief Prepares the header of an IPC command structure. + * @param cmd IPC command structure. + * @param sizeof_raw Size in bytes of the raw data structure to embed inside the IPC request + * @return Pointer to the raw embedded data structure in the request, ready to be filled out. + */ +static inline void* ipcPrepareHeader(IpcCommand* cmd, size_t sizeof_raw);//{ +// u32* buf = (u32*)armGetTls(); +// size_t i; +// *buf++ = IpcCommandType_Request | (cmd->NumStaticIn << 16) | (cmd->NumSend << 20) | (cmd->NumRecv << 24) | (cmd->NumExch << 28); +// +// u32* fill_in_size_later = buf; +// +// if (cmd->NumStaticOut > 0) { +// *buf = (cmd->NumStaticOut + 2) << 10; +// } +// else { +// *buf = 0; +// } +// +// if (cmd->SendPid || cmd->NumHandlesCopy > 0 || cmd->NumHandlesMove > 0) { +// *buf++ |= 0x80000000; +// *buf++ = (!!cmd->SendPid) | (cmd->NumHandlesCopy << 1) | (cmd->NumHandlesMove << 5); +// +// if (cmd->SendPid) +// buf += 2; +// +// for (i=0; i<(cmd->NumHandlesCopy + cmd->NumHandlesMove); i++) +// *buf++ = cmd->Handles[i]; +// } +// else { +// buf++; +// } +// +// for (i=0; iNumStaticIn; i++, buf+=2) { +// IpcStaticSendDescriptor* desc = (IpcStaticSendDescriptor*) buf; +// +// uintptr_t ptr = (uintptr_t) cmd->Statics[i]; +// desc->Addr = ptr; +// desc->Packed = cmd->StaticIndices[i] | (cmd->StaticSizes[i] << 16) | +// (((ptr >> 32) & 15) << 12) | (((ptr >> 36) & 15) << 6); +// } +// +// for (i=0; i<(cmd->NumSend + cmd->NumRecv + cmd->NumExch); i++, buf+=3) { +// IpcBufferDescriptor* desc = (IpcBufferDescriptor*) buf; +// desc->Size = cmd->BufferSizes[i]; +// +// uintptr_t ptr = (uintptr_t) cmd->Buffers[i]; +// desc->Addr = ptr; +// desc->Packed = cmd->BufferTypes[i] | +// (((ptr >> 32) & 15) << 28) | ((ptr >> 36) << 2); +// } +// +// u32 padding = ((16 - (((uintptr_t) buf) & 15)) & 15) / 4; +// u32* raw = (u32*) (buf + padding); +// +// size_t raw_size = (sizeof_raw/4) + 4; +// buf += raw_size; +// +// u16* buf_u16 = (u16*) buf; +// +// for (i=0; iNumStaticOut; i++) { +// size_t off = cmd->NumStaticIn + i; +// size_t sz = (uintptr_t) cmd->StaticSizes[off]; +// +// buf_u16[i] = (sz > 0xFFFF) ? 0 : sz; +// } +// +// size_t u16s_size = ((2*cmd->NumStaticOut) + 3)/4; +// buf += u16s_size; +// raw_size += u16s_size; +// +// *fill_in_size_later |= raw_size; +// +// for (i=0; iNumStaticOut; i++, buf+=2) { +// IpcStaticRecvDescriptor* desc = (IpcStaticRecvDescriptor*) buf; +// size_t off = cmd->NumStaticIn + i; +// +// uintptr_t ptr = (uintptr_t) cmd->Statics[off]; +// desc->Addr = ptr; +// desc->Packed = (ptr >> 32) | (cmd->StaticSizes[off] << 16); +// } +// +// return (void*) raw; +//} + +/** + * @brief Dispatches an IPC request. + * @param session IPC session handle. + * @return Result code. + */ +static inline Result ipcDispatch(Handle session);//{ +// return svcSendSyncRequest(session); +//} + +///@} + +///@name IPC response parsing +///@{ + +/// IPC parsed command (response) structure. +typedef struct { + IpcCommandType CommandType; ///< Type of the command + + bool HasPid; ///< true if the 'Pid' field is filled out. + u64 Pid; ///< PID included in the response (only if HasPid is true) + + size_t NumHandles; ///< Number of handles copied. + Handle Handles[IPC_MAX_OBJECTS]; ///< Handles. + bool WasHandleCopied[IPC_MAX_OBJECTS]; ///< true if the handle was moved, false if it was copied. + + bool IsDomainMessage; ///< true if the the message is a Domain message. + DomainMessageType MessageType; ///< Type of the domain message. + u32 MessageLength; ///< Size of rawdata (for domain messages). + u32 ThisObjectId; ///< Object ID to call the command on (for domain messages). + size_t NumObjectIds; ///< Number of object IDs (for domain messages). + u32 ObjectIds[IPC_MAX_OBJECTS]; ///< Object IDs (for domain messages). + + size_t NumBuffers; ///< Number of buffers in the response. + void* Buffers[IPC_MAX_BUFFERS]; ///< Pointers to the buffers. + size_t BufferSizes[IPC_MAX_BUFFERS]; ///< Sizes of the buffers. + BufferType BufferTypes[IPC_MAX_BUFFERS]; ///< Types of the buffers. + BufferDirection BufferDirections[IPC_MAX_BUFFERS]; ///< Direction of each buffer. + + size_t NumStatics; ///< Number of statics in the response. + void* Statics[IPC_MAX_BUFFERS]; ///< Pointers to the statics. + size_t StaticSizes[IPC_MAX_BUFFERS]; ///< Sizes of the statics. + u8 StaticIndices[IPC_MAX_BUFFERS]; ///< Indices of the statics. + + size_t NumStaticsOut; ///< Number of output statics available in the response. + + void* Raw; ///< Pointer to the raw embedded data structure in the response. + void* RawWithoutPadding; ///< Pointer to the raw embedded data structure, without padding. + size_t RawSize; ///< Size of the raw embedded data. +} IpcParsedCommand; + +/** + * @brief Parse an IPC command response into an IPC parsed command structure. + * @param IPC parsed command structure to fill in. + * @return Result code. + */ +static inline Result ipcParse(IpcParsedCommand* r);//{ +// u32* buf = (u32*)armGetTls(); +// u32 ctrl0 = *buf++; +// u32 ctrl1 = *buf++; +// size_t i; +// +// r->IsDomainMessage = false; +// +// r->CommandType = (IpcCommandType) (ctrl0 & 0xffff); +// r->HasPid = false; +// r->RawSize = (ctrl1 & 0x1ff) * 4; +// r->NumHandles = 0; +// +// r->NumStaticsOut = (ctrl1 >> 10) & 15; +// if (r->NumStaticsOut >> 1) r->NumStaticsOut--; // Value 2 -> Single descriptor +// if (r->NumStaticsOut >> 1) r->NumStaticsOut--; // Value 3+ -> (Value - 2) descriptors +// +// if (ctrl1 & 0x80000000) { +// u32 ctrl2 = *buf++; +// +// if (ctrl2 & 1) { +// r->HasPid = true; +// r->Pid = *buf++; +// r->Pid |= ((u64)(*buf++)) << 32; +// } +// +// size_t num_handles_copy = ((ctrl2 >> 1) & 15); +// size_t num_handles_move = ((ctrl2 >> 5) & 15); +// +// size_t num_handles = num_handles_copy + num_handles_move; +// u32* buf_after_handles = buf + num_handles; +// +// if (num_handles > IPC_MAX_OBJECTS) +// num_handles = IPC_MAX_OBJECTS; +// +// for (i=0; iHandles[i] = *(buf+i); +// r->WasHandleCopied[i] = (i < num_handles_copy); +// } +// +// r->NumHandles = num_handles; +// buf = buf_after_handles; +// } +// +// size_t num_statics = (ctrl0 >> 16) & 15; +// u32* buf_after_statics = buf + num_statics*2; +// +// if (num_statics > IPC_MAX_BUFFERS) +// num_statics = IPC_MAX_BUFFERS; +// +// for (i=0; iPacked; +// +// r->Statics[i] = (void*) (desc->Addr | (((packed >> 12) & 15) << 32) | (((packed >> 6) & 15) << 36)); +// r->StaticSizes[i] = packed >> 16; +// r->StaticIndices[i] = packed & 63; +// } +// +// r->NumStatics = num_statics; +// buf = buf_after_statics; +// +// size_t num_bufs_send = (ctrl0 >> 20) & 15; +// size_t num_bufs_recv = (ctrl0 >> 24) & 15; +// size_t num_bufs_exch = (ctrl0 >> 28) & 15; +// +// size_t num_bufs = num_bufs_send + num_bufs_recv + num_bufs_exch; +// r->Raw = (void*)(((uintptr_t)(buf + num_bufs*3) + 15) &~ 15); +// r->RawWithoutPadding = (void*)((uintptr_t)(buf + num_bufs*3)); +// +// if (num_bufs > IPC_MAX_BUFFERS) +// num_bufs = IPC_MAX_BUFFERS; +// +// for (i=0; iPacked; +// +// r->Buffers[i] = (void*) (desc->Addr | ((packed >> 28) << 32) | (((packed >> 2) & 15) << 36)); +// r->BufferSizes[i] = desc->Size; +// r->BufferTypes[i] = (BufferType) (packed & 3); +// +// if (i < num_bufs_send) +// r->BufferDirections[i] = BufferDirection_Send; +// else if (i < (num_bufs_send + num_bufs_recv)) +// r->BufferDirections[i] = BufferDirection_Recv; +// else +// r->BufferDirections[i] = BufferDirection_Exch; +// } +// +// r->NumBuffers = num_bufs; +// return 0; +//} + +/** + * @brief Queries the size of an IPC pointer buffer. + * @param session IPC session handle. + * @param size Output variable in which to store the size. + * @return Result code. + */ +static inline Result ipcQueryPointerBufferSize(Handle session, size_t *size);//{ +// u32* buf = (u32*)armGetTls(); +// +// buf[0] = IpcCommandType_Control; +// buf[1] = 8; +// buf[2] = 0; +// buf[3] = 0; +// buf[4] = SFCI_MAGIC; +// buf[5] = 0; +// buf[6] = 3; +// buf[7] = 0; +// +// Result rc = ipcDispatch(session); +// +// if (R_SUCCEEDED(rc)) { +// IpcParsedCommand r; +// ipcParse(&r); +// +// struct ipcQueryPointerBufferSizeResponse { +// u64 magic; +// u64 result; +// u32 size; +// } *raw = (struct ipcQueryPointerBufferSizeResponse*)r.Raw; +// +// rc = raw->result; +// +// if (R_SUCCEEDED(rc)) { +// *size = raw->size & 0xffff; +// } +// } +// +// return rc; +//} + +/** + * @brief Closes the IPC session with proper clean up. + * @param session IPC session handle. + * @return Result code. + */ +static inline Result ipcCloseSession(Handle session);//{ +// u32* buf = (u32*)armGetTls(); +// buf[0] = IpcCommandType_Close; +// buf[1] = 0; +// return ipcDispatch(session); +//} +///@} + +///@name IPC domain handling +///@{ + +/** + * @brief Converts an IPC session handle into a domain. + * @param session IPC session handle. + * @param object_id_out Output variable in which to store the object ID. + * @return Result code. + */ +static inline Result ipcConvertSessionToDomain(Handle session, u32* object_id_out);//{ +// u32* buf = (u32*)armGetTls(); +// +// buf[0] = IpcCommandType_Control; +// buf[1] = 8; +// buf[4] = SFCI_MAGIC; +// buf[5] = 0; +// buf[6] = 0; +// buf[7] = 0; +// +// Result rc = ipcDispatch(session); +// +// if (R_SUCCEEDED(rc)) { +// IpcParsedCommand r; +// ipcParse(&r); +// +// struct ipcConvertSessionToDomainResponse { +// u64 magic; +// u64 result; +// u32 object_id; +// } *raw = (struct ipcConvertSessionToDomainResponse*)r.Raw; +// +// rc = raw->result; +// +// if (R_SUCCEEDED(rc)) { +// *object_id_out = raw->object_id; +// } +// } +// +// return rc; +//} + +/** + * @brief Adds an object ID to be sent through an IPC domain command structure. + * @param cmd IPC domain command structure. + * @param object_id Object ID to send. + */ +static inline void ipcSendObjectId(IpcCommand* cmd, u32 object_id);//{ +// cmd->ObjectIds[cmd->NumObjectIds++] = object_id; +//} + +/** + * @brief Prepares the header of an IPC command structure (domain version). + * @param cmd IPC command structure. + * @param sizeof_raw Size in bytes of the raw data structure to embed inside the IPC request + * @oaram object_id Domain object ID. + * @return Pointer to the raw embedded data structure in the request, ready to be filled out. + */ +static inline void* ipcPrepareHeaderForDomain(IpcCommand* cmd, size_t sizeof_raw, u32 object_id);//{ +// void* raw = ipcPrepareHeader(cmd, sizeof_raw + sizeof(DomainMessageHeader)); +// DomainMessageHeader* hdr = (DomainMessageHeader*) raw; +// u32 *object_ids = (u32*)(((uintptr_t) raw) + sizeof(DomainMessageHeader) + sizeof_raw); +// +// hdr->Type = DomainMessageType_SendMessage; +// hdr->NumObjectIds = (u8)cmd->NumObjectIds; +// hdr->Length = sizeof_raw; +// hdr->ThisObjectId = object_id; +// hdr->Pad[0] = hdr->Pad[1] = 0; +// +// for(size_t i = 0; i < cmd->NumObjectIds; i++) +// object_ids[i] = cmd->ObjectIds[i]; +// return (void*)(((uintptr_t) raw) + sizeof(DomainMessageHeader)); +//} + +/** + * @brief Parse an IPC command response into an IPC parsed command structure (domain version). + * @param IPC parsed command structure to fill in. + * @return Result code. + */ +static inline Result ipcParseForDomain(IpcParsedCommand* r);//{ +// Result rc = ipcParse(r); +// DomainMessageHeader *hdr; +// u32 *object_ids; +// if(R_FAILED(rc)) +// return rc; +// +// hdr = (DomainMessageHeader*) r->Raw; +// object_ids = (u32*)(((uintptr_t) hdr) + sizeof(DomainMessageHeader) + hdr->Length); +// r->Raw = (void*)(((uintptr_t) r->Raw) + sizeof(DomainMessageHeader)); +// +// r->IsDomainMessage = true; +// r->MessageType = (DomainMessageType)(hdr->Type); +// switch (r->MessageType) { +// case DomainMessageType_SendMessage: +// case DomainMessageType_Close: +// break; +// default: +// return MAKERESULT(Module_Libnx, LibnxError_DomainMessageUnknownType); +// } +// r->ThisObjectId = hdr->ThisObjectId; +// r->NumObjectIds = hdr->NumObjectIds > 8 ? 8 : hdr->NumObjectIds; +// if ((uintptr_t)object_ids + sizeof(u32) * r->NumObjectIds - (uintptr_t)armGetTls() >= 0x100) { +// return MAKERESULT(Module_Libnx, LibnxError_DomainMessageTooManyObjectIds); +// } +// for(size_t i = 0; i < r->NumObjectIds; i++) +// r->ObjectIds[i] = object_ids[i]; +// +// return rc; +//} + +/** + * @brief Closes a domain object by ID. + * @param session IPC session handle. + * @param object_id ID of the object to close. + * @return Result code. + */ +static inline Result ipcCloseObjectById(Handle session, u32 object_id);//{ +// IpcCommand c; +// DomainMessageHeader* hdr; +// +// ipcInitialize(&c); +// hdr = (DomainMessageHeader*)ipcPrepareHeader(&c, sizeof(DomainMessageHeader)); +// +// hdr->Type = 2; +// hdr->NumObjectIds = 0; +// hdr->Length = 0; +// hdr->ThisObjectId = object_id; +// hdr->Pad[0] = hdr->Pad[1] = 0; +// +// return ipcDispatch(session); // this command has no associated response +//} + +///@} + diff --git a/tests/unittests/data/teststaticexpectednewlinebraces.h b/tests/unittests/data/teststaticexpectednewlinebraces.h new file mode 100644 index 0000000..3f90ab6 --- /dev/null +++ b/tests/unittests/data/teststaticexpectednewlinebraces.h @@ -0,0 +1,68 @@ +/** + * @file condvar.h + * @brief Condition variable synchronization primitive. + * @author plutoo + * @copyright libnx Authors + */ +#pragma once +#include "../types.h" +#include "../kernel/mutex.h" + +/// Condition variable structure. +typedef struct { + u32 tag; + Mutex* mutex; +} CondVar; + +/** + * @brief Initializes a condition variable. + * @param[in] c Condition variable object. + * @param[in] m Mutex object to use inside the condition variable. + */ +void condvarInit(CondVar* c, Mutex* m); + +/** + * @brief Waits on a condition variable with a timeout. + * @param[in] c Condition variable object. + * @param[in] timeout Timeout in nanoseconds. + * @return Result code (0xEA01 on timeout). + * @remark On function return, the underlying mutex is acquired. + */ +Result condvarWaitTimeout(CondVar* c, u64 timeout); + +/** + * @brief Waits on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + * @remark On function return, the underlying mutex is acquired. + */ +static inline Result condvarWait(CondVar* c);//{ +// return condvarWaitTimeout(c, -1ull); +//} + +/** + * @brief Wakes up up to the specified number of threads waiting on a condition variable. + * @param[in] c Condition variable object. + * @param[in] num Maximum number of threads to wake up (or -1 to wake them all up). + * @return Result code. + */ +Result condvarWake(CondVar* c, int num); + +/** + * @brief Wakes up a single thread waiting on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + */ +static inline Result condvarWakeOne(CondVar* c);//{ +// return condvarWake(c, 1); +//} + +/** + * @brief Wakes up all thread waiting on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + */ +static inline Result condvarWakeAll(CondVar* c);//{ +// return condvarWake(c, -1); +//} + diff --git a/tests/unittests/data/teststaticfrontbraces.h b/tests/unittests/data/teststaticfrontbraces.h new file mode 100644 index 0000000..d62f694 --- /dev/null +++ b/tests/unittests/data/teststaticfrontbraces.h @@ -0,0 +1,675 @@ +/** + * @file ipc.h + * @brief Inter-process communication handling + * @author plutoo + * @copyright libnx Authors + */ +#pragma once +#include "../result.h" +#include "../arm/tls.h" +#include "../kernel/svc.h" + +/// IPC input header magic +#define SFCI_MAGIC 0x49434653 +/// IPC output header magic +#define SFCO_MAGIC 0x4f434653 + +/// IPC invalid object ID +#define IPC_INVALID_OBJECT_ID UINT32_MAX + +///@name IPC request building +///@{ + +/// IPC command (request) structure. +#define IPC_MAX_BUFFERS 8 +#define IPC_MAX_OBJECTS 8 + +typedef enum { + BufferType_Normal=0, ///< Regular buffer. + BufferType_Type1=1, ///< Allows ProcessMemory and shared TransferMemory. + BufferType_Invalid=2, + BufferType_Type3=3 ///< Same as Type1 except remote process is not allowed to use device-mapping. +} BufferType; + +typedef enum { + BufferDirection_Send=0, + BufferDirection_Recv=1, + BufferDirection_Exch=2, +} BufferDirection; + +typedef enum { + IpcCommandType_Invalid = 0, + IpcCommandType_LegacyRequest = 1, + IpcCommandType_Close = 2, + IpcCommandType_LegacyControl = 3, + IpcCommandType_Request = 4, + IpcCommandType_Control = 5, + IpcCommandType_RequestWithContext = 6, + IpcCommandType_ControlWithContext = 7, +} IpcCommandType; + +typedef enum { + DomainMessageType_Invalid = 0, + DomainMessageType_SendMessage = 1, + DomainMessageType_Close = 2, +} DomainMessageType; + +/// IPC domain message header. +typedef struct { + u8 Type; + u8 NumObjectIds; + u16 Length; + u32 ThisObjectId; + u32 Pad[2]; +} DomainMessageHeader; + +typedef struct { + size_t NumSend; // A + size_t NumRecv; // B + size_t NumExch; // W + const void* Buffers[IPC_MAX_BUFFERS]; + size_t BufferSizes[IPC_MAX_BUFFERS]; + BufferType BufferTypes[IPC_MAX_BUFFERS]; + + size_t NumStaticIn; // X + size_t NumStaticOut; // C + const void* Statics[IPC_MAX_BUFFERS]; + size_t StaticSizes[IPC_MAX_BUFFERS]; + u8 StaticIndices[IPC_MAX_BUFFERS]; + + bool SendPid; + size_t NumHandlesCopy; + size_t NumHandlesMove; + Handle Handles[IPC_MAX_OBJECTS]; + + size_t NumObjectIds; + u32 ObjectIds[IPC_MAX_OBJECTS]; +} IpcCommand; + +/** + * @brief Initializes an IPC command structure. + * @param cmd IPC command structure. + */ +static inline void ipcInitialize(IpcCommand* cmd) { + *cmd = (IpcCommand){0}; +} + +/// IPC buffer descriptor. +typedef struct { + u32 Size; ///< Size of the buffer. + u32 Addr; ///< Lower 32-bits of the address of the buffer + u32 Packed; ///< Packed data (including higher bits of the address) +} IpcBufferDescriptor; + +/// IPC static send-buffer descriptor. +typedef struct { + u32 Packed; ///< Packed data (including higher bits of the address) + u32 Addr; ///< Lower 32-bits of the address +} IpcStaticSendDescriptor; + +/// IPC static receive-buffer descriptor. +typedef struct { + u32 Addr; ///< Lower 32-bits of the address of the buffer + u32 Packed; ///< Packed data (including higher bits of the address) +} IpcStaticRecvDescriptor; + +/** + * @brief Adds a buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param type Buffer type. + */ +static inline void ipcAddSendBuffer(IpcCommand* cmd, const void* buffer, size_t size, BufferType type) { + size_t off = cmd->NumSend; + cmd->Buffers[off] = buffer; + cmd->BufferSizes[off] = size; + cmd->BufferTypes[off] = type; + cmd->NumSend++; +} + +/** + * @brief Adds a receive-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param type Buffer type. + */ +static inline void ipcAddRecvBuffer(IpcCommand* cmd, void* buffer, size_t size, BufferType type) { + size_t off = cmd->NumSend + cmd->NumRecv; + cmd->Buffers[off] = buffer; + cmd->BufferSizes[off] = size; + cmd->BufferTypes[off] = type; + cmd->NumRecv++; +} + +/** + * @brief Adds an exchange-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param type Buffer type. + */ +static inline void ipcAddExchBuffer(IpcCommand* cmd, void* buffer, size_t size, BufferType type) { + size_t off = cmd->NumSend + cmd->NumRecv + cmd->NumExch; + cmd->Buffers[off] = buffer; + cmd->BufferSizes[off] = size; + cmd->BufferTypes[off] = type; + cmd->NumExch++; +} + +/** + * @brief Adds a static-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddSendStatic(IpcCommand* cmd, const void* buffer, size_t size, u8 index) { + size_t off = cmd->NumStaticIn; + cmd->Statics[off] = buffer; + cmd->StaticSizes[off] = size; + cmd->StaticIndices[off] = index; + cmd->NumStaticIn++; +} + +/** + * @brief Adds a static-receive-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddRecvStatic(IpcCommand* cmd, void* buffer, size_t size, u8 index) { + size_t off = cmd->NumStaticIn + cmd->NumStaticOut; + cmd->Statics[off] = buffer; + cmd->StaticSizes[off] = size; + cmd->StaticIndices[off] = index; + cmd->NumStaticOut++; +} + +/** + * @brief Adds a smart-buffer (buffer + static-buffer pair) to an IPC command structure. + * @param cmd IPC command structure. + * @param ipc_buffer_size IPC buffer size. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddSendSmart(IpcCommand* cmd, size_t ipc_buffer_size, const void* buffer, size_t size, u8 index) { + if (ipc_buffer_size != 0 && size <= ipc_buffer_size) { + ipcAddSendBuffer(cmd, NULL, 0, BufferType_Normal); + ipcAddSendStatic(cmd, buffer, size, index); + } else { + ipcAddSendBuffer(cmd, buffer, size, BufferType_Normal); + ipcAddSendStatic(cmd, NULL, 0, index); + } +} + +/** + * @brief Adds a smart-receive-buffer (buffer + static-receive-buffer pair) to an IPC command structure. + * @param cmd IPC command structure. + * @param ipc_buffer_size IPC buffer size. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddRecvSmart(IpcCommand* cmd, size_t ipc_buffer_size, void* buffer, size_t size, u8 index) { + if (ipc_buffer_size != 0 && size <= ipc_buffer_size) { + ipcAddRecvBuffer(cmd, NULL, 0, BufferType_Normal); + ipcAddRecvStatic(cmd, buffer, size, index); + } else { + ipcAddRecvBuffer(cmd, buffer, size, BufferType_Normal); + ipcAddRecvStatic(cmd, NULL, 0, index); + } +} + +/** + * @brief Tags an IPC command structure to send the PID. + * @param cmd IPC command structure. + */ +static inline void ipcSendPid(IpcCommand* cmd) { + cmd->SendPid = true; +} + +/** + * @brief Adds a copy-handle to be sent through an IPC command structure. + * @param cmd IPC command structure. + * @param h Handle to send. + * @remark The receiving process gets a copy of the handle. + */ +static inline void ipcSendHandleCopy(IpcCommand* cmd, Handle h) { + cmd->Handles[cmd->NumHandlesCopy++] = h; +} + +/** + * @brief Adds a move-handle to be sent through an IPC command structure. + * @param cmd IPC command structure. + * @param h Handle to send. + * @remark The sending process loses ownership of the handle, which is transferred to the receiving process. + */ +static inline void ipcSendHandleMove(IpcCommand* cmd, Handle h) { + cmd->Handles[cmd->NumHandlesCopy + cmd->NumHandlesMove++] = h; +} + +/** + * @brief Prepares the header of an IPC command structure. + * @param cmd IPC command structure. + * @param sizeof_raw Size in bytes of the raw data structure to embed inside the IPC request + * @return Pointer to the raw embedded data structure in the request, ready to be filled out. + */ +static inline void* ipcPrepareHeader(IpcCommand* cmd, size_t sizeof_raw) { + u32* buf = (u32*)armGetTls(); + size_t i; + *buf++ = IpcCommandType_Request | (cmd->NumStaticIn << 16) | (cmd->NumSend << 20) | (cmd->NumRecv << 24) | (cmd->NumExch << 28); + + u32* fill_in_size_later = buf; + + if (cmd->NumStaticOut > 0) { + *buf = (cmd->NumStaticOut + 2) << 10; + } + else { + *buf = 0; + } + + if (cmd->SendPid || cmd->NumHandlesCopy > 0 || cmd->NumHandlesMove > 0) { + *buf++ |= 0x80000000; + *buf++ = (!!cmd->SendPid) | (cmd->NumHandlesCopy << 1) | (cmd->NumHandlesMove << 5); + + if (cmd->SendPid) + buf += 2; + + for (i=0; i<(cmd->NumHandlesCopy + cmd->NumHandlesMove); i++) + *buf++ = cmd->Handles[i]; + } + else { + buf++; + } + + for (i=0; iNumStaticIn; i++, buf+=2) { + IpcStaticSendDescriptor* desc = (IpcStaticSendDescriptor*) buf; + + uintptr_t ptr = (uintptr_t) cmd->Statics[i]; + desc->Addr = ptr; + desc->Packed = cmd->StaticIndices[i] | (cmd->StaticSizes[i] << 16) | + (((ptr >> 32) & 15) << 12) | (((ptr >> 36) & 15) << 6); + } + + for (i=0; i<(cmd->NumSend + cmd->NumRecv + cmd->NumExch); i++, buf+=3) { + IpcBufferDescriptor* desc = (IpcBufferDescriptor*) buf; + desc->Size = cmd->BufferSizes[i]; + + uintptr_t ptr = (uintptr_t) cmd->Buffers[i]; + desc->Addr = ptr; + desc->Packed = cmd->BufferTypes[i] | + (((ptr >> 32) & 15) << 28) | ((ptr >> 36) << 2); + } + + u32 padding = ((16 - (((uintptr_t) buf) & 15)) & 15) / 4; + u32* raw = (u32*) (buf + padding); + + size_t raw_size = (sizeof_raw/4) + 4; + buf += raw_size; + + u16* buf_u16 = (u16*) buf; + + for (i=0; iNumStaticOut; i++) { + size_t off = cmd->NumStaticIn + i; + size_t sz = (uintptr_t) cmd->StaticSizes[off]; + + buf_u16[i] = (sz > 0xFFFF) ? 0 : sz; + } + + size_t u16s_size = ((2*cmd->NumStaticOut) + 3)/4; + buf += u16s_size; + raw_size += u16s_size; + + *fill_in_size_later |= raw_size; + + for (i=0; iNumStaticOut; i++, buf+=2) { + IpcStaticRecvDescriptor* desc = (IpcStaticRecvDescriptor*) buf; + size_t off = cmd->NumStaticIn + i; + + uintptr_t ptr = (uintptr_t) cmd->Statics[off]; + desc->Addr = ptr; + desc->Packed = (ptr >> 32) | (cmd->StaticSizes[off] << 16); + } + + return (void*) raw; +} + +/** + * @brief Dispatches an IPC request. + * @param session IPC session handle. + * @return Result code. + */ +static inline Result ipcDispatch(Handle session) { + return svcSendSyncRequest(session); +} + +///@} + +///@name IPC response parsing +///@{ + +/// IPC parsed command (response) structure. +typedef struct { + IpcCommandType CommandType; ///< Type of the command + + bool HasPid; ///< true if the 'Pid' field is filled out. + u64 Pid; ///< PID included in the response (only if HasPid is true) + + size_t NumHandles; ///< Number of handles copied. + Handle Handles[IPC_MAX_OBJECTS]; ///< Handles. + bool WasHandleCopied[IPC_MAX_OBJECTS]; ///< true if the handle was moved, false if it was copied. + + bool IsDomainMessage; ///< true if the the message is a Domain message. + DomainMessageType MessageType; ///< Type of the domain message. + u32 MessageLength; ///< Size of rawdata (for domain messages). + u32 ThisObjectId; ///< Object ID to call the command on (for domain messages). + size_t NumObjectIds; ///< Number of object IDs (for domain messages). + u32 ObjectIds[IPC_MAX_OBJECTS]; ///< Object IDs (for domain messages). + + size_t NumBuffers; ///< Number of buffers in the response. + void* Buffers[IPC_MAX_BUFFERS]; ///< Pointers to the buffers. + size_t BufferSizes[IPC_MAX_BUFFERS]; ///< Sizes of the buffers. + BufferType BufferTypes[IPC_MAX_BUFFERS]; ///< Types of the buffers. + BufferDirection BufferDirections[IPC_MAX_BUFFERS]; ///< Direction of each buffer. + + size_t NumStatics; ///< Number of statics in the response. + void* Statics[IPC_MAX_BUFFERS]; ///< Pointers to the statics. + size_t StaticSizes[IPC_MAX_BUFFERS]; ///< Sizes of the statics. + u8 StaticIndices[IPC_MAX_BUFFERS]; ///< Indices of the statics. + + size_t NumStaticsOut; ///< Number of output statics available in the response. + + void* Raw; ///< Pointer to the raw embedded data structure in the response. + void* RawWithoutPadding; ///< Pointer to the raw embedded data structure, without padding. + size_t RawSize; ///< Size of the raw embedded data. +} IpcParsedCommand; + +/** + * @brief Parse an IPC command response into an IPC parsed command structure. + * @param IPC parsed command structure to fill in. + * @return Result code. + */ +static inline Result ipcParse(IpcParsedCommand* r) { + u32* buf = (u32*)armGetTls(); + u32 ctrl0 = *buf++; + u32 ctrl1 = *buf++; + size_t i; + + r->IsDomainMessage = false; + + r->CommandType = (IpcCommandType) (ctrl0 & 0xffff); + r->HasPid = false; + r->RawSize = (ctrl1 & 0x1ff) * 4; + r->NumHandles = 0; + + r->NumStaticsOut = (ctrl1 >> 10) & 15; + if (r->NumStaticsOut >> 1) r->NumStaticsOut--; // Value 2 -> Single descriptor + if (r->NumStaticsOut >> 1) r->NumStaticsOut--; // Value 3+ -> (Value - 2) descriptors + + if (ctrl1 & 0x80000000) { + u32 ctrl2 = *buf++; + + if (ctrl2 & 1) { + r->HasPid = true; + r->Pid = *buf++; + r->Pid |= ((u64)(*buf++)) << 32; + } + + size_t num_handles_copy = ((ctrl2 >> 1) & 15); + size_t num_handles_move = ((ctrl2 >> 5) & 15); + + size_t num_handles = num_handles_copy + num_handles_move; + u32* buf_after_handles = buf + num_handles; + + if (num_handles > IPC_MAX_OBJECTS) + num_handles = IPC_MAX_OBJECTS; + + for (i=0; iHandles[i] = *(buf+i); + r->WasHandleCopied[i] = (i < num_handles_copy); + } + + r->NumHandles = num_handles; + buf = buf_after_handles; + } + + size_t num_statics = (ctrl0 >> 16) & 15; + u32* buf_after_statics = buf + num_statics*2; + + if (num_statics > IPC_MAX_BUFFERS) + num_statics = IPC_MAX_BUFFERS; + + for (i=0; iPacked; + + r->Statics[i] = (void*) (desc->Addr | (((packed >> 12) & 15) << 32) | (((packed >> 6) & 15) << 36)); + r->StaticSizes[i] = packed >> 16; + r->StaticIndices[i] = packed & 63; + } + + r->NumStatics = num_statics; + buf = buf_after_statics; + + size_t num_bufs_send = (ctrl0 >> 20) & 15; + size_t num_bufs_recv = (ctrl0 >> 24) & 15; + size_t num_bufs_exch = (ctrl0 >> 28) & 15; + + size_t num_bufs = num_bufs_send + num_bufs_recv + num_bufs_exch; + r->Raw = (void*)(((uintptr_t)(buf + num_bufs*3) + 15) &~ 15); + r->RawWithoutPadding = (void*)((uintptr_t)(buf + num_bufs*3)); + + if (num_bufs > IPC_MAX_BUFFERS) + num_bufs = IPC_MAX_BUFFERS; + + for (i=0; iPacked; + + r->Buffers[i] = (void*) (desc->Addr | ((packed >> 28) << 32) | (((packed >> 2) & 15) << 36)); + r->BufferSizes[i] = desc->Size; + r->BufferTypes[i] = (BufferType) (packed & 3); + + if (i < num_bufs_send) + r->BufferDirections[i] = BufferDirection_Send; + else if (i < (num_bufs_send + num_bufs_recv)) + r->BufferDirections[i] = BufferDirection_Recv; + else + r->BufferDirections[i] = BufferDirection_Exch; + } + + r->NumBuffers = num_bufs; + return 0; +} + +/** + * @brief Queries the size of an IPC pointer buffer. + * @param session IPC session handle. + * @param size Output variable in which to store the size. + * @return Result code. + */ +static inline Result ipcQueryPointerBufferSize(Handle session, size_t *size) { + u32* buf = (u32*)armGetTls(); + + buf[0] = IpcCommandType_Control; + buf[1] = 8; + buf[2] = 0; + buf[3] = 0; + buf[4] = SFCI_MAGIC; + buf[5] = 0; + buf[6] = 3; + buf[7] = 0; + + Result rc = ipcDispatch(session); + + if (R_SUCCEEDED(rc)) { + IpcParsedCommand r; + ipcParse(&r); + + struct ipcQueryPointerBufferSizeResponse { + u64 magic; + u64 result; + u32 size; + } *raw = (struct ipcQueryPointerBufferSizeResponse*)r.Raw; + + rc = raw->result; + + if (R_SUCCEEDED(rc)) { + *size = raw->size & 0xffff; + } + } + + return rc; +} + +/** + * @brief Closes the IPC session with proper clean up. + * @param session IPC session handle. + * @return Result code. + */ +static inline Result ipcCloseSession(Handle session) { + u32* buf = (u32*)armGetTls(); + buf[0] = IpcCommandType_Close; + buf[1] = 0; + return ipcDispatch(session); +} +///@} + +///@name IPC domain handling +///@{ + +/** + * @brief Converts an IPC session handle into a domain. + * @param session IPC session handle. + * @param object_id_out Output variable in which to store the object ID. + * @return Result code. + */ +static inline Result ipcConvertSessionToDomain(Handle session, u32* object_id_out) { + u32* buf = (u32*)armGetTls(); + + buf[0] = IpcCommandType_Control; + buf[1] = 8; + buf[4] = SFCI_MAGIC; + buf[5] = 0; + buf[6] = 0; + buf[7] = 0; + + Result rc = ipcDispatch(session); + + if (R_SUCCEEDED(rc)) { + IpcParsedCommand r; + ipcParse(&r); + + struct ipcConvertSessionToDomainResponse { + u64 magic; + u64 result; + u32 object_id; + } *raw = (struct ipcConvertSessionToDomainResponse*)r.Raw; + + rc = raw->result; + + if (R_SUCCEEDED(rc)) { + *object_id_out = raw->object_id; + } + } + + return rc; +} + +/** + * @brief Adds an object ID to be sent through an IPC domain command structure. + * @param cmd IPC domain command structure. + * @param object_id Object ID to send. + */ +static inline void ipcSendObjectId(IpcCommand* cmd, u32 object_id) { + cmd->ObjectIds[cmd->NumObjectIds++] = object_id; +} + +/** + * @brief Prepares the header of an IPC command structure (domain version). + * @param cmd IPC command structure. + * @param sizeof_raw Size in bytes of the raw data structure to embed inside the IPC request + * @oaram object_id Domain object ID. + * @return Pointer to the raw embedded data structure in the request, ready to be filled out. + */ +static inline void* ipcPrepareHeaderForDomain(IpcCommand* cmd, size_t sizeof_raw, u32 object_id) { + void* raw = ipcPrepareHeader(cmd, sizeof_raw + sizeof(DomainMessageHeader)); + DomainMessageHeader* hdr = (DomainMessageHeader*) raw; + u32 *object_ids = (u32*)(((uintptr_t) raw) + sizeof(DomainMessageHeader) + sizeof_raw); + + hdr->Type = DomainMessageType_SendMessage; + hdr->NumObjectIds = (u8)cmd->NumObjectIds; + hdr->Length = sizeof_raw; + hdr->ThisObjectId = object_id; + hdr->Pad[0] = hdr->Pad[1] = 0; + + for(size_t i = 0; i < cmd->NumObjectIds; i++) + object_ids[i] = cmd->ObjectIds[i]; + return (void*)(((uintptr_t) raw) + sizeof(DomainMessageHeader)); +} + +/** + * @brief Parse an IPC command response into an IPC parsed command structure (domain version). + * @param IPC parsed command structure to fill in. + * @return Result code. + */ +static inline Result ipcParseForDomain(IpcParsedCommand* r) { + Result rc = ipcParse(r); + DomainMessageHeader *hdr; + u32 *object_ids; + if(R_FAILED(rc)) + return rc; + + hdr = (DomainMessageHeader*) r->Raw; + object_ids = (u32*)(((uintptr_t) hdr) + sizeof(DomainMessageHeader) + hdr->Length); + r->Raw = (void*)(((uintptr_t) r->Raw) + sizeof(DomainMessageHeader)); + + r->IsDomainMessage = true; + r->MessageType = (DomainMessageType)(hdr->Type); + switch (r->MessageType) { + case DomainMessageType_SendMessage: + case DomainMessageType_Close: + break; + default: + return MAKERESULT(Module_Libnx, LibnxError_DomainMessageUnknownType); + } + r->ThisObjectId = hdr->ThisObjectId; + r->NumObjectIds = hdr->NumObjectIds > 8 ? 8 : hdr->NumObjectIds; + if ((uintptr_t)object_ids + sizeof(u32) * r->NumObjectIds - (uintptr_t)armGetTls() >= 0x100) { + return MAKERESULT(Module_Libnx, LibnxError_DomainMessageTooManyObjectIds); + } + for(size_t i = 0; i < r->NumObjectIds; i++) + r->ObjectIds[i] = object_ids[i]; + + return rc; +} + +/** + * @brief Closes a domain object by ID. + * @param session IPC session handle. + * @param object_id ID of the object to close. + * @return Result code. + */ +static inline Result ipcCloseObjectById(Handle session, u32 object_id) { + IpcCommand c; + DomainMessageHeader* hdr; + + ipcInitialize(&c); + hdr = (DomainMessageHeader*)ipcPrepareHeader(&c, sizeof(DomainMessageHeader)); + + hdr->Type = 2; + hdr->NumObjectIds = 0; + hdr->Length = 0; + hdr->ThisObjectId = object_id; + hdr->Pad[0] = hdr->Pad[1] = 0; + + return ipcDispatch(session); // this command has no associated response +} + +///@} + diff --git a/tests/unittests/data/teststaticnewlinebraces.h b/tests/unittests/data/teststaticnewlinebraces.h new file mode 100644 index 0000000..c72c11a --- /dev/null +++ b/tests/unittests/data/teststaticnewlinebraces.h @@ -0,0 +1,71 @@ +/** + * @file condvar.h + * @brief Condition variable synchronization primitive. + * @author plutoo + * @copyright libnx Authors + */ +#pragma once +#include "../types.h" +#include "../kernel/mutex.h" + +/// Condition variable structure. +typedef struct { + u32 tag; + Mutex* mutex; +} CondVar; + +/** + * @brief Initializes a condition variable. + * @param[in] c Condition variable object. + * @param[in] m Mutex object to use inside the condition variable. + */ +void condvarInit(CondVar* c, Mutex* m); + +/** + * @brief Waits on a condition variable with a timeout. + * @param[in] c Condition variable object. + * @param[in] timeout Timeout in nanoseconds. + * @return Result code (0xEA01 on timeout). + * @remark On function return, the underlying mutex is acquired. + */ +Result condvarWaitTimeout(CondVar* c, u64 timeout); + +/** + * @brief Waits on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + * @remark On function return, the underlying mutex is acquired. + */ +static inline Result condvarWait(CondVar* c) +{ + return condvarWaitTimeout(c, -1ull); +} + +/** + * @brief Wakes up up to the specified number of threads waiting on a condition variable. + * @param[in] c Condition variable object. + * @param[in] num Maximum number of threads to wake up (or -1 to wake them all up). + * @return Result code. + */ +Result condvarWake(CondVar* c, int num); + +/** + * @brief Wakes up a single thread waiting on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + */ +static inline Result condvarWakeOne(CondVar* c) +{ + return condvarWake(c, 1); +} + +/** + * @brief Wakes up all thread waiting on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + */ +static inline Result condvarWakeAll(CondVar* c) +{ + return condvarWake(c, -1); +} + diff --git a/tests/unittests/data/teststaticnewlinebracesreadded.h b/tests/unittests/data/teststaticnewlinebracesreadded.h new file mode 100644 index 0000000..176f4fc --- /dev/null +++ b/tests/unittests/data/teststaticnewlinebracesreadded.h @@ -0,0 +1,68 @@ +/** + * @file condvar.h + * @brief Condition variable synchronization primitive. + * @author plutoo + * @copyright libnx Authors + */ +#pragma once +#include "../types.h" +#include "../kernel/mutex.h" + +/// Condition variable structure. +typedef struct { + u32 tag; + Mutex* mutex; +} CondVar; + +/** + * @brief Initializes a condition variable. + * @param[in] c Condition variable object. + * @param[in] m Mutex object to use inside the condition variable. + */ +void condvarInit(CondVar* c, Mutex* m); + +/** + * @brief Waits on a condition variable with a timeout. + * @param[in] c Condition variable object. + * @param[in] timeout Timeout in nanoseconds. + * @return Result code (0xEA01 on timeout). + * @remark On function return, the underlying mutex is acquired. + */ +Result condvarWaitTimeout(CondVar* c, u64 timeout); + +/** + * @brief Waits on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + * @remark On function return, the underlying mutex is acquired. + */ +static inline Result condvarWait(CondVar* c) { + return condvarWaitTimeout(c, -1ull); +} + +/** + * @brief Wakes up up to the specified number of threads waiting on a condition variable. + * @param[in] c Condition variable object. + * @param[in] num Maximum number of threads to wake up (or -1 to wake them all up). + * @return Result code. + */ +Result condvarWake(CondVar* c, int num); + +/** + * @brief Wakes up a single thread waiting on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + */ +static inline Result condvarWakeOne(CondVar* c) { + return condvarWake(c, 1); +} + +/** + * @brief Wakes up all thread waiting on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + */ +static inline Result condvarWakeAll(CondVar* c) { + return condvarWake(c, -1); +} + diff --git a/tests/unittests/nim.cfg b/tests/unittests/nim.cfg new file mode 100644 index 0000000..e9b9a2e --- /dev/null +++ b/tests/unittests/nim.cfg @@ -0,0 +1 @@ +--path="../../src" diff --git a/tests/unittests/testfileops.nim b/tests/unittests/testfileops.nim new file mode 100644 index 0000000..2fff387 --- /dev/null +++ b/tests/unittests/testfileops.nim @@ -0,0 +1,211 @@ +import nimgen/fileops, common, regex, os + +import unittest + +let testFileContent = """ +this is text +this is text +replace me +prepend me +end +""" + +let prependMiddleExpected = """ +this is text +this is text +replace me +prepended data +prepend me +end +""" + +let prependEndExpected = """ +this is text +this is text +replace me +prepend me +data +end +""" + +let appendEndExpected = """ +this is text +this is text +replace me +prepend me +end +data +""" + +let appendMiddleExpected = """ +this is data + text +this is text +replace me +prepend me +end +""" + +let freplaceDefaultExpected = """ + + +replace me +prepend me +end +""" + +let freplaceWithExpected = """ +this is text +this is text +foobar +prepend me +end +""" + +let freplaceRegexExpected = """ +foobar +foobar +replace me +prepend me +end +""" + +let commentExpected = """ +this is text +this is text +//replace me +//prepend me +//end +""" + +let commentMiddleExpected = """ +this //is text +//this is text +replace me +prepend me +end +""" + + +let dataDir = currentSourcePath().splitPath().head / "data" + +let testfilename = dataDir / "testing.txt" + + +suite "test file ops": + if not dataDir.dirExists(): + dataDir.createDir() + + setup: + if testfilename.existsFile(): + removeFile(testfilename) + writeFile(testfilename, testFileContent) + + ################### Prepend ####################### + + test "prepend at beginning of file": + prepend(testfilename, "data\n") + let expected = "data\n" & testFileContent + testfilename.checkFile(expected) + + test "prepend at middle of file": + prepend(testfilename, "prepended data\n", "prepend me") + testfilename.checkFile(prependMiddleExpected) + + test "prepend at end of file": + prepend(testfilename, "data\n", "end\n") + testfilename.checkFile(prependEndExpected) + + ################### Pipe ######################### + + test "pipe command into file": + when defined(windows): + pipe(testfilename, "(ECHO foo)>>$file") + testfilename.checkFile("foo") + else: + pipe(testfilename, "cat $file | grep 'this is text'") + testfilename.checkFile("this is text\nthis is text") + + ################# Append ######################### + + test "append file end": + append(testfilename, "data\n") + testfilename.checkFile(appendEndExpected) + + test "append file middle": + append(testfilename, " data\n", "this is") + testfilename.checkFile(appendMiddleExpected) + + ################# FReplace ######################### + + test "freplace default empty": + freplace(testfilename, "this is text") + testfilename.checkFile(freplaceDefaultExpected) + + test "freplace with content": + freplace(testfilename, "replace me", "foobar") + testfilename.checkFile(freplaceWithExpected) + + test "freplace regex": + freplace(testfilename, re"this .*", "foobar") + testfilename.checkFile(freplaceRegexExpected) + + ####################### Comment ###################### + + test "comment": + comment(testfilename, "replace me", "3") + testfilename.checkFile(commentExpected) + + test "comment over length": + comment(testfilename, "replace me", "10") + testfilename.checkFile(commentExpected) + + test "comment negative": + comment(testfilename, "replace me", "-3") + testfilename.checkFile(testFileContent) + + test "comment zero": + comment(testfilename, "replace me", "0") + testfilename.checkFile(testFileContent) + + test "comment middle": + comment(testfilename, "is text", "2") + testfilename.checkFile(commentMiddleExpected) + + ############### Static inline removal ################ + + test "replace static inline with front braces at end of line": + + let + file = dataDir / "teststaticfrontbraces.h" + resFile = dataDir / "teststaticexpectedfrontbraces.h" + + test = readFile(file) + expected = readFile(resFile) + + writeFile(testfilename, test) + + removeStatic(testfilename) + testfilename.checkFile(expected) + + reAddStatic(testfilename) + testfilename.checkFile(test) + + test "replace static inline with newline before brace": + + let + file = dataDir / "teststaticnewlinebraces.h" + resFile = dataDir / "teststaticexpectednewlinebraces.h" + reAddedFile = dataDir / "teststaticnewlinebracesreadded.h" + + test = readFile(file) + expected = readFile(resFile) + reAdded = readFile(reAddedFile) + + writeFile(testfilename, test) + + removeStatic(testfilename) + testfilename.checkFile(expected) + + reAddStatic(testfilename) + testfilename.checkFile(reAdded) From 0473e79c0bbc56c41814f6fb41d91da1bf5ed79a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 14 Jul 2018 22:04:03 +0900 Subject: [PATCH 11/96] Fix miscellaneous windows issues with building --- src/nimgen/c2nim.nim | 5 ++--- src/nimgen/external.nim | 5 ++++- src/nimgen/gencore.nim | 2 +- src/nimgen/runcfg.nim | 3 +++ 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 4d721b4..bca56d1 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -38,7 +38,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = for inc in gIncludes: if inc.isAbsolute(): - passC &= ("""{.passC: "-I\"$#\"".}""" % [inc]) & "\n" + passC &= ("""{.passC: "-I\"$#\"".}""" % [inc.sanitizePath()]) & "\n" else: passC &= ( """{.passC: "-I\"" & currentSourcePath().splitPath().head & "$#\"".}""" % @@ -85,11 +85,10 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = passC &= "const header$# = currentSourcePath().splitPath().head & \"$#\"\n" % [fname, file.relativePath()] extflags = "--header:header$#" % fname - # Run c2nim on generated file var cmd = "c2nim $# $# --out:$# $#" % [c2nimConfig.flags, extflags, outfile, cfile] when defined(windows): - cmd = "cmd /c " & cmd + cmd = "cmd /c " & cmd.quoteShell discard execProc(cmd) if c2nimConfig.preprocess or c2nimConfig.ctags: diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 79426f6..096753c 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -2,6 +2,9 @@ import os, osproc, regex, ropes, streams, strutils import globals +proc sanitizePath*(path: string): string = + path.multiReplace([("\\", "/"), ("//", "/")]) + proc execProc*(cmd: string): string = result = "" var @@ -106,7 +109,7 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = cmd = "$# -E $# $#" % [pproc, ppflags, file] for inc in gIncludes: - cmd &= " -I " & inc + cmd &= " -I " & inc.quoteShell # Run preprocessor var data = execProc(cmd) diff --git a/src/nimgen/gencore.nim b/src/nimgen/gencore.nim index f794991..382fa59 100644 --- a/src/nimgen/gencore.nim +++ b/src/nimgen/gencore.nim @@ -20,7 +20,7 @@ proc addEnv*(str: string): string = discard # if there are still format args, print a warning - if newStr.contains("${"): + if newStr.contains("$") and not newStr.contains("$replace("): echo "WARNING: \"", newStr, "\" still contains an uninterpolated value!" return newStr diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 105cede..7255c52 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -196,6 +196,9 @@ proc runCfg*(cfg: string) = # Reset on a per project basis gCCompiler = getEnv(cCompilerEnv, defaultCCompiler) + gCppCompiler = gCppCompiler.quoteShell + gCCompiler = gCCompiler.quoteShell + if gConfig["n.global"].hasKey("filter"): gFilter = gConfig["n.global"]["filter"] if gConfig["n.global"].hasKey("quotes"): From ffc73c6b6d3d16b4833443a1d2b520cf572c0665 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 15 Jul 2018 10:06:46 +0900 Subject: [PATCH 12/96] Prevent processed file collisions --- src/nimgen/runcfg.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 7255c52..cafb48e 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -25,7 +25,7 @@ proc getKey(ukey: string): tuple[key: string, val: bool] = proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, string]()) = var cfg = cfgin - sfile = search(file) + sfile = search(file).sanitizePath if sfile in gDoneRecursive: return From 2d06bdcccfcc41d42949f22aa7b96624cc7df130 Mon Sep 17 00:00:00 2001 From: data-man Date: Sun, 15 Jul 2018 04:49:24 +0300 Subject: [PATCH 13/96] More multiReplaces --- src/nimgen/external.nim | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 79426f6..39c963e 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -123,18 +123,15 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = if line.strip() != "": if line[0] == '#' and not line.contains("#pragma"): start = false - if sfile in line.replace("\\", "/").replace("//", "/"): + if sfile in line.multiReplace([("\\", "/"), ("//", "/")]): start = true if not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: start = true else: if start: rdata.add( - line.replace("_Noreturn", "") - .replace("(())", "") - .replace("WINAPI", "") - .replace("__attribute__", "") - .replace("extern \"C\"", "") + line.multiReplace([("_Noreturn", ""), ("(())", ""), ("WINAPI", ""), + ("__attribute__", ""), ("extern \"C\"", "")]) .replace(re"\(\([_a-z]+?\)\)", "") .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";") & "\n" ) From f8f918642964f60cee6c8dd5ba9fe8390b8ee369 Mon Sep 17 00:00:00 2001 From: data-man Date: Sun, 15 Jul 2018 07:49:05 +0300 Subject: [PATCH 14/96] Use sanitizePath --- src/nimgen/external.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index c336f0a..350c841 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -126,7 +126,7 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = if line.strip() != "": if line[0] == '#' and not line.contains("#pragma"): start = false - if sfile in line.multiReplace([("\\", "/"), ("//", "/")]): + if sfile in line.sanitizePath: start = true if not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: start = true From 0b2fb6cfbc24041feb92221205623e0d579f0247 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 15 Jul 2018 15:40:00 +0900 Subject: [PATCH 15/96] Fix issue with sourcpath building switch on Windows --- src/nimgen/c2nim.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index bca56d1..f171e8d 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -36,12 +36,14 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = passC = "import ospaths, strutils\n" + passC &= """const sourcePath = currentSourcePath().split({'\\', '/'})[0..^2].join("/")""" & "\n" + for inc in gIncludes: if inc.isAbsolute(): passC &= ("""{.passC: "-I\"$#\"".}""" % [inc.sanitizePath()]) & "\n" else: passC &= ( - """{.passC: "-I\"" & currentSourcePath().splitPath().head & "$#\"".}""" % + """{.passC: "-I\"" & sourcePath & "$#\"".}""" % inc.relativePath() ) & "\n" @@ -82,7 +84,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = if file.isAbsolute(): passC &= "const header$# = \"$#\"\n" % [fname, file] else: - passC &= "const header$# = currentSourcePath().splitPath().head & \"$#\"\n" % + passC &= "const header$# = sourcePath & \"$#\"\n" % [fname, file.relativePath()] extflags = "--header:header$#" % fname # Run c2nim on generated file From 5e3c0999e0f969f7fb18fa14b8587b177e52666e Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 17 Jul 2018 18:11:39 +0900 Subject: [PATCH 16/96] Fix windows not processing headers --- src/nimgen/c2nim.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index f171e8d..fe257f2 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -6,7 +6,7 @@ template relativePath(path: untyped): untyped = path.multiReplace([(gOutput, ""), ("\\", "/"), ("//", "/")]) proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = - var file = search(fl) + var file = search(fl).sanitizePath if file.len() == 0: return From 6343567a9182dccac9edb61c636bd5ddf2732cfe Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 18 Jul 2018 07:52:45 +0900 Subject: [PATCH 17/96] Put sanitizePath in search() --- src/nimgen/c2nim.nim | 2 +- src/nimgen/external.nim | 2 +- src/nimgen/file.nim | 4 ++-- src/nimgen/runcfg.nim | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index fe257f2..f171e8d 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -6,7 +6,7 @@ template relativePath(path: untyped): untyped = path.multiReplace([(gOutput, ""), ("\\", "/"), ("//", "/")]) proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = - var file = search(fl).sanitizePath + var file = search(fl) if file.len() == 0: return diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 350c841..c0d5fdc 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -118,7 +118,7 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = var rdata: Rope start = false - sfile = file.replace("\\", "/") + sfile = file.sanitizePath if inline: sfile = sfile.parentDir() diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index d78c692..708ae4a 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -1,6 +1,6 @@ import os, ospaths, pegs, regex, strutils, tables -import globals +import globals, external # ### # File loction @@ -44,7 +44,7 @@ proc search*(file: string): string = quit(1) # Only keep relative directory - return result.multiReplace([("\\", $DirSep), ("//", $DirSep), (gProjectDir & $DirSep, "")]) + return result.sanitizePath.replace(gProjectDir & "/", "") proc rename*(file: string, renfile: string) = if file.splitFile().ext == ".nim": diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index cafb48e..7255c52 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -25,7 +25,7 @@ proc getKey(ukey: string): tuple[key: string, val: bool] = proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, string]()) = var cfg = cfgin - sfile = search(file).sanitizePath + sfile = search(file) if sfile in gDoneRecursive: return From f3c01e53f985a5f479ee4c58c492cc88e6d7af98 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 18 Jul 2018 12:25:13 +0900 Subject: [PATCH 18/96] Fix a few more issues with windows paths --- src/nimgen/runcfg.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 7255c52..03c0013 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -161,13 +161,13 @@ proc runCfg*(cfg: string) = echo "Config doesn't exist: " & cfg quit(1) - gProjectDir = parentDir(cfg.expandFilename()) + gProjectDir = parentDir(cfg.expandFilename()).sanitizePath gConfig = loadConfig(cfg) if gConfig.hasKey("n.global"): if gConfig["n.global"].hasKey("output"): - gOutput = gConfig["n.global"]["output"] + gOutput = gConfig["n.global"]["output"].sanitizePath if dirExists(gOutput): if "-f" in commandLineParams(): try: @@ -207,11 +207,11 @@ proc runCfg*(cfg: string) = if gConfig.hasKey("n.include"): for inc in gConfig["n.include"].keys(): - gIncludes.add(inc.addEnv()) + gIncludes.add(inc.addEnv().sanitizePath) if gConfig.hasKey("n.exclude"): for excl in gConfig["n.exclude"].keys(): - gExcludes.add(excl.addEnv()) + gExcludes.add(excl.addEnv().sanitizePath) if gConfig.hasKey("n.prepare"): for prep in gConfig["n.prepare"].keys(): From 2c3cc71540f94783a98f34a82afc5c40bf320dd3 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Aug 2018 20:49:50 -0500 Subject: [PATCH 19/96] Fix permission denied and file flushing issues --- src/nimgen/c2nim.nim | 4 ++-- src/nimgen/external.nim | 26 +++++++++----------------- src/nimgen/file.nim | 10 +++++++--- src/nimgen/runcfg.nim | 2 +- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index f171e8d..428c767 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -20,10 +20,10 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = var cfile = file if c2nimConfig.preprocess: cfile = "temp-$#.c" % [outfile.extractFilename()] - writeFile(cfile, runPreprocess(file, c2nimConfig.ppflags, c2nimConfig.flags, c2nimConfig.inline)) + writeFileFlush(cfile, runPreprocess(file, c2nimConfig.ppflags, c2nimConfig.flags, c2nimConfig.inline)) elif c2nimConfig.ctags: cfile = "temp-$#.c" % [outfile.extractFilename()] - writeFile(cfile, runCtags(file)) + writeFileFlush(cfile, runCtags(file)) if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): prepend(cfile, getDefines(file, c2nimConfig.inline)) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index c0d5fdc..9575c38 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -6,22 +6,11 @@ proc sanitizePath*(path: string): string = path.multiReplace([("\\", "/"), ("//", "/")]) proc execProc*(cmd: string): string = - result = "" - var - p = startProcess(cmd, options = {poStdErrToStdOut, poUsePath, poEvalCommand}) + var ret: int - outp = outputStream(p) - line = newStringOfCap(120).TaintedString - - while true: - if outp.readLine(line): - result.add(line) - result.add("\n") - elif not running(p): break - - var x = p.peekExitCode() - if x != 0: - echo "Command failed: " & $x + (result, ret) = execCmdEx(cmd) + if ret != 0: + echo "Command failed: " & $ret echo cmd echo result quit(1) @@ -64,12 +53,15 @@ proc gitReset*() = discard execProc("git reset --hard HEAD") proc gitCheckout*(file: string) = - echo " Resetting " & file + echo "Resetting " & file setCurrentDir(gOutput) defer: setCurrentDir(gProjectDir) - discard execProc("git checkout $#" % file.replace(gOutput & "/", "")) + let cmd = "git checkout $#" % file.replace(gOutput & "/", "") + if execCmdEx(cmd)[0].contains("Permission denied"): + sleep(500) + discard execProc(cmd) proc gitRemotePull*(url: string, pull=true) = if dirExists(gOutput/".git"): diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index 708ae4a..edf39ea 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -82,6 +82,12 @@ proc openRetry*(file: string, mode: FileMode = fmRead): File = except IOError: sleep(100) +template writeFileFlush*(file, content: string): untyped = + let f = openRetry(file, fmWrite) + f.write(content) + f.flushFile() + f.close() + template withFile*(file: string, body: untyped): untyped = if fileExists(file): var f = openRetry(file) @@ -93,9 +99,7 @@ template withFile*(file: string, body: untyped): untyped = body if content != contentOrig: - f = openRetry(file, fmWrite) - write(f, content) - f.close() + writeFileFlush(file, content) else: echo "Missing file " & file diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 03c0013..95cb1c4 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -57,7 +57,7 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str if action == "create": echo "Creating " & file createDir(file.splitPath().head) - writeFile(file, cfg[act]) + writeFileFlush(file, cfg[act]) if file in gExcludes: gExcludes.delete(gExcludes.find(file)) sfile = search(file) From 87e08bfb666789942462739e90afa3e4bd23a1de Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 3 Aug 2018 13:02:56 +0900 Subject: [PATCH 20/96] Fix unit test for Windows and quit with failure code --- tests/rununittests.nim | 4 +++- tests/unittests/testfileops.nim | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/rununittests.nim b/tests/rununittests.nim index a92ff0a..865f77d 100644 --- a/tests/rununittests.nim +++ b/tests/rununittests.nim @@ -1,8 +1,10 @@ import os, osproc, strutils proc main() = + var failures = 0 for file in walkFiles(currentSourcePath().splitPath().head / "unittests/*.nim"): let (path, fname, ext) = file.splitFile() if fname.startswith("test"): - discard execCmd "nim c -r " & file + failures += execCmd "nim c -r " & file + quit(failures) main() diff --git a/tests/unittests/testfileops.nim b/tests/unittests/testfileops.nim index 2fff387..5d2119f 100644 --- a/tests/unittests/testfileops.nim +++ b/tests/unittests/testfileops.nim @@ -120,7 +120,7 @@ suite "test file ops": test "pipe command into file": when defined(windows): - pipe(testfilename, "(ECHO foo)>>$file") + pipe(testfilename, "ECHO foo > $file") testfilename.checkFile("foo") else: pipe(testfilename, "cat $file | grep 'this is text'") From c6148571c10d073b2c4c4a6392c5a43e747315b4 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 3 Aug 2018 15:08:03 +0900 Subject: [PATCH 21/96] Fix windows echo --- tests/unittests/testfileops.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/testfileops.nim b/tests/unittests/testfileops.nim index 5d2119f..076c586 100644 --- a/tests/unittests/testfileops.nim +++ b/tests/unittests/testfileops.nim @@ -120,7 +120,7 @@ suite "test file ops": test "pipe command into file": when defined(windows): - pipe(testfilename, "ECHO foo > $file") + pipe(testfilename, "ECHO foo") testfilename.checkFile("foo") else: pipe(testfilename, "cat $file | grep 'this is text'") From 1dbff5cbd14f69529869720e72e153f6f9029957 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 3 Aug 2018 23:27:26 -0500 Subject: [PATCH 22/96] Inline fixes --- nimgen.nimble | 2 +- src/nimgen/c2nim.nim | 12 ++--------- src/nimgen/external.nim | 9 +++++--- src/nimgen/gencore.nim | 1 - src/nimgen/runcfg.nim | 47 +++++++++++++++++++++++------------------ 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/nimgen.nimble b/nimgen.nimble index 3e65545..841cb09 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.3.0" +version = "0.3.1" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 428c767..73e4ebb 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -10,12 +10,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = if file.len() == 0: return - echo " Generating " & outfile - - # Remove static inline function bodies - removeStatic(file) - - fixFuncProtos(file) + echo "Generating " & outfile var cfile = file if c2nimConfig.preprocess: @@ -121,7 +116,4 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = # Add dynamic library if outlib != "": - prepend(outfile, outlib) - - # Add back static functions for compilation - reAddStatic(file) + prepend(outfile, outlib) \ No newline at end of file diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 9575c38..571c7e0 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -50,7 +50,10 @@ proc gitReset*() = setCurrentDir(gOutput) defer: setCurrentDir(gProjectDir) - discard execProc("git reset --hard HEAD") + let cmd = "git reset --hard HEAD" + while execCmdEx(cmd)[0].contains("Permission denied"): + sleep(1000) + echo " Retrying ..." proc gitCheckout*(file: string) = echo "Resetting " & file @@ -59,9 +62,9 @@ proc gitCheckout*(file: string) = defer: setCurrentDir(gProjectDir) let cmd = "git checkout $#" % file.replace(gOutput & "/", "") - if execCmdEx(cmd)[0].contains("Permission denied"): + while execCmdEx(cmd)[0].contains("Permission denied"): sleep(500) - discard execProc(cmd) + echo " Retrying ..." proc gitRemotePull*(url: string, pull=true) = if dirExists(gOutput/".git"): diff --git a/src/nimgen/gencore.nim b/src/nimgen/gencore.nim index 382fa59..ff070df 100644 --- a/src/nimgen/gencore.nim +++ b/src/nimgen/gencore.nim @@ -87,7 +87,6 @@ proc getDefines*(file: string, inline=false): string = for incl in incls: let sincl = search(incl) if sincl != "": - echo "Inlining " & sincl result &= getDefines(sincl) withFile(file): for def in content.findAll(re"(?m)^(\s*#\s*define\s+[\w\d_]+\s+[\d\-.xf]+)(?:\r|//|/*).*?$"): diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 95cb1c4..fd65708 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -60,7 +60,7 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str writeFileFlush(file, cfg[act]) if file in gExcludes: gExcludes.delete(gExcludes.find(file)) - sfile = search(file) + sfile = file gDoneRecursive.add(sfile) elif action in @["prepend", "append", "replace", "comment", "rename", "compile", "dynlib", "pragma", @@ -127,31 +127,38 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str echo "Cannot use recurse and inline simultaneously" quit(1) - if not noprocess: - let outfile = getNimout(sfile) - c2nim(file, outfile, c2nimConfig) + removeStatic(sfile) + fixFuncProtos(sfile) - if c2nimConfig.recurse: - var - cfg = newOrderedTable[string, string]() - incls = getIncls(sfile) - incout = "" + let outfile = getNimout(sfile) + if c2nimConfig.recurse or c2nimConfig.inline: + var + cfg = newOrderedTable[string, string]() + incls = getIncls(sfile) + incout = "" - for name, value in c2nimConfig.fieldPairs: - when value is string: - cfg[name] = value - when value is bool: - cfg[name] = $value + for name, value in c2nimConfig.fieldPairs: + when value is string: + cfg[name] = value + when value is bool: + cfg[name] = $value - for i in c2nimConfig.dynlib: - cfg["dynlib." & i] = i + for i in c2nimConfig.dynlib: + cfg["dynlib." & i] = i - for inc in incls: - runFile(inc, cfg) + if c2nimConfig.inline: + cfg["noprocess"] = "true" + + for inc in incls: + runFile(inc, cfg) + if c2nimConfig.recurse: incout &= "import $#\n" % inc.search().getNimout()[0 .. ^5] - if incout.len() != 0: - prepend(outfile, incout) + if c2nimConfig.recurse and incout.len() != 0: + prepend(outfile, incout) + + if not noprocess: + c2nim(file, outfile, c2nimConfig) if reset: gitCheckout(sfile) From 32debc5534bc4a82e5f944adb2e4b42043bf5fab Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 5 Aug 2018 01:10:47 -0500 Subject: [PATCH 23/96] Regex search/replace support - #2 --- README.md | 6 ++++-- src/nimgen/c2nim.nim | 2 +- src/nimgen/fileops.nim | 8 +------- src/nimgen/gencore.nim | 22 +++++++--------------- src/nimgen/runcfg.nim | 14 ++++++++++---- 5 files changed, 23 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 5a1945d..c88af16 100644 --- a/README.md +++ b/README.md @@ -174,15 +174,17 @@ The following keys apply to library source code (before processing) and generate ```create``` = create a file at exact location with contents specified. File needs to be in the _[n.exclude]_ list in order to be created. +```pipe``` = execute a command on a file and store the output of the command as the new file contents. E.g. pipe = "cat $file | grep 'static inline'" + ```search``` = search string providing context for following prepend/append/replace directives -```pipe``` = execute a command on a file and store the output of the command as the new file contents. Ex: pipe = "cat $file | grep 'static inline'" +```regex``` = regex search string providing context for the following replace directive. Specify using """ to avoid regex parsing issues ```prepend``` = string value to prepend into file at beginning or before search ```append``` = string value to append into file at the end or after search -```replace``` = string value to replace search string in file +```replace``` = string value to replace search string in file. Regex captures can be referred to using $1, $2, etc. ```comment``` = number of lines to comment from search location diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 73e4ebb..05954e6 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -95,7 +95,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = discard # Nim doesn't like {.cdecl.} for type proc() - freplace(outfile, re"(?m)(.*? = proc.*?)\{.cdecl.\}", "$#") + freplace(outfile, re"(?m)(.*? = proc.*?)\{.cdecl.\}", "$1") freplace(outfile, " {.cdecl.})", ")") # Include {.compile.} directives diff --git a/src/nimgen/fileops.nim b/src/nimgen/fileops.nim index c4e67a0..4e8f858 100644 --- a/src/nimgen/fileops.nim +++ b/src/nimgen/fileops.nim @@ -38,13 +38,7 @@ proc freplace*(file: string, pattern: string, repl="") = proc freplace*(file: string, pattern: Regex, repl="") = withFile(file): - var m: RegexMatch - if content.find(pattern, m): - if "$#" in repl: - content = content.replace(pattern, - proc (m: RegexMatch, s: string): string = repl % s[m.group(0)[0]]) - else: - content = content.replace(pattern, repl) + content = content.replace(pattern, repl) proc comment*(file: string, pattern: string, numlines: string) = let diff --git a/src/nimgen/gencore.nim b/src/nimgen/gencore.nim index ff070df..4e79d04 100644 --- a/src/nimgen/gencore.nim +++ b/src/nimgen/gencore.nim @@ -4,24 +4,16 @@ import file, globals proc addEnv*(str: string): string = var newStr = str - for pair in envPairs(): - try: - newStr = newStr % [pair.key, pair.value.string] - except ValueError: - # Ignore if there are no values to replace. We - # want to continue anyway - discard - try: + if "$output" in newStr or "${output}" in newStr: newStr = newStr % ["output", gOutput] - except ValueError: - # Ignore if there are no values to replace. We - # want to continue anyway - discard - # if there are still format args, print a warning - if newStr.contains("$") and not newStr.contains("$replace("): - echo "WARNING: \"", newStr, "\" still contains an uninterpolated value!" + for pair in envPairs(): + if pair.key.len() == 0: + continue + + if ("$" & pair.key) in newStr or ("${" & pair.key & "}") in newStr: + newStr = newStr % [pair.key, pair.value.string] return newStr diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index fd65708..a0e51c5 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -44,6 +44,7 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str var srch = "" + rgx = "" c2nimConfig = c2nimConfigObj( flags: "--stdcall", ppflags: "", @@ -78,6 +79,8 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str elif action == "replace": if srch != "": freplace(sfile, cfg[srch], cfg[act]) + elif rgx != "": + freplace(sfile, toPattern(cfg[rgx]), cfg[act]) elif action == "comment": if srch != "": comment(sfile, cfg[srch], cfg[act]) @@ -92,8 +95,11 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str elif action == "pipe": pipe(sfile, cfg[act]) srch = "" + rgx = "" elif action == "search": srch = act + elif action == "regex": + rgx = act if file.splitFile().ext != ".nim": var @@ -131,11 +137,11 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str fixFuncProtos(sfile) let outfile = getNimout(sfile) + var incout = "" if c2nimConfig.recurse or c2nimConfig.inline: var cfg = newOrderedTable[string, string]() incls = getIncls(sfile) - incout = "" for name, value in c2nimConfig.fieldPairs: when value is string: @@ -154,12 +160,12 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str if c2nimConfig.recurse: incout &= "import $#\n" % inc.search().getNimout()[0 .. ^5] - if c2nimConfig.recurse and incout.len() != 0: - prepend(outfile, incout) - if not noprocess: c2nim(file, outfile, c2nimConfig) + if c2nimConfig.recurse and incout.len() != 0: + prepend(outfile, incout) + if reset: gitCheckout(sfile) From 667d6caf844133452f0a9145813817b08b0be896 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 8 Aug 2018 11:06:21 -0500 Subject: [PATCH 24/96] CLI support - #20 --- README.md | 35 ++++++++ src/nimgen.nim | 4 +- src/nimgen/runcfg.nim | 138 ++++++++++++++++++++++++++++---- tests/unittests/testfileops.nim | 2 - 4 files changed, 158 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index c88af16..c023d6b 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,41 @@ The following key only applies before processing and allows renaming the generat `$replace(srch1=repl1, srch2=reply2)` = rename specific portions in `$nimout` +__Command Line__ + +A subset of capabilities are available through the command line to enable quick tests using nimgen. Command line flags only apply to source files specified on the command line and do not influence any ```cfg``` files which are expected to be self-sufficient. + +``` +Usage: + nimgen [options] file.cfg|file.h ... + +Params: + -C add compile entry * + -E add n.exclude entry * + -F set c2nim flags * + -I add n.include dir * + -O set output directory + -P set preprocessor flags * + +Options: + -c set ctags = true + -d set defines = true + -i set inline = true + -n set noprocess = true + -p set preprocess = true + -r set recurse = true + +Editing: + -a append string * + -e prepend string * + -l replace string * + -o#lines comment X lines * + -s search string * + -x regex search string * + +* supports multiple instances +``` + __Feedback__ Nimgen is a work in progress and any feedback or suggestions are welcome. It is hosted on [GitHub](https://github.com/genotrance/nimgen) with an MIT license so issues, forks and PRs are most appreciated. Also join us at https://gitter.im/nimgen/Lobby to chat about nimgen and the future of Nim wrappers. diff --git a/src/nimgen.nim b/src/nimgen.nim index 5777398..69f994e 100644 --- a/src/nimgen.nim +++ b/src/nimgen.nim @@ -2,6 +2,4 @@ import os import nimgen/runcfg -for i in commandLineParams(): - if i != "-f": - runCfg(i) +runCli() \ No newline at end of file diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index a0e51c5..91e7078 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -169,6 +169,24 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str if reset: gitCheckout(sfile) +proc setOutputDir(dir: string) = + gOutput = dir.sanitizePath + if dirExists(gOutput): + if "-f" in commandLineParams(): + try: + removeDir(gOutput) + except OSError: + echo "Directory in use: " & gOutput + quit(1) + else: + for f in walkFiles(gOutput/"*.nim"): + try: + removeFile(f) + except OSError: + echo "Unable to delete: " & f + quit(1) + createDir(gOutput) + proc runCfg*(cfg: string) = if not fileExists(cfg): echo "Config doesn't exist: " & cfg @@ -180,22 +198,7 @@ proc runCfg*(cfg: string) = if gConfig.hasKey("n.global"): if gConfig["n.global"].hasKey("output"): - gOutput = gConfig["n.global"]["output"].sanitizePath - if dirExists(gOutput): - if "-f" in commandLineParams(): - try: - removeDir(gOutput) - except OSError: - echo "Directory in use: " & gOutput - quit(1) - else: - for f in walkFiles(gOutput/"*.nim"): - try: - removeFile(f) - except OSError: - echo "Unable to delete: " & f - quit(1) - createDir(gOutput) + setOutputDir(gConfig["n.global"]["output"]) if gConfig["n.global"].hasKey("cpp_compiler"): gCppCompiler = gConfig["n.global"]["cpp_compiler"] @@ -278,3 +281,106 @@ proc runCfg*(cfg: string) = gitReset() elif key == "execute": discard execProc(postVal) + +let gHelp = """ +Nimgen is a helper for c2nim to simplify and automate the wrapping of C libraries + +Usage: + nimgen [options] file.cfg|file.h ... + +Params: + -C add compile entry * + -E add n.exclude entry * + -F set c2nim flags * + -I add n.include dir * + -O set output directory + -P set preprocessor flags * + +Options: + -c set ctags = true + -d set defines = true + -i set inline = true + -n set noprocess = true + -p set preprocess = true + -r set recurse = true + +Editing: + -a append string * + -e prepend string * + -l replace string * + -o#lines comment X lines * + -s search string * + -x regex search string * + +* supports multiple instances +""" + +proc runCli*() = + var + cfg = newOrderedTable[string, string]() + files: seq[string] + uniq = 1 + + gProjectDir = getCurrentDir().sanitizePath + for param in commandLineParams(): + let flag = if param.len() <= 2: param else: param[0..<2] + + if fileExists(param): + if param.splitFile().ext.toLowerAscii() == ".cfg": + runCfg(param) + else: + files.add(param) + + elif flag == "-C": + cfg["compile." & $uniq] = param[2..^1] + elif flag == "-E": + gExcludes.add(param[2..^1].addEnv().sanitizePath) + elif flag == "-F": + if cfg.hasKey("flags"): + cfg["flags"] = cfg["flags"] & " " & param[2..^1] + else: + cfg["flags"] = param[2..^1] + elif flag == "-I": + gIncludes.add(param[2..^1].addEnv().sanitizePath) + elif flag == "-O": + setOutputDir(param[2..^1]) + elif flag == "-P": + if cfg.hasKey("ppflags"): + cfg["ppflags"] = cfg["ppflags"] & " " & param[2..^1] + else: + cfg["ppflags"] = param[2..^1] + + elif flag == "-c": + cfg["ctags"] = "true" + elif flag == "-d": + cfg["defines"] = "true" + elif flag == "-i": + cfg["inline"] = "true" + elif flag == "-n": + cfg["noprocess"] = "true" + elif flag == "-p": + cfg["preprocess"] = "true" + elif flag == "-r": + cfg["recurse"] = "true" + + elif flag == "-a": + cfg["append." & $uniq] = param[2..^1] + elif flag == "-e": + cfg["prepend." & $uniq] = param[2..^1] + elif flag == "-l": + cfg["replace." & $uniq] = param[2..^1] + elif flag == "-o": + cfg["comment." & $uniq] = param[2..^1] + elif flag == "-s": + cfg["search." & $uniq] = param[2..^1] + elif flag == "-x": + cfg["regex." & $uniq] = param[2..^1] + + elif param == "-h" or param == "-?" or param == "--help": + echo gHelp + quit(0) + + uniq += 1 + + for file in files: + runFile(file, cfg) \ No newline at end of file diff --git a/tests/unittests/testfileops.nim b/tests/unittests/testfileops.nim index 076c586..2d24d4e 100644 --- a/tests/unittests/testfileops.nim +++ b/tests/unittests/testfileops.nim @@ -97,8 +97,6 @@ suite "test file ops": dataDir.createDir() setup: - if testfilename.existsFile(): - removeFile(testfilename) writeFile(testfilename, testFileContent) ################### Prepend ####################### From bb2c9474facddbefa5b1bc4c4fa176e127aa70be Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 14 Aug 2018 11:18:31 -0500 Subject: [PATCH 25/96] Add badges --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index c023d6b..d8465cb 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +[![Chat on Gitter](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/nimgen/Lobby) +[![Build status](https://ci.appveyor.com/api/projects/status/05t5ja88lmv1rt3r/branch/master?svg=true)](https://ci.appveyor.com/project/genotrance/nimgen/branch/master) + Nimgen is a helper for [c2nim](https://github.com/nim-lang/c2nim/) to simplify and automate the wrapping of C libraries. Nimgen can be used to automate the process of manipulating C files so that c2nim can be run on them without issues. This includes adding/removing code snippets, removal of complex preprocessor definitions that c2nim doesn't yet comprehend and recursively running on #include files. From 1fb8674d07220868d9fa3015e7d7c31c593a226f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 14 Aug 2018 11:19:32 -0500 Subject: [PATCH 26/96] Add badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d8465cb..ca76311 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Chat on Gitter](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/nimgen/Lobby) -[![Build status](https://ci.appveyor.com/api/projects/status/05t5ja88lmv1rt3r/branch/master?svg=true)](https://ci.appveyor.com/project/genotrance/nimgen/branch/master) +[![Build status](https://ci.appveyor.com/api/projects/status/05t5ja88lmv1rt3r?svg=true)](https://ci.appveyor.com/project/genotrance/nimgen) Nimgen is a helper for [c2nim](https://github.com/nim-lang/c2nim/) to simplify and automate the wrapping of C libraries. From 1a29b9f0f8fc08e2d6eba283026320d1aef04cc2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 14 Aug 2018 11:21:51 -0500 Subject: [PATCH 27/96] Add badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ca76311..d8465cb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Chat on Gitter](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/nimgen/Lobby) -[![Build status](https://ci.appveyor.com/api/projects/status/05t5ja88lmv1rt3r?svg=true)](https://ci.appveyor.com/project/genotrance/nimgen) +[![Build status](https://ci.appveyor.com/api/projects/status/05t5ja88lmv1rt3r/branch/master?svg=true)](https://ci.appveyor.com/project/genotrance/nimgen/branch/master) Nimgen is a helper for [c2nim](https://github.com/nim-lang/c2nim/) to simplify and automate the wrapping of C libraries. From 1e26d3e6eb9ab1036d4992476b71ac02616af4ed Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Aug 2018 14:41:41 -0500 Subject: [PATCH 28/96] More flexibility in compile flag --- README.md | 2 +- src/nimgen/c2nim.nim | 6 +++--- src/nimgen/gencore.nim | 19 +++++++++++++++---- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d8465cb..66db546 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ The following keys apply to library source code and help with generating the .ni Multiple entries for the all following keys are possible by appending any .string to the key. E.g. dynlib.win, compile.dir -```compile``` = file or dir of files of source code to {.compile.} into generated .nim +```compile``` = file or dir of files of source code to {.compile.} into generated .nim. If directory, picks *.c if C mode and *.cxx, *.cpp, *.cc, *.c++ and *.C for cpp mode. Dir can also include wildcards. e.g. compile = """dir/A*.cxx""" ```pragma``` = pragmas to define in generated .nim file. E.g. pragma = "passL: \"-lssl\"" => {.passL: "-lssl".} diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 05954e6..e887b29 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -29,7 +29,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = outlib = "" outpragma = "" - passC = "import ospaths, strutils\n" + passC = "import strutils\n" passC &= """const sourcePath = currentSourcePath().split({'\\', '/'})[0..^2].join("/")""" & "\n" @@ -102,9 +102,9 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = for cpl in c2nimConfig.compile: let fcpl = search(cpl) if getFileInfo(fcpl).kind == pcFile: - prepend(outfile, compile(file=fcpl)) + prepend(outfile, compile(c2nimConfig.flags, file=fcpl)) else: - prepend(outfile, compile(dir=fcpl)) + prepend(outfile, compile(c2nimConfig.flags, dir=fcpl)) # Add any pragmas if outpragma != "": diff --git a/src/nimgen/gencore.nim b/src/nimgen/gencore.nim index 4e79d04..42b82c0 100644 --- a/src/nimgen/gencore.nim +++ b/src/nimgen/gencore.nim @@ -17,15 +17,26 @@ proc addEnv*(str: string): string = return newStr -proc compile*(dir="", file=""): string = +proc compile*(flags: string, dir="", file=""): string = + var data = "" + proc fcompile(file: string): string = return "{.compile: \"$#\".}" % file.replace("\\", "/") - var data = "" - if dir != "" and dirExists(dir): - for f in walkFiles(dir / "*.c"): + proc dcompile(dir: string) = + for f in walkFiles(dir): data &= fcompile(f) & "\n" + if dir != "": + if dir.contains("*") or dir.contains("?"): + dcompile(dir) + elif dirExists(dir): + if flags.contains("cpp"): + for i in @["*.C", "*.cpp", "*.c++", "*.cc", "*.cxx"]: + dcompile(dir / i) + else: + dcompile(dir / "*.c") + if file != "" and fileExists(file): data &= fcompile(file) & "\n" From 7e21b2799eafaad9662bf2d754e94d25eea25100 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Aug 2018 15:19:59 -0500 Subject: [PATCH 29/96] Improve compile --- src/nimgen/c2nim.nim | 6 +----- src/nimgen/gencore.nim | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index e887b29..30f23f0 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -100,11 +100,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = # Include {.compile.} directives for cpl in c2nimConfig.compile: - let fcpl = search(cpl) - if getFileInfo(fcpl).kind == pcFile: - prepend(outfile, compile(c2nimConfig.flags, file=fcpl)) - else: - prepend(outfile, compile(c2nimConfig.flags, dir=fcpl)) + prepend(outfile, compile(cpl, c2nimConfig.flags)) # Add any pragmas if outpragma != "": diff --git a/src/nimgen/gencore.nim b/src/nimgen/gencore.nim index 42b82c0..000651e 100644 --- a/src/nimgen/gencore.nim +++ b/src/nimgen/gencore.nim @@ -17,7 +17,7 @@ proc addEnv*(str: string): string = return newStr -proc compile*(flags: string, dir="", file=""): string = +proc compile*(cpl, flags: string): string = var data = "" proc fcompile(file: string): string = @@ -27,18 +27,18 @@ proc compile*(flags: string, dir="", file=""): string = for f in walkFiles(dir): data &= fcompile(f) & "\n" - if dir != "": - if dir.contains("*") or dir.contains("?"): - dcompile(dir) - elif dirExists(dir): + if cpl.contains("*") or cpl.contains("?"): + dcompile(cpl) + else: + let fcpl = search(cpl) + if getFileInfo(fcpl).kind == pcFile: + data &= fcompile(fcpl) & "\n" + elif getFileInfo(fcpl).kind == pcDir: if flags.contains("cpp"): for i in @["*.C", "*.cpp", "*.c++", "*.cc", "*.cxx"]: - dcompile(dir / i) + dcompile(fcpl / i) else: - dcompile(dir / "*.c") - - if file != "" and fileExists(file): - data &= fcompile(file) & "\n" + dcompile(fcpl / "*.c") return data From 1711052eae985e042ab22108f0975d9bb0906abf Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 19 Aug 2018 12:31:21 -0500 Subject: [PATCH 30/96] Add move capability, handle compile duplicate names --- README.md | 2 ++ nimgen.nimble | 2 +- src/nimgen/fileops.nim | 23 +++++++++++++++++------ src/nimgen/gencore.nim | 14 +++++++++++++- src/nimgen/globals.nim | 1 + src/nimgen/runcfg.nim | 11 ++++++++--- 6 files changed, 42 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 66db546..7cb0f1c 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,8 @@ The following keys apply to library source code (before processing) and generate ```replace``` = string value to replace search string in file. Regex captures can be referred to using $1, $2, etc. +```move``` = search string providing context for location to move the results of a preceding search or regex match + ```comment``` = number of lines to comment from search location The following key only applies before processing and allows renaming the generated .nim files as required to enable successful wrapping. This may be for organizational purposes or to prevent usage of non-nim supported strings in module names (E.g. first letter is a number). Destination is relative to output directory if defined. diff --git a/nimgen.nimble b/nimgen.nimble index 841cb09..6ca5b9e 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.3.1" +version = "0.4.0" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" diff --git a/src/nimgen/fileops.nim b/src/nimgen/fileops.nim index 4e8f858..e416cf1 100644 --- a/src/nimgen/fileops.nim +++ b/src/nimgen/fileops.nim @@ -31,15 +31,26 @@ proc append*(file: string, data: string, search="") = if idx != -1: content = content[0.. Date: Sun, 19 Aug 2018 17:05:19 -0500 Subject: [PATCH 31/96] Fix nil rope --- src/nimgen/external.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 571c7e0..5abc1c6 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -111,7 +111,7 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = # Include content only from file var - rdata: Rope + rdata = rope("") start = false sfile = file.sanitizePath From 864506afbc431ddb62fad36bda38361cfcfe9801 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 19 Aug 2018 19:25:04 -0500 Subject: [PATCH 32/96] Remove ropes dependency --- src/nimgen/external.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 5abc1c6..651ccf3 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -1,4 +1,4 @@ -import os, osproc, regex, ropes, streams, strutils +import os, osproc, regex, streams, strutils import globals @@ -111,7 +111,7 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = # Include content only from file var - rdata = rope("") + rdata: seq[string] = @[] start = false sfile = file.sanitizePath @@ -131,9 +131,9 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = line.multiReplace([("_Noreturn", ""), ("(())", ""), ("WINAPI", ""), ("__attribute__", ""), ("extern \"C\"", "")]) .replace(re"\(\([_a-z]+?\)\)", "") - .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";") & "\n" + .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";") ) - return $rdata + return rdata.join("\n") proc runCtags*(file: string): string = var From a2a2c41704beb49e69e28188b1f58283aab70bc7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 5 Sep 2018 13:18:45 -0500 Subject: [PATCH 33/96] Add nimtess2 to test suite --- README.md | 2 ++ tests/nimgentest.nims | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7cb0f1c..a8e751c 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ To see examples of nimgen in action check out the following wrappers:- * git checkout * [nimssl](https://github.com/genotrance/nimssl) - OpenSSL wrapper: [docs](http://nimgen.genotrance.com/nimssl) * git sparse checkout + * [nimtess2](https://github.com/genotrance/nimtess2) - libtess2 wrapper: [docs](http://nimgen.genotrance.com/nimtess2) + * git checkout * [libsvm](https://github.com/genotrance/libsvm) - libsvm wrapper: [docs](http://nimgen.genotrance.com/libsvm) * git sparse checkout diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 9c3e2d7..44c23c6 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -3,7 +3,7 @@ import distros, ospaths, strutils var full = true comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", - "nimfuzz", "nimpcre", "nimrax", "nimssl", "nimssh2"] + "nimfuzz", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2"] if detectOs(Windows): comps.add("nimkerberos") From 7082f897812634362a410dbe2d3d3790fd61fc89 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 5 Sep 2018 14:32:19 -0500 Subject: [PATCH 34/96] Add duktape-nim --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a8e751c..8cc8b04 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,8 @@ To see examples of nimgen in action check out the following wrappers:- * git checkout * [libsvm](https://github.com/genotrance/libsvm) - libsvm wrapper: [docs](http://nimgen.genotrance.com/libsvm) * git sparse checkout + * [duktape-nim](https://github.com/manguluka/duktape-nim) - Duktape wrapper + * git checkout * Compile in as static binary * [nimssh2](https://github.com/genotrance/nimssh2) - libssh2 wrapper: [docs](http://nimgen.genotrance.com/nimssh2) From 23bb86803db4a4b9d0481a7804a945ce591002a4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 5 Sep 2018 14:33:35 -0500 Subject: [PATCH 35/96] Add duktape-nim --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8cc8b04..be88f9e 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ To see examples of nimgen in action check out the following wrappers:- * [libsvm](https://github.com/genotrance/libsvm) - libsvm wrapper: [docs](http://nimgen.genotrance.com/libsvm) * git sparse checkout * [duktape-nim](https://github.com/manguluka/duktape-nim) - Duktape wrapper - * git checkout + * static * Compile in as static binary * [nimssh2](https://github.com/genotrance/nimssh2) - libssh2 wrapper: [docs](http://nimgen.genotrance.com/nimssh2) From 8eaa76560191901dc64e4d8aa4c7d7f3779e3e71 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 26 Sep 2018 16:44:28 -0500 Subject: [PATCH 36/96] Add nimnuklear and nimfastText --- README.md | 6 +++++- src/nimgen.nim | 2 +- tests/nimgentest.nims | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index be88f9e..121c766 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ To see examples of nimgen in action check out the following wrappers:- * [nim-clblast](https://github.com/numforge/nim-clblast) - OpenCL BLAS wrapper * static -* Compile C code into binary +* Compile C/C++ code into binary * [nim7z](https://github.com/genotrance/nim7z) - 7z decoder wrapper: [docs](http://nimgen.genotrance.com/nim7z) * git sparse checkout * [nimarchive](https://github.com/genotrance/nimarchive) - libarchive wrapper: [docs](http://nimgen.genotrance.com/nimarchive) @@ -51,10 +51,14 @@ To see examples of nimgen in action check out the following wrappers:- * git checkout * [nimclipboard](https://github.com/genotrance/nimclipboard) - libclipboard wrapper: [docs](http://nimgen.genotrance.com/nimclipboard) * git checkout + * [nimfastText](https://github.com/genotrance/nimfastText) - fastText wrapper: [docs](http://nimgen.genotrance.com/nimfastText) + * git sparse checkout * [nimfuzz](https://github.com/genotrance/nimfuzz) - fts_fuzzy_match wrapper: [docs](http://nimgen.genotrance.com/nimfuzz) * download header file * [nimkerberos](https://github.com/genotrance/nimkerberos) - WinKerberos wrapper: [docs](http://nimgen.genotrance.com/nimkerberos) * git sparse checkout + * [nimnuklear](https://github.com/genotrance/nimnuklear) - nuklear wrapper: [docs](https://nimgen.genotrance.com/nimnuklear) + * git sparse checkout * [nimpcre](https://github.com/genotrance/nimpcre) - PCRE wrapper: [docs](http://nimgen.genotrance.com/nimpcre) * git checkout * [nimrax](https://github.com/genotrance/nimrax) - Radix tree wrapper: [docs](http://nimgen.genotrance.com/nimrax) diff --git a/src/nimgen.nim b/src/nimgen.nim index 69f994e..dc98ae4 100644 --- a/src/nimgen.nim +++ b/src/nimgen.nim @@ -2,4 +2,4 @@ import os import nimgen/runcfg -runCli() \ No newline at end of file +runCli() diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 44c23c6..03c8dc9 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,8 +2,8 @@ import distros, ospaths, strutils var full = true - comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", - "nimfuzz", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2"] + comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfastText", + "nimfuzz", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2"] if detectOs(Windows): comps.add("nimkerberos") From 598214c1d2fe9aa68e584de05966173ada1edf78 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 27 Sep 2018 18:35:11 -0500 Subject: [PATCH 37/96] Save project artifacts --- appveyor.yml | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 6b5f682..0b7c921 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -31,7 +31,7 @@ for: BASE_DIR: c:\projects install: - - CD %BASE_DIR% + - CD c:\ - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - 7z x -y "%MINGW_ARCHIVE%"> nul - IF not exist "%LIBSSH2_ARCHIVE%" appveyor DownloadFile "%LIBSSH2_URL%" -FileName "%LIBSSH2_ARCHIVE%" @@ -42,7 +42,7 @@ for: - 7z x -y "%LIBCRYPTO_ARCHIVE2%"> nul - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - 7z x -y "%NIM_ARCHIVE%"> nul - - SET PATH=%BASE_DIR%\%MINGW_DIR%\bin;%BASE_DIR%\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% + - SET PATH=c:\%MINGW_DIR%\bin;c:\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - CD %BASE_DIR%\nimgen # - git clone --depth 1 https://github.com/nim-lang/nim # - cd nim @@ -59,16 +59,18 @@ for: # - cd .. on_finish: - - 7z a -r buildlogs-win.zip %USERPROFILE%\.nimble\pkgs - - appveyor PushArtifact buildlogs-win.zip + - 7z a -r buildlogs-win-pkgs.zip %USERPROFILE%\.nimble\pkgs + - appveyor PushArtifact buildlogs-win-pkgs.zip + - 7z a -r buildlogs-win-projects.zip %BASE_DIR% + - appveyor PushArtifact buildlogs-win-projects.zip - 7z a -r nimgen-docs.zip %BASE_DIR%\nimgen\web - appveyor PushArtifact nimgen-docs.zip cache: - - c:\projects\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - - c:\projects\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz - - c:\projects\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - - c:\projects\nim-0.18.0_x32.zip + - c:\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z + - c:\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz + - c:\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz + - c:\nim-0.18.0_x32.zip - matrix: @@ -84,7 +86,7 @@ for: install: - sudo apt -qq update - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev - - cd $BASE_DIR + - cd /home/appveyor - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - tar xJf $NIM_ARCHIVE - cd $NIM_VERSION @@ -92,15 +94,17 @@ for: - bin/nim c -d:release koch - ./koch boot -d:release - ./koch nimble -d:release - - export PATH=$BASE_DIR/$NIM_VERSION/bin:~/.nimble/bin:$PATH + - export PATH=/home/appveyor/$NIM_VERSION/bin:~/.nimble/bin:$PATH - cd $BASE_DIR/nimgen on_finish: - - zip -r -q buildlogs-lin.zip ~/.nimble/pkgs - - appveyor PushArtifact buildlogs-lin.zip + - zip -r -q buildlogs-lin-pkgs.zip ~/.nimble/pkgs + - appveyor PushArtifact buildlogs-lin-pkgs.zip + - zip -r -q buildlogs-lin-projects.zip $BASE_DIR + - appveyor PushArtifact buildlogs-lin-projects.zip cache: - - /home/appveyor/projects/nim-0.18.0.tar.xz + - /home/appveyor/nim-0.18.0.tar.xz build_script: - nimble install -y From 6c8bf5e8429e2d29eb909b1087a527480e413282 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 27 Sep 2018 19:06:16 -0500 Subject: [PATCH 38/96] Save artifacts --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 0b7c921..ce74b7f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -61,7 +61,7 @@ for: on_finish: - 7z a -r buildlogs-win-pkgs.zip %USERPROFILE%\.nimble\pkgs - appveyor PushArtifact buildlogs-win-pkgs.zip - - 7z a -r buildlogs-win-projects.zip %BASE_DIR% + - 7z a -r buildlogs-win-projects.zip c:\projects - appveyor PushArtifact buildlogs-win-projects.zip - 7z a -r nimgen-docs.zip %BASE_DIR%\nimgen\web - appveyor PushArtifact nimgen-docs.zip From 2f27ad0d0525a7d30ebf7fa067673e0e8c47a818 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 27 Sep 2018 19:27:22 -0500 Subject: [PATCH 39/96] Save artifacts --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index ce74b7f..c4f1d69 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -61,7 +61,7 @@ for: on_finish: - 7z a -r buildlogs-win-pkgs.zip %USERPROFILE%\.nimble\pkgs - appveyor PushArtifact buildlogs-win-pkgs.zip - - 7z a -r buildlogs-win-projects.zip c:\projects + - 7z a -r buildlogs-win-projects.zip c:\projects\* - appveyor PushArtifact buildlogs-win-projects.zip - 7z a -r nimgen-docs.zip %BASE_DIR%\nimgen\web - appveyor PushArtifact nimgen-docs.zip From 0a32e7fa2ccea82eed74d33a4e4bdc90605193be Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 2 Oct 2018 12:24:28 -0500 Subject: [PATCH 40/96] Add nimmonocrypt, remove nimfastText --- README.md | 2 ++ tests/nimgentest.nims | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 121c766..0ee6c9f 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ To see examples of nimgen in action check out the following wrappers:- * download header file * [nimkerberos](https://github.com/genotrance/nimkerberos) - WinKerberos wrapper: [docs](http://nimgen.genotrance.com/nimkerberos) * git sparse checkout + * [nimmonocypher](https://github.com/genotrance/nimmonocypher) - monocypher wrapper: [docs](http://nimgen.genotrance.com/nimmonocypher) + * git sparse checkout * [nimnuklear](https://github.com/genotrance/nimnuklear) - nuklear wrapper: [docs](https://nimgen.genotrance.com/nimnuklear) * git sparse checkout * [nimpcre](https://github.com/genotrance/nimpcre) - PCRE wrapper: [docs](http://nimgen.genotrance.com/nimpcre) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 03c8dc9..03b3666 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,8 +2,8 @@ import distros, ospaths, strutils var full = true - comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfastText", - "nimfuzz", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2"] + comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", # "nimfastText", + "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2"] if detectOs(Windows): comps.add("nimkerberos") From 6b16a83572a5a3fc4528cce630c7a4dbe9cd934a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 4 Oct 2018 17:10:43 -0500 Subject: [PATCH 41/96] Edit comps --- tests/nimgentest.nims | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 03b3666..f7e5aee 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,11 +2,15 @@ import distros, ospaths, strutils var full = true - comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", # "nimfastText", - "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2"] + comps = @["libsvm", "nim7z", "nimarchive", "nimbass", #"nimbigwig", + "nimclipboard", "nimfastText", "nimfuzz", "nimmonocypher", + "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", + "nimtess2" + ] if detectOs(Windows): comps.add("nimkerberos") + comps.add("nimzbar") if paramCount() > 2: for i in 3 .. paramCount(): From 4df59320fd43e111a952ae5be4f193bef5252ff8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 4 Oct 2018 21:17:02 -0500 Subject: [PATCH 42/96] Comp fixes --- tests/nimgentest.nims | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index f7e5aee..25eeaaf 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,15 +2,14 @@ import distros, ospaths, strutils var full = true - comps = @["libsvm", "nim7z", "nimarchive", "nimbass", #"nimbigwig", + comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfastText", "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", - "nimtess2" + "nimtess2", "nimzbar" ] if detectOs(Windows): comps.add("nimkerberos") - comps.add("nimzbar") if paramCount() > 2: for i in 3 .. paramCount(): From e04b40d6a88339f60a7099e1f9c46edc70446ea1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 5 Oct 2018 10:47:36 -0500 Subject: [PATCH 43/96] Remove nimfastText --- tests/nimgentest.nims | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 25eeaaf..364cbdd 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -3,7 +3,8 @@ import distros, ospaths, strutils var full = true comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", - "nimclipboard", "nimfastText", "nimfuzz", "nimmonocypher", + "nimclipboard", #"nimfastText", + "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2", "nimzbar" ] From efc04c38cc0b2decff014e084b7854cecc84cb0c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 12:08:38 -0500 Subject: [PATCH 44/96] Multi-version CI --- appveyor.yml | 93 ++++++++++++++++++++++--------------------- tests/nimgentest.nims | 7 +++- 2 files changed, 52 insertions(+), 48 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index c4f1d69..1673432 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,36 +14,39 @@ for: - image: Visual Studio 2017 environment: - MINGW_DIR: mingw32 - MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf/i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z/download + ARCH: 32 + MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf MINGW_ARCHIVE: i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - NIM_URL: https://nim-lang.org/download/nim-0.18.0_x32.zip - NIM_ARCHIVE: nim-0.18.0_x32.zip - NIM_VERSION: nim-0.18.0 - LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz - LIBSSH2_ARCHIVE2: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar - LIBSSH2_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz/download - LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - LIBCRYPTO_ARCHIVE2: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar - LIBCRYPTO_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz/download - DLLS_URL: http://nim-lang.org/download/dlls.zip - DLLS_ARCHIVE: dlls.zip - BASE_DIR: c:\projects + SFNET_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686 + LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg + LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg + matrix: + - NIM_VERSION: 0.18.0 + - NIM_VERSION: 0.19.0 install: - CD c:\ - - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - - 7z x -y "%MINGW_ARCHIVE%"> nul - - IF not exist "%LIBSSH2_ARCHIVE%" appveyor DownloadFile "%LIBSSH2_URL%" -FileName "%LIBSSH2_ARCHIVE%" - - 7z x -y "%LIBSSH2_ARCHIVE%"> nul - - 7z x -y "%LIBSSH2_ARCHIVE2%"> nul - - IF not exist "%LIBCRYPTO_ARCHIVE%" appveyor DownloadFile "%LIBCRYPTO_URL%" -FileName "%LIBCRYPTO_ARCHIVE%" - - 7z x -y "%LIBCRYPTO_ARCHIVE%"> nul - - 7z x -y "%LIBCRYPTO_ARCHIVE2%"> nul - - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - - 7z x -y "%NIM_ARCHIVE%"> nul - - SET PATH=c:\%MINGW_DIR%\bin;c:\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - - CD %BASE_DIR%\nimgen + - IF not exist "binaries" ( + MKDIR binaries && + CD binaries && + appveyor DownloadFile "%MINGW_URL%/%MINGW_ARCHIVE%/download" -FileName "%MINGW_ARCHIVE%" && + 7z x -y "%MINGW_ARCHIVE%"> nul && + del "%MINGW_ARCHIVE%" && + appveyor DownloadFile "%SFNET_URL%/%LIBSSH2_ARCHIVE%.tar.xz/download" -FileName "%LIBSSH2_ARCHIVE%.tar.xz" && + 7z x -y "%LIBSSH2_ARCHIVE%.tar.xz"> nul && + del "%LIBSSH2_ARCHIVE%.tar.xz" && + 7z x -y "%LIBSSH2_ARCHIVE%.tar"> nul && + del "%LIBSSH2_ARCHIVE%.tar" && + appveyor DownloadFile "%SFNET_URL%/%LIBCRYPTO_ARCHIVE%.tar.xz/download" -FileName "%LIBCRYPTO_ARCHIVE%.tar.xz" && + 7z x -y "%LIBCRYPTO_ARCHIVE%.tar.xz"> nul && + del "%LIBCRYPTO_ARCHIVE%.tar.xz" + 7z x -y "%LIBCRYPTO_ARCHIVE2%.tar"> nul && + del "%LIBCRYPTO_ARCHIVE%.tar" + appveyor DownloadFile "https://nim-lang.org/download/nim-%NIM_VERSION%_x%ARCH%.zip" -FileName "nim-%NIM_VERSION%_x%ARCH%.zip" && + 7z x -y "nim-%NIM_VERSION%_x%ARCH%.zip"> nul && + del "nim-%NIM_VERSION%_x%ARCH%.zip") + - SET PATH=c:\binaries\mingw%ARCH%\bin;c:\nim-%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% + - CD c:\projects\nimgen # - git clone --depth 1 https://github.com/nim-lang/nim # - cd nim # - git clone --depth 1 https://github.com/nim-lang/csources @@ -67,10 +70,7 @@ for: - appveyor PushArtifact nimgen-docs.zip cache: - - c:\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - - c:\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz - - c:\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - - c:\nim-0.18.0_x32.zip + - c:\binaries - matrix: @@ -78,33 +78,34 @@ for: - image: Ubuntu environment: - NIM_URL: https://nim-lang.org/download/nim-0.18.0.tar.xz - NIM_ARCHIVE: nim-0.18.0.tar.xz - NIM_VERSION: nim-0.18.0 - BASE_DIR: /home/appveyor/projects + matrix: + - NIM_VERSION: 0.18.0 + - NIM_VERSION: 0.19.0 install: - sudo apt -qq update - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev - - cd /home/appveyor - - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - - tar xJf $NIM_ARCHIVE - - cd $NIM_VERSION - - sh build.sh - - bin/nim c -d:release koch - - ./koch boot -d:release - - ./koch nimble -d:release - - export PATH=/home/appveyor/$NIM_VERSION/bin:~/.nimble/bin:$PATH - - cd $BASE_DIR/nimgen + - cd /home/appveyor/binaries + - if [ ! -e nim-$NIM_VERSION ]; then + curl -s -o nim-$nim_version.tar.xz https://nim-lang.org/download/nim-$nim_version.tar.xz && + tar xjf nim-$nim_version.tar.xz && + cd nim-$NIM_VERSION && + sh build.sh && + bin/nim c -d:release koch && + ./koch boot -d:release && + ./koch nimble -d:release; + fi + - export PATH=/home/appveyor/binaries/nim-$NIM_VERSION/bin:~/.nimble/bin:$PATH + - cd /home/appveyor/projects/nimgen on_finish: - zip -r -q buildlogs-lin-pkgs.zip ~/.nimble/pkgs - appveyor PushArtifact buildlogs-lin-pkgs.zip - - zip -r -q buildlogs-lin-projects.zip $BASE_DIR + - zip -r -q buildlogs-lin-projects.zip /home/appveyor/projects - appveyor PushArtifact buildlogs-lin-projects.zip cache: - - /home/appveyor/nim-0.18.0.tar.xz + - /home/appveyor/binaries build_script: - nimble install -y diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 364cbdd..134e94b 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -3,12 +3,15 @@ import distros, ospaths, strutils var full = true comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", - "nimclipboard", #"nimfastText", - "nimfuzz", "nimmonocypher", + "nimclipboard", "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2", "nimzbar" ] +var nimver = staticExec("nim -v").split("\n")[0].split(" ")[3] +if nimver >= "0.19.0": + comps.add("nimfastText") + if detectOs(Windows): comps.add("nimkerberos") From 83465b25de63f0a2bb5982c9aa17652c6071b1a6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 12:15:01 -0500 Subject: [PATCH 45/96] Multi-version CI --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 1673432..e9b7001 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -85,6 +85,7 @@ for: install: - sudo apt -qq update - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev + - mkdir /home/appveyor/binaries - cd /home/appveyor/binaries - if [ ! -e nim-$NIM_VERSION ]; then curl -s -o nim-$nim_version.tar.xz https://nim-lang.org/download/nim-$nim_version.tar.xz && From db26f9580d84758cc7bd71b5de95bfd7551e7a67 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 12:21:49 -0500 Subject: [PATCH 46/96] Multi-version CI --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e9b7001..af39ed8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -88,8 +88,8 @@ for: - mkdir /home/appveyor/binaries - cd /home/appveyor/binaries - if [ ! -e nim-$NIM_VERSION ]; then - curl -s -o nim-$nim_version.tar.xz https://nim-lang.org/download/nim-$nim_version.tar.xz && - tar xjf nim-$nim_version.tar.xz && + curl -s -o nim-$NIM_VERSION.tar.xz https://nim-lang.org/download/nim-$NIM_VERSION.tar.xz && + tar xJf nim-$nim_version.tar.xz && cd nim-$NIM_VERSION && sh build.sh && bin/nim c -d:release koch && From 35765f046b1a0b1785067d45fc128057cb7a2725 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 12:28:12 -0500 Subject: [PATCH 47/96] Multi-version CI --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index af39ed8..8daa767 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -89,7 +89,7 @@ for: - cd /home/appveyor/binaries - if [ ! -e nim-$NIM_VERSION ]; then curl -s -o nim-$NIM_VERSION.tar.xz https://nim-lang.org/download/nim-$NIM_VERSION.tar.xz && - tar xJf nim-$nim_version.tar.xz && + tar xJf nim-$NIM_VERSION.tar.xz && cd nim-$NIM_VERSION && sh build.sh && bin/nim c -d:release koch && From c4d45b318bbb9049cb856101fdb5a5f76b4b1779 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 17:17:33 -0500 Subject: [PATCH 48/96] Multi-version CI --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 8daa767..d80a59c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -85,10 +85,12 @@ for: install: - sudo apt -qq update - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev + - echo $NIM_VERSION - mkdir /home/appveyor/binaries - cd /home/appveyor/binaries - if [ ! -e nim-$NIM_VERSION ]; then curl -s -o nim-$NIM_VERSION.tar.xz https://nim-lang.org/download/nim-$NIM_VERSION.tar.xz && + ls -l && tar xJf nim-$NIM_VERSION.tar.xz && cd nim-$NIM_VERSION && sh build.sh && From b5030a8bd54198e9973385b98477c861f30c765c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 17:27:07 -0500 Subject: [PATCH 49/96] Multi-version CI --- appveyor.yml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d80a59c..52dee07 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,6 +7,11 @@ image: matrix: fast_finish: true +environment: + matrix: + - NIM_VERSION: 0.18.0 + - NIM_VERSION: 0.19.0 + for: - matrix: @@ -20,13 +25,11 @@ for: SFNET_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686 LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg - matrix: - - NIM_VERSION: 0.18.0 - - NIM_VERSION: 0.19.0 - install: + install: - CD c:\ - IF not exist "binaries" ( + echo $NIM_VERSION && MKDIR binaries && CD binaries && appveyor DownloadFile "%MINGW_URL%/%MINGW_ARCHIVE%/download" -FileName "%MINGW_ARCHIVE%" && @@ -77,20 +80,14 @@ for: only: - image: Ubuntu - environment: - matrix: - - NIM_VERSION: 0.18.0 - - NIM_VERSION: 0.19.0 - install: - sudo apt -qq update - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev - - echo $NIM_VERSION - - mkdir /home/appveyor/binaries - - cd /home/appveyor/binaries - - if [ ! -e nim-$NIM_VERSION ]; then + - if [ ! -e /home/appveyor/binaries ]; then + echo $NIM_VERSION && + mkdir /home/appveyor/binaries && + cd /home/appveyor/binaries && curl -s -o nim-$NIM_VERSION.tar.xz https://nim-lang.org/download/nim-$NIM_VERSION.tar.xz && - ls -l && tar xJf nim-$NIM_VERSION.tar.xz && cd nim-$NIM_VERSION && sh build.sh && From da865618be271f0cbf4fd8a66c03a74986f23b74 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 17:28:58 -0500 Subject: [PATCH 50/96] Multi-version CI --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 52dee07..a7337a1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -26,7 +26,7 @@ for: LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg - install: + install: - CD c:\ - IF not exist "binaries" ( echo $NIM_VERSION && From caba6fcc50629456ac06dbe306f765d617ced14c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 17:54:43 -0500 Subject: [PATCH 51/96] Fix regex crash --- nimgen.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimgen.nimble b/nimgen.nimble index 6ca5b9e..cb85eec 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -11,7 +11,7 @@ skipDirs = @["nimgen", "tests"] # Dependencies -requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex >= 0.7.1" +requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex <= 0.7.4" task test, "Test nimgen": exec "nim c -r tests/rununittests.nim" From 1c057f2520dd912176d2559e0761b060029fd217 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Oct 2018 21:55:17 -0500 Subject: [PATCH 52/96] OS specific sections, execAction, nowildcard, multi git repos, branch/tag/commit --- README.md | 20 +++- appveyor.yml | 2 +- nimgen.nimble | 4 +- src/nimgen/c2nim.nim | 9 +- src/nimgen/external.nim | 57 ++++++++---- src/nimgen/file.nim | 3 +- src/nimgen/globals.nim | 26 ++++-- src/nimgen/runcfg.nim | 202 ++++++++++++++++++++++------------------ 8 files changed, 189 insertions(+), 134 deletions(-) diff --git a/README.md b/README.md index 0ee6c9f..153fe98 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,8 @@ In all sections below, environment variables are supported via Nim's string inte "${output}/library/include" "${MY_INCLUDE_PATH}/include" +Append -win, -lin and -mac/osx for OS specific sections and tasks. E.g. n.global-win, n.post-lin, download-win, execute-lin-mac.unique1. -unix can be used as a shortcut for -lin-mac. + _[n.global]_ ```output``` = name of the Nimble project once installed, also location to place generated .nim files @@ -119,17 +121,21 @@ List of all directories or files to exclude from all parsing. If an entry here m _[n.prepare]_ -The following keys can be used to prepare dependencies such as downloading ZIP files, cloning Git repositories, etc. Multiple entries are possible by appending any .string to the key. E.g. download.file1. -win, -lin and -mac/osx can be used for OS specific tasks. E.g. download-win, execute-lin,mac.unique1 +The following keys can be used to prepare dependencies such as downloading ZIP files, cloning Git repositories, etc. Multiple entries are possible by appending any .string to the key. E.g. download.file1. ```download``` = url to download to the output directory. ZIP files are automatically extracted. Files are not redownloaded if already present but re-extracted ```extract``` = ZIP file to extract in case they are local and don't need to be downloaded. Path is relative to output directory. -```git``` = url of Git repository to clone. Full repo is pulled so gitremote + gitsparse is preferable. Resets to HEAD if already present +```gitcheckout``` = branch, commit or tag of repository to checkout in following Git command, resets after each use. Use "-b name" for branches + +```gitoutput`` = directory for all following Git commands relative to `n.global:output` [default: `n.global:output` directory] + +```git``` = url of Git repository to clone. Full repo is pulled so gitremote + gitsparse is preferable. Resets if already present ```gitremote``` = url of Git repository to partially checkout. Use with gitsparse to pull only files and dirs of interest -```gitsparse``` = list of files and/or dirs to include in partial checkout, one per line. Resets to HEAD if already present +```gitsparse``` = list of files and/or dirs to include in partial checkout, one per line. Resets if already present ```execute``` = command to run during preparation @@ -139,7 +145,9 @@ _[n.post]_ This section is the same as the prepare section, but for performing actions after the project has been processed. -```reset``` = whether or not to perform a git reset on all files after processing [default: false] +```gitoutput`` = output directory for Git reset [default: `n.global:output` directory] + +```reset``` = perform a Git reset on all files after processing [default: false] ```execute``` = command to run after processing @@ -155,7 +163,7 @@ This section allows selection of multiple sourcefiles without requiring a detail _[sourcefile]_ -The following keys apply to library source code and help with generating the .nim files. -win, -lin and -osx can be used for OS specific tasks. E.g. dynlib-win, pragma-win +The following keys apply to library source code and help with generating the .nim files. -win, -lin and -osx can be used for OS specific tasks. E.g. dynlib-win, pragma-win. ```recurse``` = find #include files and process them [default: false] @@ -173,6 +181,8 @@ The following keys apply to library source code and help with generating the .ni ```noprocess``` = do not process this source file with c2nim [default: false] - this is useful if a file only needs to be manipulated +```nowildcard``` = ignore any wildcard definitions for this sourcefile + ```reset``` = reset the file back to original state after all processing [default: false] Multiple entries for the all following keys are possible by appending any .string to the key. E.g. dynlib.win, compile.dir diff --git a/appveyor.yml b/appveyor.yml index a7337a1..531866f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,7 +10,7 @@ matrix: environment: matrix: - NIM_VERSION: 0.18.0 - - NIM_VERSION: 0.19.0 + # - NIM_VERSION: 0.19.0 for: - diff --git a/nimgen.nimble b/nimgen.nimble index cb85eec..c99b7a6 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.4.0" +version = "0.5.0" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" @@ -11,7 +11,7 @@ skipDirs = @["nimgen", "tests"] # Dependencies -requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex <= 0.7.4" +requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex >= 0.10.0" task test, "Test nimgen": exec "nim c -r tests/rununittests.nim" diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 30f23f0..0464c6a 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -2,8 +2,13 @@ import os, ospaths, regex, strutils import external, file, fileops, gencore, globals -template relativePath(path: untyped): untyped = - path.multiReplace([(gOutput, ""), ("\\", "/"), ("//", "/")]) +proc relativePath(path: string): string = + if gOutput.len() == 0: + result = path + else: + # multiReplace() bug - #9557 + result = path.replace(gOutput, "") + return result.multiReplace([("\\", "/"), ("//", "/")]) proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = var file = search(fl) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 651ccf3..a34ea69 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -15,6 +15,16 @@ proc execProc*(cmd: string): string = echo result quit(1) +proc execAction*(cmd: string): string = + var ccmd = "" + when defined(Windows): + ccmd = "cmd /c " & cmd.replace("/", "\\") + when defined(Linux) or defined(MacOSX): + ccmd = "bash -c '" & cmd & "'" + + echo "Running '" & ccmd[0..min(64, len(ccmd))] & "'" + return execProc(ccmd) + proc extractZip*(zipfile: string) = var cmd = "unzip -o $#" if defined(Windows): @@ -44,13 +54,16 @@ proc downloadUrl*(url: string) = if ext == ".zip": extractZip(file) -proc gitReset*() = - echo "Resetting Git repo" - - setCurrentDir(gOutput) +template setGitDir() = + setCurrentDir(gGitOutput) defer: setCurrentDir(gProjectDir) - let cmd = "git reset --hard HEAD" +proc gitReset*() = + echo "Resetting " & gGitOutput + + setGitDir() + + let cmd = "git reset --hard" while execCmdEx(cmd)[0].contains("Permission denied"): sleep(1000) echo " Retrying ..." @@ -58,45 +71,51 @@ proc gitReset*() = proc gitCheckout*(file: string) = echo "Resetting " & file - setCurrentDir(gOutput) - defer: setCurrentDir(gProjectDir) + setGitDir() - let cmd = "git checkout $#" % file.replace(gOutput & "/", "") + let cmd = "git checkout $#" % file.replace(gGitOutput & "/", "") while execCmdEx(cmd)[0].contains("Permission denied"): sleep(500) echo " Retrying ..." +proc gitPull() = + if gGitCheckout.len() != 0: + echo "Checking out " & gGitCheckout + #discard execProc("git fetch --depth=1 origin " & gGitCheckout) + discard execProc("git pull --tags origin master") + discard execProc("git checkout " & gGitCheckout) + gGitCheckout = "" + else: + echo "Pulling repository" + discard execProc("git pull --depth=1 origin master") + proc gitRemotePull*(url: string, pull=true) = - if dirExists(gOutput/".git"): + if dirExists(gGitOutput/".git"): if pull: gitReset() return - setCurrentDir(gOutput) - defer: setCurrentDir(gProjectDir) + setGitDir() - echo "Setting up Git repo" + echo "Setting up Git repo: " & url discard execProc("git init .") discard execProc("git remote add origin " & url) if pull: - echo "Checking out artifacts" - discard execProc("git pull --depth=1 origin master") + gitPull() proc gitSparseCheckout*(plist: string) = let sparsefile = ".git/info/sparse-checkout" - if fileExists(gOutput/sparsefile): + if fileExists(gGitOutput/sparsefile): gitReset() return - setCurrentDir(gOutput) - defer: setCurrentDir(gProjectDir) + setGitDir() discard execProc("git config core.sparsecheckout true") writeFile(sparsefile, plist) - echo "Checking out artifacts" - discard execProc("git pull --depth=1 origin master") + gitPull() proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = var diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index edf39ea..769c914 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -16,8 +16,7 @@ proc getNimout*(file: string, rename=true): string = if gRenames.hasKey(file): result = gRenames[file] - if not dirExists(parentDir(result)): - createDir(parentDir(result)) + createDir(parentDir(result)) proc exclude*(file: string): bool = for excl in gExcludes: diff --git a/src/nimgen/globals.nim b/src/nimgen/globals.nim index a83460f..9892b99 100644 --- a/src/nimgen/globals.nim +++ b/src/nimgen/globals.nim @@ -7,21 +7,27 @@ const defaultCppCompiler* = "g++" var - gDoneRecursive*: seq[string] = @[] - gDoneInline*: seq[string] = @[] - - gProjectDir* = "" + # Config gConfig*: Config - gFilter* = "" - gQuotes* = true - gCppCompiler* = getEnv(cppCompilerEnv, defaultCCompiler) - gCCompiler* = getEnv(cCompilerEnv, defaultCppCompiler) - gOutput* = "" - gIncludes*: seq[string] = @[] gExcludes*: seq[string] = @[] + gIncludes*: seq[string] = @[] gRenames* = initTable[string, string]() gWildcards* = newConfig() + + # n.global + gOutput* = "." + gQuotes* = true + gFilter* = "" + gCppCompiler* = getEnv(cppCompilerEnv, defaultCCompiler) + gCCompiler* = getEnv(cCompilerEnv, defaultCppCompiler) + + # State tracking + gGitCheckout* = "" + gGitOutput* = "" + gProjectDir* = "" gCompile*: seq[string] = @[] + gDoneInline*: seq[string] = @[] + gDoneRecursive*: seq[string] = @[] type c2nimConfigObj* = object diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 5f04b04..f2310ff 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -6,18 +6,19 @@ proc `[]`*(table: OrderedTableRef[string, string], key: string): string = ## Gets table values with env vars inserted tables.`[]`(table, key).addEnv -proc getKey(ukey: string): tuple[key: string, val: bool] = - var kv = ukey.replace(re"\..*", "").split("-", 1) +proc getKey(ukey: string, section = false): tuple[key: string, val: bool] = + var kv = if not section: ukey.replace(re"\..*", "").split("-", 1) else: ukey.split("-", 1) if kv.len() == 1: kv.add("") if kv[1] == "": return (kv[0], true) - for ostyp in kv[1].split(","): + for ostyp in kv[1].split("-"): if (ostyp == "win" and defined(Windows)) or (ostyp == "lin" and defined(Linux)) or - ((ostyp == "osx" or ostyp == "mac") and defined(MacOSX)): + ((ostyp == "osx" or ostyp == "mac") and defined(MacOSX)) or + (ostyp == "unix" and (defined(Linux) or defined(MacOSX))): return (kv[0], true) return (kv[0], false) @@ -26,6 +27,7 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str var cfg = cfgin sfile = search(file) + nowildcard = false if sfile in gDoneRecursive: return @@ -34,13 +36,20 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str echo "Processing " & sfile gDoneRecursive.add(sfile) - for pattern in gWildcards.keys(): - var m: RegexMatch - let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") - if file.find(toPattern(pat), m): - echo " Appending keys for wildcard " & pattern - for key in gWildcards[pattern].keys(): - cfg[key & "." & pattern] = gWildcards[pattern][key] + for act in cfg.keys(): + let (action, val) = getKey(act) + if val == true and action == "nowildcard" and cfg[act] == "true": + nowildcard = true + break + + if not nowildcard: + for pattern in gWildcards.keys(): + var m: RegexMatch + let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") + if file.find(toPattern(pat), m): + echo " Appending keys for wildcard " & pattern + for key in gWildcards[pattern].keys(): + cfg[key & "." & pattern] = gWildcards[pattern][key] var srch = "" @@ -200,92 +209,99 @@ proc runCfg*(cfg: string) = gProjectDir = parentDir(cfg.expandFilename()).sanitizePath gConfig = loadConfig(cfg) + gCppCompiler = getEnv(cppCompilerEnv, defaultCppCompiler).quoteShell + gCCompiler = getEnv(cCompilerEnv, defaultCCompiler).quoteShell + gGitOutput = gOutput - if gConfig.hasKey("n.global"): - if gConfig["n.global"].hasKey("output"): - setOutputDir(gConfig["n.global"]["output"]) - - if gConfig["n.global"].hasKey("cpp_compiler"): - gCppCompiler = gConfig["n.global"]["cpp_compiler"] - else: - # Reset on a per project basis - gCppCompiler = getEnv(cppCompilerEnv, defaultCppCompiler) - - if gConfig["n.global"].hasKey("c_compiler"): - gCCompiler = gConfig["n.global"]["c_compiler"] - else: - # Reset on a per project basis - gCCompiler = getEnv(cCompilerEnv, defaultCCompiler) - - gCppCompiler = gCppCompiler.quoteShell - gCCompiler = gCCompiler.quoteShell - - if gConfig["n.global"].hasKey("filter"): - gFilter = gConfig["n.global"]["filter"] - if gConfig["n.global"].hasKey("quotes"): - if gConfig["n.global"]["quotes"] == "false": - gQuotes = false - - if gConfig.hasKey("n.include"): - for inc in gConfig["n.include"].keys(): - gIncludes.add(inc.addEnv().sanitizePath) - - if gConfig.hasKey("n.exclude"): - for excl in gConfig["n.exclude"].keys(): - gExcludes.add(excl.addEnv().sanitizePath) - - if gConfig.hasKey("n.prepare"): - for prep in gConfig["n.prepare"].keys(): - let (key, val) = getKey(prep) - if val == true: - let prepVal = gConfig["n.prepare"][prep] - if key == "download": - downloadUrl(prepVal) - elif key == "extract": - extractZip(prepVal) - elif key == "git": - gitRemotePull(prepVal) - elif key == "gitremote": - gitRemotePull(prepVal, false) - elif key == "gitsparse": - gitSparseCheckout(prepVal) - elif key == "execute": - discard execProc(prepVal) - elif key == "copy": - doCopy(prepVal) - - if gConfig.hasKey("n.wildcard"): - var wildcard = "" - for wild in gConfig["n.wildcard"].keys(): - let (key, val) = getKey(wild) - if val == true: - if key == "wildcard": - wildcard = gConfig["n.wildcard"][wild] - else: - gWildcards.setSectionKey(wildcard, wild, - gConfig["n.wildcard"][wild]) - - for file in gConfig.keys(): - if file in @["n.global", "n.include", "n.exclude", - "n.prepare", "n.wildcard", "n.post"]: + for section in gConfig.keys(): + let (sname, sval) = getKey(section, true) + if not sval: continue - if file == "n.sourcefile": - for pattern in gConfig["n.sourcefile"].keys(): - for file in walkFiles(pattern.addEnv): - runFile(file) - else: - runFile(file, gConfig[file]) + case sname: + of "n.global": + for glob in gConfig[section].keys(): + let (key, val) = getKey(glob) + if val == true: + let globVal = gConfig[section][glob] + case key: + of "output": + setOutputDir(globVal) + of "cpp_compiler": + gCppCompiler = globVal.quoteShell + of "c_compiler": + gCCompiler = globVal.quoteShell + of "filter": + gFilter = globVal + of "quotes": + if globVal == "false": + gQuotes = false - if gConfig.hasKey("n.post"): - for post in gConfig["n.post"].keys(): - let (key, val) = getKey(post) - if val == true: - let postVal = gConfig["n.post"][post] - if key == "reset": - gitReset() - elif key == "execute": - discard execProc(postVal) + of "n.include": + for inc in gConfig[section].keys(): + gIncludes.add(inc.addEnv().sanitizePath) + + of "n.exclude": + for excl in gConfig[section].keys(): + gExcludes.add(excl.addEnv().sanitizePath) + + of "n.prepare": + for prep in gConfig[section].keys(): + let (key, val) = getKey(prep) + if val == true: + let prepVal = gConfig[section][prep] + case key: + of "download": + downloadUrl(prepVal) + of "extract": + extractZip(prepVal) + of "gitcheckout": + gGitCheckout = prepVal + of "gitoutput": + gGitOutput = gOutput/prepVal + createDir(gGitOutput) + of "git": + gitRemotePull(prepVal) + of "gitremote": + gitRemotePull(prepVal, false) + of "gitsparse": + gitSparseCheckout(prepVal) + of "execute": + discard execAction(prepVal) + of "copy": + doCopy(prepVal) + + of "n.wildcard": + var wildcard = "" + for wild in gConfig[section].keys(): + let (key, val) = getKey(wild) + if val == true: + if key == "wildcard": + wildcard = gConfig[section][wild] + else: + gWildcards.setSectionKey(wildcard, wild, + gConfig[section][wild]) + + of "n.sourcefile": + for pattern in gConfig[section].keys(): + for file in walkFiles(pattern.addEnv): + runFile(file) + + of "n.post": + for post in gConfig[section].keys(): + let (key, val) = getKey(post) + if val == true: + let postVal = gConfig[section][post] + case key: + of "gitoutput": + gGitOutput = gOutput/postVal + of "reset": + gitReset() + of "execute": + discard execAction(postVal) + + else: + runFile(section, gConfig[section]) let gHelp = """ Nimgen is a helper for c2nim to simplify and automate the wrapping of C libraries From 1f3ae34dc0fd9959cdd187a57cc4854cb9ca9b67 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Oct 2018 22:16:29 -0500 Subject: [PATCH 53/96] Fix gGitOutput set --- src/nimgen/external.nim | 1 - src/nimgen/runcfg.nim | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index a34ea69..8656147 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -81,7 +81,6 @@ proc gitCheckout*(file: string) = proc gitPull() = if gGitCheckout.len() != 0: echo "Checking out " & gGitCheckout - #discard execProc("git fetch --depth=1 origin " & gGitCheckout) discard execProc("git pull --tags origin master") discard execProc("git checkout " & gGitCheckout) gGitCheckout = "" diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index f2310ff..a924734 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -201,6 +201,8 @@ proc setOutputDir(dir: string) = quit(1) createDir(gOutput) + gGitOutput = gOutput + proc runCfg*(cfg: string) = if not fileExists(cfg): echo "Config doesn't exist: " & cfg From 9ec27678ea8acccce9089f4736defad97631713d Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Oct 2018 22:43:55 -0500 Subject: [PATCH 54/96] Appveyor bugfix --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 531866f..4df9fe9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -42,9 +42,9 @@ for: del "%LIBSSH2_ARCHIVE%.tar" && appveyor DownloadFile "%SFNET_URL%/%LIBCRYPTO_ARCHIVE%.tar.xz/download" -FileName "%LIBCRYPTO_ARCHIVE%.tar.xz" && 7z x -y "%LIBCRYPTO_ARCHIVE%.tar.xz"> nul && - del "%LIBCRYPTO_ARCHIVE%.tar.xz" + del "%LIBCRYPTO_ARCHIVE%.tar.xz" && 7z x -y "%LIBCRYPTO_ARCHIVE2%.tar"> nul && - del "%LIBCRYPTO_ARCHIVE%.tar" + del "%LIBCRYPTO_ARCHIVE%.tar" && appveyor DownloadFile "https://nim-lang.org/download/nim-%NIM_VERSION%_x%ARCH%.zip" -FileName "nim-%NIM_VERSION%_x%ARCH%.zip" && 7z x -y "nim-%NIM_VERSION%_x%ARCH%.zip"> nul && del "nim-%NIM_VERSION%_x%ARCH%.zip") From e8faaf8d0cf0619dca0d9ab0a53563cc66d572b1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Oct 2018 22:54:08 -0500 Subject: [PATCH 55/96] Appveyor bugfix --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 4df9fe9..2cfa477 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -43,7 +43,7 @@ for: appveyor DownloadFile "%SFNET_URL%/%LIBCRYPTO_ARCHIVE%.tar.xz/download" -FileName "%LIBCRYPTO_ARCHIVE%.tar.xz" && 7z x -y "%LIBCRYPTO_ARCHIVE%.tar.xz"> nul && del "%LIBCRYPTO_ARCHIVE%.tar.xz" && - 7z x -y "%LIBCRYPTO_ARCHIVE2%.tar"> nul && + 7z x -y "%LIBCRYPTO_ARCHIVE%.tar"> nul && del "%LIBCRYPTO_ARCHIVE%.tar" && appveyor DownloadFile "https://nim-lang.org/download/nim-%NIM_VERSION%_x%ARCH%.zip" -FileName "nim-%NIM_VERSION%_x%ARCH%.zip" && 7z x -y "nim-%NIM_VERSION%_x%ARCH%.zip"> nul && From ebd362d934af435bb6c329a08afd8ddbdb53e553 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Oct 2018 23:04:28 -0500 Subject: [PATCH 56/96] Appveyor bugfix --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 2cfa477..a9b8462 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -48,7 +48,7 @@ for: appveyor DownloadFile "https://nim-lang.org/download/nim-%NIM_VERSION%_x%ARCH%.zip" -FileName "nim-%NIM_VERSION%_x%ARCH%.zip" && 7z x -y "nim-%NIM_VERSION%_x%ARCH%.zip"> nul && del "nim-%NIM_VERSION%_x%ARCH%.zip") - - SET PATH=c:\binaries\mingw%ARCH%\bin;c:\nim-%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% + - SET PATH=c:\binaries\mingw%ARCH%\bin;c:\binaries\nim-%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - CD c:\projects\nimgen # - git clone --depth 1 https://github.com/nim-lang/nim # - cd nim From ad499177337fc8b3f682f5ea91648191300cfc56 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Oct 2018 23:25:43 -0500 Subject: [PATCH 57/96] Appveyor bugfix --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index a9b8462..5d3cd62 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -69,7 +69,7 @@ for: - appveyor PushArtifact buildlogs-win-pkgs.zip - 7z a -r buildlogs-win-projects.zip c:\projects\* - appveyor PushArtifact buildlogs-win-projects.zip - - 7z a -r nimgen-docs.zip %BASE_DIR%\nimgen\web + - 7z a -r nimgen-docs.zip c:\projects\nimgen\web - appveyor PushArtifact nimgen-docs.zip cache: From 8188c7511858c28259f0244056f886c25a8f0cc5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 1 Nov 2018 14:31:09 -0500 Subject: [PATCH 58/96] Enable 0.19.0 --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 5d3cd62..35c49fb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,8 @@ version: '{build}' image: - - Ubuntu - Visual Studio 2017 + - Ubuntu matrix: fast_finish: true @@ -10,7 +10,7 @@ matrix: environment: matrix: - NIM_VERSION: 0.18.0 - # - NIM_VERSION: 0.19.0 + - NIM_VERSION: 0.19.0 for: - From f1a8d178910e150746e6cc630cc0f08eb6e86a34 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 1 Nov 2018 15:37:56 -0500 Subject: [PATCH 59/96] Fix version print --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 35c49fb..a4d7c74 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -29,7 +29,7 @@ for: install: - CD c:\ - IF not exist "binaries" ( - echo $NIM_VERSION && + echo %NIM_VERSION% && MKDIR binaries && CD binaries && appveyor DownloadFile "%MINGW_URL%/%MINGW_ARCHIVE%/download" -FileName "%MINGW_ARCHIVE%" && From ee7a55f2ecf801d411f34e284ef531dffe22de6e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 10:38:38 -0500 Subject: [PATCH 60/96] Add Travis for devel/OSX --- .travis.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..9c56b1f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,22 @@ +sudo: false +language: c +os: + - linux + - osx +dist: trusty +before_script: + - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` + - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` + - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` + - curl -L --silent -o $TXZ https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ + - tar xf $TXZ + - cd nim-$VERSION + - sh build.sh + - bin/nim c koch + - ./koch boot -d:release + - ./koch nimble + - export PATH=$(pwd)/bin:$PATH + - cd .. +script: + - nimble install -y + - nimble tests From 713703caae174888a10b27144a967a936de02b21 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 10:55:05 -0500 Subject: [PATCH 61/96] Travis fixes --- .travis.yml | 12 ++++++++++-- README.md | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c56b1f..720d143 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,19 @@ os: - linux - osx dist: trusty +addons: + apt: + packages: + - python-pygments + - libssh2-1-dev + - libgcrypt20-dev + - libgpg-error-dev before_script: - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - - curl -L --silent -o $TXZ https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ + - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" + - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - tar xf $TXZ - cd nim-$VERSION - sh build.sh @@ -19,4 +27,4 @@ before_script: - cd .. script: - nimble install -y - - nimble tests + - nimble test diff --git a/README.md b/README.md index 153fe98..88bed4d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Chat on Gitter](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/nimgen/Lobby) [![Build status](https://ci.appveyor.com/api/projects/status/05t5ja88lmv1rt3r/branch/master?svg=true)](https://ci.appveyor.com/project/genotrance/nimgen/branch/master) +[![Build Status](https://travis-ci.org/genotrance/nimgen.svg?branch=master)](https://travis-ci.org/genotrance/nimgen) Nimgen is a helper for [c2nim](https://github.com/nim-lang/c2nim/) to simplify and automate the wrapping of C libraries. From c40919b87cf16c1066ea028914a828f6d1c04398 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 10:58:38 -0500 Subject: [PATCH 62/96] Fix deps --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 720d143..ad9de5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,6 @@ addons: packages: - python-pygments - libssh2-1-dev - - libgcrypt20-dev - - libgpg-error-dev before_script: - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` From 715b502274da13adb810891d7a493e4ee73ba8dc Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 11:25:48 -0500 Subject: [PATCH 63/96] Fix path --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ad9de5c..340e9a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ before_script: - bin/nim c koch - ./koch boot -d:release - ./koch nimble - - export PATH=$(pwd)/bin:$PATH + - export PATH=$(pwd)/bin:~/.nimble/bin:$PATH - cd .. script: - nimble install -y From 34efd6619ffdb4d52047d13ca05000a75941f605 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 14:14:15 -0500 Subject: [PATCH 64/96] Docgen only on Windows --- .travis.yml | 2 +- appveyor.yml | 2 +- tests/nimgentest.nims | 29 +++++++++++++++-------------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 340e9a3..9e2b6dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,12 @@ dist: trusty addons: apt: packages: - - python-pygments - libssh2-1-dev before_script: - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` + - gcc -v - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - tar xf $TXZ diff --git a/appveyor.yml b/appveyor.yml index a4d7c74..ff7f260 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -82,7 +82,7 @@ for: install: - sudo apt -qq update - - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev + - sudo apt -qq install --yes libssh2-1-dev libgcrypt20-dev libgpg-error-dev - if [ ! -e /home/appveyor/binaries ]; then echo $NIM_VERSION && mkdir /home/appveyor/binaries && diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 134e94b..f3ab526 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -40,20 +40,21 @@ for comp in comps: exec "nimble install -y" exec "nimble test" - if dirExists("web"/comp): - rmDir("web"/comp) + when defined(windows): + if dirExists("web"/comp): + rmDir("web"/comp) - mkDir("web"/comp) - for file in listFiles(".."/comp/comp) & listFiles(".."/comp): - if file.splitFile().ext == ".nim": - cpFile(file, "web"/comp/extractFilename(file)) - - cpFile("web"/"nimdoc.cfg", "web"/comp/"nimdoc.cfg") - withDir("web"/comp): - for file in listFiles("."): + mkDir("web"/comp) + for file in listFiles(".."/comp/comp) & listFiles(".."/comp): if file.splitFile().ext == ".nim": - exec "nim doc --git.url:. --index:on -o:" & file.changeFileExt("html") & " " & file - exec "pygmentize -f html -O full,linenos=1,anchorlinenos=True,lineanchors=L,style=vs -o " & file & ".html " & file + cpFile(file, "web"/comp/extractFilename(file)) - exec "nim buildIndex -o:index.html ." - rmFile("web"/comp/"nimdoc.cfg") + cpFile("web"/"nimdoc.cfg", "web"/comp/"nimdoc.cfg") + withDir("web"/comp): + for file in listFiles("."): + if file.splitFile().ext == ".nim": + exec "nim doc --git.url:. --index:on -o:" & file.changeFileExt("html") & " " & file + exec "pygmentize -f html -O full,linenos=1,anchorlinenos=True,lineanchors=L,style=vs -o " & file & ".html " & file + + exec "nim buildIndex -o:index.html ." + rmFile("web"/comp/"nimdoc.cfg") From 5549505f61dc5f2a6b1da5cd3b12130dafe5bae8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 15:09:28 -0500 Subject: [PATCH 65/96] Cleanup --- .travis.yml | 6 +++--- appveyor.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e2b6dd..fc24973 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,10 +9,10 @@ addons: packages: - libssh2-1-dev before_script: - - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` + - export RELEASE=`curl --silent "https://api.github.com/repos/nim-lang/nightlies/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` + - export TXZ=`curl --silent "https://api.github.com/repos/nim-lang/nightlies/releases/latest" | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - - gcc -v + - curl --silent "https://api.github.com/repos/nim-lang/nightlies/releases/latest" - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - tar xf $TXZ diff --git a/appveyor.yml b/appveyor.yml index ff7f260..e4e062e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,8 @@ version: '{build}' image: - - Visual Studio 2017 - Ubuntu + - Visual Studio 2017 matrix: fast_finish: true From c7c419e6fad121dee205146f60dd451789aa6a5a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 17:12:37 -0500 Subject: [PATCH 66/96] Backrev c2nim --- nimgen.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimgen.nimble b/nimgen.nimble index c99b7a6..c6a2625 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -11,7 +11,7 @@ skipDirs = @["nimgen", "tests"] # Dependencies -requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex >= 0.10.0" +requires "nim >= 0.17.0", "c2nim#3ec45c24585ebaed", "regex >= 0.10.0" task test, "Test nimgen": exec "nim c -r tests/rununittests.nim" From d27d28f0d94bbd495cc912fae78300712c71a970 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 17:15:35 -0500 Subject: [PATCH 67/96] Fix travis again --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index fc24973..79889fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,10 +9,10 @@ addons: packages: - libssh2-1-dev before_script: - - export RELEASE=`curl --silent "https://api.github.com/repos/nim-lang/nightlies/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - - export TXZ=`curl --silent "https://api.github.com/repos/nim-lang/nightlies/releases/latest" | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` + - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` + - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - - curl --silent "https://api.github.com/repos/nim-lang/nightlies/releases/latest" + - curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - tar xf $TXZ From d080be55a548049b170c88a356724d505401d545 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 21:56:14 -0500 Subject: [PATCH 68/96] Github download --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 79889fb..ebf0bd4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,12 +9,11 @@ addons: packages: - libssh2-1-dev before_script: - - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` + - curl -u $TOKEN --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest -o latest.json + - export RELEASE=`echo latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` + - export TXZ=`echo latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - - curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - tar xf $TXZ - cd nim-$VERSION - sh build.sh From 03c38a244838809fb630431ce20b7183a6d316ca Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 22:04:09 -0500 Subject: [PATCH 69/96] Github download --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ebf0bd4..4687afe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,9 @@ addons: packages: - libssh2-1-dev before_script: - - curl -u $TOKEN --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest -o latest.json - - export RELEASE=`echo latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - - export TXZ=`echo latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` + - curl -u $TOKEN -o latest.json https://api.github.com/repos/nim-lang/nightlies/releases/latest + - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` + - export TXZ=`cat latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - tar xf $TXZ From d6c827bb281f24899a16aa60e0b5cd0a1c3c61dc Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 22:18:05 -0500 Subject: [PATCH 70/96] Github download --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4687afe..7b3241e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,9 @@ addons: packages: - libssh2-1-dev before_script: - - curl -u $TOKEN -o latest.json https://api.github.com/repos/nim-lang/nightlies/releases/latest + - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest + - curl -u $TOKEN https://api.github.com/repos/nim-lang/nightlies/releases/latest + - cat latest.json - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - export TXZ=`cat latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` From 81fa6d15bae2f5529556d917706a50d8df26ea8d Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 22:26:46 -0500 Subject: [PATCH 71/96] Github download --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7b3241e..cddc6a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,12 +10,11 @@ addons: - libssh2-1-dev before_script: - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - - curl -u $TOKEN https://api.github.com/repos/nim-lang/nightlies/releases/latest - - cat latest.json - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - export TXZ=`cat latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" + - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - tar xf $TXZ - cd nim-$VERSION - sh build.sh From a8b836656ef1d3147d16931daa0ddf53e6a925fb Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 23:04:58 -0500 Subject: [PATCH 72/96] libssh2 for OSX --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index cddc6a0..66e3685 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,9 @@ addons: apt: packages: - libssh2-1-dev + brew: + packages: + - libssh2 before_script: - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` From 5be86fcd990162724298b4d3c9692fd3acd9f8ed Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 23:24:45 -0500 Subject: [PATCH 73/96] Brew install --- .travis.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66e3685..27b2cb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,10 @@ addons: apt: packages: - libssh2-1-dev - brew: - packages: - - libssh2 + +before_install: + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libssh2; fi + before_script: - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` @@ -26,6 +27,7 @@ before_script: - ./koch nimble - export PATH=$(pwd)/bin:~/.nimble/bin:$PATH - cd .. + script: - nimble install -y - nimble test From c60a8cd415a3f495242968fb43e6211ff80effd5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 3 Nov 2018 21:14:13 -0500 Subject: [PATCH 74/96] Remove nimzbar from OSX --- tests/nimgentest.nims | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index f3ab526..cb70a63 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -5,7 +5,7 @@ var comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", - "nimtess2", "nimzbar" + "nimtess2" ] var nimver = staticExec("nim -v").split("\n")[0].split(" ")[3] @@ -15,6 +15,9 @@ if nimver >= "0.19.0": if detectOs(Windows): comps.add("nimkerberos") +if not detectOs(MacOSX): + comps.add("nimzbar") + if paramCount() > 2: for i in 3 .. paramCount(): if paramStr(i) == "--full": From 47ca2cffd6fa0f91e390110b4b3b4a7736c5310c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 3 Nov 2018 23:20:28 -0500 Subject: [PATCH 75/96] Brew no autoupdate --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 27b2cb1..988a833 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ addons: - libssh2-1-dev before_install: - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libssh2; fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install libssh2; fi before_script: - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest From ae6d4aff28559b13cec28c878209591ea3b57314 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 3 Nov 2018 23:38:04 -0500 Subject: [PATCH 76/96] Last nightlies release --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 988a833..9b889c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ before_install: before_script: - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - - export TXZ=`cat latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` + - export TXZ=`cat latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux | tail -n 1` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" From 44fc30a9cb00f6cb2537cdc2a4fad9a091b7681c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 4 Nov 2018 14:27:27 -0600 Subject: [PATCH 77/96] Add nimdoc.cfg --- nimgen.nimble | 2 +- web/CNAME | 1 + web/nimdoc.cfg | 1401 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1403 insertions(+), 1 deletion(-) create mode 100644 web/CNAME create mode 100644 web/nimdoc.cfg diff --git a/nimgen.nimble b/nimgen.nimble index c6a2625..085580a 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -7,7 +7,7 @@ license = "MIT" bin = @["nimgen"] srcDir = "src" -skipDirs = @["nimgen", "tests"] +skipDirs = @["tests", "web"] # Dependencies diff --git a/web/CNAME b/web/CNAME new file mode 100644 index 0000000..2c6879b --- /dev/null +++ b/web/CNAME @@ -0,0 +1 @@ +nimgen.genotrance.com \ No newline at end of file diff --git a/web/nimdoc.cfg b/web/nimdoc.cfg new file mode 100644 index 0000000..55c2827 --- /dev/null +++ b/web/nimdoc.cfg @@ -0,0 +1,1401 @@ +# This is the config file for the documentation generator. +# (c) 2016 Andreas Rumpf +# Feel free to edit the templates as you need. If you modify this file, it +# might be worth updating the hardcoded values in packages/docutils/rstgen.nim + +split.item.toc = "20" +# too long entries in the table of contents wrap around +# after this number of characters + +doc.section = """ +
+

$sectionTitle

+
+$content +
+""" + +doc.section.toc = """ +
  • + $sectionTitle +
      + $content +
    +
    • +""" + +# Chunk of HTML emitted for each entry in the HTML table of contents. +# Available variables are: +# * $desc: the actual docstring of the item. +# * $header: the full version of name, including types, pragmas, tags, etc. +# * $header_plain: like header but without HTML, for attribute embedding. +# * $itemID: numerical unique entry of the item in the HTML. +# * $itemSym: short symbolic name of the item for easier hyperlinking. +# * $itemSymEnc: quoted version for URLs or attributes. +# * $itemSymOrID: the symbolic name or the ID if that is not unique. +# * $itemSymOrIDEnc: quoted version for URLs or attributes. +# * $name: reduced name of the item. +# * $seeSrc: generated HTML from doc.item.seesrc (if some switches are used). + +doc.item = """ +
    $header
    +
    +$desc +$seeSrc +
    +""" + +# Chunk of HTML emitted for each entry in the HTML table of contents. +# See doc.item for available substitution variables. +doc.item.toc = """ +
  • $name
    • +""" + +# HTML rendered for doc.item's seeSrc variable. Note that this will render to +# the empty string if you don't pass anything through --docSeeSrcURL. Available +# substitutaion variables here are: +# * $path: relative path to the file being processed. +# * $line: line of the item in the original source file. +# * $url: whatever you did pass through the --docSeeSrcUrl switch (which also +# gets variables path/line replaced!) +doc.item.seesrc = """  Source +""" + +doc.toc = """ +
      +$content +
    +""" + +doc.body_toc = """ +
    +
    +
    + Search: +
    + $tableofcontents +
    +
    +
    +

    $moduledesc

    + $content +
    +
    +""" + +@if boot: +# This is enabled with the "boot" directive to generate +# the compiler documentation. +# As a user, tweak the block below instead. +# You can add your own global-links entries +doc.body_toc_group = """ +
    +
    + +
    + Search: +
    +
    + Group by: + +
    + $tableofcontents +
    +
    +
    +

    $moduledesc

    + $content +
    +
    +""" + +@else + +doc.body_toc_group = """ +
    +
    + +
    + Search: +
    +
    + Group by: + +
    + $tableofcontents +
    +
    +
    +

    $moduledesc

    + $content +
    +
    +""" +@end + +doc.body_no_toc = """ +$moduledesc +$content +""" + +doc.listing_start = "
    "
+doc.listing_end = "
    " + +# * $analytics: Google analytics location, includes + + + + + +
    +
    +

    $title

    + $content +
    +
    + +
    + Made with Nim. Generated: $date $time UTC +
    +
    +
    +
    +$analytics + + +""" From b78f6d38fdfabe7fddc5aa1abe7cae41d77e46ad Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 4 Nov 2018 14:34:50 -0600 Subject: [PATCH 78/96] Fix nimgen dir regression --- nimgen.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimgen.nimble b/nimgen.nimble index 085580a..8c44026 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -7,7 +7,7 @@ license = "MIT" bin = @["nimgen"] srcDir = "src" -skipDirs = @["tests", "web"] +skipDirs = @["nimgen", "tests", "web"] # Dependencies From 02459f3d82eda6a79f58eed8b50526089a9cb47c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 4 Nov 2018 15:06:37 -0600 Subject: [PATCH 79/96] Remove nimfastText from gcc < 5.x --- tests/nimgentest.nims | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index cb70a63..e544f43 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -8,8 +8,11 @@ var "nimtess2" ] -var nimver = staticExec("nim -v").split("\n")[0].split(" ")[3] -if nimver >= "0.19.0": +let + gccver = staticExec("gcc --version").split("\n")[0].split(" ")[^1] + nimver = staticExec("nim -v").split("\n")[0].split(" ")[3] + +if nimver >= "0.19.0" and gccver >= "5.0.0": comps.add("nimfastText") if detectOs(Windows): @@ -18,6 +21,10 @@ if detectOs(Windows): if not detectOs(MacOSX): comps.add("nimzbar") +echo "Testing comps:" +for comp in comps: + echo " " & comp + if paramCount() > 2: for i in 3 .. paramCount(): if paramStr(i) == "--full": From d6241b1da4571c846e321b008826400fefe75112 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 4 Nov 2018 16:19:54 -0600 Subject: [PATCH 80/96] Pygmentize only --- tests/nimgentest.nims | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index e544f43..0f1810c 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -1,7 +1,7 @@ import distros, ospaths, strutils var - full = true + pygonly = false comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", @@ -27,44 +27,47 @@ for comp in comps: if paramCount() > 2: for i in 3 .. paramCount(): - if paramStr(i) == "--full": - full = true + if paramStr(i) == "--pygonly": + pygonly = true elif paramStr(i).len() > 10 and "--comps=" in paramStr(i)[0 ..< 8]: comps = paramStr(i)[8 .. ^1].split(",") for comp in comps: - if not dirExists(".."/comp): - withDir(".."): - exec "git clone --depth=1 https://github.com/genotrance/" & comp + if not pygonly: + if not dirExists(".."/comp): + withDir(".."): + exec "git clone --depth=1 https://github.com/genotrance/" & comp - exec "nimble uninstall -y " & comp, "", "" - withDir(".."/comp): - exec "git pull" + exec "nimble uninstall -y " & comp, "", "" + withDir(".."/comp): + exec "git pull" - if full: rmDir(comp) exec "nimble install -y" exec "nimble test" - exec "nimble install -y" - exec "nimble test" + exec "nimble install -y" + exec "nimble test" when defined(windows): - if dirExists("web"/comp): - rmDir("web"/comp) + if not pygonly: + if dirExists("web"/comp): + rmDir("web"/comp) - mkDir("web"/comp) - for file in listFiles(".."/comp/comp) & listFiles(".."/comp): - if file.splitFile().ext == ".nim": - cpFile(file, "web"/comp/extractFilename(file)) + mkDir("web"/comp) + for file in listFiles(".."/comp/comp) & listFiles(".."/comp): + if file.splitFile().ext == ".nim": + cpFile(file, "web"/comp/extractFilename(file)) cpFile("web"/"nimdoc.cfg", "web"/comp/"nimdoc.cfg") withDir("web"/comp): for file in listFiles("."): if file.splitFile().ext == ".nim": - exec "nim doc --git.url:. --index:on -o:" & file.changeFileExt("html") & " " & file + if not pygonly: + exec "nim doc --git.url:. --index:on -o:" & file.changeFileExt("html") & " " & file exec "pygmentize -f html -O full,linenos=1,anchorlinenos=True,lineanchors=L,style=vs -o " & file & ".html " & file - exec "nim buildIndex -o:index.html ." + if not pygonly: + exec "nim buildIndex -o:index.html ." rmFile("web"/comp/"nimdoc.cfg") From 86df7b19f1ddc1939dfdc0d2851b064fd66712c2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 8 Nov 2018 21:39:20 -0600 Subject: [PATCH 81/96] Fix README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 88bed4d..4df389c 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ The following keys can be used to prepare dependencies such as downloading ZIP f ```gitcheckout``` = branch, commit or tag of repository to checkout in following Git command, resets after each use. Use "-b name" for branches -```gitoutput`` = directory for all following Git commands relative to `n.global:output` [default: `n.global:output` directory] +```gitoutput``` = directory for all following Git commands relative to `n.global:output` [default: `n.global:output` directory] ```git``` = url of Git repository to clone. Full repo is pulled so gitremote + gitsparse is preferable. Resets if already present @@ -146,7 +146,7 @@ _[n.post]_ This section is the same as the prepare section, but for performing actions after the project has been processed. -```gitoutput`` = output directory for Git reset [default: `n.global:output` directory] +```gitoutput``` = output directory for Git reset [default: `n.global:output` directory] ```reset``` = perform a Git reset on all files after processing [default: false] From 24c62dbc188fe638f7640b50a3767e5c09c9f9d7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 7 Dec 2018 17:58:46 -0600 Subject: [PATCH 82/96] Enable nimfastText on OSX --- tests/nimgentest.nims | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 0f1810c..7df6419 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -12,7 +12,7 @@ let gccver = staticExec("gcc --version").split("\n")[0].split(" ")[^1] nimver = staticExec("nim -v").split("\n")[0].split(" ")[3] -if nimver >= "0.19.0" and gccver >= "5.0.0": +if nimver >= "0.19.0" and (gccver >= "5.0.0" or detectOs(MacOSX)): comps.add("nimfastText") if detectOs(Windows): From 2f1a62a03023981f14a8a3cdfbd1d5238a388cb2 Mon Sep 17 00:00:00 2001 From: data-man Date: Sun, 9 Dec 2018 05:54:36 +0500 Subject: [PATCH 83/96] Don't use ospaths on Nim-devel --- src/nimgen/c2nim.nim | 5 ++++- src/nimgen/file.nim | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 0464c6a..f3b5c33 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -1,4 +1,7 @@ -import os, ospaths, regex, strutils +import os, regex, strutils + +when (NimMajor, NimMinor, NimPatch) < (0, 19, 9): + import ospaths import external, file, fileops, gencore, globals diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index 769c914..81e1d86 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -1,4 +1,7 @@ -import os, ospaths, pegs, regex, strutils, tables +import os, pegs, regex, strutils, tables + +when (NimMajor, NimMinor, NimPatch) < (0, 19, 9): + import ospaths import globals, external From 46cce39e64a9056d2d6d494047be7dbb31c204f0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 10 Dec 2018 09:48:46 -0600 Subject: [PATCH 84/96] Use choosenim for Travis --- .travis.yml | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9b889c5..1ed925a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,12 @@ -sudo: false -language: c os: - linux - osx -dist: trusty + +language: c + +env: + - BRANCH=devel + addons: apt: packages: @@ -12,21 +15,12 @@ addons: before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install libssh2; fi -before_script: - - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - - export TXZ=`cat latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux | tail -n 1` - - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - - tar xf $TXZ - - cd nim-$VERSION - - sh build.sh - - bin/nim c koch - - ./koch boot -d:release - - ./koch nimble - - export PATH=$(pwd)/bin:~/.nimble/bin:$PATH - - cd .. +install: + - export CHOOSENIM_CHOOSE_VERSION=$BRANCH + - | + curl https://nim-lang.org/choosenim/init.sh -sSf > init.sh + sh init.sh -y + - export PATH=$HOME/.nimble/bin:$PATH script: - nimble install -y From 31b63a6ea6d035d23b009d8b18d6f74c004c1ce5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 10 Dec 2018 12:00:53 -0600 Subject: [PATCH 85/96] Update nimfuzz to nimfuzzy --- README.md | 4 ++-- tests/nimgentest.nims | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4df389c..6a64779 100644 --- a/README.md +++ b/README.md @@ -54,14 +54,14 @@ To see examples of nimgen in action check out the following wrappers:- * git checkout * [nimfastText](https://github.com/genotrance/nimfastText) - fastText wrapper: [docs](http://nimgen.genotrance.com/nimfastText) * git sparse checkout - * [nimfuzz](https://github.com/genotrance/nimfuzz) - fts_fuzzy_match wrapper: [docs](http://nimgen.genotrance.com/nimfuzz) + * [nimfuzzy](https://github.com/genotrance/nimfuzzy) - fts_fuzzy_match wrapper: [docs](http://nimgen.genotrance.com/nimfuzzy) * download header file * [nimkerberos](https://github.com/genotrance/nimkerberos) - WinKerberos wrapper: [docs](http://nimgen.genotrance.com/nimkerberos) * git sparse checkout * [nimmonocypher](https://github.com/genotrance/nimmonocypher) - monocypher wrapper: [docs](http://nimgen.genotrance.com/nimmonocypher) * git sparse checkout * [nimnuklear](https://github.com/genotrance/nimnuklear) - nuklear wrapper: [docs](https://nimgen.genotrance.com/nimnuklear) - * git sparse checkout + * git sparse checkout * [nimpcre](https://github.com/genotrance/nimpcre) - PCRE wrapper: [docs](http://nimgen.genotrance.com/nimpcre) * git checkout * [nimrax](https://github.com/genotrance/nimrax) - Radix tree wrapper: [docs](http://nimgen.genotrance.com/nimrax) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 7df6419..8a5cbd6 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -3,7 +3,7 @@ import distros, ospaths, strutils var pygonly = false comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", - "nimclipboard", "nimfuzz", "nimmonocypher", + "nimclipboard", "nimfuzzy", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2" ] @@ -21,6 +21,9 @@ if detectOs(Windows): if not detectOs(MacOSX): comps.add("nimzbar") +echo "Nim version: " & nimver +echo "GCC version: " & gccver + echo "Testing comps:" for comp in comps: echo " " & comp From cf132951bdc326602532a740791041c139895b5e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 31 Dec 2018 12:33:09 -0600 Subject: [PATCH 86/96] Update appveyor to 0.19.2 --- appveyor.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e4e062e..2e382be 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,7 +10,7 @@ matrix: environment: matrix: - NIM_VERSION: 0.18.0 - - NIM_VERSION: 0.19.0 + - NIM_VERSION: 0.19.2 for: - @@ -50,19 +50,6 @@ for: del "nim-%NIM_VERSION%_x%ARCH%.zip") - SET PATH=c:\binaries\mingw%ARCH%\bin;c:\binaries\nim-%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - CD c:\projects\nimgen - # - git clone --depth 1 https://github.com/nim-lang/nim - # - cd nim - # - git clone --depth 1 https://github.com/nim-lang/csources - # - cd csources - # - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat ) - # - cd .. - # - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%" - # - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\bin"> nul - # - bin\nim c -d:release koch - # - koch boot -d:release - # - koch nimble -d:release - # - SET PATH=%CD%\bin;%PATH% - # - cd .. on_finish: - 7z a -r buildlogs-win-pkgs.zip %USERPROFILE%\.nimble\pkgs From 293476c0a967e07b9b277ced2437a821a5604921 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 6 Mar 2019 00:34:11 -0600 Subject: [PATCH 87/96] Remove nimarchive since it is currently broken --- tests/nimgentest.nims | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 8a5cbd6..d8e5737 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,7 +2,8 @@ import distros, ospaths, strutils var pygonly = false - comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", + comps = @["libsvm", "nim7z", #"nimarchive", + "nimbass", "nimbigwig", "nimclipboard", "nimfuzzy", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2" From 69b9ee9083d1ef81764141407b6b114e43abdc37 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 22 Mar 2019 09:15:49 -0600 Subject: [PATCH 88/96] Re-enable remove static --- src/nimgen/c2nim.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index f3b5c33..3d15243 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -31,6 +31,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): prepend(cfile, getDefines(file, c2nimConfig.inline)) + removeStatic(cfile) var extflags = "" passC = "" @@ -101,6 +102,8 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = removeFile(cfile) except: discard + else: + reAddStatic(cfile) # Nim doesn't like {.cdecl.} for type proc() freplace(outfile, re"(?m)(.*? = proc.*?)\{.cdecl.\}", "$1") @@ -120,4 +123,4 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = # Add dynamic library if outlib != "": - prepend(outfile, outlib) \ No newline at end of file + prepend(outfile, outlib) From d6f8d140ccd25567996e018d3cebe4f698b63eaa Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 22 Mar 2019 09:53:23 -0600 Subject: [PATCH 89/96] Add function to remove function bodies --- src/nimgen/c2nim.nim | 5 +++++ src/nimgen/fileops.nim | 30 ++++++++++++++++++++++++++++++ src/nimgen/globals.nim | 2 +- src/nimgen/runcfg.nim | 4 +++- 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 3d15243..c9ccff7 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -28,6 +28,9 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = cfile = "temp-$#.c" % [outfile.extractFilename()] writeFileFlush(cfile, runCtags(file)) + if c2nimConfig.removeBodies: + removeBodies(cfile) + if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): prepend(cfile, getDefines(file, c2nimConfig.inline)) @@ -103,6 +106,8 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = except: discard else: + if c2nimConfig.removeBodies: + reAddBodies(cfile) reAddStatic(cfile) # Nim doesn't like {.cdecl.} for type proc() diff --git a/src/nimgen/fileops.nim b/src/nimgen/fileops.nim index e416cf1..7f16139 100644 --- a/src/nimgen/fileops.nim +++ b/src/nimgen/fileops.nim @@ -91,6 +91,36 @@ proc removeStatic*(filename: string) = result.add(body.replace(re"(?m)^(.*\n?)", "//$1")) ) +proc removeBodies*(filename: string) = + ## Replace function bodies with a semicolon and commented + ## out body + withFile(filename): + content = content.replace( + re"(?m)(.*?\))(\s*\{(\s*?.*?$)*?[\n\r]\})", + proc (m: RegexMatch, s: string): string = + let funcDecl = s[m.group(0)[0]] + let body = s[m.group(1)[0]].strip() + result = "" + + result.add("$#;" % [funcDecl]) + result.add(body.replace(re"(?m)^(.*\n?)", "//$1")) + ) + +proc reAddBodies*(filename: string) = + ## Uncomment out the body and remove the semicolon. Undoes + ## removeBodies + withFile(filename): + content = content.replace( + re"(?m)(.*?\))(\s*\{(\s*?.*?$)*?[\n\r]\})", + proc (m: RegexMatch, s: string): string = + let funcDecl = s[m.group(0)[0]] + let body = s[m.group(1)[0]].strip() + result = "" + + result.add("$# " % [funcDecl]) + result.add(body.replace(re"(?m)^\/\/(.*\n?)", "$1")) + ) + proc reAddStatic*(filename: string) = ## Uncomment out the body and remove the semicolon. Undoes ## removeStatic diff --git a/src/nimgen/globals.nim b/src/nimgen/globals.nim index 9892b99..da90635 100644 --- a/src/nimgen/globals.nim +++ b/src/nimgen/globals.nim @@ -32,7 +32,7 @@ var type c2nimConfigObj* = object flags*, ppflags*: string - recurse*, inline*, preprocess*, ctags*, defines*: bool + recurse*, inline*, preprocess*, removeBodies*, ctags*, defines*: bool dynlib*, compile*, pragma*: seq[string] const gDoc* = """ diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index a924734..c582c2c 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -130,6 +130,8 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str c2nimConfig.inline = true elif action == "preprocess": c2nimConfig.preprocess = true + elif action == "removeBodies": + c2nimConfig.removeBodies = true elif action == "ctags": c2nimConfig.ctags = true elif action == "defines": @@ -406,4 +408,4 @@ proc runCli*() = uniq += 1 for file in files: - runFile(file, cfg) \ No newline at end of file + runFile(file, cfg) From 97141f523fbdf5d5b0b4594ea615f05faf7efcfb Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 9 Jul 2019 23:58:01 -0500 Subject: [PATCH 90/96] Bump to latest --- appveyor.yml | 4 ++-- nimgen.nimble | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 2e382be..7d8dabf 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,8 +9,8 @@ matrix: environment: matrix: - - NIM_VERSION: 0.18.0 - - NIM_VERSION: 0.19.2 + - NIM_VERSION: 0.20.0 + - NIM_VERSION: 0.19.6 for: - diff --git a/nimgen.nimble b/nimgen.nimble index 8c44026..0427d5a 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -11,7 +11,7 @@ skipDirs = @["nimgen", "tests", "web"] # Dependencies -requires "nim >= 0.17.0", "c2nim#3ec45c24585ebaed", "regex >= 0.10.0" +requires "nim >= 0.17.0", "c2nim >= 0.9.14", "regex >= 0.10.0" task test, "Test nimgen": exec "nim c -r tests/rununittests.nim" From 4ab5ff93b8c18ea63deefa5a947950756d215a24 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 15 Jul 2019 22:49:04 -0500 Subject: [PATCH 91/96] Fix c2nim dependency --- nimgen.nimble | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nimgen.nimble b/nimgen.nimble index 0427d5a..10bddf5 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.0" +version = "0.5.1" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" @@ -11,7 +11,12 @@ skipDirs = @["nimgen", "tests", "web"] # Dependencies -requires "nim >= 0.17.0", "c2nim >= 0.9.14", "regex >= 0.10.0" +requires "nim >= 0.19.0", "regex >= 0.10.0" + +when NimVersion < "0.20.0": + requires "c2nim#8f1705509084ae47319f6fcfb131e515b134e0f1" +else: + requires "c2nim >= 0.9.14" task test, "Test nimgen": exec "nim c -r tests/rununittests.nim" From b54feea3fcdae07fb0462327807e0632a5892123 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jul 2019 13:04:05 -0500 Subject: [PATCH 92/96] Remove nimnuklear --- tests/nimgentest.nims | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index d8e5737..b1816cd 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -5,7 +5,8 @@ var comps = @["libsvm", "nim7z", #"nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfuzzy", "nimmonocypher", - "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", + #"nimnuklear", + "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2" ] From fabd61537939b0e1a30d784a4761714a93d1abb6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jul 2019 14:06:02 -0500 Subject: [PATCH 93/96] C2nim is now backwards compatible --- nimgen.nimble | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nimgen.nimble b/nimgen.nimble index 10bddf5..fc7409e 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -11,12 +11,7 @@ skipDirs = @["nimgen", "tests", "web"] # Dependencies -requires "nim >= 0.19.0", "regex >= 0.10.0" - -when NimVersion < "0.20.0": - requires "c2nim#8f1705509084ae47319f6fcfb131e515b134e0f1" -else: - requires "c2nim >= 0.9.14" +requires "nim >= 0.19.0", "c2nim >= 0.9.14", "regex >= 0.10.0" task test, "Test nimgen": exec "nim c -r tests/rununittests.nim" From 39f5f320a627a895d3000673a0fb29603354061f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jul 2019 22:06:06 -0500 Subject: [PATCH 94/96] Remove libsvm --- README.md | 2 -- tests/nimgentest.nims | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 6a64779..818b8cd 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,6 @@ To see examples of nimgen in action check out the following wrappers:- * git sparse checkout * [nimtess2](https://github.com/genotrance/nimtess2) - libtess2 wrapper: [docs](http://nimgen.genotrance.com/nimtess2) * git checkout - * [libsvm](https://github.com/genotrance/libsvm) - libsvm wrapper: [docs](http://nimgen.genotrance.com/libsvm) - * git sparse checkout * [duktape-nim](https://github.com/manguluka/duktape-nim) - Duktape wrapper * static diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index b1816cd..0de436a 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,7 +2,7 @@ import distros, ospaths, strutils var pygonly = false - comps = @["libsvm", "nim7z", #"nimarchive", + comps = @["nim7z", #"nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfuzzy", "nimmonocypher", #"nimnuklear", From 2be81614a9851ec13547bffcfe73c20fec6f7708 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jul 2019 22:17:32 -0500 Subject: [PATCH 95/96] Fix #42 - curl should follow redirects, powershell wget backport --- src/nimgen/external.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 8656147..cc7a5e8 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -43,9 +43,10 @@ proc downloadUrl*(url: string) = file = url.extractFilename() ext = file.splitFile().ext.toLowerAscii() - var cmd = "curl $# -o $#" - if defined(Windows): - cmd = "powershell wget $# -OutFile $#" + var cmd = if defined(Windows): + "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" + else: + "curl -L $# -o $#" if not (ext == ".zip" and fileExists(gOutput/file)): echo "Downloading " & file From 638c9b4a5ae4d09c9451feaca363840e130ffbb7 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 7 Sep 2019 09:12:32 -0600 Subject: [PATCH 96/96] Fix bug with length --- src/nimgen/external.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index cc7a5e8..a887e20 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -22,7 +22,7 @@ proc execAction*(cmd: string): string = when defined(Linux) or defined(MacOSX): ccmd = "bash -c '" & cmd & "'" - echo "Running '" & ccmd[0..min(64, len(ccmd))] & "'" + echo "Running '" & ccmd[0..