From 7fbc8fbcdeb7460c13073ced6b86227175a79b6a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 11 Jun 2018 01:47:08 -0500 Subject: [PATCH 001/174] Style fixes --- README.md | 2 + nimgen.nim | 1098 +++++++++++++++++++++-------------------- tests/nimgentest.nims | 7 +- 3 files changed, 561 insertions(+), 546 deletions(-) diff --git a/README.md b/README.md index 129ce56..aadfd90 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,8 @@ To see examples of nimgen in action check out the following wrappers:- * download header file * [nimkerberos](https://github.com/genotrance/nimkerberos) - WinKerberos wrapper: [docs](http://nimgen.genotrance.com/nimkerberos) * git sparse checkout + * [nimpcre](https://github.com/genotrance/nimpcre) - PCRE wrapper: [docs](http://nimgen.genotrance.com/nimpcre) + * git checkout * [nimrax](https://github.com/genotrance/nimrax) - Radix tree wrapper: [docs](http://nimgen.genotrance.com/nimrax) * git checkout * [nimssl](https://github.com/genotrance/nimssl) - OpenSSL wrapper: [docs](http://nimgen.genotrance.com/nimssl) diff --git a/nimgen.nim b/nimgen.nim index 9771683..4766bb7 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -1,26 +1,17 @@ -import nre -import os -import ospaths -import osproc -import parsecfg -import pegs -import ropes -import sequtils -import streams -import strutils -import tables +import nre, os, ospaths, osproc, parsecfg, pegs, ropes, sequtils, streams, strutils, tables -var DONE: seq[string] = @[] -var DONE_INLINE: seq[string] = @[] +var + DONE: seq[string] = @[] + DONE_INLINE: seq[string] = @[] -var CONFIG: Config -var FILTER = "" -var QUOTES = true -var OUTPUT = "" -var INCLUDES: seq[string] = @[] -var EXCLUDES: seq[string] = @[] -var RENAMES = initTable[string, string]() -var WILDCARDS = newConfig() + CONFIG: Config + FILTER = "" + QUOTES = true + OUTPUT = "" + INCLUDES: seq[string] = @[] + EXCLUDES: seq[string] = @[] + RENAMES = initTable[string, string]() + WILDCARDS = newConfig() const DOC = """ Nimgen is a helper for c2nim to simpilfy and automate the wrapping of C libraries @@ -29,672 +20,691 @@ Usage: nimgen [options] ... Options: - -f delete all artifacts and regenerate + -f delete all artifacts and regenerate """ # ### # Helpers proc execProc(cmd: string): string = - var p = startProcess(cmd, options = {poStdErrToStdOut, poUsePath, poEvalCommand}) + result = "" + var + p = startProcess(cmd, options = {poStdErrToStdOut, poUsePath, poEvalCommand}) - result = "" - var outp = outputStream(p) - var line = newStringOfCap(120).TaintedString - while true: - if outp.readLine(line): - result.add(line) - result.add("\n") - elif not running(p): break + outp = outputStream(p) + line = newStringOfCap(120).TaintedString - var x = p.peekExitCode() - if x != 0: - echo "Command failed: " & $x - echo cmd - echo result - quit(1) + while true: + if outp.readLine(line): + result.add(line) + result.add("\n") + elif not running(p): break + + var x = p.peekExitCode() + if x != 0: + echo "Command failed: " & $x + echo cmd + echo result + quit(1) proc extractZip(zipfile: string) = - var cmd = "unzip -o $#" - if defined(Windows): - cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" + var cmd = "unzip -o $#" + if defined(Windows): + cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" - setCurrentDir(OUTPUT) - defer: setCurrentDir("..") + setCurrentDir(OUTPUT) + defer: setCurrentDir("..") - echo "Extracting " & zipfile - discard execProc(cmd % zipfile) + echo "Extracting " & zipfile + discard execProc(cmd % zipfile) proc downloadUrl(url: string) = - let file = url.extractFilename() - let ext = file.splitFile().ext.toLowerAscii() + let + file = url.extractFilename() + ext = file.splitFile().ext.toLowerAscii() - var cmd = "curl $# -o $#" - if defined(Windows): - cmd = "powershell wget $# -OutFile $#" + var cmd = "curl $# -o $#" + if defined(Windows): + cmd = "powershell wget $# -OutFile $#" - if not (ext == ".zip" and fileExists(OUTPUT/file)): - echo "Downloading " & file - discard execProc(cmd % [url, OUTPUT/file]) + if not (ext == ".zip" and fileExists(OUTPUT/file)): + echo "Downloading " & file + discard execProc(cmd % [url, OUTPUT/file]) - if ext == ".zip": - extractZip(file) + if ext == ".zip": + extractZip(file) proc gitReset() = - echo "Resetting Git repo" + echo "Resetting Git repo" - setCurrentDir(OUTPUT) - defer: setCurrentDir("..") + setCurrentDir(OUTPUT) + defer: setCurrentDir("..") - discard execProc("git reset --hard HEAD") + discard execProc("git reset --hard HEAD") proc gitRemotePull(url: string, pull=true) = - if dirExists(OUTPUT/".git"): - if pull: - gitReset() - return - - setCurrentDir(OUTPUT) - defer: setCurrentDir("..") - - echo "Setting up Git repo" - discard execProc("git init .") - discard execProc("git remote add origin " & url) - + if dirExists(OUTPUT/".git"): if pull: - echo "Checking out artifacts" - discard execProc("git pull --depth=1 origin master") + gitReset() + return -proc gitSparseCheckout(plist: string) = - let sparsefile = ".git/info/sparse-checkout" - if fileExists(OUTPUT/sparsefile): - gitReset() - return + setCurrentDir(OUTPUT) + defer: setCurrentDir("..") - setCurrentDir(OUTPUT) - defer: setCurrentDir("..") - - discard execProc("git config core.sparsecheckout true") - writeFile(sparsefile, plist) + echo "Setting up Git repo" + discard execProc("git init .") + discard execProc("git remote add origin " & url) + if pull: echo "Checking out artifacts" discard execProc("git pull --depth=1 origin master") +proc gitSparseCheckout(plist: string) = + let sparsefile = ".git/info/sparse-checkout" + if fileExists(OUTPUT/sparsefile): + gitReset() + return + + setCurrentDir(OUTPUT) + defer: setCurrentDir("..") + + discard execProc("git config core.sparsecheckout true") + writeFile(sparsefile, plist) + + echo "Checking out artifacts" + discard execProc("git pull --depth=1 origin master") + proc getKey(ukey: string): tuple[key: string, val: bool] = - var kv = ukey.replace(re"\..*", "").split("-", 1) - if kv.len() == 1: - kv.add("") + var kv = ukey.replace(re"\..*", "").split("-", 1) + if kv.len() == 1: + kv.add("") - if (kv[1] == "") or - (kv[1] == "win" and defined(Windows)) or - (kv[1] == "lin" and defined(Linux)) or - (kv[1] == "osx" and defined(MacOSX)): - return (kv[0], true) + if (kv[1] == "") or + (kv[1] == "win" and defined(Windows)) or + (kv[1] == "lin" and defined(Linux)) or + (kv[1] == "osx" and defined(MacOSX)): + return (kv[0], true) - return (kv[0], false) + return (kv[0], false) # ### # File loction proc getNimout(file: string, rename=true): string = - result = file.splitFile().name.replace(re"[\-\.]", "_") & ".nim" - if OUTPUT != "": - result = OUTPUT/result + result = file.splitFile().name.replace(re"[\-\.]", "_") & ".nim" + if OUTPUT != "": + result = OUTPUT/result - if not rename: - return + if not rename: + return - if RENAMES.hasKey(file): - result = RENAMES[file] + if RENAMES.hasKey(file): + result = RENAMES[file] - if not dirExists(parentDir(result)): - createDir(parentDir(result)) + if not dirExists(parentDir(result)): + createDir(parentDir(result)) proc exclude(file: string): bool = - for excl in EXCLUDES: - if excl in file: - return true - return false + for excl in EXCLUDES: + if excl in file: + return true + return false proc search(file: string): string = - if exclude(file): - return "" + if exclude(file): + return "" - result = file - if file.splitFile().ext == ".nim": - result = getNimout(file) - elif not fileExists(result) and not dirExists(result): - var found = false - for inc in INCLUDES: - result = inc/file - if fileExists(result) or dirExists(result): - found = true - break - if not found: - echo "File doesn't exist: " & file - quit(1) + result = file + if file.splitFile().ext == ".nim": + result = getNimout(file) + elif not fileExists(result) and not dirExists(result): + var found = false + for inc in INCLUDES: + result = inc/file + if fileExists(result) or dirExists(result): + found = true + break + if not found: + echo "File doesn't exist: " & file + quit(1) - return result.replace(re"[\\/]", $DirSep) + return result.replace(re"[\\/]", $DirSep) # ### # Loading / unloading template withFile(file: string, body: untyped): untyped = - if fileExists(file): - var f: File - while true: - try: - f = open(file) - break - except: - sleep(100) + if fileExists(file): + var f: File + while true: + try: + f = open(file) + break + except: + sleep(100) - var contentOrig = f.readAll() - f.close() - var content {.inject.} = contentOrig + var contentOrig = f.readAll() + f.close() + var content {.inject.} = contentOrig - body + body - if content != contentOrig: - var f = open(file, fmWrite) - write(f, content) - f.close() - else: - echo "Missing file " & file + if content != contentOrig: + var f = open(file, fmWrite) + write(f, content) + f.close() + else: + echo "Missing file " & file # ### # Manipulating content proc prepend(file: string, data: string, search="") = - withFile(file): - if search == "": - content = data & content - else: - let idx = content.find(search) - if idx != -1: - content = content[0..= content.len(): - break - content = content[0..= content.len(): + break + content = content[0.."]""", "").replace(re"\/[\*\/].*$", "").strip()) - - result = result.deduplicate() - - DONE_INLINE.add(file) - - if inline: - var sres = newSeq[string]() - for incl in result: - let sincl = search(incl) - if sincl == "": - continue - - sres.add(getIncls(sincl, inline)) - result.add(sres) + withFile(file): + for f in content.findIter(re"(?m)^\s*#\s*include\s+(.*?)$"): + var inc = f.captures[0].strip() + if ((QUOTES and inc.contains("\"")) or (FILTER != "" and FILTER in inc)) and (not exclude(inc)): + result.add( + inc.replace(re"""[<>"]""", "").replace(re"\/[\*\/].*$", "").strip()) result = result.deduplicate() + DONE_INLINE.add(file) + + if inline: + var sres = newSeq[string]() + for incl in result: + let sincl = search(incl) + if sincl == "": + continue + + sres.add(getIncls(sincl, inline)) + result.add(sres) + + result = result.deduplicate() + proc getDefines(file: string, inline=false): string = - result = "" - if inline: - var incls = getIncls(file, inline) - for incl in incls: - let sincl = search(incl) - if sincl != "": - echo "Inlining " & sincl - result &= getDefines(sincl) - withFile(file): - for def in content.findIter(re"(?m)^(\s*#\s*define\s+[\w\d_]+\s+[\d\-.xf]+)(?:\r|//|/*).*?$"): - result &= def.captures[0] & "\n" + result = "" + if inline: + var incls = getIncls(file, inline) + for incl in incls: + let sincl = search(incl) + if sincl != "": + echo "Inlining " & sincl + result &= getDefines(sincl) + withFile(file): + for def in content.findIter(re"(?m)^(\s*#\s*define\s+[\w\d_]+\s+[\d\-.xf]+)(?:\r|//|/*).*?$"): + result &= def.captures[0] & "\n" proc runPreprocess(file, ppflags, flags: string, inline: bool): string = - var pproc = "gcc" - if flags.contains("cpp"): - pproc = "g++" - var cmd = "$# -E $# $#" % [pproc, ppflags, file] - for inc in INCLUDES: - cmd &= " -I " & inc + var + pproc = if flags.contains("cpp"): "g++" else: "gcc" + cmd = "$# -E $# $#" % [pproc, ppflags, file] - # Run preprocessor - var data = execProc(cmd) + for inc in INCLUDES: + cmd &= " -I " & inc - # Include content only from file - var rdata: Rope - var start = false - var sfile = file.replace("\\", "/") - if inline: - sfile = sfile.parentDir() - for line in data.splitLines(): - if line.strip() != "": - if line[0] == '#' and not line.contains("#pragma"): - start = false - if sfile in line.replace("\\", "/").replace("//", "/"): - start = true - if not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: - start = true - else: - if start: - rdata.add( - line.replace("_Noreturn", "") - .replace("(())", "") - .replace("WINAPI", "") - .replace("__attribute__", "") - .replace("extern \"C\"", "") - .replace(re"\(\([_a-z]+?\)\)", "") - .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";") & "\n" - ) - return $rdata + # Run preprocessor + var data = execProc(cmd) + + # Include content only from file + var + rdata: Rope + start = false + sfile = file.replace("\\", "/") + + if inline: + sfile = sfile.parentDir() + for line in data.splitLines(): + if line.strip() != "": + if line[0] == '#' and not line.contains("#pragma"): + start = false + if sfile in line.replace("\\", "/").replace("//", "/"): + start = true + if not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: + start = true + else: + if start: + rdata.add( + line.replace("_Noreturn", "") + .replace("(())", "") + .replace("WINAPI", "") + .replace("__attribute__", "") + .replace("extern \"C\"", "") + .replace(re"\(\([_a-z]+?\)\)", "") + .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";") & "\n" + ) + return $rdata proc runCtags(file: string): string = - var cmd = "ctags -o - --fields=+S+K --c-kinds=p --file-scope=no " & file - var fps = execProc(cmd) + var + cmd = "ctags -o - --fields=+S+K --c-kinds=p --file-scope=no " & file + fps = execProc(cmd) + fdata = "" - var fdata = "" - for line in fps.splitLines(): - var spl = line.split(re"\t") - if spl.len() > 4: - if spl[0] != "main": - var fn = "" - var match = spl[2].find(re"/\^(.*?)\(") - if match.isSome(): - fn = match.get().captures[0] - fn &= spl[4].replace("signature:", "") & ";" - fdata &= fn & "\n" + for line in fps.splitLines(): + var spl = line.split(re"\t") + if spl.len() > 4: + if spl[0] != "main": + var fn = "" + var match = spl[2].find(re"/\^(.*?)\(") + if match.isSome(): + fn = match.get().captures[0] + fn &= spl[4].replace("signature:", "") & ";" + fdata &= fn & "\n" - return fdata + return fdata proc runFile(file: string, cfgin: OrderedTableRef) proc c2nim(fl, outfile, flags, ppflags: string, recurse, inline, preprocess, ctags, defines: bool, dynlib, compile, pragma: seq[string] = @[]) = - var file = search(fl) - if file == "": - return + var file = search(fl) + if file == "": + return - if file in DONE: - return + if file in DONE: + return - echo "Processing $# => $#" % [file, outfile] - DONE.add(file) + echo "Processing $# => $#" % [file, outfile] + DONE.add(file) - fixFuncProtos(file) + fixFuncProtos(file) - var incout = "" - if recurse: - var incls = getIncls(file) - for inc in incls: - var cfg = newOrderedTable[string, string]() - if flags != "": cfg["flags"] = flags - if ppflags != "": cfg["ppflags"] = ppflags - if recurse: cfg["recurse"] = $recurse - if preprocess: cfg["preprocess"] = $preprocess - if ctags: cfg["ctags"] = $ctags - if defines: cfg["defines"] = $defines - for i in dynlib: - cfg["dynlib." & i] = i - runFile(inc, cfg) + var incout = "" + if recurse: + var incls = getIncls(file) + for inc in incls: + var cfg = newOrderedTable[string, string]() + if flags != "": cfg["flags"] = flags + if ppflags != "": cfg["ppflags"] = ppflags + if recurse: cfg["recurse"] = $recurse + if preprocess: cfg["preprocess"] = $preprocess + if ctags: cfg["ctags"] = $ctags + if defines: cfg["defines"] = $defines + for i in dynlib: + cfg["dynlib." & i] = i + runFile(inc, cfg) - incout &= "import $#\n" % inc.search().getNimout()[0 .. ^5] #inc.splitFile().name.replace(re"[\-\.]", "_") & "\n" + incout &= "import $#\n" % inc.search().getNimout()[0 .. ^5] #inc.splitFile().name.replace(re"[\-\.]", "_") & "\n" - var cfile = file - if preprocess: - cfile = "temp-$#.c" % [outfile.extractFilename()] - writeFile(cfile, runPreprocess(file, ppflags, flags, inline)) - elif ctags: - cfile = "temp-$#.c" % [outfile.extractFilename()] - writeFile(cfile, runCtags(file)) + var cfile = file + if preprocess: + cfile = "temp-$#.c" % [outfile.extractFilename()] + writeFile(cfile, runPreprocess(file, ppflags, flags, inline)) + elif ctags: + cfile = "temp-$#.c" % [outfile.extractFilename()] + writeFile(cfile, runCtags(file)) - if defines and (preprocess or ctags): - prepend(cfile, getDefines(file, inline)) + if defines and (preprocess or ctags): + prepend(cfile, getDefines(file, inline)) - var extflags = "" - var passC = "" - var outlib = "" - var outpragma = "" + var + extflags = "" + passC = "" + outlib = "" + outpragma = "" - passC = "import strutils\n" - for inc in INCLUDES: - passC &= ("""{.passC: "-I\"" & gorge("nimble path $#").strip() & "/$#\"".}""" % [OUTPUT, inc]) & "\n" + passC = "import strutils\n" + for inc in INCLUDES: + passC &= ("""{.passC: "-I\"" & gorge("nimble path $#").strip() & "/$#\"".}""" % [OUTPUT, inc]) & "\n" - for prag in pragma: - outpragma &= "{." & prag & ".}\n" + for prag in pragma: + outpragma &= "{." & prag & ".}\n" - let fname = file.splitFile().name.replace(re"[\.\-]", "_") - if dynlib.len() != 0: - let win = "when defined(Windows):\n" - let lin = "when defined(Linux):\n" - let osx = "when defined(MacOSX):\n" - var winlib, linlib, osxlib: string = "" - for dl in dynlib: - let lib = " const dynlib$# = \"$#\"\n" % [fname, dl] - let ext = dl.splitFile().ext - if ext == ".dll": - winlib &= lib - elif ext == ".so": - linlib &= lib - elif ext == ".dylib": - osxlib &= lib + let fname = file.splitFile().name.replace(re"[\.\-]", "_") + if dynlib.len() != 0: + let + win = "when defined(Windows):\n" + lin = "when defined(Linux):\n" + osx = "when defined(MacOSX):\n" - if winlib != "": - outlib &= win & winlib & "\n" - if linlib != "": - outlib &= lin & linlib & "\n" - if osxlib != "": - outlib &= osx & osxlib & "\n" + var winlib, linlib, osxlib: string = "" + for dl in dynlib: + let + lib = " const dynlib$# = \"$#\"\n" % [fname, dl] + ext = dl.splitFile().ext - if outlib != "": - extflags &= " --dynlib:dynlib$#" % fname - else: - passC &= "const header$# = \"$#\"\n" % [fname, fl] - extflags = "--header:header$#" % fname + if ext == ".dll": + winlib &= lib + elif ext == ".so": + linlib &= lib + elif ext == ".dylib": + osxlib &= lib - # Run c2nim on generated file - var cmd = "c2nim $# $# --out:$# $#" % [flags, extflags, outfile, cfile] - when defined(windows): - cmd = "cmd /c " & cmd - discard execProc(cmd) + if winlib != "": + outlib &= win & winlib & "\n" + if linlib != "": + outlib &= lin & linlib & "\n" + if osxlib != "": + outlib &= osx & osxlib & "\n" - if preprocess or ctags: - try: - removeFile(cfile) - except: - discard - - # Import nim modules - if recurse: - prepend(outfile, incout) - - # Nim doesn't like {.cdecl.} for type proc() - freplace(outfile, re"(?m)(.*? = proc.*?){.cdecl.}", "$#") - freplace(outfile, " {.cdecl.})", ")") - - # Include {.compile.} directives - for cpl in compile: - let fcpl = search(cpl) - if getFileInfo(fcpl).kind == pcFile: - prepend(outfile, compile(file=fcpl)) - else: - prepend(outfile, compile(dir=fcpl)) - - # Add any pragmas - if outpragma != "": - prepend(outfile, outpragma) - - # Add header file and include paths - if passC != "": - prepend(outfile, passC) - - # Add dynamic library if outlib != "": - prepend(outfile, outlib) + extflags &= " --dynlib:dynlib$#" % fname + else: + passC &= "const header$# = \"$#\"\n" % [fname, fl] + extflags = "--header:header$#" % fname + + # Run c2nim on generated file + var cmd = "c2nim $# $# --out:$# $#" % [flags, extflags, outfile, cfile] + when defined(windows): + cmd = "cmd /c " & cmd + discard execProc(cmd) + + if preprocess or ctags: + try: + removeFile(cfile) + except: + discard + + # Import nim modules + if recurse: + prepend(outfile, incout) + + # Nim doesn't like {.cdecl.} for type proc() + freplace(outfile, re"(?m)(.*? = proc.*?){.cdecl.}", "$#") + freplace(outfile, " {.cdecl.})", ")") + + # Include {.compile.} directives + for cpl in compile: + let fcpl = search(cpl) + if getFileInfo(fcpl).kind == pcFile: + prepend(outfile, compile(file=fcpl)) + else: + prepend(outfile, compile(dir=fcpl)) + + # Add any pragmas + if outpragma != "": + prepend(outfile, outpragma) + + # Add header file and include paths + if passC != "": + prepend(outfile, passC) + + # Add dynamic library + if outlib != "": + prepend(outfile, outlib) # ### # Processor proc runFile(file: string, cfgin: OrderedTableRef) = - var cfg = cfgin - var sfile = search(file) + var + cfg = cfgin + sfile = search(file) - for pattern in WILDCARDS.keys(): - let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") - if file.find(re(pat)).isSome(): - echo "Appending " & file & " " & pattern - for key in WILDCARDS[pattern].keys(): - cfg[key & "." & pattern] = WILDCARDS[pattern][key] + for pattern in WILDCARDS.keys(): + let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") + if file.find(re(pat)).isSome(): + echo "Appending " & file & " " & pattern + for key in WILDCARDS[pattern].keys(): + cfg[key & "." & pattern] = WILDCARDS[pattern][key] + + var + srch = "" + compile: seq[string] = @[] + dynlib: seq[string] = @[] + pragma: seq[string] = @[] + + for act in cfg.keys(): + let (action, val) = getKey(act) + if val == true: + if action == "create": + createDir(file.splitPath().head) + writeFile(file, cfg[act]) + elif action in @["prepend", "append", "replace", "comment", "rename", "compile", "dynlib", "pragma"] and sfile != "": + if action == "prepend": + if srch != "": + prepend(sfile, cfg[act], cfg[srch]) + else: + prepend(sfile, cfg[act]) + elif action == "append": + if srch != "": + append(sfile, cfg[act], cfg[srch]) + else: + append(sfile, cfg[act]) + elif action == "replace": + if srch != "": + freplace(sfile, cfg[srch], cfg[act]) + elif action == "comment": + if srch != "": + comment(sfile, cfg[srch], cfg[act]) + elif action == "rename": + rename(sfile, cfg[act]) + elif action == "compile": + compile.add(cfg[act]) + elif action == "dynlib": + dynlib.add(cfg[act]) + elif action == "pragma": + pragma.add(cfg[act]) + srch = "" + elif action == "search": + srch = act + + if file.splitFile().ext != ".nim": + var + recurse = false + inline = false + preprocess = false + ctags = false + defines = false + noprocess = false + flags = "--stdcall" + ppflags = "" - var srch = "" - var compile: seq[string] = @[] - var dynlib: seq[string] = @[] - var pragma: seq[string] = @[] for act in cfg.keys(): - let (action, val) = getKey(act) - if val == true: - if action == "create": - createDir(file.splitPath().head) - writeFile(file, cfg[act]) - elif action in @["prepend", "append", "replace", "comment", "rename", "compile", "dynlib", "pragma"] and sfile != "": - if action == "prepend": - if srch != "": - prepend(sfile, cfg[act], cfg[srch]) - else: - prepend(sfile, cfg[act]) - elif action == "append": - if srch != "": - append(sfile, cfg[act], cfg[srch]) - else: - append(sfile, cfg[act]) - elif action == "replace": - if srch != "": - freplace(sfile, cfg[srch], cfg[act]) - elif action == "comment": - if srch != "": - comment(sfile, cfg[srch], cfg[act]) - elif action == "rename": - rename(sfile, cfg[act]) - elif action == "compile": - compile.add(cfg[act]) - elif action == "dynlib": - dynlib.add(cfg[act]) - elif action == "pragma": - pragma.add(cfg[act]) - srch = "" - elif action == "search": - srch = act + if cfg[act] == "true": + if act == "recurse": + recurse = true + elif act == "inline": + inline = true + elif act == "preprocess": + preprocess = true + elif act == "ctags": + ctags = true + elif act == "defines": + defines = true + elif act == "noprocess": + noprocess = true + elif act == "flags": + flags = cfg[act] + elif act == "ppflags": + ppflags = cfg[act] - if file.splitFile().ext != ".nim": - var recurse = false - var inline = false - var preprocess = false - var ctags = false - var defines = false - var noprocess = false - var flags = "--stdcall" - var ppflags = "" + if recurse and inline: + echo "Cannot use recurse and inline simultaneously" + quit(1) - for act in cfg.keys(): - if cfg[act] == "true": - if act == "recurse": - recurse = true - elif act == "inline": - inline = true - elif act == "preprocess": - preprocess = true - elif act == "ctags": - ctags = true - elif act == "defines": - defines = true - elif act == "noprocess": - noprocess = true - elif act == "flags": - flags = cfg[act] - elif act == "ppflags": - ppflags = cfg[act] - - if recurse and inline: - echo "Cannot use recurse and inline simultaneously" - quit(1) - - if not noprocess: - c2nim(file, getNimout(sfile), flags, ppflags, recurse, inline, preprocess, ctags, defines, dynlib, compile, pragma) + if not noprocess: + c2nim(file, getNimout(sfile), flags, ppflags, recurse, inline, preprocess, ctags, defines, dynlib, compile, pragma) proc runCfg(cfg: string) = - if not fileExists(cfg): - echo "Config doesn't exist: " & cfg - quit(1) + if not fileExists(cfg): + echo "Config doesn't exist: " & cfg + quit(1) - CONFIG = loadConfig(cfg) + CONFIG = loadConfig(cfg) - if CONFIG.hasKey("n.global"): - if CONFIG["n.global"].hasKey("output"): - OUTPUT = CONFIG["n.global"]["output"] - if dirExists(OUTPUT): - if "-f" in commandLineParams(): - try: - removeDir(OUTPUT) - except OSError: - echo "Directory in use: " & OUTPUT - quit(1) - else: - for f in walkFiles(OUTPUT/"*.nim"): - try: - removeFile(f) - except OSError: - echo "Unable to delete: " & f - quit(1) - createDir(OUTPUT) + if CONFIG.hasKey("n.global"): + if CONFIG["n.global"].hasKey("output"): + OUTPUT = CONFIG["n.global"]["output"] + if dirExists(OUTPUT): + if "-f" in commandLineParams(): + try: + removeDir(OUTPUT) + except OSError: + echo "Directory in use: " & OUTPUT + quit(1) + else: + for f in walkFiles(OUTPUT/"*.nim"): + try: + removeFile(f) + except OSError: + echo "Unable to delete: " & f + quit(1) + createDir(OUTPUT) - if CONFIG["n.global"].hasKey("filter"): - FILTER = CONFIG["n.global"]["filter"] - if CONFIG["n.global"].hasKey("quotes"): - if CONFIG["n.global"]["quotes"] == "false": - QUOTES = false + if CONFIG["n.global"].hasKey("filter"): + FILTER = CONFIG["n.global"]["filter"] + if CONFIG["n.global"].hasKey("quotes"): + if CONFIG["n.global"]["quotes"] == "false": + QUOTES = false - if CONFIG.hasKey("n.include"): - for inc in CONFIG["n.include"].keys(): - INCLUDES.add(inc) + if CONFIG.hasKey("n.include"): + for inc in CONFIG["n.include"].keys(): + INCLUDES.add(inc) - if CONFIG.hasKey("n.exclude"): - for excl in CONFIG["n.exclude"].keys(): - EXCLUDES.add(excl) + if CONFIG.hasKey("n.exclude"): + for excl in CONFIG["n.exclude"].keys(): + EXCLUDES.add(excl) - if CONFIG.hasKey("n.prepare"): - for prep in CONFIG["n.prepare"].keys(): - let (key, val) = getKey(prep) - if val == true: - if key == "download": - downloadUrl(CONFIG["n.prepare"][prep]) - elif key == "extract": - extractZip(CONFIG["n.prepare"][prep]) - elif key == "git": - gitRemotePull(CONFIG["n.prepare"][prep]) - elif key == "gitremote": - gitRemotePull(CONFIG["n.prepare"][prep], false) - elif key == "gitsparse": - gitSparseCheckout(CONFIG["n.prepare"][prep]) - elif key == "execute": - discard execProc(CONFIG["n.prepare"][prep]) + if CONFIG.hasKey("n.prepare"): + for prep in CONFIG["n.prepare"].keys(): + let (key, val) = getKey(prep) + if val == true: + if key == "download": + downloadUrl(CONFIG["n.prepare"][prep]) + elif key == "extract": + extractZip(CONFIG["n.prepare"][prep]) + elif key == "git": + gitRemotePull(CONFIG["n.prepare"][prep]) + elif key == "gitremote": + gitRemotePull(CONFIG["n.prepare"][prep], false) + elif key == "gitsparse": + gitSparseCheckout(CONFIG["n.prepare"][prep]) + elif key == "execute": + discard execProc(CONFIG["n.prepare"][prep]) - if CONFIG.hasKey("n.wildcard"): - var wildcard = "" - for wild in CONFIG["n.wildcard"].keys(): - let (key, val) = getKey(wild) - if val == true: - if key == "wildcard": - wildcard = CONFIG["n.wildcard"][wild] - else: - WILDCARDS.setSectionKey(wildcard, wild, CONFIG["n.wildcard"][wild]) + if CONFIG.hasKey("n.wildcard"): + var wildcard = "" + for wild in CONFIG["n.wildcard"].keys(): + let (key, val) = getKey(wild) + if val == true: + if key == "wildcard": + wildcard = CONFIG["n.wildcard"][wild] + else: + WILDCARDS.setSectionKey(wildcard, wild, CONFIG["n.wildcard"][wild]) - for file in CONFIG.keys(): - if file in @["n.global", "n.include", "n.exclude", "n.prepare", "n.wildcard"]: - continue + for file in CONFIG.keys(): + if file in @["n.global", "n.include", "n.exclude", "n.prepare", "n.wildcard"]: + continue - runFile(file, CONFIG[file]) + runFile(file, CONFIG[file]) # ### # Main loop for i in commandLineParams(): - if i != "-f": - runCfg(i) + if i != "-f": + runCfg(i) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 2001ef2..f5136c6 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,8 +2,11 @@ import distros import ospaths import strutils -var full = false -var comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimfuzz", "nimrax", "nimssl", "nimssh2"] +var + full = false + comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimfuzz", + "nimpcre", "nimrax", "nimssl", "nimssh2"] + if detectOs(Windows): comps.add("nimkerberos") From 4b91f1bbbe186e8e87b701c1542441f348dbc230 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 11 Jun 2018 09:42:41 -0500 Subject: [PATCH 002/174] Copy files during prepare, NEP1 conformance --- README.md | 2 + nimgen.nim | 162 ++++++++++++++++++++++++++++---------------------- nimgen.nimble | 2 +- 3 files changed, 93 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index aadfd90..d47239e 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,8 @@ The following keys can be used to prepare dependencies such as downloading ZIP f ```execute``` = command to run during preparation +```copy``` = copy a file to another location. Preferred over moving to preserve original. Comma separate for multiple entries. E.g. copy = "output/config.h.in=output/config.h" + _[n.wildcard]_ File wildcards such as *.nim, ssl*.h, etc. can be used to perform tasks across a group of files. This is useful to define common operations such as global text replacements without having to specify an explicit section for every single file. These operations will be performed on every matching file that is defined as a _sourcefile_ or recursed files. Only applies on source files following the wildcard declarations. diff --git a/nimgen.nim b/nimgen.nim index 4766bb7..fc3e8ce 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -1,17 +1,17 @@ import nre, os, ospaths, osproc, parsecfg, pegs, ropes, sequtils, streams, strutils, tables var - DONE: seq[string] = @[] - DONE_INLINE: seq[string] = @[] + gDoneRecursive: seq[string] = @[] + gDoneInline: seq[string] = @[] - CONFIG: Config - FILTER = "" - QUOTES = true - OUTPUT = "" - INCLUDES: seq[string] = @[] - EXCLUDES: seq[string] = @[] - RENAMES = initTable[string, string]() - WILDCARDS = newConfig() + gConfig: Config + gFilter = "" + gQuotes = true + gOutput = "" + gIncludes: seq[string] = @[] + gExcludes: seq[string] = @[] + gRenames = initTable[string, string]() + gWildcards = newConfig() const DOC = """ Nimgen is a helper for c2nim to simpilfy and automate the wrapping of C libraries @@ -52,7 +52,7 @@ proc extractZip(zipfile: string) = if defined(Windows): cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" - setCurrentDir(OUTPUT) + setCurrentDir(gOutput) defer: setCurrentDir("..") echo "Extracting " & zipfile @@ -67,9 +67,9 @@ proc downloadUrl(url: string) = if defined(Windows): cmd = "powershell wget $# -OutFile $#" - if not (ext == ".zip" and fileExists(OUTPUT/file)): + if not (ext == ".zip" and fileExists(gOutput/file)): echo "Downloading " & file - discard execProc(cmd % [url, OUTPUT/file]) + discard execProc(cmd % [url, gOutput/file]) if ext == ".zip": extractZip(file) @@ -77,18 +77,18 @@ proc downloadUrl(url: string) = proc gitReset() = echo "Resetting Git repo" - setCurrentDir(OUTPUT) + setCurrentDir(gOutput) defer: setCurrentDir("..") discard execProc("git reset --hard HEAD") proc gitRemotePull(url: string, pull=true) = - if dirExists(OUTPUT/".git"): + if dirExists(gOutput/".git"): if pull: gitReset() return - setCurrentDir(OUTPUT) + setCurrentDir(gOutput) defer: setCurrentDir("..") echo "Setting up Git repo" @@ -101,11 +101,11 @@ proc gitRemotePull(url: string, pull=true) = proc gitSparseCheckout(plist: string) = let sparsefile = ".git/info/sparse-checkout" - if fileExists(OUTPUT/sparsefile): + if fileExists(gOutput/sparsefile): gitReset() return - setCurrentDir(OUTPUT) + setCurrentDir(gOutput) defer: setCurrentDir("..") discard execProc("git config core.sparsecheckout true") @@ -114,6 +114,20 @@ proc gitSparseCheckout(plist: string) = echo "Checking out artifacts" discard execProc("git pull --depth=1 origin master") +proc doCopy(flist: string) = + for pair in flist.split(","): + let spl = pair.split("=") + if spl.len() != 2: + echo "Bad copy syntax: " & flist + quit(1) + + let + lfile = spl[0].strip() + rfile = spl[1].strip() + + copyFile(lfile, rfile) + echo "Copied $# to $#" % [lfile, rfile] + proc getKey(ukey: string): tuple[key: string, val: bool] = var kv = ukey.replace(re"\..*", "").split("-", 1) if kv.len() == 1: @@ -132,20 +146,20 @@ proc getKey(ukey: string): tuple[key: string, val: bool] = proc getNimout(file: string, rename=true): string = result = file.splitFile().name.replace(re"[\-\.]", "_") & ".nim" - if OUTPUT != "": - result = OUTPUT/result + if gOutput != "": + result = gOutput/result if not rename: return - if RENAMES.hasKey(file): - result = RENAMES[file] + if gRenames.hasKey(file): + result = gRenames[file] if not dirExists(parentDir(result)): createDir(parentDir(result)) proc exclude(file: string): bool = - for excl in EXCLUDES: + for excl in gExcludes: if excl in file: return true return false @@ -159,7 +173,7 @@ proc search(file: string): string = result = getNimout(file) elif not fileExists(result) and not dirExists(result): var found = false - for inc in INCLUDES: + for inc in gIncludes: result = inc/file if fileExists(result) or dirExists(result): found = true @@ -265,7 +279,6 @@ proc rename(file: string, renfile: string) = nimout = getNimout(file, false) newname = renfile.replace("$nimout", extractFilename(nimout)) - #if newname =~ peg"(!\$.)*{'$replace'\s*'('\s*{(!\,\S)+}\s*','\s*{(!\)\S)+}\s*')'}": if newname =~ peg"(!\$.)*{'$replace'\s*'('\s*{(!\)\S)+}')'}": var final = nimout.extractFilename() for entry in matches[1].split(","): @@ -274,14 +287,14 @@ proc rename(file: string, renfile: string) = echo "Bad replace syntax: " & renfile quit(1) - var + let srch = spl[0].strip() repl = spl[1].strip() final = final.replace(srch, repl) newname = newname.replace(matches[0], final) - RENAMES[file] = OUTPUT/newname + gRenames[file] = gOutput/newname proc compile(dir="", file=""): string = proc fcompile(file: string): string = @@ -309,19 +322,19 @@ proc fixFuncProtos(file: string) = proc getIncls(file: string, inline=false): seq[string] = result = @[] - if inline and file in DONE_INLINE: + if inline and file in gDoneInline: return withFile(file): for f in content.findIter(re"(?m)^\s*#\s*include\s+(.*?)$"): var inc = f.captures[0].strip() - if ((QUOTES and inc.contains("\"")) or (FILTER != "" and FILTER in inc)) and (not exclude(inc)): + if ((gQuotes and inc.contains("\"")) or (gFilter != "" and gFilter in inc)) and (not exclude(inc)): result.add( inc.replace(re"""[<>"]""", "").replace(re"\/[\*\/].*$", "").strip()) result = result.deduplicate() - DONE_INLINE.add(file) + gDoneInline.add(file) if inline: var sres = newSeq[string]() @@ -353,7 +366,7 @@ proc runPreprocess(file, ppflags, flags: string, inline: bool): string = pproc = if flags.contains("cpp"): "g++" else: "gcc" cmd = "$# -E $# $#" % [pproc, ppflags, file] - for inc in INCLUDES: + for inc in gIncludes: cmd &= " -I " & inc # Run preprocessor @@ -414,11 +427,11 @@ proc c2nim(fl, outfile, flags, ppflags: string, recurse, inline, preprocess, cta if file == "": return - if file in DONE: + if file in gDoneRecursive: return echo "Processing $# => $#" % [file, outfile] - DONE.add(file) + gDoneRecursive.add(file) fixFuncProtos(file) @@ -455,10 +468,11 @@ proc c2nim(fl, outfile, flags, ppflags: string, recurse, inline, preprocess, cta passC = "" outlib = "" outpragma = "" + outcompile = "" passC = "import strutils\n" - for inc in INCLUDES: - passC &= ("""{.passC: "-I\"" & gorge("nimble path $#").strip() & "/$#\"".}""" % [OUTPUT, inc]) & "\n" + for inc in gIncludes: + passC &= ("""{.passC: "-I\"" & gorge("nimble path $#").strip() & "/$#\"".}""" % [gOutput, inc]) & "\n" for prag in pragma: outpragma &= "{." & prag & ".}\n" @@ -517,6 +531,8 @@ proc c2nim(fl, outfile, flags, ppflags: string, recurse, inline, preprocess, cta freplace(outfile, " {.cdecl.})", ")") # Include {.compile.} directives + outcompile = compile(compile) + for cpl in compile: let fcpl = search(cpl) if getFileInfo(fcpl).kind == pcFile: @@ -544,12 +560,12 @@ proc runFile(file: string, cfgin: OrderedTableRef) = cfg = cfgin sfile = search(file) - for pattern in WILDCARDS.keys(): + for pattern in gWildcards.keys(): let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") if file.find(re(pat)).isSome(): echo "Appending " & file & " " & pattern - for key in WILDCARDS[pattern].keys(): - cfg[key & "." & pattern] = WILDCARDS[pattern][key] + for key in gWildcards[pattern].keys(): + cfg[key & "." & pattern] = gWildcards[pattern][key] var srch = "" @@ -634,73 +650,75 @@ proc runCfg(cfg: string) = echo "Config doesn't exist: " & cfg quit(1) - CONFIG = loadConfig(cfg) + gConfig = loadConfig(cfg) - if CONFIG.hasKey("n.global"): - if CONFIG["n.global"].hasKey("output"): - OUTPUT = CONFIG["n.global"]["output"] - if dirExists(OUTPUT): + if gConfig.hasKey("n.global"): + if gConfig["n.global"].hasKey("output"): + gOutput = gConfig["n.global"]["output"] + if dirExists(gOutput): if "-f" in commandLineParams(): try: - removeDir(OUTPUT) + removeDir(gOutput) except OSError: - echo "Directory in use: " & OUTPUT + echo "Directory in use: " & gOutput quit(1) else: - for f in walkFiles(OUTPUT/"*.nim"): + for f in walkFiles(gOutput/"*.nim"): try: removeFile(f) except OSError: echo "Unable to delete: " & f quit(1) - createDir(OUTPUT) + createDir(gOutput) - if CONFIG["n.global"].hasKey("filter"): - FILTER = CONFIG["n.global"]["filter"] - if CONFIG["n.global"].hasKey("quotes"): - if CONFIG["n.global"]["quotes"] == "false": - QUOTES = false + if gConfig["n.global"].hasKey("filter"): + gFilter = gConfig["n.global"]["filter"] + if gConfig["n.global"].hasKey("quotes"): + if gConfig["n.global"]["quotes"] == "false": + gQuotes = false - if CONFIG.hasKey("n.include"): - for inc in CONFIG["n.include"].keys(): - INCLUDES.add(inc) + if gConfig.hasKey("n.include"): + for inc in gConfig["n.include"].keys(): + gIncludes.add(inc) - if CONFIG.hasKey("n.exclude"): - for excl in CONFIG["n.exclude"].keys(): - EXCLUDES.add(excl) + if gConfig.hasKey("n.exclude"): + for excl in gConfig["n.exclude"].keys(): + gExcludes.add(excl) - if CONFIG.hasKey("n.prepare"): - for prep in CONFIG["n.prepare"].keys(): + if gConfig.hasKey("n.prepare"): + for prep in gConfig["n.prepare"].keys(): let (key, val) = getKey(prep) if val == true: if key == "download": - downloadUrl(CONFIG["n.prepare"][prep]) + downloadUrl(gConfig["n.prepare"][prep]) elif key == "extract": - extractZip(CONFIG["n.prepare"][prep]) + extractZip(gConfig["n.prepare"][prep]) elif key == "git": - gitRemotePull(CONFIG["n.prepare"][prep]) + gitRemotePull(gConfig["n.prepare"][prep]) elif key == "gitremote": - gitRemotePull(CONFIG["n.prepare"][prep], false) + gitRemotePull(gConfig["n.prepare"][prep], false) elif key == "gitsparse": - gitSparseCheckout(CONFIG["n.prepare"][prep]) + gitSparseCheckout(gConfig["n.prepare"][prep]) elif key == "execute": - discard execProc(CONFIG["n.prepare"][prep]) + discard execProc(gConfig["n.prepare"][prep]) + elif key == "copy": + doCopy(gConfig["n.prepare"][prep]) - if CONFIG.hasKey("n.wildcard"): + if gConfig.hasKey("n.wildcard"): var wildcard = "" - for wild in CONFIG["n.wildcard"].keys(): + for wild in gConfig["n.wildcard"].keys(): let (key, val) = getKey(wild) if val == true: if key == "wildcard": - wildcard = CONFIG["n.wildcard"][wild] + wildcard = gConfig["n.wildcard"][wild] else: - WILDCARDS.setSectionKey(wildcard, wild, CONFIG["n.wildcard"][wild]) + gWildcards.setSectionKey(wildcard, wild, gConfig["n.wildcard"][wild]) - for file in CONFIG.keys(): + for file in gConfig.keys(): if file in @["n.global", "n.include", "n.exclude", "n.prepare", "n.wildcard"]: continue - runFile(file, CONFIG[file]) + runFile(file, gConfig[file]) # ### # Main loop diff --git a/nimgen.nimble b/nimgen.nimble index 3e458ba..4d7106a 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.2.0" +version = "0.2.1" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" From e3a3fe12e66252d94f62ec8338072f60e2d4acd4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 11 Jun 2018 09:50:30 -0500 Subject: [PATCH 003/174] Bugfix for compile --- nimgen.nim | 3 --- 1 file changed, 3 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index fc3e8ce..ffe2cbe 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -468,7 +468,6 @@ proc c2nim(fl, outfile, flags, ppflags: string, recurse, inline, preprocess, cta passC = "" outlib = "" outpragma = "" - outcompile = "" passC = "import strutils\n" for inc in gIncludes: @@ -531,8 +530,6 @@ proc c2nim(fl, outfile, flags, ppflags: string, recurse, inline, preprocess, cta freplace(outfile, " {.cdecl.})", ")") # Include {.compile.} directives - outcompile = compile(compile) - for cpl in compile: let fcpl = search(cpl) if getFileInfo(fcpl).kind == pcFile: From 9805bdbc801c5d87dcda6570cb6d932fe2d56f54 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 11 Jun 2018 10:38:29 -0500 Subject: [PATCH 004/174] Internal cleanup --- nimgen.nim | 105 +++++++++++++++++++++++++++-------------------------- 1 file changed, 54 insertions(+), 51 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index ffe2cbe..11dd8ae 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -13,6 +13,12 @@ var gRenames = initTable[string, string]() gWildcards = newConfig() +type + c2nimConfigObj = object + flags, ppflags: string + recurse, inline, preprocess, ctags, defines: bool + dynlib, compile, pragma: seq[string] + const DOC = """ Nimgen is a helper for c2nim to simpilfy and automate the wrapping of C libraries @@ -422,7 +428,7 @@ proc runCtags(file: string): string = proc runFile(file: string, cfgin: OrderedTableRef) -proc c2nim(fl, outfile, flags, ppflags: string, recurse, inline, preprocess, ctags, defines: bool, dynlib, compile, pragma: seq[string] = @[]) = +proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = var file = search(fl) if file == "": return @@ -436,32 +442,34 @@ proc c2nim(fl, outfile, flags, ppflags: string, recurse, inline, preprocess, cta fixFuncProtos(file) var incout = "" - if recurse: - var incls = getIncls(file) - for inc in incls: - var cfg = newOrderedTable[string, string]() - if flags != "": cfg["flags"] = flags - if ppflags != "": cfg["ppflags"] = ppflags - if recurse: cfg["recurse"] = $recurse - if preprocess: cfg["preprocess"] = $preprocess - if ctags: cfg["ctags"] = $ctags - if defines: cfg["defines"] = $defines - for i in dynlib: - cfg["dynlib." & i] = i - runFile(inc, cfg) + if c2nimConfig.recurse: + var + incls = getIncls(file) + cfg = newOrderedTable[string, string]() - incout &= "import $#\n" % inc.search().getNimout()[0 .. ^5] #inc.splitFile().name.replace(re"[\-\.]", "_") & "\n" + for name, value in c2nimConfig.fieldPairs: + when value is string: + cfg[name] = value + when value is bool: + cfg[name] = $value + + for i in c2nimConfig.dynlib: + cfg["dynlib." & i] = i + + for inc in incls: + runFile(inc, cfg) + incout &= "import $#\n" % inc.search().getNimout()[0 .. ^5] var cfile = file - if preprocess: + if c2nimConfig.preprocess: cfile = "temp-$#.c" % [outfile.extractFilename()] - writeFile(cfile, runPreprocess(file, ppflags, flags, inline)) - elif ctags: + writeFile(cfile, runPreprocess(file, c2nimConfig.ppflags, c2nimConfig.flags, c2nimConfig.inline)) + elif c2nimConfig.ctags: cfile = "temp-$#.c" % [outfile.extractFilename()] writeFile(cfile, runCtags(file)) - if defines and (preprocess or ctags): - prepend(cfile, getDefines(file, inline)) + if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): + prepend(cfile, getDefines(file, c2nimConfig.inline)) var extflags = "" @@ -473,18 +481,18 @@ proc c2nim(fl, outfile, flags, ppflags: string, recurse, inline, preprocess, cta for inc in gIncludes: passC &= ("""{.passC: "-I\"" & gorge("nimble path $#").strip() & "/$#\"".}""" % [gOutput, inc]) & "\n" - for prag in pragma: + for prag in c2nimConfig.pragma: outpragma &= "{." & prag & ".}\n" let fname = file.splitFile().name.replace(re"[\.\-]", "_") - if dynlib.len() != 0: + if c2nimConfig.dynlib.len() != 0: let win = "when defined(Windows):\n" lin = "when defined(Linux):\n" osx = "when defined(MacOSX):\n" var winlib, linlib, osxlib: string = "" - for dl in dynlib: + for dl in c2nimConfig.dynlib: let lib = " const dynlib$# = \"$#\"\n" % [fname, dl] ext = dl.splitFile().ext @@ -510,19 +518,19 @@ proc c2nim(fl, outfile, flags, ppflags: string, recurse, inline, preprocess, cta extflags = "--header:header$#" % fname # Run c2nim on generated file - var cmd = "c2nim $# $# --out:$# $#" % [flags, extflags, outfile, cfile] + var cmd = "c2nim $# $# --out:$# $# --nep1" % [c2nimConfig.flags, extflags, outfile, cfile] when defined(windows): cmd = "cmd /c " & cmd discard execProc(cmd) - if preprocess or ctags: + if c2nimConfig.preprocess or c2nimConfig.ctags: try: removeFile(cfile) except: discard # Import nim modules - if recurse: + if c2nimConfig.recurse: prepend(outfile, incout) # Nim doesn't like {.cdecl.} for type proc() @@ -530,7 +538,7 @@ proc c2nim(fl, outfile, flags, ppflags: string, recurse, inline, preprocess, cta freplace(outfile, " {.cdecl.})", ")") # Include {.compile.} directives - for cpl in compile: + for cpl in c2nimConfig.compile: let fcpl = search(cpl) if getFileInfo(fcpl).kind == pcFile: prepend(outfile, compile(file=fcpl)) @@ -566,9 +574,12 @@ proc runFile(file: string, cfgin: OrderedTableRef) = var srch = "" - compile: seq[string] = @[] - dynlib: seq[string] = @[] - pragma: seq[string] = @[] + + c2nimConfig = c2nimConfigObj( + flags: "--stdcall", ppflags: "", + recurse: false, inline: false, preprocess: false, ctags: false, defines: false, + dynlib: @[], compile: @[], pragma: @[] + ) for act in cfg.keys(): let (action, val) = getKey(act) @@ -596,51 +607,43 @@ proc runFile(file: string, cfgin: OrderedTableRef) = elif action == "rename": rename(sfile, cfg[act]) elif action == "compile": - compile.add(cfg[act]) + c2nimConfig.compile.add(cfg[act]) elif action == "dynlib": - dynlib.add(cfg[act]) + c2nimConfig.dynlib.add(cfg[act]) elif action == "pragma": - pragma.add(cfg[act]) + c2nimConfig.pragma.add(cfg[act]) srch = "" elif action == "search": srch = act if file.splitFile().ext != ".nim": - var - recurse = false - inline = false - preprocess = false - ctags = false - defines = false - noprocess = false - flags = "--stdcall" - ppflags = "" + var noprocess = false for act in cfg.keys(): if cfg[act] == "true": if act == "recurse": - recurse = true + c2nimConfig.recurse = true elif act == "inline": - inline = true + c2nimConfig.inline = true elif act == "preprocess": - preprocess = true + c2nimConfig.preprocess = true elif act == "ctags": - ctags = true + c2nimConfig.ctags = true elif act == "defines": - defines = true + c2nimConfig.defines = true elif act == "noprocess": noprocess = true elif act == "flags": - flags = cfg[act] + c2nimConfig.flags = cfg[act] elif act == "ppflags": - ppflags = cfg[act] + c2nimConfig.ppflags = cfg[act] - if recurse and inline: + if c2nimConfig.recurse and c2nimConfig.inline: echo "Cannot use recurse and inline simultaneously" quit(1) if not noprocess: - c2nim(file, getNimout(sfile), flags, ppflags, recurse, inline, preprocess, ctags, defines, dynlib, compile, pragma) + c2nim(file, getNimout(sfile), c2nimConfig) proc runCfg(cfg: string) = if not fileExists(cfg): From a8bcbf8e89460a460cfed6842b10eedcda9f50b2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 11 Jun 2018 11:00:51 -0500 Subject: [PATCH 005/174] Revert nep1 for c2nim --- nimgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimgen.nim b/nimgen.nim index 11dd8ae..d804657 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -518,7 +518,7 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = extflags = "--header:header$#" % fname # Run c2nim on generated file - var cmd = "c2nim $# $# --out:$# $# --nep1" % [c2nimConfig.flags, extflags, outfile, cfile] + var cmd = "c2nim $# $# --out:$# $#" % [c2nimConfig.flags, extflags, outfile, cfile] when defined(windows): cmd = "cmd /c " & cmd discard execProc(cmd) From 0f73ad6d3f9ec081792e03967b49895a1d53b9b8 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 14 Jun 2018 17:13:46 +0900 Subject: [PATCH 006/174] Add configurable compiler to global section --- nimgen.nim | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nimgen.nim b/nimgen.nim index d804657..397ece4 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -7,6 +7,8 @@ var gConfig: Config gFilter = "" gQuotes = true + gCppCompiler = "g++" + gCCompiler = "gcc" gOutput = "" gIncludes: seq[string] = @[] gExcludes: seq[string] = @[] @@ -369,7 +371,7 @@ proc getDefines(file: string, inline=false): string = proc runPreprocess(file, ppflags, flags: string, inline: bool): string = var - pproc = if flags.contains("cpp"): "g++" else: "gcc" + pproc = if flags.contains("cpp"): gCppCompiler else: gCCompiler cmd = "$# -E $# $#" % [pproc, ppflags, file] for inc in gIncludes: @@ -671,6 +673,11 @@ proc runCfg(cfg: string) = quit(1) createDir(gOutput) + if gConfig["n.global"].hasKey("cpp_compiler"): + gCppCompiler = gConfig["n.global"]["cpp_compiler"] + if gConfig["n.global"].hasKey("c_compiler"): + gCCompiler = gConfig["n.global"]["c_compiler"] + if gConfig["n.global"].hasKey("filter"): gFilter = gConfig["n.global"]["filter"] if gConfig["n.global"].hasKey("quotes"): From f275bc0e352734cf6e8bbe125d56355b7949e432 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 16 Jun 2018 15:17:51 +0900 Subject: [PATCH 007/174] Update readme with compiler spec --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index d47239e..9fc8618 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,10 @@ _[n.global]_ ```filter``` = string to identify and recurse into library .h files in #include statements and exclude standard headers +```cpp_compiler``` = string to specify a CPP compiler executable. [default: g++] + +```c_compiler``` = string to specify a C compiler executable. [default: gcc] + _[n.include]_ List of all directories, one per line, to include in the search path. This is used by:- From 71699332be05b7a5aed93c4d9d80c4333f7692c3 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 16 Jun 2018 15:22:16 +0900 Subject: [PATCH 008/174] Allow for deeper than one directory paths --- nimgen.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index d804657..e601153 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -4,6 +4,7 @@ var gDoneRecursive: seq[string] = @[] gDoneInline: seq[string] = @[] + gProjectDir = getCurrentDir() gConfig: Config gFilter = "" gQuotes = true @@ -59,7 +60,7 @@ proc extractZip(zipfile: string) = cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" setCurrentDir(gOutput) - defer: setCurrentDir("..") + defer: setCurrentDir(gProjectDir) echo "Extracting " & zipfile discard execProc(cmd % zipfile) @@ -84,7 +85,7 @@ proc gitReset() = echo "Resetting Git repo" setCurrentDir(gOutput) - defer: setCurrentDir("..") + defer: setCurrentDir(gProjectDir) discard execProc("git reset --hard HEAD") @@ -95,7 +96,7 @@ proc gitRemotePull(url: string, pull=true) = return setCurrentDir(gOutput) - defer: setCurrentDir("..") + defer: setCurrentDir(gProjectDir) echo "Setting up Git repo" discard execProc("git init .") @@ -112,7 +113,7 @@ proc gitSparseCheckout(plist: string) = return setCurrentDir(gOutput) - defer: setCurrentDir("..") + defer: setCurrentDir(gProjectDir) discard execProc("git config core.sparsecheckout true") writeFile(sparsefile, plist) From e371546d522b1ce39a1009552a43de614368279a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 16 Jun 2018 15:33:47 +0900 Subject: [PATCH 009/174] Add support for executing a command on a file. $file in the command string references the file name. The stdout of the command will become the file's new contents. This is useful for doing things that nimgen might not support, like using sed or grep or some other command line tool. --- README.md | 2 ++ nimgen.nim | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d47239e..0c4bb89 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,8 @@ The following keys apply to library source code (before processing) and generate ```search``` = search string providing context for following prepend/append/replace directives +```execute``` = execute a command on a file and store the output of the command as the new file contents. Ex: execute = "cat $file | grep 'static inline'" + ```prepend``` = string value to prepend into file at beginning or before search ```append``` = string value to append into file at the end or after search diff --git a/nimgen.nim b/nimgen.nim index d804657..cff48e3 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -228,6 +228,12 @@ proc prepend(file: string, data: string, search="") = if idx != -1: content = content[0.. Date: Sat, 23 Jun 2018 12:08:48 -0500 Subject: [PATCH 010/174] Fix #2 - ctags not working --- nimgen.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index d804657..9535dc6 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -409,14 +409,14 @@ proc runPreprocess(file, ppflags, flags: string, inline: bool): string = proc runCtags(file: string): string = var - cmd = "ctags -o - --fields=+S+K --c-kinds=p --file-scope=no " & file + cmd = "ctags -o - --fields=+S+K --c-kinds=+p --file-scope=no " & file fps = execProc(cmd) fdata = "" for line in fps.splitLines(): var spl = line.split(re"\t") if spl.len() > 4: - if spl[0] != "main": + if spl[0] != "main" and spl[3] != "member": var fn = "" var match = spl[2].find(re"/\^(.*?)\(") if match.isSome(): From 2f2b26d269ba690ffe90708d1cb16f415cc673cb Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 24 Jun 2018 07:50:47 +0900 Subject: [PATCH 011/174] Change project dir based on nimgen cfg file --- nimgen.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nimgen.nim b/nimgen.nim index e601153..14ece01 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -4,7 +4,7 @@ var gDoneRecursive: seq[string] = @[] gDoneInline: seq[string] = @[] - gProjectDir = getCurrentDir() + gProjectDir = "" gConfig: Config gFilter = "" gQuotes = true @@ -651,6 +651,8 @@ proc runCfg(cfg: string) = echo "Config doesn't exist: " & cfg quit(1) + gProjectDir = parentDir(cfg) + gConfig = loadConfig(cfg) if gConfig.hasKey("n.global"): From 4a83f6d5a1177396cd9f0287bc3f99e9ca0a7e87 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 24 Jun 2018 08:21:50 +0900 Subject: [PATCH 012/174] Fix issue if cfg file is local gProject dir will be an empty string --- nimgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimgen.nim b/nimgen.nim index 14ece01..376e0eb 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -651,7 +651,7 @@ proc runCfg(cfg: string) = echo "Config doesn't exist: " & cfg quit(1) - gProjectDir = parentDir(cfg) + gProjectDir = parentDir(cfg.expandFilename()) gConfig = loadConfig(cfg) From a7edd8525d93ef07b3d1134067c3ab42a311d1bc Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 27 Jun 2018 22:22:38 +0900 Subject: [PATCH 013/174] Rename execute to pipe --- README.md | 2 +- nimgen.nim | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 0c4bb89..a92a5dc 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ The following keys apply to library source code (before processing) and generate ```search``` = search string providing context for following prepend/append/replace directives -```execute``` = execute a command on a file and store the output of the command as the new file contents. Ex: execute = "cat $file | grep 'static inline'" +```pipe``` = execute a command on a file and store the output of the command as the new file contents. Ex: pipe = "cat $file | grep 'static inline'" ```prepend``` = string value to prepend into file at beginning or before search diff --git a/nimgen.nim b/nimgen.nim index cff48e3..82b072a 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -228,11 +228,12 @@ proc prepend(file: string, data: string, search="") = if idx != -1: content = content[0.. Date: Sat, 7 Jul 2018 15:03:14 -0500 Subject: [PATCH 014/174] Fix IOErrors --- nimgen.nim | 6 ++++++ nimgen.nimble | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/nimgen.nim b/nimgen.nim index 545c3be..ac61767 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -476,6 +476,9 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = cfile = "temp-$#.c" % [outfile.extractFilename()] writeFile(cfile, runCtags(file)) + while not fileExists(cfile): + sleep(10) + if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): prepend(cfile, getDefines(file, c2nimConfig.inline)) @@ -531,6 +534,9 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = cmd = "cmd /c " & cmd discard execProc(cmd) + while not fileExists(outfile): + sleep(10) + if c2nimConfig.preprocess or c2nimConfig.ctags: try: removeFile(cfile) diff --git a/nimgen.nimble b/nimgen.nimble index 4d7106a..e0833ef 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -9,7 +9,7 @@ skipDirs = @["tests"] # Dependencies -requires "nim >= 0.17.2", "c2nim >= 0.9.13" +requires "nim >= 0.17.0", "c2nim >= 0.9.13" bin = @["nimgen"] From a0369758afda24a0332f776c7a9d52a4ea9275ce Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 7 Jul 2018 16:57:23 -0500 Subject: [PATCH 015/174] Support 0.17.0 onwards, fix IOError --- nimgen.nim | 24 ++++++++++-------------- nimgen.nimble | 2 +- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index ac61767..acf94da 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -194,15 +194,17 @@ proc search(file: string): string = # ### # Loading / unloading +proc openRetry(file: string, mode: FileMode = fmRead): File = + while true: + try: + result = open(file, mode) + break + except IOError: + sleep(100) + template withFile(file: string, body: untyped): untyped = if fileExists(file): - var f: File - while true: - try: - f = open(file) - break - except: - sleep(100) + var f = openRetry(file) var contentOrig = f.readAll() f.close() @@ -211,7 +213,7 @@ template withFile(file: string, body: untyped): untyped = body if content != contentOrig: - var f = open(file, fmWrite) + f = openRetry(file, fmWrite) write(f, content) f.close() else: @@ -476,9 +478,6 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = cfile = "temp-$#.c" % [outfile.extractFilename()] writeFile(cfile, runCtags(file)) - while not fileExists(cfile): - sleep(10) - if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): prepend(cfile, getDefines(file, c2nimConfig.inline)) @@ -534,9 +533,6 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = cmd = "cmd /c " & cmd discard execProc(cmd) - while not fileExists(outfile): - sleep(10) - if c2nimConfig.preprocess or c2nimConfig.ctags: try: removeFile(cfile) diff --git a/nimgen.nimble b/nimgen.nimble index e0833ef..2a048b9 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.2.1" +version = "0.2.2" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" From 44294a7513985dafd5aabca790648e85fa5789ef Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 7 Jul 2018 17:47:55 -0500 Subject: [PATCH 016/174] Readme fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a92a5dc..1e95d0a 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Nimble already requires Git so those commands can be assumed to be present to do __Capabilities & Limitations__ -Nimgen supports compiling in C/C++ sources as well as loading in dynamic libraries at this time. Support for static libraries (.a, .lib) are still to come. +Nimgen supports compiling in C/C++ sources and static libraries as well as loading in dynamic libraries. To see examples of nimgen in action check out the following wrappers:- * Link with a dynamic library From 467d0c4eae83d49e677bc03f8c3300a71bd9fcf1 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 20 Jun 2018 13:59:46 +0900 Subject: [PATCH 017/174] Add ability to use environment vars --- nimgen.nim | 54 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index 3f9a104..e1e16b7 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -35,6 +35,33 @@ Options: # ### # Helpers +proc addEnv(str: string): string = + var newStr = str + for pair in envPairs(): + try: + newStr = newStr % [pair.key, pair.value.string] + except ValueError: + # Ignore if there are no values to replace. We + # want to continue anyway + discard + + try: + newStr = newStr % ["output", gOutput] + except ValueError: + # Ignore if there are no values to replace. We + # want to continue anyway + discard + + # if there are still format args, print a warning + if newStr.contains("${"): + echo "WARNING: \"", newStr, "\" still contains an uninterpolated value!" + + return newStr + +proc `[]`(table: OrderedTableRef[string, string], key: string): string = + ## Gets table values with env vars inserted + tables.`[]`(table, key).addEnv + proc execProc(cmd: string): string = result = "" var @@ -497,6 +524,7 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = outpragma &= "{." & prag & ".}\n" let fname = file.splitFile().name.replace(re"[\.\-]", "_") + if c2nimConfig.dynlib.len() != 0: let win = "when defined(Windows):\n" @@ -599,7 +627,9 @@ proc runFile(file: string, cfgin: OrderedTableRef) = if action == "create": createDir(file.splitPath().head) writeFile(file, cfg[act]) - elif action in @["prepend", "append", "replace", "comment", "rename", "compile", "dynlib", "pragma", "pipe"] and sfile != "": + elif action in @["prepend", "append", "replace", "comment", + "rename", "compile", "dynlib", "pragma", + "pipe"] and sfile != "": if action == "prepend": if srch != "": prepend(sfile, cfg[act], cfg[srch]) @@ -700,30 +730,31 @@ proc runCfg(cfg: string) = if gConfig.hasKey("n.include"): for inc in gConfig["n.include"].keys(): - gIncludes.add(inc) + gIncludes.add(inc.addEnv()) if gConfig.hasKey("n.exclude"): for excl in gConfig["n.exclude"].keys(): - gExcludes.add(excl) + gExcludes.add(excl.addEnv()) if gConfig.hasKey("n.prepare"): for prep in gConfig["n.prepare"].keys(): let (key, val) = getKey(prep) if val == true: + let prepVal = gConfig["n.prepare"][prep] if key == "download": - downloadUrl(gConfig["n.prepare"][prep]) + downloadUrl(prepVal) elif key == "extract": - extractZip(gConfig["n.prepare"][prep]) + extractZip(prepVal) elif key == "git": - gitRemotePull(gConfig["n.prepare"][prep]) + gitRemotePull(prepVal) elif key == "gitremote": - gitRemotePull(gConfig["n.prepare"][prep], false) + gitRemotePull(prepVal, false) elif key == "gitsparse": - gitSparseCheckout(gConfig["n.prepare"][prep]) + gitSparseCheckout(prepVal) elif key == "execute": - discard execProc(gConfig["n.prepare"][prep]) + discard execProc(prepVal) elif key == "copy": - doCopy(gConfig["n.prepare"][prep]) + doCopy(prepVal) if gConfig.hasKey("n.wildcard"): var wildcard = "" @@ -733,7 +764,8 @@ proc runCfg(cfg: string) = if key == "wildcard": wildcard = gConfig["n.wildcard"][wild] else: - gWildcards.setSectionKey(wildcard, wild, gConfig["n.wildcard"][wild]) + gWildcards.setSectionKey(wildcard, wild, + gConfig["n.wildcard"][wild]) for file in gConfig.keys(): if file in @["n.global", "n.include", "n.exclude", "n.prepare", "n.wildcard"]: From 2fc12b8577c42d7fb23e8f7d1fea2591ef0e5da9 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 18 Jun 2018 19:53:54 +0900 Subject: [PATCH 018/174] Only include necessary file for include libs The previous behavior caused a "Too many files open" error when referrencing lots of libs because of the call to "gorge". This modification retains the old behavior and also works if the library is compiled locally. Fixes #12 --- nimgen.nim | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index acf94da..7137dcd 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -488,13 +488,14 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = outpragma = "" passC = "import strutils\n" - for inc in gIncludes: - passC &= ("""{.passC: "-I\"" & gorge("nimble path $#").strip() & "/$#\"".}""" % [gOutput, inc]) & "\n" + passC &= "import ospaths\n" for prag in c2nimConfig.pragma: outpragma &= "{." & prag & ".}\n" let fname = file.splitFile().name.replace(re"[\.\-]", "_") + let fincl = file.replace(gOutput, "") + if c2nimConfig.dynlib.len() != 0: let win = "when defined(Windows):\n" @@ -524,7 +525,12 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = if outlib != "": extflags &= " --dynlib:dynlib$#" % fname else: - passC &= "const header$# = \"$#\"\n" % [fname, fl] + if file.isAbsolute(): + passC &= "const header$# = \"$#\"\n" % [fname, fincl] + else: + # based on the current source directory, get the include path + # works for nimble installations and local repo clones + passC &= "const header$# = currentSourcePath().splitPath().head & \"/$#\"\n" % [fname, fincl] extflags = "--header:header$#" % fname # Run c2nim on generated file From 76020e8f219bda1af3f7e1bc7974ffb1d6b2a6df Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 20 Jun 2018 14:04:14 +0900 Subject: [PATCH 019/174] Add env vars to readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 765b6da..f74950b 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,14 @@ __Capabilities & Limitations__ Nimgen supports compiling in C/C++ sources and static libraries as well as loading in dynamic libraries. +Environment variables are supported via Nim's string interpolation `%` symbol imported from the `strutils` module. Simply use double quotes to enclose any value and put `${}` around the environment variable name. In addition, the `output` var from the n.global section is available as ${output}. For example: + + [n.global] + output="src/path" + + [n.include] + "${output}/library/include" + To see examples of nimgen in action check out the following wrappers:- * Link with a dynamic library * [nimbass](https://github.com/genotrance/nimbass) - BASS audio wrapper: [docs](http://nimgen.genotrance.com/nimbass) From 4c45a5f5b2a7e75da707f450c330a20d8126ae9b Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 8 Jul 2018 22:43:17 +0900 Subject: [PATCH 020/174] Combine passC and absolute imports --- nimgen.nim | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nimgen.nim b/nimgen.nim index 7137dcd..8693a7c 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -490,11 +490,18 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = passC = "import strutils\n" passC &= "import ospaths\n" + for inc in gIncludes: + let relativeInc = inc.replace(gOutput & $DirSep, "") + passC &= ( + """{.passC: "-I\"" & currentSourcePath().splitPath().head & "/$#\"".}""" % + [relativeInc] + ) & "\n" + for prag in c2nimConfig.pragma: outpragma &= "{." & prag & ".}\n" let fname = file.splitFile().name.replace(re"[\.\-]", "_") - let fincl = file.replace(gOutput, "") + let fincl = file.replace(gOutput & $DirSep, "") if c2nimConfig.dynlib.len() != 0: let From 15d417d2c5f6b3a5d25aaa5506a60201dc4f36cc Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 8 Jul 2018 22:50:34 +0900 Subject: [PATCH 021/174] Remove dir sep from replace --- nimgen.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index 8693a7c..a432ab6 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -491,7 +491,7 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = passC &= "import ospaths\n" for inc in gIncludes: - let relativeInc = inc.replace(gOutput & $DirSep, "") + let relativeInc = inc.replace(gOutput, "") passC &= ( """{.passC: "-I\"" & currentSourcePath().splitPath().head & "/$#\"".}""" % [relativeInc] @@ -501,7 +501,7 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = outpragma &= "{." & prag & ".}\n" let fname = file.splitFile().name.replace(re"[\.\-]", "_") - let fincl = file.replace(gOutput & $DirSep, "") + let fincl = file.replace(gOutput, "") if c2nimConfig.dynlib.len() != 0: let From 512cdd3a9f64d2be1a6fd338fe6d7fddeceaebae Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 8 Jul 2018 22:56:36 +0900 Subject: [PATCH 022/174] Add check for absolute directories --- nimgen.nim | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index a432ab6..75461bc 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -491,11 +491,14 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = passC &= "import ospaths\n" for inc in gIncludes: - let relativeInc = inc.replace(gOutput, "") - passC &= ( - """{.passC: "-I\"" & currentSourcePath().splitPath().head & "/$#\"".}""" % - [relativeInc] - ) & "\n" + if inc.isAbsolute: + passC &= ("""{.passC: "-I\"$#\"".}""" % [inc]) & "\n" + else: + let relativeInc = inc.replace(gOutput, "") + passC &= ( + """{.passC: "-I\"" & currentSourcePath().splitPath().head & "/$#\"".}""" % + [relativeInc] + ) & "\n" for prag in c2nimConfig.pragma: outpragma &= "{." & prag & ".}\n" From c846bf106a2b32ab6a0113fd8ce31f2d31463280 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Jul 2018 16:01:54 -0500 Subject: [PATCH 023/174] Fixes to work with all wrappers --- nimgen.nim | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index 75461bc..2bb3e16 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -438,6 +438,9 @@ proc runCtags(file: string): string = proc runFile(file: string, cfgin: OrderedTableRef) +template relativePath(path: untyped): untyped = + path.multiReplace([(gOutput, ""), ("\\", "/"), ("//", "/")]) + proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = var file = search(fl) if file == "": @@ -487,24 +490,18 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = outlib = "" outpragma = "" - passC = "import strutils\n" - passC &= "import ospaths\n" + passC = "import ospaths, strutils\n" for inc in gIncludes: - if inc.isAbsolute: - passC &= ("""{.passC: "-I\"$#\"".}""" % [inc]) & "\n" - else: - let relativeInc = inc.replace(gOutput, "") - passC &= ( - """{.passC: "-I\"" & currentSourcePath().splitPath().head & "/$#\"".}""" % - [relativeInc] - ) & "\n" + passC &= ( + """{.passC: "-I\"" & currentSourcePath().splitPath().head & "$#\"".}""" % + inc.relativePath() + ) & "\n" for prag in c2nimConfig.pragma: outpragma &= "{." & prag & ".}\n" let fname = file.splitFile().name.replace(re"[\.\-]", "_") - let fincl = file.replace(gOutput, "") if c2nimConfig.dynlib.len() != 0: let @@ -535,12 +532,8 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = if outlib != "": extflags &= " --dynlib:dynlib$#" % fname else: - if file.isAbsolute(): - passC &= "const header$# = \"$#\"\n" % [fname, fincl] - else: - # based on the current source directory, get the include path - # works for nimble installations and local repo clones - passC &= "const header$# = currentSourcePath().splitPath().head & \"/$#\"\n" % [fname, fincl] + passC &= "const header$# = currentSourcePath().splitPath().head & \"$#\"\n" % + [fname, file.relativePath()] extflags = "--header:header$#" % fname # Run c2nim on generated file From a717b9d140b1f6b77e998d020360cffa11ae21eb Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Jul 2018 16:03:13 -0500 Subject: [PATCH 024/174] Update version --- nimgen.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimgen.nimble b/nimgen.nimble index 2a048b9..043fc86 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.2.2" +version = "0.2.3" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" From 60d94e9114832ee8f46ca73dccb761706aed364d Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Jul 2018 19:11:41 -0500 Subject: [PATCH 025/174] Fix for absolute paths in includes and headers --- nimgen.nim | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index 2bb3e16..f4631c3 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -493,10 +493,13 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = passC = "import ospaths, strutils\n" for inc in gIncludes: - passC &= ( - """{.passC: "-I\"" & currentSourcePath().splitPath().head & "$#\"".}""" % - inc.relativePath() - ) & "\n" + if inc.isAbsolute(): + passC &= ("""{.passC: "-I\"$#\"".}""" % [inc]) & "\n" + else: + passC &= ( + """{.passC: "-I\"" & currentSourcePath().splitPath().head & "$#\"".}""" % + inc.relativePath() + ) & "\n" for prag in c2nimConfig.pragma: outpragma &= "{." & prag & ".}\n" @@ -532,8 +535,11 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = if outlib != "": extflags &= " --dynlib:dynlib$#" % fname else: - passC &= "const header$# = currentSourcePath().splitPath().head & \"$#\"\n" % - [fname, file.relativePath()] + if file.isAbsolute(): + passC &= "const header$# = \"$#\"\n" % [fname, file] + else: + passC &= "const header$# = currentSourcePath().splitPath().head & \"$#\"\n" % + [fname, file.relativePath()] extflags = "--header:header$#" % fname # Run c2nim on generated file From 17e54d3b8786422c8643cbe2f405f30ef324ed38 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 9 Jul 2018 09:16:44 +0900 Subject: [PATCH 026/174] Make c/cpp compiler priority .cfg > CC/CXX > gcc/g++ --- nimgen.nim | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index e1e16b7..4f2c251 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -1,5 +1,11 @@ import nre, os, ospaths, osproc, parsecfg, pegs, ropes, sequtils, streams, strutils, tables +const + C_COMPILER_ENV = "CC" + CPP_COMPILER_ENV = "CPP" + DEFAULT_C_COMPILER = "gcc" + DEFAULT_CPP_COMPILER = "g++" + var gDoneRecursive: seq[string] = @[] gDoneInline: seq[string] = @[] @@ -8,8 +14,8 @@ var gConfig: Config gFilter = "" gQuotes = true - gCppCompiler = "g++" - gCCompiler = "gcc" + gCppCompiler = getEnv(CPP_COMPILER_ENV, DEFAULT_C_COMPILER) + gCCompiler = getEnv(C_COMPILER_ENV, DEFAULT_CPP_COMPILER) gOutput = "" gIncludes: seq[string] = @[] gExcludes: seq[string] = @[] @@ -719,8 +725,15 @@ proc runCfg(cfg: string) = if gConfig["n.global"].hasKey("cpp_compiler"): gCppCompiler = gConfig["n.global"]["cpp_compiler"] + else: + # Reset on a per project basis + gCppCompiler = getEnv(CPP_COMPILER_ENV, DEFAULT_CPP_COMPILER) + if gConfig["n.global"].hasKey("c_compiler"): gCCompiler = gConfig["n.global"]["c_compiler"] + else: + # Reset on a per project basis + gCCompiler = getEnv(C_COMPILER_ENV, DEFAULT_C_COMPILER) if gConfig["n.global"].hasKey("filter"): gFilter = gConfig["n.global"]["filter"] From 9c41b5a17644726ff982b65e2740f2cc3e57c54a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 9 Jul 2018 09:21:14 +0900 Subject: [PATCH 027/174] Add better example to readme for env vars --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f74950b..78ad979 100644 --- a/README.md +++ b/README.md @@ -30,13 +30,16 @@ __Capabilities & Limitations__ Nimgen supports compiling in C/C++ sources and static libraries as well as loading in dynamic libraries. -Environment variables are supported via Nim's string interpolation `%` symbol imported from the `strutils` module. Simply use double quotes to enclose any value and put `${}` around the environment variable name. In addition, the `output` var from the n.global section is available as ${output}. For example: +Environment variables are supported via Nim's string interpolation `%` symbol imported from the `strutils` module. Simply use double quotes to enclose any value and put `$` or `${}` around the environment variable name. In addition, the `output` var from the n.global section is available as ${output}. For example: [n.global] + c_compiler="$CC" + cpp_compiler="${CPP}-arm" output="src/path" [n.include] "${output}/library/include" + "${MY_INCLUDE_PATH}/include" To see examples of nimgen in action check out the following wrappers:- * Link with a dynamic library From 3b60a0e4b10be17d00e39afbc163702555555d62 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 9 Jul 2018 09:51:44 +0900 Subject: [PATCH 028/174] Move env var section in readme --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 78ad979..40f7556 100644 --- a/README.md +++ b/README.md @@ -30,17 +30,6 @@ __Capabilities & Limitations__ Nimgen supports compiling in C/C++ sources and static libraries as well as loading in dynamic libraries. -Environment variables are supported via Nim's string interpolation `%` symbol imported from the `strutils` module. Simply use double quotes to enclose any value and put `$` or `${}` around the environment variable name. In addition, the `output` var from the n.global section is available as ${output}. For example: - - [n.global] - c_compiler="$CC" - cpp_compiler="${CPP}-arm" - output="src/path" - - [n.include] - "${output}/library/include" - "${MY_INCLUDE_PATH}/include" - To see examples of nimgen in action check out the following wrappers:- * Link with a dynamic library * [nimbass](https://github.com/genotrance/nimbass) - BASS audio wrapper: [docs](http://nimgen.genotrance.com/nimbass) @@ -76,6 +65,17 @@ Nimgen only supports the ```gcc``` preprocessor at this time. Support for detect __Config file__ +In all sections below, environment variables are supported via Nim's string interpolation `%` symbol imported from the `strutils` module. Simply use double quotes to enclose any value and put `$` or `${}` around the environment variable name. In addition, the `output` var from the n.global section is available as ${output}. For example: + + [n.global] + c_compiler="$CC" + cpp_compiler="${CPP}-arm" + output="src/path" + + [n.include] + "${output}/library/include" + "${MY_INCLUDE_PATH}/include" + _[n.global]_ ```output``` = name of the Nimble project once installed, also location to place generated .nim files From 8e8e5fbd09de38207d13ac0e79b4716308b1d828 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 9 Jul 2018 09:52:23 +0900 Subject: [PATCH 029/174] Change consts to camel case for nep1 --- nimgen.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index 4f2c251..d618142 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -1,10 +1,10 @@ import nre, os, ospaths, osproc, parsecfg, pegs, ropes, sequtils, streams, strutils, tables const - C_COMPILER_ENV = "CC" - CPP_COMPILER_ENV = "CPP" - DEFAULT_C_COMPILER = "gcc" - DEFAULT_CPP_COMPILER = "g++" + cCompilerEnv = "CC" + cppCompilerEnv = "CPP" + defaultCCompiler = "gcc" + defaultCppCompiler = "g++" var gDoneRecursive: seq[string] = @[] @@ -14,8 +14,8 @@ var gConfig: Config gFilter = "" gQuotes = true - gCppCompiler = getEnv(CPP_COMPILER_ENV, DEFAULT_C_COMPILER) - gCCompiler = getEnv(C_COMPILER_ENV, DEFAULT_CPP_COMPILER) + gCppCompiler = getEnv(cppCompilerEnv, defaultCCompiler) + gCCompiler = getEnv(cCompilerEnv, defaultCppCompiler) gOutput = "" gIncludes: seq[string] = @[] gExcludes: seq[string] = @[] @@ -727,13 +727,13 @@ proc runCfg(cfg: string) = gCppCompiler = gConfig["n.global"]["cpp_compiler"] else: # Reset on a per project basis - gCppCompiler = getEnv(CPP_COMPILER_ENV, DEFAULT_CPP_COMPILER) + gCppCompiler = getEnv(cppCompilerEnv, defaultCppCompiler) if gConfig["n.global"].hasKey("c_compiler"): gCCompiler = gConfig["n.global"]["c_compiler"] else: # Reset on a per project basis - gCCompiler = getEnv(C_COMPILER_ENV, DEFAULT_C_COMPILER) + gCCompiler = getEnv(cCompilerEnv, defaultCCompiler) if gConfig["n.global"].hasKey("filter"): gFilter = gConfig["n.global"]["filter"] From 0226dba4276b7e60bd3f25fdc222e3289b93c302 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Jul 2018 21:06:28 -0500 Subject: [PATCH 030/174] Setup CI --- appveyor.yml | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..476e6d7 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,44 @@ +version: '{build}' + +cache: +- i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z + +matrix: + fast_finish: true + +environment: + matrix: + - MINGW_DIR: mingw32 + MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/4.9.2/threads-win32/dwarf/i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z/download + MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z + platform: x86 + +install: + - MKDIR %CD%\DIST + - MKDIR %CD%\DIST\PCRE + - nuget install pcre -Verbosity quiet -Version 8.33.0.1 -OutputDirectory %CD%\DIST\PCRE + - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" + - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul + - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH% + - IF "%PLATFORM%" == "x64" ( copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay64.dll & copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl64.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl32.dll ) + ELSE ( copy C:\OpenSSL-Win32\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win32\libssl32.dll %CD%\BIN\libssl32.dll ) + - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\x64\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre64.dll ) ELSE ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\Win32\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre32.dll ) + - git clone --depth 1 https://github.com/nim-lang/nim + - cd nim + - git clone --depth 1 https://github.com/nim-lang/csources + - cd csources + - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat ) + - cd .. + - bin\nim c -d:release koch + - koch boot -d:release + - koch nimble -d:release + - SET PATH=%CD%\bin;%PATH% + - cd .. + +build_script: + - nimble install -y + +test_script: + - nimble test + +deploy: off From ad7df8d34e49b8a14f8b7bb3e6127d3fe4fa1ef9 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Jul 2018 21:12:52 -0500 Subject: [PATCH 031/174] Setup CI --- appveyor.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 476e6d7..2a1c748 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,6 +2,7 @@ version: '{build}' cache: - i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z +- dlls.zip matrix: fast_finish: true @@ -11,6 +12,8 @@ environment: - MINGW_DIR: mingw32 MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/4.9.2/threads-win32/dwarf/i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z/download MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z + DLLS_URL: http://nim-lang.org/download/dlls.zip + DLLS_ARCHIVE: dlls.zip platform: x86 install: @@ -20,8 +23,6 @@ install: - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH% - - IF "%PLATFORM%" == "x64" ( copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay64.dll & copy C:\OpenSSL-Win64\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl64.dll & copy C:\OpenSSL-Win64\libssl32.dll %CD%\BIN\libssl32.dll ) - ELSE ( copy C:\OpenSSL-Win32\libeay32.dll %CD%\BIN\libeay32.dll & copy C:\OpenSSL-Win32\libssl32.dll %CD%\BIN\libssl32.dll ) - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\x64\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre64.dll ) ELSE ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\Win32\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre32.dll ) - git clone --depth 1 https://github.com/nim-lang/nim - cd nim @@ -29,6 +30,8 @@ install: - cd csources - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat ) - cd .. + - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%" + - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\bin" - bin\nim c -d:release koch - koch boot -d:release - koch nimble -d:release From 30e5da82a0ef2ee42ef790c5b0d457e09da0a4bd Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Jul 2018 21:14:23 -0500 Subject: [PATCH 032/174] Setup CI --- appveyor.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 2a1c748..b6f9f61 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -18,12 +18,9 @@ environment: install: - MKDIR %CD%\DIST - - MKDIR %CD%\DIST\PCRE - - nuget install pcre -Verbosity quiet -Version 8.33.0.1 -OutputDirectory %CD%\DIST\PCRE - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH% - - IF "%PLATFORM%" == "x64" ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\x64\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre64.dll ) ELSE ( copy %CD%\DIST\PCRE\pcre.redist.8.33.0.1\build\native\bin\v100\Win32\Release\dynamic\utf8\pcre8.dll %CD%\bin\pcre32.dll ) - git clone --depth 1 https://github.com/nim-lang/nim - cd nim - git clone --depth 1 https://github.com/nim-lang/csources From 1aab352f035a91784d4ee10875357f8f7c74b712 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Jul 2018 21:31:09 -0500 Subject: [PATCH 033/174] Setup CI --- appveyor.yml | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b6f9f61..bfc1906 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,7 +2,8 @@ version: '{build}' cache: - i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z -- dlls.zip +- nim-0.18.0_x32.zip +# - dlls.zip matrix: fast_finish: true @@ -12,6 +13,9 @@ environment: - MINGW_DIR: mingw32 MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/4.9.2/threads-win32/dwarf/i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z/download MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z + NIM_URL: https://nim-lang.org/download/nim-0.18.0_x32.zip + NIM_ARCHIVE: nim-0.18.0_x32.zip + NIM_VERSION: nim-0.18.0 DLLS_URL: http://nim-lang.org/download/dlls.zip DLLS_ARCHIVE: dlls.zip platform: x86 @@ -20,20 +24,22 @@ install: - MKDIR %CD%\DIST - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul - - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH% - - git clone --depth 1 https://github.com/nim-lang/nim - - cd nim - - git clone --depth 1 https://github.com/nim-lang/csources - - cd csources - - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat ) - - cd .. - - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%" - - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\bin" - - bin\nim c -d:release koch - - koch boot -d:release - - koch nimble -d:release - - SET PATH=%CD%\bin;%PATH% - - cd .. + - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" + - 7z x -y "%NIM_ARCHIVE%" -o"%CD%\DIST"> nul + - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\DIST\%NIM_VERSION%;%USERPROFILE%\.nimble\bin;%PATH% + # - git clone --depth 1 https://github.com/nim-lang/nim + # - cd nim + # - git clone --depth 1 https://github.com/nim-lang/csources + # - cd csources + # - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat ) + # - cd .. + # - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%" + # - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\bin"> nul + # - bin\nim c -d:release koch + # - koch boot -d:release + # - koch nimble -d:release + # - SET PATH=%CD%\bin;%PATH% + # - cd .. build_script: - nimble install -y From 8892eabbe6598410d6c5f53771f62cd61648d5c3 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Jul 2018 21:33:01 -0500 Subject: [PATCH 034/174] Setup CI --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index bfc1906..1cd28d2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -26,7 +26,7 @@ install: - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - 7z x -y "%NIM_ARCHIVE%" -o"%CD%\DIST"> nul - - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\DIST\%NIM_VERSION%;%USERPROFILE%\.nimble\bin;%PATH% + - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\DIST\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% # - git clone --depth 1 https://github.com/nim-lang/nim # - cd nim # - git clone --depth 1 https://github.com/nim-lang/csources From fdcbec8d5bb4994603a0e0798c33d9d9190026a8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Jul 2018 21:44:36 -0500 Subject: [PATCH 035/174] Setup CI --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 1cd28d2..834d1ed 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,7 +1,7 @@ version: '{build}' cache: -- i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z +- i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - nim-0.18.0_x32.zip # - dlls.zip @@ -11,8 +11,8 @@ matrix: environment: matrix: - MINGW_DIR: mingw32 - MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/4.9.2/threads-win32/dwarf/i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z/download - MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z + MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf/i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z/download + MINGW_ARCHIVE: i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z NIM_URL: https://nim-lang.org/download/nim-0.18.0_x32.zip NIM_ARCHIVE: nim-0.18.0_x32.zip NIM_VERSION: nim-0.18.0 From 395422f1e5e5ff506986d0667391e7f4a3011e7b Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Jul 2018 22:09:41 -0500 Subject: [PATCH 036/174] Setup CI --- appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 834d1ed..123e007 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -47,4 +47,8 @@ build_script: test_script: - nimble test +after_build: + - 7z a -r buildlogs.zip %APPVEYOR_BUILD_FOLDER% + - appveyor PushArtifact buildlogs.zip + deploy: off From c7012369beffb33a9dbcc834b49c2cb065953489 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 8 Jul 2018 22:14:22 -0500 Subject: [PATCH 037/174] Setup CI --- appveyor.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 123e007..b13e213 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -47,8 +47,8 @@ build_script: test_script: - nimble test -after_build: - - 7z a -r buildlogs.zip %APPVEYOR_BUILD_FOLDER% - - appveyor PushArtifact buildlogs.zip - deploy: off + +on_finish: + - 7z a -r buildlogs.zip %USERPROFILE%\.nimble\pkgs + - appveyor PushArtifact buildlogs.zip From b8670b1bf1ccfca0b07ae360dfa93f3980927b7c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 09:11:25 -0500 Subject: [PATCH 038/174] Setup CI --- appveyor.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b13e213..c090ead 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -18,15 +18,16 @@ environment: NIM_VERSION: nim-0.18.0 DLLS_URL: http://nim-lang.org/download/dlls.zip DLLS_ARCHIVE: dlls.zip + BASE_DIR: c:\projects platform: x86 install: - - MKDIR %CD%\DIST + - CD %BASE_DIR% - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul + - 7z x -y "%MINGW_ARCHIVE%"> nul - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - - 7z x -y "%NIM_ARCHIVE%" -o"%CD%\DIST"> nul - - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\DIST\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% + - 7z x -y "%NIM_ARCHIVE%"> nul + - SET PATH=%BASE_DIR%\%MINGW_DIR%\BIN;%BASE_DIR%\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% # - git clone --depth 1 https://github.com/nim-lang/nim # - cd nim # - git clone --depth 1 https://github.com/nim-lang/csources From 941df2327835ae45bc1d991add3b607d4d1f9ef5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 09:13:15 -0500 Subject: [PATCH 039/174] Setup CI --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index c090ead..95e0d59 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,6 +28,7 @@ install: - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - 7z x -y "%NIM_ARCHIVE%"> nul - SET PATH=%BASE_DIR%\%MINGW_DIR%\BIN;%BASE_DIR%\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% + - CD %BASE_DIR%\nimgen # - git clone --depth 1 https://github.com/nim-lang/nim # - cd nim # - git clone --depth 1 https://github.com/nim-lang/csources From 23d0016d1a22c05824da26aa1b6abdaac570e623 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 09:42:55 -0500 Subject: [PATCH 040/174] Setup CI --- appveyor.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 95e0d59..58eeb2e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,6 +3,7 @@ version: '{build}' cache: - i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - nim-0.18.0_x32.zip +- libssh2-1.8.0-1-i686.pkg.tar.xz # - dlls.zip matrix: @@ -16,6 +17,10 @@ environment: NIM_URL: https://nim-lang.org/download/nim-0.18.0_x32.zip NIM_ARCHIVE: nim-0.18.0_x32.zip NIM_VERSION: nim-0.18.0 + LIBSSH2_ARCHIVE: libssh2-1.8.0-1-i686.pkg.tar.xz + LIBSSH2_URL: http://repo.msys2.org/msys/i686/libssh2-1.8.0-1-i686.pkg.tar.xz + LIBCRYPTO_ARCHIVE: libopenssl-1.0.2.m-1-i686.pkg.tar.xz + LIBCRYPTO_URL: http://repo.msys2.org/msys/i686/libopenssl-1.0.2.m-1-i686.pkg.tar.xz DLLS_URL: http://nim-lang.org/download/dlls.zip DLLS_ARCHIVE: dlls.zip BASE_DIR: c:\projects @@ -27,7 +32,12 @@ install: - 7z x -y "%MINGW_ARCHIVE%"> nul - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - 7z x -y "%NIM_ARCHIVE%"> nul - - SET PATH=%BASE_DIR%\%MINGW_DIR%\BIN;%BASE_DIR%\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% + - CD %MINGW_DIR% + - IF not exist "%LIBSSH2_ARCHIVE%" appveyor DownloadFile "%LIBSSH2_URL%" -FileName "%LIBSSH2_ARCHIVE%" + - 7z x -y "%LIBSSH2_ARCHIVE%"> nul + - IF not exist "%LIBCRYPTO_ARCHIVE%" appveyor DownloadFile "%LIBCRYPTO_URL%" -FileName "%LIBCRYPTO_ARCHIVE%" + - 7z x -y "%LIBCRYPTO_ARCHIVE%"> nul + - SET PATH=%BASE_DIR%\%MINGW_DIR%\bin;%BASE_DIR%\%MINGW_DIR%\usr\bin;%BASE_DIR%\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - CD %BASE_DIR%\nimgen # - git clone --depth 1 https://github.com/nim-lang/nim # - cd nim From 7979cec722eb82fc0b78b3896821ab294bbe50b8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 10:13:12 -0500 Subject: [PATCH 041/174] Setup CI --- appveyor.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 58eeb2e..9ac633c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,10 +17,10 @@ environment: NIM_URL: https://nim-lang.org/download/nim-0.18.0_x32.zip NIM_ARCHIVE: nim-0.18.0_x32.zip NIM_VERSION: nim-0.18.0 - LIBSSH2_ARCHIVE: libssh2-1.8.0-1-i686.pkg.tar.xz - LIBSSH2_URL: http://repo.msys2.org/msys/i686/libssh2-1.8.0-1-i686.pkg.tar.xz - LIBCRYPTO_ARCHIVE: libopenssl-1.0.2.m-1-i686.pkg.tar.xz - LIBCRYPTO_URL: http://repo.msys2.org/msys/i686/libopenssl-1.0.2.m-1-i686.pkg.tar.xz + LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz + LIBSSH2_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz/download + LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz + LIBCRYPTO_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz/download DLLS_URL: http://nim-lang.org/download/dlls.zip DLLS_ARCHIVE: dlls.zip BASE_DIR: c:\projects @@ -32,7 +32,6 @@ install: - 7z x -y "%MINGW_ARCHIVE%"> nul - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - 7z x -y "%NIM_ARCHIVE%"> nul - - CD %MINGW_DIR% - IF not exist "%LIBSSH2_ARCHIVE%" appveyor DownloadFile "%LIBSSH2_URL%" -FileName "%LIBSSH2_ARCHIVE%" - 7z x -y "%LIBSSH2_ARCHIVE%"> nul - IF not exist "%LIBCRYPTO_ARCHIVE%" appveyor DownloadFile "%LIBCRYPTO_URL%" -FileName "%LIBCRYPTO_ARCHIVE%" From 9821b228afff997d1924d23e479fcd2b66cf93ee Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 10:26:18 -0500 Subject: [PATCH 042/174] Setup CI --- appveyor.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 9ac633c..438d80b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,9 +17,9 @@ environment: NIM_URL: https://nim-lang.org/download/nim-0.18.0_x32.zip NIM_ARCHIVE: nim-0.18.0_x32.zip NIM_VERSION: nim-0.18.0 - LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz + LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar LIBSSH2_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz/download - LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz + LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar LIBCRYPTO_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz/download DLLS_URL: http://nim-lang.org/download/dlls.zip DLLS_ARCHIVE: dlls.zip @@ -33,10 +33,12 @@ install: - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - 7z x -y "%NIM_ARCHIVE%"> nul - IF not exist "%LIBSSH2_ARCHIVE%" appveyor DownloadFile "%LIBSSH2_URL%" -FileName "%LIBSSH2_ARCHIVE%" + - 7z x -y "%LIBSSH2_ARCHIVE%".xz> nul - 7z x -y "%LIBSSH2_ARCHIVE%"> nul - IF not exist "%LIBCRYPTO_ARCHIVE%" appveyor DownloadFile "%LIBCRYPTO_URL%" -FileName "%LIBCRYPTO_ARCHIVE%" + - 7z x -y "%LIBCRYPTO_ARCHIVE%".xz> nul - 7z x -y "%LIBCRYPTO_ARCHIVE%"> nul - - SET PATH=%BASE_DIR%\%MINGW_DIR%\bin;%BASE_DIR%\%MINGW_DIR%\usr\bin;%BASE_DIR%\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% + - SET PATH=%BASE_DIR%\%MINGW_DIR%\bin;%BASE_DIR%\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - CD %BASE_DIR%\nimgen # - git clone --depth 1 https://github.com/nim-lang/nim # - cd nim From 37fdefa4b834c071811ebf4c6c7568da8d922ce1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 10:29:50 -0500 Subject: [PATCH 043/174] Setup CI --- appveyor.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 438d80b..2615890 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,9 +17,11 @@ environment: NIM_URL: https://nim-lang.org/download/nim-0.18.0_x32.zip NIM_ARCHIVE: nim-0.18.0_x32.zip NIM_VERSION: nim-0.18.0 - LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar + LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz + LIBSSH2_ARCHIVE2: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar LIBSSH2_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz/download - LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar + LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz + LIBCRYPTO_ARCHIVE2: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar LIBCRYPTO_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz/download DLLS_URL: http://nim-lang.org/download/dlls.zip DLLS_ARCHIVE: dlls.zip @@ -33,11 +35,11 @@ install: - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - 7z x -y "%NIM_ARCHIVE%"> nul - IF not exist "%LIBSSH2_ARCHIVE%" appveyor DownloadFile "%LIBSSH2_URL%" -FileName "%LIBSSH2_ARCHIVE%" - - 7z x -y "%LIBSSH2_ARCHIVE%".xz> nul - 7z x -y "%LIBSSH2_ARCHIVE%"> nul + - 7z x -y "%LIBSSH2_ARCHIVE2%"> nul - IF not exist "%LIBCRYPTO_ARCHIVE%" appveyor DownloadFile "%LIBCRYPTO_URL%" -FileName "%LIBCRYPTO_ARCHIVE%" - - 7z x -y "%LIBCRYPTO_ARCHIVE%".xz> nul - 7z x -y "%LIBCRYPTO_ARCHIVE%"> nul + - 7z x -y "%LIBCRYPTO_ARCHIVE2%"> nul - SET PATH=%BASE_DIR%\%MINGW_DIR%\bin;%BASE_DIR%\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - CD %BASE_DIR%\nimgen # - git clone --depth 1 https://github.com/nim-lang/nim From 88d8ee6b848d98ebe7b27fdf619d6f4833fce358 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 11:18:14 -0500 Subject: [PATCH 044/174] Setup CI --- appveyor.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 2615890..f3de341 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,10 +1,10 @@ version: '{build}' cache: -- i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z -- nim-0.18.0_x32.zip -- libssh2-1.8.0-1-i686.pkg.tar.xz -# - dlls.zip +- c:\projects\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z +- c:\projects\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz +- c:\projects\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz +- c:\projects\nim-0.18.0_x32.zip matrix: fast_finish: true @@ -32,14 +32,14 @@ install: - CD %BASE_DIR% - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - 7z x -y "%MINGW_ARCHIVE%"> nul - - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - - 7z x -y "%NIM_ARCHIVE%"> nul - IF not exist "%LIBSSH2_ARCHIVE%" appveyor DownloadFile "%LIBSSH2_URL%" -FileName "%LIBSSH2_ARCHIVE%" - 7z x -y "%LIBSSH2_ARCHIVE%"> nul - 7z x -y "%LIBSSH2_ARCHIVE2%"> nul - IF not exist "%LIBCRYPTO_ARCHIVE%" appveyor DownloadFile "%LIBCRYPTO_URL%" -FileName "%LIBCRYPTO_ARCHIVE%" - 7z x -y "%LIBCRYPTO_ARCHIVE%"> nul - 7z x -y "%LIBCRYPTO_ARCHIVE2%"> nul + - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" + - 7z x -y "%NIM_ARCHIVE%"> nul - SET PATH=%BASE_DIR%\%MINGW_DIR%\bin;%BASE_DIR%\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - CD %BASE_DIR%\nimgen # - git clone --depth 1 https://github.com/nim-lang/nim From 7c325683cc26ed6b96f1e5d5265813b14ec141c2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 11:42:01 -0500 Subject: [PATCH 045/174] Setup CI --- appveyor.yml | 115 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 78 insertions(+), 37 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index f3de341..15b6a10 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,17 +1,16 @@ version: '{build}' -cache: -- c:\projects\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z -- c:\projects\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz -- c:\projects\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz -- c:\projects\nim-0.18.0_x32.zip +image: + - Visual Studio 2017 + - Ubuntu matrix: fast_finish: true environment: matrix: - - MINGW_DIR: mingw32 + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + MINGW_DIR: mingw32 MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf/i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z/download MINGW_ARCHIVE: i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z NIM_URL: https://nim-lang.org/download/nim-0.18.0_x32.zip @@ -27,34 +26,80 @@ environment: DLLS_ARCHIVE: dlls.zip BASE_DIR: c:\projects platform: x86 + - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu + NIM_URL: https://nim-lang.org/download/nim-0.18.0.tar.xz + NIM_ARCHIVE: nim-0.18.0.tar.xz + NIM_VERSION: nim-0.18.0 -install: - - CD %BASE_DIR% - - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - - 7z x -y "%MINGW_ARCHIVE%"> nul - - IF not exist "%LIBSSH2_ARCHIVE%" appveyor DownloadFile "%LIBSSH2_URL%" -FileName "%LIBSSH2_ARCHIVE%" - - 7z x -y "%LIBSSH2_ARCHIVE%"> nul - - 7z x -y "%LIBSSH2_ARCHIVE2%"> nul - - IF not exist "%LIBCRYPTO_ARCHIVE%" appveyor DownloadFile "%LIBCRYPTO_URL%" -FileName "%LIBCRYPTO_ARCHIVE%" - - 7z x -y "%LIBCRYPTO_ARCHIVE%"> nul - - 7z x -y "%LIBCRYPTO_ARCHIVE2%"> nul - - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - - 7z x -y "%NIM_ARCHIVE%"> nul - - SET PATH=%BASE_DIR%\%MINGW_DIR%\bin;%BASE_DIR%\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - - CD %BASE_DIR%\nimgen - # - git clone --depth 1 https://github.com/nim-lang/nim - # - cd nim - # - git clone --depth 1 https://github.com/nim-lang/csources - # - cd csources - # - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat ) - # - cd .. - # - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%" - # - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\bin"> nul - # - bin\nim c -d:release koch - # - koch boot -d:release - # - koch nimble -d:release - # - SET PATH=%CD%\bin;%PATH% - # - cd .. +for: +- + matrix: + only: + - image: Visual Studio 2017 + + install: + - CD %BASE_DIR% + - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" + - 7z x -y "%MINGW_ARCHIVE%"> nul + - IF not exist "%LIBSSH2_ARCHIVE%" appveyor DownloadFile "%LIBSSH2_URL%" -FileName "%LIBSSH2_ARCHIVE%" + - 7z x -y "%LIBSSH2_ARCHIVE%"> nul + - 7z x -y "%LIBSSH2_ARCHIVE2%"> nul + - IF not exist "%LIBCRYPTO_ARCHIVE%" appveyor DownloadFile "%LIBCRYPTO_URL%" -FileName "%LIBCRYPTO_ARCHIVE%" + - 7z x -y "%LIBCRYPTO_ARCHIVE%"> nul + - 7z x -y "%LIBCRYPTO_ARCHIVE2%"> nul + - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" + - 7z x -y "%NIM_ARCHIVE%"> nul + - SET PATH=%BASE_DIR%\%MINGW_DIR%\bin;%BASE_DIR%\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% + - CD %BASE_DIR%\nimgen + # - git clone --depth 1 https://github.com/nim-lang/nim + # - cd nim + # - git clone --depth 1 https://github.com/nim-lang/csources + # - cd csources + # - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat ) + # - cd .. + # - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%" + # - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\bin"> nul + # - bin\nim c -d:release koch + # - koch boot -d:release + # - koch nimble -d:release + # - SET PATH=%CD%\bin;%PATH% + # - cd .. + + on_finish: + - 7z a -r buildlogs-win.zip %USERPROFILE%\.nimble\pkgs + - appveyor PushArtifact buildlogs-win.zip + + cache: + - c:\projects\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z + - c:\projects\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz + - c:\projects\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz + - c:\projects\nim-0.18.0_x32.zip + +for: +- + matrix: + only: + - image: Ubuntu + + install: + - if [ ! $NIM_ARCHIVE ]; then curl -o $NIM_ARCHIVE $NIM_URL; fi + - tar xJf $NIM_ARCHIVE + - cd $NIM_VERSION + - sh build.sh + - bin/nim -d:release koch + - ./koch boot -d:release + - ./koch nimble -d:release + - set PATH=`pwd`/bin:~/.nimble/bin:$PATH + - cd .. + - echo `pwd` + - ls + + on_finish: + - zip buildlogs-lin.zip ~/.nimble/pkgs + - appveyor PushArtifact buildlogs-lin.zip + + cache: + - nim-0.18.0.tar.xz build_script: - nimble install -y @@ -63,7 +108,3 @@ test_script: - nimble test deploy: off - -on_finish: - - 7z a -r buildlogs.zip %USERPROFILE%\.nimble\pkgs - - appveyor PushArtifact buildlogs.zip From 6f36dcb99d83bd99c4d1b6c462bd3d1c487326aa Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 11:45:09 -0500 Subject: [PATCH 046/174] Setup CI --- appveyor.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 15b6a10..63609de 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -75,8 +75,6 @@ for: - c:\projects\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - c:\projects\nim-0.18.0_x32.zip -for: -- matrix: only: - image: Ubuntu From 894818a0db0af2630383c85cdd2e1eb757e0fb2e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 11:47:13 -0500 Subject: [PATCH 047/174] Setup CI --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 63609de..dbc300f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -75,6 +75,7 @@ for: - c:\projects\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - c:\projects\nim-0.18.0_x32.zip +- matrix: only: - image: Ubuntu From 7ae458419e5c69a1198b6c68b57a9f982ff35176 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 11:51:08 -0500 Subject: [PATCH 048/174] Setup CI --- appveyor.yml | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index dbc300f..b12579b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,36 +7,29 @@ image: matrix: fast_finish: true -environment: - matrix: - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - MINGW_DIR: mingw32 - MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf/i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z/download - MINGW_ARCHIVE: i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - NIM_URL: https://nim-lang.org/download/nim-0.18.0_x32.zip - NIM_ARCHIVE: nim-0.18.0_x32.zip - NIM_VERSION: nim-0.18.0 - LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz - LIBSSH2_ARCHIVE2: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar - LIBSSH2_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz/download - LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - LIBCRYPTO_ARCHIVE2: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar - LIBCRYPTO_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz/download - DLLS_URL: http://nim-lang.org/download/dlls.zip - DLLS_ARCHIVE: dlls.zip - BASE_DIR: c:\projects - platform: x86 - - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu - NIM_URL: https://nim-lang.org/download/nim-0.18.0.tar.xz - NIM_ARCHIVE: nim-0.18.0.tar.xz - NIM_VERSION: nim-0.18.0 - for: - matrix: only: - image: Visual Studio 2017 + environment: + MINGW_DIR: mingw32 + MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf/i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z/download + MINGW_ARCHIVE: i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z + NIM_URL: https://nim-lang.org/download/nim-0.18.0_x32.zip + NIM_ARCHIVE: nim-0.18.0_x32.zip + NIM_VERSION: nim-0.18.0 + LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz + LIBSSH2_ARCHIVE2: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar + LIBSSH2_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz/download + LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz + LIBCRYPTO_ARCHIVE2: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar + LIBCRYPTO_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz/download + DLLS_URL: http://nim-lang.org/download/dlls.zip + DLLS_ARCHIVE: dlls.zip + BASE_DIR: c:\projects + install: - CD %BASE_DIR% - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" @@ -80,6 +73,11 @@ for: only: - image: Ubuntu + environment: + NIM_URL: https://nim-lang.org/download/nim-0.18.0.tar.xz + NIM_ARCHIVE: nim-0.18.0.tar.xz + NIM_VERSION: nim-0.18.0 + install: - if [ ! $NIM_ARCHIVE ]; then curl -o $NIM_ARCHIVE $NIM_URL; fi - tar xJf $NIM_ARCHIVE From 9257a0a81bad916a51fc434cc1039cda030abf6b Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 12:06:07 -0500 Subject: [PATCH 049/174] Setup CI --- appveyor.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b12579b..9cccdd5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,8 @@ version: '{build}' image: - - Visual Studio 2017 - Ubuntu + - Visual Studio 2017 matrix: fast_finish: true @@ -79,7 +79,9 @@ for: NIM_VERSION: nim-0.18.0 install: - - if [ ! $NIM_ARCHIVE ]; then curl -o $NIM_ARCHIVE $NIM_URL; fi + - echo $NIM_ARCHIVE + - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi + - echo $? - tar xJf $NIM_ARCHIVE - cd $NIM_VERSION - sh build.sh From cc7b6cb726d8d22995a69236b93d2e8b456b0eae Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 12:09:55 -0500 Subject: [PATCH 050/174] Setup CI --- appveyor.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 9cccdd5..f36fe84 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -79,26 +79,23 @@ for: NIM_VERSION: nim-0.18.0 install: - - echo $NIM_ARCHIVE + - cd ~/projects - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - - echo $? - tar xJf $NIM_ARCHIVE - cd $NIM_VERSION - sh build.sh - - bin/nim -d:release koch + - bin/nim c -d:release koch - ./koch boot -d:release - ./koch nimble -d:release - set PATH=`pwd`/bin:~/.nimble/bin:$PATH - - cd .. - - echo `pwd` - - ls + - cd ~/projects/nimgen on_finish: - zip buildlogs-lin.zip ~/.nimble/pkgs - appveyor PushArtifact buildlogs-lin.zip cache: - - nim-0.18.0.tar.xz + - ~/projects/nim-0.18.0.tar.xz build_script: - nimble install -y From 9fcd0ddd0f278bacfa2b4dcfb0373d953f17f68a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 12:14:55 -0500 Subject: [PATCH 051/174] Setup CI --- appveyor.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index f36fe84..1e24408 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -87,7 +87,9 @@ for: - bin/nim c -d:release koch - ./koch boot -d:release - ./koch nimble -d:release - - set PATH=`pwd`/bin:~/.nimble/bin:$PATH + - set PATH=~/projects/$NIM_VERSION/bin:~/.nimble/bin:$PATH + - echo $PATH + - ls bin - cd ~/projects/nimgen on_finish: From 6aa9b4290ac3491736289f5cf96dc642f7aa80da Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 12:18:08 -0500 Subject: [PATCH 052/174] Setup CI --- appveyor.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 1e24408..86c217a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -87,9 +87,7 @@ for: - bin/nim c -d:release koch - ./koch boot -d:release - ./koch nimble -d:release - - set PATH=~/projects/$NIM_VERSION/bin:~/.nimble/bin:$PATH - - echo $PATH - - ls bin + - export PATH=~/projects/$NIM_VERSION/bin:~/.nimble/bin:$PATH - cd ~/projects/nimgen on_finish: From 217161bb4dd2217f2baf40feceba4cfc63fd49c8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 12:26:58 -0500 Subject: [PATCH 053/174] Setup CI --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 86c217a..0e5df1e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -79,6 +79,8 @@ for: NIM_VERSION: nim-0.18.0 install: + - sudo apt update + - sudo apt install --yes python-pygments - cd ~/projects - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - tar xJf $NIM_ARCHIVE From b894dbd17ac94af6e7e83407910de8750a8d9545 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 14:02:38 -0500 Subject: [PATCH 054/174] Allow ppflags and related to be OS specific --- nimgen.nim | 36 +++++++++++++++++++----------------- nimgen.nimble | 2 +- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index f4631c3..328dd2d 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -647,23 +647,25 @@ proc runFile(file: string, cfgin: OrderedTableRef) = var noprocess = false for act in cfg.keys(): - if cfg[act] == "true": - if act == "recurse": - c2nimConfig.recurse = true - elif act == "inline": - c2nimConfig.inline = true - elif act == "preprocess": - c2nimConfig.preprocess = true - elif act == "ctags": - c2nimConfig.ctags = true - elif act == "defines": - c2nimConfig.defines = true - elif act == "noprocess": - noprocess = true - elif act == "flags": - c2nimConfig.flags = cfg[act] - elif act == "ppflags": - c2nimConfig.ppflags = cfg[act] + let (action, val) = getKey(act) + if val == true: + if cfg[act] == "true": + if action == "recurse": + c2nimConfig.recurse = true + elif action == "inline": + c2nimConfig.inline = true + elif action == "preprocess": + c2nimConfig.preprocess = true + elif action == "ctags": + c2nimConfig.ctags = true + elif action == "defines": + c2nimConfig.defines = true + elif action == "noprocess": + noprocess = true + elif action == "flags": + c2nimConfig.flags = cfg[act] + elif action == "ppflags": + c2nimConfig.ppflags = cfg[act] if c2nimConfig.recurse and c2nimConfig.inline: echo "Cannot use recurse and inline simultaneously" diff --git a/nimgen.nimble b/nimgen.nimble index 043fc86..28967d4 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.2.3" +version = "0.2.4" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" From 1f2bd20da1a546c867e2347ac4ad35a1e2d791d8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 14:09:38 -0500 Subject: [PATCH 055/174] Setup CI --- appveyor.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0e5df1e..cb3f926 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -79,8 +79,8 @@ for: NIM_VERSION: nim-0.18.0 install: - - sudo apt update - - sudo apt install --yes python-pygments + - sudo apt update -qq + - sudo apt install -qq --yes python-pygments - cd ~/projects - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - tar xJf $NIM_ARCHIVE @@ -90,10 +90,11 @@ for: - ./koch boot -d:release - ./koch nimble -d:release - export PATH=~/projects/$NIM_VERSION/bin:~/.nimble/bin:$PATH + - alias g++="g++ -std=c++11" - cd ~/projects/nimgen on_finish: - - zip buildlogs-lin.zip ~/.nimble/pkgs + - zip -r buildlogs-lin.zip ~/.nimble/pkgs - appveyor PushArtifact buildlogs-lin.zip cache: From 942f194945cf979527859a76e4e6f15fafb95802 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 14:27:20 -0500 Subject: [PATCH 056/174] Setup CI --- appveyor.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index cb3f926..24cdba4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -90,7 +90,6 @@ for: - ./koch boot -d:release - ./koch nimble -d:release - export PATH=~/projects/$NIM_VERSION/bin:~/.nimble/bin:$PATH - - alias g++="g++ -std=c++11" - cd ~/projects/nimgen on_finish: From 5621d59e3ce4a95dc14127042cc4a842e63bbc8b Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 15:01:58 -0500 Subject: [PATCH 057/174] Setup CI --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 24cdba4..be2fa30 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -80,7 +80,7 @@ for: install: - sudo apt update -qq - - sudo apt install -qq --yes python-pygments + - sudo apt install -qq --yes python-pygments libssh2 - cd ~/projects - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - tar xJf $NIM_ARCHIVE @@ -93,7 +93,7 @@ for: - cd ~/projects/nimgen on_finish: - - zip -r buildlogs-lin.zip ~/.nimble/pkgs + - zip -r -q buildlogs-lin.zip ~/.nimble/pkgs - appveyor PushArtifact buildlogs-lin.zip cache: From 0e6a7b8025cfe44740d3fb9302e2648ad33f6652 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 15:03:41 -0500 Subject: [PATCH 058/174] Setup CI --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index be2fa30..acefe6b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -79,8 +79,8 @@ for: NIM_VERSION: nim-0.18.0 install: - - sudo apt update -qq - - sudo apt install -qq --yes python-pygments libssh2 + - sudo apt -qq update + - sudo apt -qq install --yes python-pygments libssh2-1 - cd ~/projects - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - tar xJf $NIM_ARCHIVE From a24d0d043d96a2f65c1bf72cbeccbce195376118 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 15:10:02 -0500 Subject: [PATCH 059/174] Setup CI --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index acefe6b..91a9a07 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -80,7 +80,7 @@ for: install: - sudo apt -qq update - - sudo apt -qq install --yes python-pygments libssh2-1 + - sudo apt -qq install --yes python-pygments libssh2-1 libgcrypt20 libgpg-error0 - cd ~/projects - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - tar xJf $NIM_ARCHIVE From adaa7e997f34e3374c60c72206cd61f396f95075 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 9 Jul 2018 15:20:31 -0500 Subject: [PATCH 060/174] Setup CI --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 91a9a07..a54b7ce 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -80,7 +80,7 @@ for: install: - sudo apt -qq update - - sudo apt -qq install --yes python-pygments libssh2-1 libgcrypt20 libgpg-error0 + - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev - cd ~/projects - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - tar xJf $NIM_ARCHIVE From b32d39ad08ec630f288958c4c9e0878ab2e8677f Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 7 Jul 2018 19:16:16 +0900 Subject: [PATCH 061/174] Add preliminary support for removing static funcs --- nimgen.nim | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index c132fe1..7a053bb 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -25,7 +25,7 @@ var type c2nimConfigObj = object flags, ppflags: string - recurse, inline, preprocess, ctags, defines: bool + recurse, inline, preprocess, ctags, defines, remove_static: bool dynlib, compile, pragma: seq[string] const DOC = """ @@ -92,7 +92,9 @@ proc execProc(cmd: string): string = proc extractZip(zipfile: string) = var cmd = "unzip -o $#" if defined(Windows): - cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" + cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A " & + "'System.IO.Compression.FileSystem'; " & + "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" setCurrentDir(gOutput) defer: setCurrentDir(gProjectDir) @@ -322,6 +324,67 @@ proc comment(file: string, pattern: string, numlines: string) = idx += 1 break +proc removeStatic(filename: string) = + + if not fileExists(filename): + echo "Missing file: " & filename + return + + # This function should ideally be a regex. However, I could not + # get it to work as intended with the current re implementation + # in Nim. + # + # withFile(filename): + # content = content.replacef( + # re"(static inline.*?\))([ \r\n]*?\{([ \r\n]*?.*?)*[\n\r]\})", "$1;") + # ) + # + # This currently won't even run, but if the replacef function is modified + # to not have nil checks, it will run on 1/3 of the input file. Maybe there's + # a buffer length issue. + + var + file = open(filename) + stack: seq[string] = @[] + foundBrace = false + foundStatic = false + writeOutput = true + output = newStringofCap(getFileSize(filename)) + + for line in file.lines(): + var modLine = line + + if not foundStatic: + writeOutput = true + if line.startswith("static inline"): + foundStatic = true + let index = modLine.find("{") + if index != -1: + foundBrace = true + modLine.setLen(index) + elif not foundBrace: + writeOutput = true + if modLine.strip().startswith("{"): + foundBrace = true + writeOutput = false + else: + if modLine.startswith("}"): + foundBrace = false + foundStatic = false + output[^1] = ';' + output &= "\n" + + if writeOutput: + output &= modLine + output &= "\n" + writeOutput = false + + file.close() + + var f = open(filename, fmWrite) + write(f, output) + f.close() + proc rename(file: string, renfile: string) = if file.splitFile().ext == ".nim": return @@ -519,6 +582,9 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): prepend(cfile, getDefines(file, c2nimConfig.inline)) + if c2nimConfig.remove_static: + removeStatic(cfile) + var extflags = "" passC = "" @@ -697,6 +763,8 @@ proc runFile(file: string, cfgin: OrderedTableRef) = c2nimConfig.ctags = true elif action == "defines": c2nimConfig.defines = true + elif act == "remove_static": + c2nimConfig.remove_static = true elif action == "noprocess": noprocess = true elif action == "flags": From 5321014983a5dff827cecf37392b7ee99985b738 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 8 Jul 2018 10:03:33 +0900 Subject: [PATCH 062/174] Add readme section about remove_static --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 40f7556..360bec5 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,8 @@ The following keys apply to library source code and help with generating the .ni ```defines``` = pulls out simple #defines of ints, floats and hex values for separate conversion [default: false] - works only when preprocess or ctags is used and helps include useful definitions in generated .nim file +```remove_static``` = pulls out the bodies of inline static functions [default: false] + ```flags``` = flags to pass to the c2nim process in "quotes" [default: --stdcall]. --cdecl, --assumedef, --assumendef may be useful ```ppflags``` = flags to pass to the preprocessor [default: ""]. -D for gcc and others may be useful From 2e894dc90702b92290c226dd683a198b3d092634 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 18 Jun 2018 19:44:45 +0900 Subject: [PATCH 063/174] Implement relative import support in header files. Fixes #17 --- nimgen.nim | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index c132fe1..6baea3b 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -224,6 +224,8 @@ proc search(file: string): string = echo "File doesn't exist: " & file quit(1) + # Only keep relative directory + result = result.replace(gProjectDir & $DirSep, "") return result.replace(re"[\\/]", $DirSep) # ### @@ -376,12 +378,25 @@ proc getIncls(file: string, inline=false): seq[string] = if inline and file in gDoneInline: return + let curPath = splitFile(expandFileName(file)).dir withFile(file): for f in content.findIter(re"(?m)^\s*#\s*include\s+(.*?)$"): var inc = f.captures[0].strip() if ((gQuotes and inc.contains("\"")) or (gFilter != "" and gFilter in inc)) and (not exclude(inc)): - result.add( - inc.replace(re"""[<>"]""", "").replace(re"\/[\*\/].*$", "").strip()) + let addInc = inc.replace(re"""[<>"]""", "").replace(re"\/[\*\/].*$", "").strip() + try: + # Try searching for a local library. expandFilename will throw + # OSError if the file does not exist + let + finc = expandFileName(curPath / addInc) + fname = extractFileName(finc).search() + + if fname.len() > 0: + # only add if the file is non-empty + result.add(fname.search()) + except OSError: + # If it's a system library + result.add(addInc) result = result.deduplicate() From ab3185b611c138ad32c7149ba5260ba67bf0fb4a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 8 Jul 2018 13:05:09 +0900 Subject: [PATCH 064/174] Add n.post section --- README.md | 8 ++++++++ nimgen.nim | 13 ++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 40f7556..6bc1cd1 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,14 @@ The following keys can be used to prepare dependencies such as downloading ZIP f ```copy``` = copy a file to another location. Preferred over moving to preserve original. Comma separate for multiple entries. E.g. copy = "output/config.h.in=output/config.h" +_[n.post]_ + +This section is the same as the prepare section, but for performing actions after the project has been processed. + +```reset``` = whether or not to perform a git reset on all files after processing [default: false] + +```execute``` = command to run after processing + _[n.wildcard]_ File wildcards such as *.nim, ssl*.h, etc. can be used to perform tasks across a group of files. This is useful to define common operations such as global text replacements without having to specify an explicit section for every single file. These operations will be performed on every matching file that is defined as a _sourcefile_ or recursed files. Only applies on source files following the wildcard declarations. diff --git a/nimgen.nim b/nimgen.nim index c132fe1..fedbee5 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -797,11 +797,22 @@ proc runCfg(cfg: string) = gConfig["n.wildcard"][wild]) for file in gConfig.keys(): - if file in @["n.global", "n.include", "n.exclude", "n.prepare", "n.wildcard"]: + if file in @["n.global", "n.include", "n.exclude", + "n.prepare", "n.wildcard", "n.post"]: continue runFile(file, gConfig[file]) + if gConfig.hasKey("n.post"): + for post in gConfig["n.post"].keys(): + let (key, val) = getKey(post) + if val == true: + let postVal = gConfig["n.post"][post] + if key == "reset": + gitReset() + elif key == "execute": + discard execProc(postVal) + # ### # Main loop From f5e8535ce83c79d66e32e1ebe2d5f1a945f61c77 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 10 Jul 2018 13:53:53 +0900 Subject: [PATCH 065/174] Fix windows paths --- nimgen.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index 6baea3b..94fa48c 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -225,8 +225,7 @@ proc search(file: string): string = quit(1) # Only keep relative directory - result = result.replace(gProjectDir & $DirSep, "") - return result.replace(re"[\\/]", $DirSep) + return result.multiReplace([("\\", $DirSep), ("//", $DirSep), (gProjectDir & $DirSep, "")]) # ### # Loading / unloading @@ -389,7 +388,7 @@ proc getIncls(file: string, inline=false): seq[string] = # OSError if the file does not exist let finc = expandFileName(curPath / addInc) - fname = extractFileName(finc).search() + fname = finc.replace(curPath & $DirSep, "") if fname.len() > 0: # only add if the file is non-empty From 0d0506675c6fc91de1fd6ac59faeece0f248ac4a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 10 Jul 2018 20:09:51 +0900 Subject: [PATCH 066/174] Make static bodies replacement use regex --- nimgen.nim | 64 +++++------------------------------------------------- 1 file changed, 5 insertions(+), 59 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index 7a053bb..e520309 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -325,65 +325,11 @@ proc comment(file: string, pattern: string, numlines: string) = break proc removeStatic(filename: string) = - - if not fileExists(filename): - echo "Missing file: " & filename - return - - # This function should ideally be a regex. However, I could not - # get it to work as intended with the current re implementation - # in Nim. - # - # withFile(filename): - # content = content.replacef( - # re"(static inline.*?\))([ \r\n]*?\{([ \r\n]*?.*?)*[\n\r]\})", "$1;") - # ) - # - # This currently won't even run, but if the replacef function is modified - # to not have nil checks, it will run on 1/3 of the input file. Maybe there's - # a buffer length issue. - - var - file = open(filename) - stack: seq[string] = @[] - foundBrace = false - foundStatic = false - writeOutput = true - output = newStringofCap(getFileSize(filename)) - - for line in file.lines(): - var modLine = line - - if not foundStatic: - writeOutput = true - if line.startswith("static inline"): - foundStatic = true - let index = modLine.find("{") - if index != -1: - foundBrace = true - modLine.setLen(index) - elif not foundBrace: - writeOutput = true - if modLine.strip().startswith("{"): - foundBrace = true - writeOutput = false - else: - if modLine.startswith("}"): - foundBrace = false - foundStatic = false - output[^1] = ';' - output &= "\n" - - if writeOutput: - output &= modLine - output &= "\n" - writeOutput = false - - file.close() - - var f = open(filename, fmWrite) - write(f, output) - f.close() + ## Replace static function bodies with a semicolon + withFile(filename): + content = content.replace( + re"(?m)(static inline.*?\))(\s*\{(\s*?.*?$)*[\n\r]\})", "$1;" + ) proc rename(file: string, renfile: string) = if file.splitFile().ext == ".nim": From 14b119f433f7584f2b9a529f388b4f298f3da874 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 11 Jul 2018 07:02:36 +0900 Subject: [PATCH 067/174] Move remove static and reset after processing --- nimgen.nim | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index e520309..32e640b 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -25,7 +25,7 @@ var type c2nimConfigObj = object flags, ppflags: string - recurse, inline, preprocess, ctags, defines, remove_static: bool + recurse, inline, preprocess, ctags, defines, removeStatic: bool dynlib, compile, pragma: seq[string] const DOC = """ @@ -126,6 +126,16 @@ proc gitReset() = discard execProc("git reset --hard HEAD") +proc gitCheckout(filename: string) = + echo "Resetting file: $#" % [filename] + + setCurrentDir(gOutput) + defer: setCurrentDir(gProjectDir) + + let adjustedFile = filename.replace(gOutput & $DirSep, "") + + discard execProc("git checkout $#" % [adjustedFile]) + proc gitRemotePull(url: string, pull=true) = if dirExists(gOutput/".git"): if pull: @@ -528,7 +538,7 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): prepend(cfile, getDefines(file, c2nimConfig.inline)) - if c2nimConfig.remove_static: + if c2nimConfig.removeStatic: removeStatic(cfile) var @@ -659,6 +669,9 @@ proc runFile(file: string, cfgin: OrderedTableRef) = if action == "create": createDir(file.splitPath().head) writeFile(file, cfg[act]) + elif action == "removestatic": + removeStatic(sfile) + c2nimConfig.removeStatic = true elif action in @["prepend", "append", "replace", "comment", "rename", "compile", "dynlib", "pragma", "pipe"] and sfile != "": @@ -709,8 +722,6 @@ proc runFile(file: string, cfgin: OrderedTableRef) = c2nimConfig.ctags = true elif action == "defines": c2nimConfig.defines = true - elif act == "remove_static": - c2nimConfig.remove_static = true elif action == "noprocess": noprocess = true elif action == "flags": @@ -725,6 +736,9 @@ proc runFile(file: string, cfgin: OrderedTableRef) = if not noprocess: c2nim(file, getNimout(sfile), c2nimConfig) + if c2nimConfig.removeStatic: + gitCheckout(sfile) + proc runCfg(cfg: string) = if not fileExists(cfg): echo "Config doesn't exist: " & cfg From f62125a4aaed2a15970bdd186dddb7cb9a4bd862 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 11 Jul 2018 07:53:05 +0900 Subject: [PATCH 068/174] Update readme with removestatic --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 360bec5..7f3f256 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ The following keys apply to library source code and help with generating the .ni ```defines``` = pulls out simple #defines of ints, floats and hex values for separate conversion [default: false] - works only when preprocess or ctags is used and helps include useful definitions in generated .nim file -```remove_static``` = pulls out the bodies of inline static functions [default: false] +```removestatic``` = pulls out the bodies of inline static functions [default: false] ```flags``` = flags to pass to the c2nim process in "quotes" [default: --stdcall]. --cdecl, --assumedef, --assumendef may be useful From 685081b6a99c0022e29863e8d70bd873d7fe5876 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 11 Jul 2018 13:36:45 +0900 Subject: [PATCH 069/174] Modify removeStatic to comment out body, then re-comment --- nimgen.nim | 44 ++++++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index 32e640b..d613386 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -25,7 +25,7 @@ var type c2nimConfigObj = object flags, ppflags: string - recurse, inline, preprocess, ctags, defines, removeStatic: bool + recurse, inline, preprocess, ctags, defines: bool dynlib, compile, pragma: seq[string] const DOC = """ @@ -126,7 +126,7 @@ proc gitReset() = discard execProc("git reset --hard HEAD") -proc gitCheckout(filename: string) = +proc gitCheckout(filename: string) {.used.} = echo "Resetting file: $#" % [filename] setCurrentDir(gOutput) @@ -335,10 +335,33 @@ proc comment(file: string, pattern: string, numlines: string) = break proc removeStatic(filename: string) = - ## Replace static function bodies with a semicolon + ## Replace static function bodies with a semicolon and commented + ## out body withFile(filename): content = content.replace( - re"(?m)(static inline.*?\))(\s*\{(\s*?.*?$)*[\n\r]\})", "$1;" + re"(?m)(static inline.*?\))(\s*\{(\s*?.*?$)*[\n\r]\})", + proc (match: RegexMatch): string = + let funcDecl = match.captures[0] + let body = match.captures[1].strip() + result = "" + + result.add("$#;" % [funcDecl]) + result.add(body.replace(re"(?m)^", "//")) + ) + +proc reAddStatic(filename: string) = + ## Uncomment out the body and remove the semicolon. Undoes + ## removeStatic + withFile(filename): + content = content.replace( + re"(?m)(static inline.*?\));(\/\/\s*\{(\s*?.*?$)*[\n\r]\/\/\})", + proc (match: RegexMatch): string = + let funcDecl = match.captures[0] + let body = match.captures[1].strip() + result = "" + + result.add("$# " % [funcDecl]) + result.add(body.replace(re"(?m)^\/\/", "")) ) proc rename(file: string, renfile: string) = @@ -538,9 +561,6 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): prepend(cfile, getDefines(file, c2nimConfig.inline)) - if c2nimConfig.removeStatic: - removeStatic(cfile) - var extflags = "" passC = "" @@ -669,9 +689,6 @@ proc runFile(file: string, cfgin: OrderedTableRef) = if action == "create": createDir(file.splitPath().head) writeFile(file, cfg[act]) - elif action == "removestatic": - removeStatic(sfile) - c2nimConfig.removeStatic = true elif action in @["prepend", "append", "replace", "comment", "rename", "compile", "dynlib", "pragma", "pipe"] and sfile != "": @@ -733,11 +750,14 @@ proc runFile(file: string, cfgin: OrderedTableRef) = echo "Cannot use recurse and inline simultaneously" quit(1) + # Remove static inline function bodies + removeStatic(sfile) + if not noprocess: c2nim(file, getNimout(sfile), c2nimConfig) - if c2nimConfig.removeStatic: - gitCheckout(sfile) + # Add them back for compilation + reAddStatic(sfile) proc runCfg(cfg: string) = if not fileExists(cfg): From 15ba47946822a80d034c247c3f22b350d4651d74 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 11 Jul 2018 13:54:37 +0900 Subject: [PATCH 070/174] Remove removestatic from readme --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 7f3f256..40f7556 100644 --- a/README.md +++ b/README.md @@ -138,8 +138,6 @@ The following keys apply to library source code and help with generating the .ni ```defines``` = pulls out simple #defines of ints, floats and hex values for separate conversion [default: false] - works only when preprocess or ctags is used and helps include useful definitions in generated .nim file -```removestatic``` = pulls out the bodies of inline static functions [default: false] - ```flags``` = flags to pass to the c2nim process in "quotes" [default: --stdcall]. --cdecl, --assumedef, --assumendef may be useful ```ppflags``` = flags to pass to the preprocessor [default: ""]. -D for gcc and others may be useful From e32acea78e3a1c30e73059cd131129ea8706b4f4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 11 Jul 2018 22:37:06 -0500 Subject: [PATCH 071/174] Move from nre to regex --- nimgen.nim | 37 ++++++++++++++++++++----------------- nimgen.nimble | 2 +- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index 328dd2d..12380d5 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -1,4 +1,4 @@ -import nre, os, ospaths, osproc, parsecfg, pegs, ropes, sequtils, streams, strutils, tables +import os, ospaths, osproc, parsecfg, pegs, regex, ropes, sequtils, streams, strutils, tables var gDoneRecursive: seq[string] = @[] @@ -255,10 +255,11 @@ proc freplace(file: string, pattern: string, repl="") = proc freplace(file: string, pattern: Regex, repl="") = withFile(file): - if content.find(pattern).isSome(): + var m: RegexMatch + if content.find(pattern, m): if "$#" in repl: - for m in content.findIter(pattern): - content = content.replace(m.match, repl % m.captures[0]) + content = content.replace(pattern, + proc (m: RegexMatch, s: string): string = repl % s[m.group(0)[0]]) else: content = content.replace(pattern, repl) @@ -328,10 +329,11 @@ proc compile(dir="", file=""): string = proc fixFuncProtos(file: string) = withFile(file): - for fp in content.findIter(re"(?m)(^.*?)[ ]*\(\*(.*?)\((.*?)\)\)[ \r\n]*\((.*?[\r\n]*.*?)\);"): - var tdout = "typedef $# (*type_$#)($#);\n" % [fp.captures[0], fp.captures[1], fp.captures[3]] & - "type_$# $#($#);" % [fp.captures[1], fp.captures[1], fp.captures[2]] - content = content.replace(fp.match, tdout) + content = content.replace(re"(?m)(^.*?)[ ]*\(\*(.*?)\((.*?)\)\)[ \r\n]*\((.*?[\r\n]*.*?)\);", + proc (m: RegexMatch, s: string): string = + (("typedef $# (*type_$#)($#);\n" % [s[m.group(0)[0]], s[m.group(1)[0]], s[m.group(3)[0]]]) & + ("type_$# $#($#);" % [s[m.group(1)[0]], s[m.group(1)[0]], s[m.group(2)[0]]])) + ) # ### # Convert to Nim @@ -342,8 +344,8 @@ proc getIncls(file: string, inline=false): seq[string] = return withFile(file): - for f in content.findIter(re"(?m)^\s*#\s*include\s+(.*?)$"): - var inc = f.captures[0].strip() + for f in content.findAll(re"(?m)^\s*#\s*include\s+(.*?)$"): + var inc = content[f.group(0)[0]].strip() if ((gQuotes and inc.contains("\"")) or (gFilter != "" and gFilter in inc)) and (not exclude(inc)): result.add( inc.replace(re"""[<>"]""", "").replace(re"\/[\*\/].*$", "").strip()) @@ -374,8 +376,8 @@ proc getDefines(file: string, inline=false): string = echo "Inlining " & sincl result &= getDefines(sincl) withFile(file): - for def in content.findIter(re"(?m)^(\s*#\s*define\s+[\w\d_]+\s+[\d\-.xf]+)(?:\r|//|/*).*?$"): - result &= def.captures[0] & "\n" + for def in content.findAll(re"(?m)^(\s*#\s*define\s+[\w\d_]+\s+[\d\-.xf]+)(?:\r|//|/*).*?$"): + result &= content[def.group(0)[0]] & "\n" proc runPreprocess(file, ppflags, flags: string, inline: bool): string = var @@ -428,9 +430,9 @@ proc runCtags(file: string): string = if spl.len() > 4: if spl[0] != "main" and spl[3] != "member": var fn = "" - var match = spl[2].find(re"/\^(.*?)\(") - if match.isSome(): - fn = match.get().captures[0] + var match: RegexMatch + if spl[2].find(re"/\^(.*?)\(", match): + fn = spl[2][match.group(0)[0]] fn &= spl[4].replace("signature:", "") & ";" fdata &= fn & "\n" @@ -559,7 +561,7 @@ proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = prepend(outfile, incout) # Nim doesn't like {.cdecl.} for type proc() - freplace(outfile, re"(?m)(.*? = proc.*?){.cdecl.}", "$#") + freplace(outfile, re"(?m)(.*? = proc.*?)\{.cdecl.\}", "$#") freplace(outfile, " {.cdecl.})", ")") # Include {.compile.} directives @@ -591,8 +593,9 @@ proc runFile(file: string, cfgin: OrderedTableRef) = sfile = search(file) for pattern in gWildcards.keys(): + var match: RegexMatch let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") - if file.find(re(pat)).isSome(): + if file.find(toPattern(pat), match): echo "Appending " & file & " " & pattern for key in gWildcards[pattern].keys(): cfg[key & "." & pattern] = gWildcards[pattern][key] diff --git a/nimgen.nimble b/nimgen.nimble index 28967d4..03dfb78 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -9,7 +9,7 @@ skipDirs = @["tests"] # Dependencies -requires "nim >= 0.17.0", "c2nim >= 0.9.13" +requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex >= 0.7.1" bin = @["nimgen"] From bb4073ba97e5d1e22d42bf5e326bbffd0512bfb4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 11 Jul 2018 23:03:14 -0500 Subject: [PATCH 072/174] Merge latest --- nimgen.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nimgen.nim b/nimgen.nim index 4e540d8..e3a0883 100644 --- a/nimgen.nim +++ b/nimgen.nim @@ -342,9 +342,9 @@ proc removeStatic(filename: string) = withFile(filename): content = content.replace( re"(?m)(static inline.*?\))(\s*\{(\s*?.*?$)*[\n\r]\})", - proc (match: RegexMatch): string = - let funcDecl = match.captures[0] - let body = match.captures[1].strip() + proc (m: RegexMatch, s: string): string = + let funcDecl = s[m.group(0)[0]] + let body = s[m.group(1)[0]].strip() result = "" result.add("$#;" % [funcDecl]) @@ -357,9 +357,9 @@ proc reAddStatic(filename: string) = withFile(filename): content = content.replace( re"(?m)(static inline.*?\));(\/\/\s*\{(\s*?.*?$)*[\n\r]\/\/\})", - proc (match: RegexMatch): string = - let funcDecl = match.captures[0] - let body = match.captures[1].strip() + proc (m: RegexMatch, s: string): string = + let funcDecl = s[m.group(0)[0]] + let body = s[m.group(1)[0]].strip() result = "" result.add("$# " % [funcDecl]) From 293e8cca850be17b1a27c47daedfbbc322c2f2d5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 11 Jul 2018 23:18:00 -0500 Subject: [PATCH 073/174] Minor updates --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6bc1cd1..8b4be53 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ To see examples of nimgen in action check out the following wrappers:- * download ZIP with headers and binary * [nimlibxlsxwriter](https://github.com/KeepCoolWithCoolidge/nimlibxlsxwriter) - libxlsxwriter wrapper * git checkout + * [nim-clblast](https://github.com/numforge/nim-clblast) - OpenCL BLAS wrapper + * static * Compile C code into binary * [nim7z](https://github.com/genotrance/nim7z) - 7z decoder wrapper: [docs](http://nimgen.genotrance.com/nim7z) @@ -188,4 +190,10 @@ The following key only applies before processing and allows renaming the generat __Feedback__ -Nimgen is a work in progress and any feedback or suggestions are welcome. It is hosted on [GitHub](https://github.com/genotrance/nimgen) with an MIT license so issues, forks and PRs are most appreciated. +Nimgen is a work in progress and any feedback or suggestions are welcome. It is hosted on [GitHub](https://github.com/genotrance/nimgen) with an MIT license so issues, forks and PRs are most appreciated. Also join us at https://gitter.im/nimgen/Lobby to chat about nimgen and the future of Nim wrappers. + +_Credits_ + +Thank you to the following contributors for their hard work! + +https://github.com/jyapayne From cf629494a40fa569ab4316b417dfb27856ea2034 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 12 Jul 2018 01:36:15 -0500 Subject: [PATCH 074/174] Split into multiple files --- nimgen.nim | 886 ----------------------------------------- nimgen.nimble | 4 +- src/nimgen.nim | 7 + src/nimgen/config.nim | 229 +++++++++++ src/nimgen/file.nim | 100 +++++ src/nimgen/fileops.nim | 110 +++++ src/nimgen/gencore.nim | 295 ++++++++++++++ src/nimgen/globals.nim | 40 ++ src/nimgen/prepare.nim | 117 ++++++ 9 files changed, 900 insertions(+), 888 deletions(-) delete mode 100644 nimgen.nim create mode 100644 src/nimgen.nim create mode 100644 src/nimgen/config.nim create mode 100644 src/nimgen/file.nim create mode 100644 src/nimgen/fileops.nim create mode 100644 src/nimgen/gencore.nim create mode 100644 src/nimgen/globals.nim create mode 100644 src/nimgen/prepare.nim diff --git a/nimgen.nim b/nimgen.nim deleted file mode 100644 index e3a0883..0000000 --- a/nimgen.nim +++ /dev/null @@ -1,886 +0,0 @@ -import os, ospaths, osproc, parsecfg, pegs, regex, ropes, sequtils, streams, strutils, tables - -const - cCompilerEnv = "CC" - cppCompilerEnv = "CPP" - defaultCCompiler = "gcc" - defaultCppCompiler = "g++" - -var - gDoneRecursive: seq[string] = @[] - gDoneInline: seq[string] = @[] - - gProjectDir = "" - gConfig: Config - gFilter = "" - gQuotes = true - gCppCompiler = getEnv(cppCompilerEnv, defaultCCompiler) - gCCompiler = getEnv(cCompilerEnv, defaultCppCompiler) - gOutput = "" - gIncludes: seq[string] = @[] - gExcludes: seq[string] = @[] - gRenames = initTable[string, string]() - gWildcards = newConfig() - -type - c2nimConfigObj = object - flags, ppflags: string - recurse, inline, preprocess, ctags, defines: bool - dynlib, compile, pragma: seq[string] - -const DOC = """ -Nimgen is a helper for c2nim to simpilfy and automate the wrapping of C libraries - -Usage: - nimgen [options] ... - -Options: - -f delete all artifacts and regenerate -""" - -# ### -# Helpers - -proc addEnv(str: string): string = - var newStr = str - for pair in envPairs(): - try: - newStr = newStr % [pair.key, pair.value.string] - except ValueError: - # Ignore if there are no values to replace. We - # want to continue anyway - discard - - try: - newStr = newStr % ["output", gOutput] - except ValueError: - # Ignore if there are no values to replace. We - # want to continue anyway - discard - - # if there are still format args, print a warning - if newStr.contains("${"): - echo "WARNING: \"", newStr, "\" still contains an uninterpolated value!" - - return newStr - -proc `[]`(table: OrderedTableRef[string, string], key: string): string = - ## Gets table values with env vars inserted - tables.`[]`(table, key).addEnv - -proc execProc(cmd: string): string = - result = "" - var - p = startProcess(cmd, options = {poStdErrToStdOut, poUsePath, poEvalCommand}) - - outp = outputStream(p) - line = newStringOfCap(120).TaintedString - - while true: - if outp.readLine(line): - result.add(line) - result.add("\n") - elif not running(p): break - - var x = p.peekExitCode() - if x != 0: - echo "Command failed: " & $x - echo cmd - echo result - quit(1) - -proc extractZip(zipfile: string) = - var cmd = "unzip -o $#" - if defined(Windows): - cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A " & - "'System.IO.Compression.FileSystem'; " & - "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" - - setCurrentDir(gOutput) - defer: setCurrentDir(gProjectDir) - - echo "Extracting " & zipfile - discard execProc(cmd % zipfile) - -proc downloadUrl(url: string) = - let - file = url.extractFilename() - ext = file.splitFile().ext.toLowerAscii() - - var cmd = "curl $# -o $#" - if defined(Windows): - cmd = "powershell wget $# -OutFile $#" - - if not (ext == ".zip" and fileExists(gOutput/file)): - echo "Downloading " & file - discard execProc(cmd % [url, gOutput/file]) - - if ext == ".zip": - extractZip(file) - -proc gitReset() = - echo "Resetting Git repo" - - setCurrentDir(gOutput) - defer: setCurrentDir(gProjectDir) - - discard execProc("git reset --hard HEAD") - -proc gitCheckout(filename: string) {.used.} = - echo "Resetting file: $#" % [filename] - - setCurrentDir(gOutput) - defer: setCurrentDir(gProjectDir) - - let adjustedFile = filename.replace(gOutput & $DirSep, "") - - discard execProc("git checkout $#" % [adjustedFile]) - -proc gitRemotePull(url: string, pull=true) = - if dirExists(gOutput/".git"): - if pull: - gitReset() - return - - setCurrentDir(gOutput) - defer: setCurrentDir(gProjectDir) - - echo "Setting up Git repo" - discard execProc("git init .") - discard execProc("git remote add origin " & url) - - if pull: - echo "Checking out artifacts" - discard execProc("git pull --depth=1 origin master") - -proc gitSparseCheckout(plist: string) = - let sparsefile = ".git/info/sparse-checkout" - if fileExists(gOutput/sparsefile): - gitReset() - return - - setCurrentDir(gOutput) - defer: setCurrentDir(gProjectDir) - - discard execProc("git config core.sparsecheckout true") - writeFile(sparsefile, plist) - - echo "Checking out artifacts" - discard execProc("git pull --depth=1 origin master") - -proc doCopy(flist: string) = - for pair in flist.split(","): - let spl = pair.split("=") - if spl.len() != 2: - echo "Bad copy syntax: " & flist - quit(1) - - let - lfile = spl[0].strip() - rfile = spl[1].strip() - - copyFile(lfile, rfile) - echo "Copied $# to $#" % [lfile, rfile] - -proc getKey(ukey: string): tuple[key: string, val: bool] = - var kv = ukey.replace(re"\..*", "").split("-", 1) - if kv.len() == 1: - kv.add("") - - if (kv[1] == "") or - (kv[1] == "win" and defined(Windows)) or - (kv[1] == "lin" and defined(Linux)) or - (kv[1] == "osx" and defined(MacOSX)): - return (kv[0], true) - - return (kv[0], false) - -# ### -# File loction - -proc getNimout(file: string, rename=true): string = - result = file.splitFile().name.replace(re"[\-\.]", "_") & ".nim" - if gOutput != "": - result = gOutput/result - - if not rename: - return - - if gRenames.hasKey(file): - result = gRenames[file] - - if not dirExists(parentDir(result)): - createDir(parentDir(result)) - -proc exclude(file: string): bool = - for excl in gExcludes: - if excl in file: - return true - return false - -proc search(file: string): string = - if exclude(file): - return "" - - result = file - if file.splitFile().ext == ".nim": - result = getNimout(file) - elif not fileExists(result) and not dirExists(result): - var found = false - for inc in gIncludes: - result = inc/file - if fileExists(result) or dirExists(result): - found = true - break - if not found: - echo "File doesn't exist: " & file - quit(1) - - # Only keep relative directory - return result.multiReplace([("\\", $DirSep), ("//", $DirSep), (gProjectDir & $DirSep, "")]) - -# ### -# Loading / unloading - -proc openRetry(file: string, mode: FileMode = fmRead): File = - while true: - try: - result = open(file, mode) - break - except IOError: - sleep(100) - -template withFile(file: string, body: untyped): untyped = - if fileExists(file): - var f = openRetry(file) - - var contentOrig = f.readAll() - f.close() - var content {.inject.} = contentOrig - - body - - if content != contentOrig: - f = openRetry(file, fmWrite) - write(f, content) - f.close() - else: - echo "Missing file " & file - -# ### -# Manipulating content - -proc prepend(file: string, data: string, search="") = - withFile(file): - if search == "": - content = data & content - else: - let idx = content.find(search) - if idx != -1: - content = content[0..= content.len(): - break - content = content[0.."]""", "").replace(re"\/[\*\/].*$", "").strip() - try: - # Try searching for a local library. expandFilename will throw - # OSError if the file does not exist - let - finc = expandFileName(curPath / addInc) - fname = finc.replace(curPath & $DirSep, "") - - if fname.len() > 0: - # only add if the file is non-empty - result.add(fname.search()) - except OSError: - # If it's a system library - result.add(addInc) - - result = result.deduplicate() - - gDoneInline.add(file) - - if inline: - var sres = newSeq[string]() - for incl in result: - let sincl = search(incl) - if sincl == "": - continue - - sres.add(getIncls(sincl, inline)) - result.add(sres) - - result = result.deduplicate() - -proc getDefines(file: string, inline=false): string = - result = "" - if inline: - var incls = getIncls(file, inline) - for incl in incls: - let sincl = search(incl) - if sincl != "": - echo "Inlining " & sincl - result &= getDefines(sincl) - withFile(file): - for def in content.findAll(re"(?m)^(\s*#\s*define\s+[\w\d_]+\s+[\d\-.xf]+)(?:\r|//|/*).*?$"): - result &= content[def.group(0)[0]] & "\n" - -proc runPreprocess(file, ppflags, flags: string, inline: bool): string = - var - pproc = if flags.contains("cpp"): gCppCompiler else: gCCompiler - cmd = "$# -E $# $#" % [pproc, ppflags, file] - - for inc in gIncludes: - cmd &= " -I " & inc - - # Run preprocessor - var data = execProc(cmd) - - # Include content only from file - var - rdata: Rope - start = false - sfile = file.replace("\\", "/") - - if inline: - sfile = sfile.parentDir() - for line in data.splitLines(): - if line.strip() != "": - if line[0] == '#' and not line.contains("#pragma"): - start = false - if sfile in line.replace("\\", "/").replace("//", "/"): - start = true - if not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: - start = true - else: - if start: - rdata.add( - line.replace("_Noreturn", "") - .replace("(())", "") - .replace("WINAPI", "") - .replace("__attribute__", "") - .replace("extern \"C\"", "") - .replace(re"\(\([_a-z]+?\)\)", "") - .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";") & "\n" - ) - return $rdata - -proc runCtags(file: string): string = - var - cmd = "ctags -o - --fields=+S+K --c-kinds=+p --file-scope=no " & file - fps = execProc(cmd) - fdata = "" - - for line in fps.splitLines(): - var spl = line.split(re"\t") - if spl.len() > 4: - if spl[0] != "main" and spl[3] != "member": - var fn = "" - var match: RegexMatch - if spl[2].find(re"/\^(.*?)\(", match): - fn = spl[2][match.group(0)[0]] - fn &= spl[4].replace("signature:", "") & ";" - fdata &= fn & "\n" - - return fdata - -proc runFile(file: string, cfgin: OrderedTableRef) - -template relativePath(path: untyped): untyped = - path.multiReplace([(gOutput, ""), ("\\", "/"), ("//", "/")]) - -proc c2nim(fl, outfile: string, c2nimConfig: c2nimConfigObj) = - var file = search(fl) - if file == "": - return - - if file in gDoneRecursive: - return - - echo "Processing $# => $#" % [file, outfile] - gDoneRecursive.add(file) - - fixFuncProtos(file) - - var incout = "" - if c2nimConfig.recurse: - var - incls = getIncls(file) - cfg = newOrderedTable[string, string]() - - for name, value in c2nimConfig.fieldPairs: - when value is string: - cfg[name] = value - when value is bool: - cfg[name] = $value - - for i in c2nimConfig.dynlib: - cfg["dynlib." & i] = i - - for inc in incls: - runFile(inc, cfg) - incout &= "import $#\n" % inc.search().getNimout()[0 .. ^5] - - var cfile = file - if c2nimConfig.preprocess: - cfile = "temp-$#.c" % [outfile.extractFilename()] - writeFile(cfile, runPreprocess(file, c2nimConfig.ppflags, c2nimConfig.flags, c2nimConfig.inline)) - elif c2nimConfig.ctags: - cfile = "temp-$#.c" % [outfile.extractFilename()] - writeFile(cfile, runCtags(file)) - - if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): - prepend(cfile, getDefines(file, c2nimConfig.inline)) - - var - extflags = "" - passC = "" - outlib = "" - outpragma = "" - - passC = "import ospaths, strutils\n" - - for inc in gIncludes: - if inc.isAbsolute(): - passC &= ("""{.passC: "-I\"$#\"".}""" % [inc]) & "\n" - else: - passC &= ( - """{.passC: "-I\"" & currentSourcePath().splitPath().head & "$#\"".}""" % - inc.relativePath() - ) & "\n" - - for prag in c2nimConfig.pragma: - outpragma &= "{." & prag & ".}\n" - - let fname = file.splitFile().name.replace(re"[\.\-]", "_") - - if c2nimConfig.dynlib.len() != 0: - let - win = "when defined(Windows):\n" - lin = "when defined(Linux):\n" - osx = "when defined(MacOSX):\n" - - var winlib, linlib, osxlib: string = "" - for dl in c2nimConfig.dynlib: - let - lib = " const dynlib$# = \"$#\"\n" % [fname, dl] - ext = dl.splitFile().ext - - if ext == ".dll": - winlib &= lib - elif ext == ".so": - linlib &= lib - elif ext == ".dylib": - osxlib &= lib - - if winlib != "": - outlib &= win & winlib & "\n" - if linlib != "": - outlib &= lin & linlib & "\n" - if osxlib != "": - outlib &= osx & osxlib & "\n" - - if outlib != "": - extflags &= " --dynlib:dynlib$#" % fname - else: - if file.isAbsolute(): - passC &= "const header$# = \"$#\"\n" % [fname, file] - else: - passC &= "const header$# = currentSourcePath().splitPath().head & \"$#\"\n" % - [fname, file.relativePath()] - extflags = "--header:header$#" % fname - - # Run c2nim on generated file - var cmd = "c2nim $# $# --out:$# $#" % [c2nimConfig.flags, extflags, outfile, cfile] - when defined(windows): - cmd = "cmd /c " & cmd - discard execProc(cmd) - - if c2nimConfig.preprocess or c2nimConfig.ctags: - try: - removeFile(cfile) - except: - discard - - # Import nim modules - if c2nimConfig.recurse: - prepend(outfile, incout) - - # Nim doesn't like {.cdecl.} for type proc() - freplace(outfile, re"(?m)(.*? = proc.*?)\{.cdecl.\}", "$#") - freplace(outfile, " {.cdecl.})", ")") - - # Include {.compile.} directives - for cpl in c2nimConfig.compile: - let fcpl = search(cpl) - if getFileInfo(fcpl).kind == pcFile: - prepend(outfile, compile(file=fcpl)) - else: - prepend(outfile, compile(dir=fcpl)) - - # Add any pragmas - if outpragma != "": - prepend(outfile, outpragma) - - # Add header file and include paths - if passC != "": - prepend(outfile, passC) - - # Add dynamic library - if outlib != "": - prepend(outfile, outlib) - -# ### -# Processor - -proc runFile(file: string, cfgin: OrderedTableRef) = - var - cfg = cfgin - sfile = search(file) - - for pattern in gWildcards.keys(): - var match: RegexMatch - let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") - if file.find(toPattern(pat), match): - echo "Appending " & file & " " & pattern - for key in gWildcards[pattern].keys(): - cfg[key & "." & pattern] = gWildcards[pattern][key] - - var - srch = "" - - c2nimConfig = c2nimConfigObj( - flags: "--stdcall", ppflags: "", - recurse: false, inline: false, preprocess: false, ctags: false, defines: false, - dynlib: @[], compile: @[], pragma: @[] - ) - - for act in cfg.keys(): - let (action, val) = getKey(act) - if val == true: - if action == "create": - createDir(file.splitPath().head) - writeFile(file, cfg[act]) - elif action in @["prepend", "append", "replace", "comment", - "rename", "compile", "dynlib", "pragma", - "pipe"] and sfile != "": - if action == "prepend": - if srch != "": - prepend(sfile, cfg[act], cfg[srch]) - else: - prepend(sfile, cfg[act]) - elif action == "append": - if srch != "": - append(sfile, cfg[act], cfg[srch]) - else: - append(sfile, cfg[act]) - elif action == "replace": - if srch != "": - freplace(sfile, cfg[srch], cfg[act]) - elif action == "comment": - if srch != "": - comment(sfile, cfg[srch], cfg[act]) - elif action == "rename": - rename(sfile, cfg[act]) - elif action == "compile": - c2nimConfig.compile.add(cfg[act]) - elif action == "dynlib": - c2nimConfig.dynlib.add(cfg[act]) - elif action == "pragma": - c2nimConfig.pragma.add(cfg[act]) - elif action == "pipe": - pipe(sfile, cfg[act]) - srch = "" - elif action == "search": - srch = act - - if file.splitFile().ext != ".nim": - var noprocess = false - - for act in cfg.keys(): - let (action, val) = getKey(act) - if val == true: - if cfg[act] == "true": - if action == "recurse": - c2nimConfig.recurse = true - elif action == "inline": - c2nimConfig.inline = true - elif action == "preprocess": - c2nimConfig.preprocess = true - elif action == "ctags": - c2nimConfig.ctags = true - elif action == "defines": - c2nimConfig.defines = true - elif action == "noprocess": - noprocess = true - elif action == "flags": - c2nimConfig.flags = cfg[act] - elif action == "ppflags": - c2nimConfig.ppflags = cfg[act] - - if c2nimConfig.recurse and c2nimConfig.inline: - echo "Cannot use recurse and inline simultaneously" - quit(1) - - # Remove static inline function bodies - removeStatic(sfile) - - if not noprocess: - c2nim(file, getNimout(sfile), c2nimConfig) - - # Add them back for compilation - reAddStatic(sfile) - -proc runCfg(cfg: string) = - if not fileExists(cfg): - echo "Config doesn't exist: " & cfg - quit(1) - - gProjectDir = parentDir(cfg.expandFilename()) - - gConfig = loadConfig(cfg) - - if gConfig.hasKey("n.global"): - if gConfig["n.global"].hasKey("output"): - gOutput = gConfig["n.global"]["output"] - if dirExists(gOutput): - if "-f" in commandLineParams(): - try: - removeDir(gOutput) - except OSError: - echo "Directory in use: " & gOutput - quit(1) - else: - for f in walkFiles(gOutput/"*.nim"): - try: - removeFile(f) - except OSError: - echo "Unable to delete: " & f - quit(1) - createDir(gOutput) - - if gConfig["n.global"].hasKey("cpp_compiler"): - gCppCompiler = gConfig["n.global"]["cpp_compiler"] - else: - # Reset on a per project basis - gCppCompiler = getEnv(cppCompilerEnv, defaultCppCompiler) - - if gConfig["n.global"].hasKey("c_compiler"): - gCCompiler = gConfig["n.global"]["c_compiler"] - else: - # Reset on a per project basis - gCCompiler = getEnv(cCompilerEnv, defaultCCompiler) - - if gConfig["n.global"].hasKey("filter"): - gFilter = gConfig["n.global"]["filter"] - if gConfig["n.global"].hasKey("quotes"): - if gConfig["n.global"]["quotes"] == "false": - gQuotes = false - - if gConfig.hasKey("n.include"): - for inc in gConfig["n.include"].keys(): - gIncludes.add(inc.addEnv()) - - if gConfig.hasKey("n.exclude"): - for excl in gConfig["n.exclude"].keys(): - gExcludes.add(excl.addEnv()) - - if gConfig.hasKey("n.prepare"): - for prep in gConfig["n.prepare"].keys(): - let (key, val) = getKey(prep) - if val == true: - let prepVal = gConfig["n.prepare"][prep] - if key == "download": - downloadUrl(prepVal) - elif key == "extract": - extractZip(prepVal) - elif key == "git": - gitRemotePull(prepVal) - elif key == "gitremote": - gitRemotePull(prepVal, false) - elif key == "gitsparse": - gitSparseCheckout(prepVal) - elif key == "execute": - discard execProc(prepVal) - elif key == "copy": - doCopy(prepVal) - - if gConfig.hasKey("n.wildcard"): - var wildcard = "" - for wild in gConfig["n.wildcard"].keys(): - let (key, val) = getKey(wild) - if val == true: - if key == "wildcard": - wildcard = gConfig["n.wildcard"][wild] - else: - gWildcards.setSectionKey(wildcard, wild, - gConfig["n.wildcard"][wild]) - - for file in gConfig.keys(): - if file in @["n.global", "n.include", "n.exclude", - "n.prepare", "n.wildcard", "n.post"]: - continue - - runFile(file, gConfig[file]) - - if gConfig.hasKey("n.post"): - for post in gConfig["n.post"].keys(): - let (key, val) = getKey(post) - if val == true: - let postVal = gConfig["n.post"][post] - if key == "reset": - gitReset() - elif key == "execute": - discard execProc(postVal) - -# ### -# Main loop - -for i in commandLineParams(): - if i != "-f": - runCfg(i) diff --git a/nimgen.nimble b/nimgen.nimble index 03dfb78..b1590df 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -5,13 +5,13 @@ author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" +bin = @["nimgen"] +srcDir = "src" skipDirs = @["tests"] # Dependencies requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex >= 0.7.1" -bin = @["nimgen"] - task test, "Test nimgen": exec "nim e tests/nimgentest.nims" diff --git a/src/nimgen.nim b/src/nimgen.nim new file mode 100644 index 0000000..44ee88b --- /dev/null +++ b/src/nimgen.nim @@ -0,0 +1,7 @@ +import os + +import nimgen/config + +for i in commandLineParams(): + if i != "-f": + runCfg(i) diff --git a/src/nimgen/config.nim b/src/nimgen/config.nim new file mode 100644 index 0000000..3a0f3b0 --- /dev/null +++ b/src/nimgen/config.nim @@ -0,0 +1,229 @@ +import os, parsecfg, regex, strutils, tables + +import file, fileops, gencore, globals, prepare + +proc `[]`*(table: OrderedTableRef[string, string], key: string): string = + ## Gets table values with env vars inserted + tables.`[]`(table, key).addEnv + +proc getKey(ukey: string): tuple[key: string, val: bool] = + var kv = ukey.replace(re"\..*", "").split("-", 1) + if kv.len() == 1: + kv.add("") + + if (kv[1] == "") or + (kv[1] == "win" and defined(Windows)) or + (kv[1] == "lin" and defined(Linux)) or + (kv[1] == "osx" and defined(MacOSX)): + return (kv[0], true) + + return (kv[0], false) + +proc runFile*(file: string, cfgin: OrderedTableRef) = + var + cfg = cfgin + sfile = search(file) + + for pattern in gWildcards.keys(): + var match: RegexMatch + let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") + if file.find(toPattern(pat), match): + echo "Appending " & file & " " & pattern + for key in gWildcards[pattern].keys(): + cfg[key & "." & pattern] = gWildcards[pattern][key] + + var + srch = "" + + c2nimConfig = c2nimConfigObj( + flags: "--stdcall", ppflags: "", + recurse: false, inline: false, preprocess: false, ctags: false, defines: false, + dynlib: @[], compile: @[], pragma: @[] + ) + + for act in cfg.keys(): + let (action, val) = getKey(act) + if val == true: + if action == "create": + createDir(file.splitPath().head) + writeFile(file, cfg[act]) + elif action in @["prepend", "append", "replace", "comment", + "rename", "compile", "dynlib", "pragma", + "pipe"] and sfile != "": + if action == "prepend": + if srch != "": + prepend(sfile, cfg[act], cfg[srch]) + else: + prepend(sfile, cfg[act]) + elif action == "append": + if srch != "": + append(sfile, cfg[act], cfg[srch]) + else: + append(sfile, cfg[act]) + elif action == "replace": + if srch != "": + freplace(sfile, cfg[srch], cfg[act]) + elif action == "comment": + if srch != "": + comment(sfile, cfg[srch], cfg[act]) + elif action == "rename": + rename(sfile, cfg[act]) + elif action == "compile": + c2nimConfig.compile.add(cfg[act]) + elif action == "dynlib": + c2nimConfig.dynlib.add(cfg[act]) + elif action == "pragma": + c2nimConfig.pragma.add(cfg[act]) + elif action == "pipe": + pipe(sfile, cfg[act]) + srch = "" + elif action == "search": + srch = act + + if file.splitFile().ext != ".nim": + var noprocess = false + + for act in cfg.keys(): + let (action, val) = getKey(act) + if val == true: + if cfg[act] == "true": + if action == "recurse": + c2nimConfig.recurse = true + elif action == "inline": + c2nimConfig.inline = true + elif action == "preprocess": + c2nimConfig.preprocess = true + elif action == "ctags": + c2nimConfig.ctags = true + elif action == "defines": + c2nimConfig.defines = true + elif action == "noprocess": + noprocess = true + elif action == "flags": + c2nimConfig.flags = cfg[act] + elif action == "ppflags": + c2nimConfig.ppflags = cfg[act] + + if c2nimConfig.recurse and c2nimConfig.inline: + echo "Cannot use recurse and inline simultaneously" + quit(1) + + if not noprocess: + var incls = c2nim(file, getNimout(sfile), c2nimConfig) + if c2nimConfig.recurse and incls.len() != 0: + var + cfg = newOrderedTable[string, string]() + + for name, value in c2nimConfig.fieldPairs: + when value is string: + cfg[name] = value + when value is bool: + cfg[name] = $value + + for i in c2nimConfig.dynlib: + cfg["dynlib." & i] = i + + for inc in incls: + runFile(inc, cfg) + +proc runCfg*(cfg: string) = + if not fileExists(cfg): + echo "Config doesn't exist: " & cfg + quit(1) + + gProjectDir = parentDir(cfg.expandFilename()) + + gConfig = loadConfig(cfg) + + if gConfig.hasKey("n.global"): + if gConfig["n.global"].hasKey("output"): + gOutput = gConfig["n.global"]["output"] + if dirExists(gOutput): + if "-f" in commandLineParams(): + try: + removeDir(gOutput) + except OSError: + echo "Directory in use: " & gOutput + quit(1) + else: + for f in walkFiles(gOutput/"*.nim"): + try: + removeFile(f) + except OSError: + echo "Unable to delete: " & f + quit(1) + createDir(gOutput) + + if gConfig["n.global"].hasKey("cpp_compiler"): + gCppCompiler = gConfig["n.global"]["cpp_compiler"] + else: + # Reset on a per project basis + gCppCompiler = getEnv(cppCompilerEnv, defaultCppCompiler) + + if gConfig["n.global"].hasKey("c_compiler"): + gCCompiler = gConfig["n.global"]["c_compiler"] + else: + # Reset on a per project basis + gCCompiler = getEnv(cCompilerEnv, defaultCCompiler) + + if gConfig["n.global"].hasKey("filter"): + gFilter = gConfig["n.global"]["filter"] + if gConfig["n.global"].hasKey("quotes"): + if gConfig["n.global"]["quotes"] == "false": + gQuotes = false + + if gConfig.hasKey("n.include"): + for inc in gConfig["n.include"].keys(): + gIncludes.add(inc.addEnv()) + + if gConfig.hasKey("n.exclude"): + for excl in gConfig["n.exclude"].keys(): + gExcludes.add(excl.addEnv()) + + if gConfig.hasKey("n.prepare"): + for prep in gConfig["n.prepare"].keys(): + let (key, val) = getKey(prep) + if val == true: + let prepVal = gConfig["n.prepare"][prep] + if key == "download": + downloadUrl(prepVal) + elif key == "extract": + extractZip(prepVal) + elif key == "git": + gitRemotePull(prepVal) + elif key == "gitremote": + gitRemotePull(prepVal, false) + elif key == "gitsparse": + gitSparseCheckout(prepVal) + elif key == "execute": + discard execProc(prepVal) + elif key == "copy": + doCopy(prepVal) + + if gConfig.hasKey("n.wildcard"): + var wildcard = "" + for wild in gConfig["n.wildcard"].keys(): + let (key, val) = getKey(wild) + if val == true: + if key == "wildcard": + wildcard = gConfig["n.wildcard"][wild] + else: + gWildcards.setSectionKey(wildcard, wild, + gConfig["n.wildcard"][wild]) + + for file in gConfig.keys(): + if file in @["n.global", "n.include", "n.exclude", + "n.prepare", "n.wildcard", "n.post"]: + continue + + runFile(file, gConfig[file]) + + if gConfig.hasKey("n.post"): + for post in gConfig["n.post"].keys(): + let (key, val) = getKey(post) + if val == true: + let postVal = gConfig["n.post"][post] + if key == "reset": + gitReset() + elif key == "execute": + discard execProc(postVal) diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim new file mode 100644 index 0000000..27296d2 --- /dev/null +++ b/src/nimgen/file.nim @@ -0,0 +1,100 @@ +import os, ospaths, pegs, regex, strutils, tables + +import globals + +# ### +# File loction + +proc getNimout*(file: string, rename=true): string = + result = file.splitFile().name.replace(re"[\-\.]", "_") & ".nim" + if gOutput != "": + result = gOutput/result + + if not rename: + return + + if gRenames.hasKey(file): + result = gRenames[file] + + if not dirExists(parentDir(result)): + createDir(parentDir(result)) + +proc exclude*(file: string): bool = + for excl in gExcludes: + if excl in file: + return true + return false + +proc search*(file: string): string = + if exclude(file): + return "" + + result = file + if file.splitFile().ext == ".nim": + result = getNimout(file) + elif not fileExists(result) and not dirExists(result): + var found = false + for inc in gIncludes: + result = inc/file + if fileExists(result) or dirExists(result): + found = true + break + if not found: + echo "File doesn't exist: " & file + quit(1) + + # Only keep relative directory + return result.multiReplace([("\\", $DirSep), ("//", $DirSep), (gProjectDir & $DirSep, "")]) + +# ### +# Loading / unloading + +proc openRetry*(file: string, mode: FileMode = fmRead): File = + while true: + try: + result = open(file, mode) + break + except IOError: + sleep(100) + +template withFile*(file: string, body: untyped): untyped = + if fileExists(file): + var f = openRetry(file) + + var contentOrig = f.readAll() + f.close() + var content {.inject.} = contentOrig + + body + + if content != contentOrig: + f = openRetry(file, fmWrite) + write(f, content) + f.close() + else: + echo "Missing file " & file + +proc rename*(file: string, renfile: string) = + if file.splitFile().ext == ".nim": + return + + var + nimout = getNimout(file, false) + newname = renfile.replace("$nimout", extractFilename(nimout)) + + if newname =~ peg"(!\$.)*{'$replace'\s*'('\s*{(!\)\S)+}')'}": + var final = nimout.extractFilename() + for entry in matches[1].split(","): + let spl = entry.split("=") + if spl.len() != 2: + echo "Bad replace syntax: " & renfile + quit(1) + + let + srch = spl[0].strip() + repl = spl[1].strip() + + final = final.replace(srch, repl) + newname = newname.replace(matches[0], final) + + gRenames[file] = gOutput/newname diff --git a/src/nimgen/fileops.nim b/src/nimgen/fileops.nim new file mode 100644 index 0000000..d7ecd62 --- /dev/null +++ b/src/nimgen/fileops.nim @@ -0,0 +1,110 @@ +import os, regex, strutils + +import file, prepare + +# ### +# Manipulating content + +proc prepend*(file: string, data: string, search="") = + withFile(file): + if search == "": + content = data & content + else: + let idx = content.find(search) + if idx != -1: + content = content[0..= content.len(): + break + content = content[0.."]""", "").replace(re"\/[\*\/].*$", "").strip() + try: + # Try searching for a local library. expandFilename will throw + # OSError if the file does not exist + let + finc = expandFileName(curPath / addInc) + fname = finc.replace(curPath & $DirSep, "") + + if fname.len() > 0: + # only add if the file is non-empty + result.add(fname.search()) + except OSError: + # If it's a system library + result.add(addInc) + + result = result.deduplicate() + + gDoneInline.add(file) + + if inline: + var sres = newSeq[string]() + for incl in result: + let sincl = search(incl) + if sincl == "": + continue + + sres.add(getIncls(sincl, inline)) + result.add(sres) + + result = result.deduplicate() + +proc getDefines*(file: string, inline=false): string = + result = "" + if inline: + var incls = getIncls(file, inline) + for incl in incls: + let sincl = search(incl) + if sincl != "": + echo "Inlining " & sincl + result &= getDefines(sincl) + withFile(file): + for def in content.findAll(re"(?m)^(\s*#\s*define\s+[\w\d_]+\s+[\d\-.xf]+)(?:\r|//|/*).*?$"): + result &= content[def.group(0)[0]] & "\n" + +proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = + var + pproc = if flags.contains("cpp"): gCppCompiler else: gCCompiler + cmd = "$# -E $# $#" % [pproc, ppflags, file] + + for inc in gIncludes: + cmd &= " -I " & inc + + # Run preprocessor + var data = execProc(cmd) + + # Include content only from file + var + rdata: Rope + start = false + sfile = file.replace("\\", "/") + + if inline: + sfile = sfile.parentDir() + for line in data.splitLines(): + if line.strip() != "": + if line[0] == '#' and not line.contains("#pragma"): + start = false + if sfile in line.replace("\\", "/").replace("//", "/"): + start = true + if not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: + start = true + else: + if start: + rdata.add( + line.replace("_Noreturn", "") + .replace("(())", "") + .replace("WINAPI", "") + .replace("__attribute__", "") + .replace("extern \"C\"", "") + .replace(re"\(\([_a-z]+?\)\)", "") + .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";") & "\n" + ) + return $rdata + +proc runCtags*(file: string): string = + var + cmd = "ctags -o - --fields=+S+K --c-kinds=+p --file-scope=no " & file + fps = execProc(cmd) + fdata = "" + + for line in fps.splitLines(): + var spl = line.split(re"\t") + if spl.len() > 4: + if spl[0] != "main" and spl[3] != "member": + var fn = "" + var match: RegexMatch + if spl[2].find(re"/\^(.*?)\(", match): + fn = spl[2][match.group(0)[0]] + fn &= spl[4].replace("signature:", "") & ";" + fdata &= fn & "\n" + + return fdata + +template relativePath(path: untyped): untyped = + path.multiReplace([(gOutput, ""), ("\\", "/"), ("//", "/")]) + +proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj): seq[string] = + var + incls: seq[string] = @[] + incout = "" + file = search(fl) + + if file == "": + return + + if file in gDoneRecursive: + return + + echo "Processing $# => $#" % [file, outfile] + gDoneRecursive.add(file) + + # Remove static inline function bodies + removeStatic(file) + + fixFuncProtos(file) + + if c2nimConfig.recurse: + incls = getIncls(file) + for inc in incls: + incout &= "import $#\n" % inc.search().getNimout()[0 .. ^5] + + var cfile = file + if c2nimConfig.preprocess: + cfile = "temp-$#.c" % [outfile.extractFilename()] + writeFile(cfile, runPreprocess(file, c2nimConfig.ppflags, c2nimConfig.flags, c2nimConfig.inline)) + elif c2nimConfig.ctags: + cfile = "temp-$#.c" % [outfile.extractFilename()] + writeFile(cfile, runCtags(file)) + + if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): + prepend(cfile, getDefines(file, c2nimConfig.inline)) + + var + extflags = "" + passC = "" + outlib = "" + outpragma = "" + + passC = "import ospaths, strutils\n" + + for inc in gIncludes: + if inc.isAbsolute(): + passC &= ("""{.passC: "-I\"$#\"".}""" % [inc]) & "\n" + else: + passC &= ( + """{.passC: "-I\"" & currentSourcePath().splitPath().head & "$#\"".}""" % + inc.relativePath() + ) & "\n" + + for prag in c2nimConfig.pragma: + outpragma &= "{." & prag & ".}\n" + + let fname = file.splitFile().name.replace(re"[\.\-]", "_") + + if c2nimConfig.dynlib.len() != 0: + let + win = "when defined(Windows):\n" + lin = "when defined(Linux):\n" + osx = "when defined(MacOSX):\n" + + var winlib, linlib, osxlib: string = "" + for dl in c2nimConfig.dynlib: + let + lib = " const dynlib$# = \"$#\"\n" % [fname, dl] + ext = dl.splitFile().ext + + if ext == ".dll": + winlib &= lib + elif ext == ".so": + linlib &= lib + elif ext == ".dylib": + osxlib &= lib + + if winlib != "": + outlib &= win & winlib & "\n" + if linlib != "": + outlib &= lin & linlib & "\n" + if osxlib != "": + outlib &= osx & osxlib & "\n" + + if outlib != "": + extflags &= " --dynlib:dynlib$#" % fname + else: + if file.isAbsolute(): + passC &= "const header$# = \"$#\"\n" % [fname, file] + else: + passC &= "const header$# = currentSourcePath().splitPath().head & \"$#\"\n" % + [fname, file.relativePath()] + extflags = "--header:header$#" % fname + + # Run c2nim on generated file + var cmd = "c2nim $# $# --out:$# $#" % [c2nimConfig.flags, extflags, outfile, cfile] + when defined(windows): + cmd = "cmd /c " & cmd + discard execProc(cmd) + + if c2nimConfig.preprocess or c2nimConfig.ctags: + try: + removeFile(cfile) + except: + discard + + # Import nim modules + if c2nimConfig.recurse: + prepend(outfile, incout) + + # Nim doesn't like {.cdecl.} for type proc() + freplace(outfile, re"(?m)(.*? = proc.*?)\{.cdecl.\}", "$#") + freplace(outfile, " {.cdecl.})", ")") + + # Include {.compile.} directives + for cpl in c2nimConfig.compile: + let fcpl = search(cpl) + if getFileInfo(fcpl).kind == pcFile: + prepend(outfile, compile(file=fcpl)) + else: + prepend(outfile, compile(dir=fcpl)) + + # Add any pragmas + if outpragma != "": + prepend(outfile, outpragma) + + # Add header file and include paths + if passC != "": + prepend(outfile, passC) + + # Add dynamic library + if outlib != "": + prepend(outfile, outlib) + + # Add back static functions for compilation + reAddStatic(file) + + return incls diff --git a/src/nimgen/globals.nim b/src/nimgen/globals.nim new file mode 100644 index 0000000..c371fe2 --- /dev/null +++ b/src/nimgen/globals.nim @@ -0,0 +1,40 @@ +import os, parsecfg, tables + +const + cCompilerEnv* = "CC" + cppCompilerEnv* = "CPP" + defaultCCompiler* = "gcc" + defaultCppCompiler* = "g++" + +var + gDoneRecursive*: seq[string] = @[] + gDoneInline*: seq[string] = @[] + + gProjectDir* = "" + gConfig*: Config + gFilter* = "" + gQuotes* = true + gCppCompiler* = getEnv(cppCompilerEnv, defaultCCompiler) + gCCompiler* = getEnv(cCompilerEnv, defaultCppCompiler) + gOutput* = "" + gIncludes*: seq[string] = @[] + gExcludes*: seq[string] = @[] + gRenames* = initTable[string, string]() + gWildcards* = newConfig() + +type + c2nimConfigObj* = object + flags*, ppflags*: string + recurse*, inline*, preprocess*, ctags*, defines*: bool + dynlib*, compile*, pragma*: seq[string] + +const gDoc* = """ +Nimgen is a helper for c2nim to simpilfy and automate the wrapping of C libraries + +Usage: + nimgen [options] ... + +Options: + -f delete all artifacts and regenerate +""" + diff --git a/src/nimgen/prepare.nim b/src/nimgen/prepare.nim new file mode 100644 index 0000000..a1857a5 --- /dev/null +++ b/src/nimgen/prepare.nim @@ -0,0 +1,117 @@ +import os, osproc, streams, strutils + +import globals + +proc execProc*(cmd: string): string = + result = "" + var + p = startProcess(cmd, options = {poStdErrToStdOut, poUsePath, poEvalCommand}) + + outp = outputStream(p) + line = newStringOfCap(120).TaintedString + + while true: + if outp.readLine(line): + result.add(line) + result.add("\n") + elif not running(p): break + + var x = p.peekExitCode() + if x != 0: + echo "Command failed: " & $x + echo cmd + echo result + quit(1) + +proc extractZip*(zipfile: string) = + var cmd = "unzip -o $#" + if defined(Windows): + cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A " & + "'System.IO.Compression.FileSystem'; " & + "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" + + setCurrentDir(gOutput) + defer: setCurrentDir(gProjectDir) + + echo "Extracting " & zipfile + discard execProc(cmd % zipfile) + +proc downloadUrl*(url: string) = + let + file = url.extractFilename() + ext = file.splitFile().ext.toLowerAscii() + + var cmd = "curl $# -o $#" + if defined(Windows): + cmd = "powershell wget $# -OutFile $#" + + if not (ext == ".zip" and fileExists(gOutput/file)): + echo "Downloading " & file + discard execProc(cmd % [url, gOutput/file]) + + if ext == ".zip": + extractZip(file) + +proc gitReset*() = + echo "Resetting Git repo" + + setCurrentDir(gOutput) + defer: setCurrentDir(gProjectDir) + + discard execProc("git reset --hard HEAD") + +proc gitCheckout*(filename: string) {.used.} = + echo "Resetting file: $#" % [filename] + + setCurrentDir(gOutput) + defer: setCurrentDir(gProjectDir) + + let adjustedFile = filename.replace(gOutput & $DirSep, "") + + discard execProc("git checkout $#" % [adjustedFile]) + +proc gitRemotePull*(url: string, pull=true) = + if dirExists(gOutput/".git"): + if pull: + gitReset() + return + + setCurrentDir(gOutput) + defer: setCurrentDir(gProjectDir) + + echo "Setting up Git repo" + discard execProc("git init .") + discard execProc("git remote add origin " & url) + + if pull: + echo "Checking out artifacts" + discard execProc("git pull --depth=1 origin master") + +proc gitSparseCheckout*(plist: string) = + let sparsefile = ".git/info/sparse-checkout" + if fileExists(gOutput/sparsefile): + gitReset() + return + + setCurrentDir(gOutput) + defer: setCurrentDir(gProjectDir) + + discard execProc("git config core.sparsecheckout true") + writeFile(sparsefile, plist) + + echo "Checking out artifacts" + discard execProc("git pull --depth=1 origin master") + +proc doCopy*(flist: string) = + for pair in flist.split(","): + let spl = pair.split("=") + if spl.len() != 2: + echo "Bad copy syntax: " & flist + quit(1) + + let + lfile = spl[0].strip() + rfile = spl[1].strip() + + copyFile(lfile, rfile) + echo "Copied $# to $#" % [lfile, rfile] From 562f3fd7ebbbef9302d62a3ef578a614fa345a2a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 12 Jul 2018 01:47:22 -0500 Subject: [PATCH 075/174] Fix directory exists error --- nimgen.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimgen.nimble b/nimgen.nimble index b1590df..3fddd8c 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -7,7 +7,7 @@ license = "MIT" bin = @["nimgen"] srcDir = "src" -skipDirs = @["tests"] +skipDirs = @["nimgen", "tests"] # Dependencies From 4b761bebd4528d771cfb4b56d5b2e73f34e9a6d5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 12 Jul 2018 14:39:24 -0500 Subject: [PATCH 076/174] Fix rename bug, additional cleanup --- src/nimgen.nim | 2 +- src/nimgen/c2nim.nim | 130 ++++++++++++++ src/nimgen/{prepare.nim => external.nim} | 69 ++++++-- src/nimgen/file.nim | 60 ++++--- src/nimgen/fileops.nim | 2 +- src/nimgen/gencore.nim | 205 +---------------------- src/nimgen/{config.nim => runcfg.nim} | 14 +- 7 files changed, 239 insertions(+), 243 deletions(-) create mode 100644 src/nimgen/c2nim.nim rename src/nimgen/{prepare.nim => external.nim} (57%) rename src/nimgen/{config.nim => runcfg.nim} (95%) diff --git a/src/nimgen.nim b/src/nimgen.nim index 44ee88b..5777398 100644 --- a/src/nimgen.nim +++ b/src/nimgen.nim @@ -1,6 +1,6 @@ import os -import nimgen/config +import nimgen/runcfg for i in commandLineParams(): if i != "-f": diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim new file mode 100644 index 0000000..07b15e0 --- /dev/null +++ b/src/nimgen/c2nim.nim @@ -0,0 +1,130 @@ +import os, ospaths, regex, strutils + +import external, file, fileops, gencore, globals + +template relativePath(path: untyped): untyped = + path.multiReplace([(gOutput, ""), ("\\", "/"), ("//", "/")]) + +proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = + var file = search(fl) + if file == "": + return + + if file in gDoneRecursive: + return + + echo "Processing $# => $#" % [file, outfile] + gDoneRecursive.add(file) + + # Remove static inline function bodies + removeStatic(file) + + fixFuncProtos(file) + + var cfile = file + if c2nimConfig.preprocess: + cfile = "temp-$#.c" % [outfile.extractFilename()] + writeFile(cfile, runPreprocess(file, c2nimConfig.ppflags, c2nimConfig.flags, c2nimConfig.inline)) + elif c2nimConfig.ctags: + cfile = "temp-$#.c" % [outfile.extractFilename()] + writeFile(cfile, runCtags(file)) + + if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): + prepend(cfile, getDefines(file, c2nimConfig.inline)) + + var + extflags = "" + passC = "" + outlib = "" + outpragma = "" + + passC = "import ospaths, strutils\n" + + for inc in gIncludes: + if inc.isAbsolute(): + passC &= ("""{.passC: "-I\"$#\"".}""" % [inc]) & "\n" + else: + passC &= ( + """{.passC: "-I\"" & currentSourcePath().splitPath().head & "$#\"".}""" % + inc.relativePath() + ) & "\n" + + for prag in c2nimConfig.pragma: + outpragma &= "{." & prag & ".}\n" + + let fname = file.splitFile().name.replace(re"[\.\-]", "_") + + if c2nimConfig.dynlib.len() != 0: + let + win = "when defined(Windows):\n" + lin = "when defined(Linux):\n" + osx = "when defined(MacOSX):\n" + + var winlib, linlib, osxlib: string = "" + for dl in c2nimConfig.dynlib: + let + lib = " const dynlib$# = \"$#\"\n" % [fname, dl] + ext = dl.splitFile().ext + + if ext == ".dll": + winlib &= lib + elif ext == ".so": + linlib &= lib + elif ext == ".dylib": + osxlib &= lib + + if winlib != "": + outlib &= win & winlib & "\n" + if linlib != "": + outlib &= lin & linlib & "\n" + if osxlib != "": + outlib &= osx & osxlib & "\n" + + if outlib != "": + extflags &= " --dynlib:dynlib$#" % fname + else: + if file.isAbsolute(): + passC &= "const header$# = \"$#\"\n" % [fname, file] + else: + passC &= "const header$# = currentSourcePath().splitPath().head & \"$#\"\n" % + [fname, file.relativePath()] + extflags = "--header:header$#" % fname + + # Run c2nim on generated file + var cmd = "c2nim $# $# --out:$# $#" % [c2nimConfig.flags, extflags, outfile, cfile] + when defined(windows): + cmd = "cmd /c " & cmd + discard execProc(cmd) + + if c2nimConfig.preprocess or c2nimConfig.ctags: + try: + removeFile(cfile) + except: + discard + + # Nim doesn't like {.cdecl.} for type proc() + freplace(outfile, re"(?m)(.*? = proc.*?)\{.cdecl.\}", "$#") + freplace(outfile, " {.cdecl.})", ")") + + # Include {.compile.} directives + for cpl in c2nimConfig.compile: + let fcpl = search(cpl) + if getFileInfo(fcpl).kind == pcFile: + prepend(outfile, compile(file=fcpl)) + else: + prepend(outfile, compile(dir=fcpl)) + + # Add any pragmas + if outpragma != "": + prepend(outfile, outpragma) + + # Add header file and include paths + if passC != "": + prepend(outfile, passC) + + # Add dynamic library + if outlib != "": + prepend(outfile, outlib) + + # Add back static functions for compilation + reAddStatic(file) diff --git a/src/nimgen/prepare.nim b/src/nimgen/external.nim similarity index 57% rename from src/nimgen/prepare.nim rename to src/nimgen/external.nim index a1857a5..f642cfb 100644 --- a/src/nimgen/prepare.nim +++ b/src/nimgen/external.nim @@ -1,4 +1,4 @@ -import os, osproc, streams, strutils +import os, osproc, regex, ropes, streams, strutils import globals @@ -102,16 +102,61 @@ proc gitSparseCheckout*(plist: string) = echo "Checking out artifacts" discard execProc("git pull --depth=1 origin master") -proc doCopy*(flist: string) = - for pair in flist.split(","): - let spl = pair.split("=") - if spl.len() != 2: - echo "Bad copy syntax: " & flist - quit(1) +proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = + var + pproc = if flags.contains("cpp"): gCppCompiler else: gCCompiler + cmd = "$# -E $# $#" % [pproc, ppflags, file] - let - lfile = spl[0].strip() - rfile = spl[1].strip() + for inc in gIncludes: + cmd &= " -I " & inc - copyFile(lfile, rfile) - echo "Copied $# to $#" % [lfile, rfile] + # Run preprocessor + var data = execProc(cmd) + + # Include content only from file + var + rdata: Rope + start = false + sfile = file.replace("\\", "/") + + if inline: + sfile = sfile.parentDir() + for line in data.splitLines(): + if line.strip() != "": + if line[0] == '#' and not line.contains("#pragma"): + start = false + if sfile in line.replace("\\", "/").replace("//", "/"): + start = true + if not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: + start = true + else: + if start: + rdata.add( + line.replace("_Noreturn", "") + .replace("(())", "") + .replace("WINAPI", "") + .replace("__attribute__", "") + .replace("extern \"C\"", "") + .replace(re"\(\([_a-z]+?\)\)", "") + .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";") & "\n" + ) + return $rdata + +proc runCtags*(file: string): string = + var + cmd = "ctags -o - --fields=+S+K --c-kinds=+p --file-scope=no " & file + fps = execProc(cmd) + fdata = "" + + for line in fps.splitLines(): + var spl = line.split(re"\t") + if spl.len() > 4: + if spl[0] != "main" and spl[3] != "member": + var fn = "" + var match: RegexMatch + if spl[2].find(re"/\^(.*?)\(", match): + fn = spl[2][match.group(0)[0]] + fn &= spl[4].replace("signature:", "") & ";" + fdata &= fn & "\n" + + return fdata diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index 27296d2..1a316bf 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -46,8 +46,33 @@ proc search*(file: string): string = # Only keep relative directory return result.multiReplace([("\\", $DirSep), ("//", $DirSep), (gProjectDir & $DirSep, "")]) +proc rename*(file: string, renfile: string) = + if file.splitFile().ext == ".nim": + return + + var + nimout = getNimout(file, false) + newname = renfile.replace("$nimout", extractFilename(nimout)) + + if newname =~ peg"(!\$.)*{'$replace'\s*'('\s*{(!\)\S)+}')'}": + var final = nimout.extractFilename() + for entry in matches[1].split(","): + let spl = entry.split("=") + if spl.len() != 2: + echo "Bad replace syntax: " & renfile + quit(1) + + let + srch = spl[0].strip() + repl = spl[1].strip() + + final = final.replace(srch, repl) + newname = newname.replace(matches[0], final) + + gRenames[file] = gOutput/newname + # ### -# Loading / unloading +# Actions proc openRetry*(file: string, mode: FileMode = fmRead): File = while true: @@ -74,27 +99,16 @@ template withFile*(file: string, body: untyped): untyped = else: echo "Missing file " & file -proc rename*(file: string, renfile: string) = - if file.splitFile().ext == ".nim": - return +proc doCopy*(flist: string) = + for pair in flist.split(","): + let spl = pair.split("=") + if spl.len() != 2: + echo "Bad copy syntax: " & flist + quit(1) - var - nimout = getNimout(file, false) - newname = renfile.replace("$nimout", extractFilename(nimout)) + let + lfile = spl[0].strip() + rfile = spl[1].strip() - if newname =~ peg"(!\$.)*{'$replace'\s*'('\s*{(!\)\S)+}')'}": - var final = nimout.extractFilename() - for entry in matches[1].split(","): - let spl = entry.split("=") - if spl.len() != 2: - echo "Bad replace syntax: " & renfile - quit(1) - - let - srch = spl[0].strip() - repl = spl[1].strip() - - final = final.replace(srch, repl) - newname = newname.replace(matches[0], final) - - gRenames[file] = gOutput/newname + copyFile(lfile, rfile) + echo "Copied $# to $#" % [lfile, rfile] diff --git a/src/nimgen/fileops.nim b/src/nimgen/fileops.nim index d7ecd62..2e51d09 100644 --- a/src/nimgen/fileops.nim +++ b/src/nimgen/fileops.nim @@ -1,6 +1,6 @@ import os, regex, strutils -import file, prepare +import external, file # ### # Manipulating content diff --git a/src/nimgen/gencore.nim b/src/nimgen/gencore.nim index 59e86fe..f794991 100644 --- a/src/nimgen/gencore.nim +++ b/src/nimgen/gencore.nim @@ -1,6 +1,6 @@ -import os, regex, ropes, sequtils, strutils, tables +import os, regex, sequtils, strutils -import file, fileops, globals, prepare +import file, globals proc addEnv*(str: string): string = var newStr = str @@ -92,204 +92,3 @@ proc getDefines*(file: string, inline=false): string = withFile(file): for def in content.findAll(re"(?m)^(\s*#\s*define\s+[\w\d_]+\s+[\d\-.xf]+)(?:\r|//|/*).*?$"): result &= content[def.group(0)[0]] & "\n" - -proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = - var - pproc = if flags.contains("cpp"): gCppCompiler else: gCCompiler - cmd = "$# -E $# $#" % [pproc, ppflags, file] - - for inc in gIncludes: - cmd &= " -I " & inc - - # Run preprocessor - var data = execProc(cmd) - - # Include content only from file - var - rdata: Rope - start = false - sfile = file.replace("\\", "/") - - if inline: - sfile = sfile.parentDir() - for line in data.splitLines(): - if line.strip() != "": - if line[0] == '#' and not line.contains("#pragma"): - start = false - if sfile in line.replace("\\", "/").replace("//", "/"): - start = true - if not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: - start = true - else: - if start: - rdata.add( - line.replace("_Noreturn", "") - .replace("(())", "") - .replace("WINAPI", "") - .replace("__attribute__", "") - .replace("extern \"C\"", "") - .replace(re"\(\([_a-z]+?\)\)", "") - .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";") & "\n" - ) - return $rdata - -proc runCtags*(file: string): string = - var - cmd = "ctags -o - --fields=+S+K --c-kinds=+p --file-scope=no " & file - fps = execProc(cmd) - fdata = "" - - for line in fps.splitLines(): - var spl = line.split(re"\t") - if spl.len() > 4: - if spl[0] != "main" and spl[3] != "member": - var fn = "" - var match: RegexMatch - if spl[2].find(re"/\^(.*?)\(", match): - fn = spl[2][match.group(0)[0]] - fn &= spl[4].replace("signature:", "") & ";" - fdata &= fn & "\n" - - return fdata - -template relativePath(path: untyped): untyped = - path.multiReplace([(gOutput, ""), ("\\", "/"), ("//", "/")]) - -proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj): seq[string] = - var - incls: seq[string] = @[] - incout = "" - file = search(fl) - - if file == "": - return - - if file in gDoneRecursive: - return - - echo "Processing $# => $#" % [file, outfile] - gDoneRecursive.add(file) - - # Remove static inline function bodies - removeStatic(file) - - fixFuncProtos(file) - - if c2nimConfig.recurse: - incls = getIncls(file) - for inc in incls: - incout &= "import $#\n" % inc.search().getNimout()[0 .. ^5] - - var cfile = file - if c2nimConfig.preprocess: - cfile = "temp-$#.c" % [outfile.extractFilename()] - writeFile(cfile, runPreprocess(file, c2nimConfig.ppflags, c2nimConfig.flags, c2nimConfig.inline)) - elif c2nimConfig.ctags: - cfile = "temp-$#.c" % [outfile.extractFilename()] - writeFile(cfile, runCtags(file)) - - if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): - prepend(cfile, getDefines(file, c2nimConfig.inline)) - - var - extflags = "" - passC = "" - outlib = "" - outpragma = "" - - passC = "import ospaths, strutils\n" - - for inc in gIncludes: - if inc.isAbsolute(): - passC &= ("""{.passC: "-I\"$#\"".}""" % [inc]) & "\n" - else: - passC &= ( - """{.passC: "-I\"" & currentSourcePath().splitPath().head & "$#\"".}""" % - inc.relativePath() - ) & "\n" - - for prag in c2nimConfig.pragma: - outpragma &= "{." & prag & ".}\n" - - let fname = file.splitFile().name.replace(re"[\.\-]", "_") - - if c2nimConfig.dynlib.len() != 0: - let - win = "when defined(Windows):\n" - lin = "when defined(Linux):\n" - osx = "when defined(MacOSX):\n" - - var winlib, linlib, osxlib: string = "" - for dl in c2nimConfig.dynlib: - let - lib = " const dynlib$# = \"$#\"\n" % [fname, dl] - ext = dl.splitFile().ext - - if ext == ".dll": - winlib &= lib - elif ext == ".so": - linlib &= lib - elif ext == ".dylib": - osxlib &= lib - - if winlib != "": - outlib &= win & winlib & "\n" - if linlib != "": - outlib &= lin & linlib & "\n" - if osxlib != "": - outlib &= osx & osxlib & "\n" - - if outlib != "": - extflags &= " --dynlib:dynlib$#" % fname - else: - if file.isAbsolute(): - passC &= "const header$# = \"$#\"\n" % [fname, file] - else: - passC &= "const header$# = currentSourcePath().splitPath().head & \"$#\"\n" % - [fname, file.relativePath()] - extflags = "--header:header$#" % fname - - # Run c2nim on generated file - var cmd = "c2nim $# $# --out:$# $#" % [c2nimConfig.flags, extflags, outfile, cfile] - when defined(windows): - cmd = "cmd /c " & cmd - discard execProc(cmd) - - if c2nimConfig.preprocess or c2nimConfig.ctags: - try: - removeFile(cfile) - except: - discard - - # Import nim modules - if c2nimConfig.recurse: - prepend(outfile, incout) - - # Nim doesn't like {.cdecl.} for type proc() - freplace(outfile, re"(?m)(.*? = proc.*?)\{.cdecl.\}", "$#") - freplace(outfile, " {.cdecl.})", ")") - - # Include {.compile.} directives - for cpl in c2nimConfig.compile: - let fcpl = search(cpl) - if getFileInfo(fcpl).kind == pcFile: - prepend(outfile, compile(file=fcpl)) - else: - prepend(outfile, compile(dir=fcpl)) - - # Add any pragmas - if outpragma != "": - prepend(outfile, outpragma) - - # Add header file and include paths - if passC != "": - prepend(outfile, passC) - - # Add dynamic library - if outlib != "": - prepend(outfile, outlib) - - # Add back static functions for compilation - reAddStatic(file) - - return incls diff --git a/src/nimgen/config.nim b/src/nimgen/runcfg.nim similarity index 95% rename from src/nimgen/config.nim rename to src/nimgen/runcfg.nim index 3a0f3b0..96e067e 100644 --- a/src/nimgen/config.nim +++ b/src/nimgen/runcfg.nim @@ -1,6 +1,6 @@ import os, parsecfg, regex, strutils, tables -import file, fileops, gencore, globals, prepare +import c2nim, external, file, fileops, gencore, globals proc `[]`*(table: OrderedTableRef[string, string], key: string): string = ## Gets table values with env vars inserted @@ -109,10 +109,14 @@ proc runFile*(file: string, cfgin: OrderedTableRef) = quit(1) if not noprocess: - var incls = c2nim(file, getNimout(sfile), c2nimConfig) - if c2nimConfig.recurse and incls.len() != 0: + let outfile = getNimout(sfile) + c2nim(file, outfile, c2nimConfig) + + if c2nimConfig.recurse: var cfg = newOrderedTable[string, string]() + incls = getIncls(sfile) + incout = "" for name, value in c2nimConfig.fieldPairs: when value is string: @@ -125,6 +129,10 @@ proc runFile*(file: string, cfgin: OrderedTableRef) = for inc in incls: runFile(inc, cfg) + incout &= "import $#\n" % inc.search().getNimout()[0 .. ^5] + + if incout.len() != 0: + prepend(outfile, incout) proc runCfg*(cfg: string) = if not fileExists(cfg): From 7d60e2eea78169d7237594f6ab77ed02c968acc3 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 12 Jul 2018 16:37:31 -0500 Subject: [PATCH 077/174] Update version to v0.3.0 --- nimgen.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimgen.nimble b/nimgen.nimble index 3fddd8c..59de8ed 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.2.4" +version = "0.3.0" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" From 59c64782bd1f70688d3105024bc2c9cffb8bf313 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 13 Jul 2018 08:17:10 +0900 Subject: [PATCH 078/174] Workaround regex replace issue --- src/nimgen/fileops.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nimgen/fileops.nim b/src/nimgen/fileops.nim index 2e51d09..c4e67a0 100644 --- a/src/nimgen/fileops.nim +++ b/src/nimgen/fileops.nim @@ -76,14 +76,14 @@ proc removeStatic*(filename: string) = ## out body withFile(filename): content = content.replace( - re"(?m)(static inline.*?\))(\s*\{(\s*?.*?$)*[\n\r]\})", + re"(?m)(static inline.*?\))(\s*\{(\s*?.*?$)*?[\n\r]\})", proc (m: RegexMatch, s: string): string = let funcDecl = s[m.group(0)[0]] let body = s[m.group(1)[0]].strip() result = "" result.add("$#;" % [funcDecl]) - result.add(body.replace(re"(?m)^", "//")) + result.add(body.replace(re"(?m)^(.*\n?)", "//$1")) ) proc reAddStatic*(filename: string) = @@ -91,14 +91,14 @@ proc reAddStatic*(filename: string) = ## removeStatic withFile(filename): content = content.replace( - re"(?m)(static inline.*?\));(\/\/\s*\{(\s*?.*?$)*[\n\r]\/\/\})", + re"(?m)(static inline.*?\));(\/\/\s*\{(\s*?.*?$)*?[\n\r]\/\/\})", proc (m: RegexMatch, s: string): string = let funcDecl = s[m.group(0)[0]] let body = s[m.group(1)[0]].strip() result = "" result.add("$# " % [funcDecl]) - result.add(body.replace(re"(?m)^\/\/", "")) + result.add(body.replace(re"(?m)^\/\/(.*\n?)", "$1")) ) proc fixFuncProtos*(file: string) = From e46d3b2332d439cda073a34947cdaa803926e948 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 00:19:50 -0500 Subject: [PATCH 079/174] Implement #7 - [n.sourcefile] section, add nimclipboard --- README.md | 6 ++++++ appveyor.yml | 2 ++ src/nimgen/runcfg.nim | 13 +++++++++---- tests/nimgentest.nims | 10 ++++------ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 8b4be53..1d7b574 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ To see examples of nimgen in action check out the following wrappers:- * git sparse checkout * [nimbigwig](https://github.com/genotrance/nimbigwig) - libbigWig wrapper: [docs](http://nimgen.genotrance.com/nimbigwig) * git checkout + * [nimclipboard](https://github.com/genotrance/nimclipboard) - libclipboard wrapper: [docs](http://nimgen.genotrance.com/nimclipboard) + * git checkout * [nimfuzz](https://github.com/genotrance/nimfuzz) - fts_fuzzy_match wrapper: [docs](http://nimgen.genotrance.com/nimfuzz) * download header file * [nimkerberos](https://github.com/genotrance/nimkerberos) - WinKerberos wrapper: [docs](http://nimgen.genotrance.com/nimkerberos) @@ -134,6 +136,10 @@ File wildcards such as *.nim, ssl*.h, etc. can be used to perform tasks across a ```wildcard``` = pattern to match against. All keys following the wildcard declaration will apply to matched files +_[n.sourcefile]_ + +This section allows selection of multiple sourcefiles without requiring a detailed section for each file. Each specific file can be listed one line at a time and file wildcards can be used to include multiple source files. E.g. `$output/include/*/h`. `[n.wildcard]` definitions can be used to perform common operations on these source files if required. + _[sourcefile]_ The following keys apply to library source code and help with generating the .nim files. -win, -lin and -osx can be used for OS specific tasks. E.g. dynlib-win, pragma-win diff --git a/appveyor.yml b/appveyor.yml index a54b7ce..bcadda0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -61,6 +61,8 @@ for: on_finish: - 7z a -r buildlogs-win.zip %USERPROFILE%\.nimble\pkgs - appveyor PushArtifact buildlogs-win.zip + - 7z a -r nimgen-docs.zip %BASE_DIR%\nimgen\web + - appveyor PushArtifact nimgen-docs.zip cache: - c:\projects\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 96e067e..cc48c01 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -19,15 +19,15 @@ proc getKey(ukey: string): tuple[key: string, val: bool] = return (kv[0], false) -proc runFile*(file: string, cfgin: OrderedTableRef) = +proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, string]()) = var cfg = cfgin sfile = search(file) for pattern in gWildcards.keys(): - var match: RegexMatch + var m: RegexMatch let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") - if file.find(toPattern(pat), match): + if file.find(toPattern(pat), m): echo "Appending " & file & " " & pattern for key in gWildcards[pattern].keys(): cfg[key & "." & pattern] = gWildcards[pattern][key] @@ -224,7 +224,12 @@ proc runCfg*(cfg: string) = "n.prepare", "n.wildcard", "n.post"]: continue - runFile(file, gConfig[file]) + if file == "n.sourcefile": + for pattern in gConfig["n.sourcefile"].keys(): + for file in walkFiles(pattern.addEnv): + runFile(file) + else: + runFile(file, gConfig[file]) if gConfig.hasKey("n.post"): for post in gConfig["n.post"].keys(): diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index f5136c6..9c3e2d7 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -1,11 +1,9 @@ -import distros -import ospaths -import strutils +import distros, ospaths, strutils var - full = false - comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimfuzz", - "nimpcre", "nimrax", "nimssl", "nimssh2"] + full = true + comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", + "nimfuzz", "nimpcre", "nimrax", "nimssl", "nimssh2"] if detectOs(Windows): comps.add("nimkerberos") From 1a2ef870873f134e0df767d729ba89ecd7e1f01c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 01:37:50 -0500 Subject: [PATCH 080/174] Skip files already done --- src/nimgen/c2nim.nim | 8 ++------ src/nimgen/file.nim | 6 +++--- src/nimgen/runcfg.nim | 8 +++++++- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 07b15e0..28a3b77 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -7,14 +7,10 @@ template relativePath(path: untyped): untyped = proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = var file = search(fl) - if file == "": + if file.len() == 0: return - if file in gDoneRecursive: - return - - echo "Processing $# => $#" % [file, outfile] - gDoneRecursive.add(file) + echo " Generating " & outfile # Remove static inline function bodies removeStatic(file) diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index 1a316bf..b46f7be 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -8,7 +8,7 @@ import globals proc getNimout*(file: string, rename=true): string = result = file.splitFile().name.replace(re"[\-\.]", "_") & ".nim" if gOutput != "": - result = gOutput/result + result = gOutput & "/" & result if not rename: return @@ -35,7 +35,7 @@ proc search*(file: string): string = elif not fileExists(result) and not dirExists(result): var found = false for inc in gIncludes: - result = inc/file + result = inc & "/" & file if fileExists(result) or dirExists(result): found = true break @@ -69,7 +69,7 @@ proc rename*(file: string, renfile: string) = final = final.replace(srch, repl) newname = newname.replace(matches[0], final) - gRenames[file] = gOutput/newname + gRenames[file] = gOutput & "/" & newname # ### # Actions diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index cc48c01..a702e47 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -24,11 +24,17 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str cfg = cfgin sfile = search(file) + if sfile.len() == 0 or sfile in gDoneRecursive: + return + + echo "Processing " & sfile + gDoneRecursive.add(sfile) + for pattern in gWildcards.keys(): var m: RegexMatch let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") if file.find(toPattern(pat), m): - echo "Appending " & file & " " & pattern + echo " Appending keys for wildcard " & pattern for key in gWildcards[pattern].keys(): cfg[key & "." & pattern] = gWildcards[pattern][key] From 9e0eb256857b9dce6895b3dcb0e3f6a9dd2b5def Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 02:16:01 -0500 Subject: [PATCH 081/174] Implement #10 - per file reset --- README.md | 2 ++ src/nimgen/external.nim | 8 +++----- src/nimgen/runcfg.nim | 17 +++++++++++++---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1d7b574..df949b8 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,8 @@ The following keys apply to library source code and help with generating the .ni ```noprocess``` = do not process this source file with c2nim [default: false] - this is useful if a file only needs to be manipulated +```reset``` = reset the file back to original state after all processing [default: false] + Multiple entries for the all following keys are possible by appending any .string to the key. E.g. dynlib.win, compile.dir ```compile``` = file or dir of files of source code to {.compile.} into generated .nim diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index f642cfb..2c4a12e 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -60,15 +60,13 @@ proc gitReset*() = discard execProc("git reset --hard HEAD") -proc gitCheckout*(filename: string) {.used.} = - echo "Resetting file: $#" % [filename] +proc gitCheckout*(file: string) = + echo " Resetting " & file setCurrentDir(gOutput) defer: setCurrentDir(gProjectDir) - let adjustedFile = filename.replace(gOutput & $DirSep, "") - - discard execProc("git checkout $#" % [adjustedFile]) + discard execProc("git checkout $#" % file.replace(gOutput & "/", "")) proc gitRemotePull*(url: string, pull=true) = if dirExists(gOutput/".git"): diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index a702e47..a641150 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -24,11 +24,12 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str cfg = cfgin sfile = search(file) - if sfile.len() == 0 or sfile in gDoneRecursive: + if sfile in gDoneRecursive: return - echo "Processing " & sfile - gDoneRecursive.add(sfile) + if sfile.len() != 0: + echo "Processing " & sfile + gDoneRecursive.add(sfile) for pattern in gWildcards.keys(): var m: RegexMatch @@ -51,6 +52,7 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str let (action, val) = getKey(act) if val == true: if action == "create": + echo "Creating " & file createDir(file.splitPath().head) writeFile(file, cfg[act]) elif action in @["prepend", "append", "replace", "comment", @@ -87,7 +89,9 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str srch = act if file.splitFile().ext != ".nim": - var noprocess = false + var + noprocess = false + reset = false for act in cfg.keys(): let (action, val) = getKey(act) @@ -105,6 +109,8 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str c2nimConfig.defines = true elif action == "noprocess": noprocess = true + elif action == "reset": + reset = true elif action == "flags": c2nimConfig.flags = cfg[act] elif action == "ppflags": @@ -140,6 +146,9 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str if incout.len() != 0: prepend(outfile, incout) + if reset: + gitCheckout(sfile) + proc runCfg*(cfg: string) = if not fileExists(cfg): echo "Config doesn't exist: " & cfg From 04cd7055cf3398fd13d39bb04a549791b82b12ab Mon Sep 17 00:00:00 2001 From: data-man Date: Fri, 13 Jul 2018 12:52:11 +0300 Subject: [PATCH 082/174] Use strutils.split --- src/nimgen/external.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 2c4a12e..79426f6 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -147,7 +147,7 @@ proc runCtags*(file: string): string = fdata = "" for line in fps.splitLines(): - var spl = line.split(re"\t") + var spl = line.split('\t') if spl.len() > 4: if spl[0] != "main" and spl[3] != "member": var fn = "" From 2e332b29af2ce5e30f740aff47011744803e52f9 Mon Sep 17 00:00:00 2001 From: data-man Date: Fri, 13 Jul 2018 13:33:43 +0300 Subject: [PATCH 083/174] Use multiReplace in some places --- src/nimgen/c2nim.nim | 2 +- src/nimgen/file.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 28a3b77..4d721b4 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -48,7 +48,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = for prag in c2nimConfig.pragma: outpragma &= "{." & prag & ".}\n" - let fname = file.splitFile().name.replace(re"[\.\-]", "_") + let fname = file.splitFile().name.multiReplace([(".", "_"), ("-", "_")]) if c2nimConfig.dynlib.len() != 0: let diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index b46f7be..d78c692 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -6,7 +6,7 @@ import globals # File loction proc getNimout*(file: string, rename=true): string = - result = file.splitFile().name.replace(re"[\-\.]", "_") & ".nim" + result = file.splitFile().name.multiReplace([("-", "_"), (".", "_")]) & ".nim" if gOutput != "": result = gOutput & "/" & result From 2cfad92753025f5be7323af7e4fa58e3633e2eff Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 12:51:27 -0500 Subject: [PATCH 084/174] Fix file creation issue --- src/nimgen/runcfg.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index a641150..32146b8 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -55,6 +55,10 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str echo "Creating " & file createDir(file.splitPath().head) writeFile(file, cfg[act]) + if file in gExcludes: + gExcludes.delete(gExcludes.find(file)) + sfile = search(file) + gDoneRecursive.add(sfile) elif action in @["prepend", "append", "replace", "comment", "rename", "compile", "dynlib", "pragma", "pipe"] and sfile != "": From d1aaf70e25d00e33d6978a8c225defff25efb3b4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 13:12:46 -0500 Subject: [PATCH 085/174] Cache Linux binary --- appveyor.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index bcadda0..f0d5fac 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -65,10 +65,10 @@ for: - appveyor PushArtifact nimgen-docs.zip cache: - - c:\projects\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - - c:\projects\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz - - c:\projects\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - - c:\projects\nim-0.18.0_x32.zip + - %BASE_DIR%\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z + - %BASE_DIR%\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz + - %BASE_DIR%\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz + - %BASE_DIR%\nim-0.18.0_x32.zip - matrix: @@ -79,11 +79,12 @@ for: NIM_URL: https://nim-lang.org/download/nim-0.18.0.tar.xz NIM_ARCHIVE: nim-0.18.0.tar.xz NIM_VERSION: nim-0.18.0 + BASE_DIR: /home/appveyor/projects install: - sudo apt -qq update - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev - - cd ~/projects + - cd $BASE_DIR - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - tar xJf $NIM_ARCHIVE - cd $NIM_VERSION @@ -91,15 +92,15 @@ for: - bin/nim c -d:release koch - ./koch boot -d:release - ./koch nimble -d:release - - export PATH=~/projects/$NIM_VERSION/bin:~/.nimble/bin:$PATH - - cd ~/projects/nimgen + - export PATH=$BASE_DIR/$NIM_VERSION/bin:~/.nimble/bin:$PATH + - cd $BASE_DIR/nimgen on_finish: - zip -r -q buildlogs-lin.zip ~/.nimble/pkgs - appveyor PushArtifact buildlogs-lin.zip cache: - - ~/projects/nim-0.18.0.tar.xz + - $BASE_DIR/nim-0.18.0.tar.xz build_script: - nimble install -y From 0eaaf2e1fe94d6184d5f1473715100b9d7f40086 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 13:36:15 -0500 Subject: [PATCH 086/174] Fix base dir --- appveyor.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index f0d5fac..6b5f682 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -65,10 +65,10 @@ for: - appveyor PushArtifact nimgen-docs.zip cache: - - %BASE_DIR%\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - - %BASE_DIR%\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz - - %BASE_DIR%\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - - %BASE_DIR%\nim-0.18.0_x32.zip + - c:\projects\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z + - c:\projects\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz + - c:\projects\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz + - c:\projects\nim-0.18.0_x32.zip - matrix: @@ -100,7 +100,7 @@ for: - appveyor PushArtifact buildlogs-lin.zip cache: - - $BASE_DIR/nim-0.18.0.tar.xz + - /home/appveyor/projects/nim-0.18.0.tar.xz build_script: - nimble install -y From 29d694d630090e410a65f396adf5e91ced619d29 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 13 Jul 2018 20:06:09 -0500 Subject: [PATCH 087/174] Implement #11 - comma separated for OS specific tasks --- README.md | 2 +- src/nimgen/runcfg.nim | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index df949b8..5a1945d 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ List of all directories or files to exclude from all parsing. If an entry here m _[n.prepare]_ -The following keys can be used to prepare dependencies such as downloading ZIP files, cloning Git repositories, etc. Multiple entries are possible by appending any .string to the key. E.g. download.file1. -win, -lin and -osx can be used for OS specific tasks. E.g. download-win +The following keys can be used to prepare dependencies such as downloading ZIP files, cloning Git repositories, etc. Multiple entries are possible by appending any .string to the key. E.g. download.file1. -win, -lin and -mac/osx can be used for OS specific tasks. E.g. download-win, execute-lin,mac.unique1 ```download``` = url to download to the output directory. ZIP files are automatically extracted. Files are not redownloaded if already present but re-extracted diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 32146b8..105cede 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -11,12 +11,15 @@ proc getKey(ukey: string): tuple[key: string, val: bool] = if kv.len() == 1: kv.add("") - if (kv[1] == "") or - (kv[1] == "win" and defined(Windows)) or - (kv[1] == "lin" and defined(Linux)) or - (kv[1] == "osx" and defined(MacOSX)): + if kv[1] == "": return (kv[0], true) + for ostyp in kv[1].split(","): + if (ostyp == "win" and defined(Windows)) or + (ostyp == "lin" and defined(Linux)) or + ((ostyp == "osx" or ostyp == "mac") and defined(MacOSX)): + return (kv[0], true) + return (kv[0], false) proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, string]()) = From de4cb3f7831101c91e5d450e6a44388da878a8a1 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 14 Jul 2018 10:24:53 +0900 Subject: [PATCH 088/174] Add tests for fileops --- nimgen.nimble | 1 + tests/rununittests.nim | 8 + tests/unittests/common.nim | 5 + .../data/teststaticexpectedfrontbraces.h | 675 ++++++++++++++++++ .../data/teststaticexpectednewlinebraces.h | 68 ++ tests/unittests/data/teststaticfrontbraces.h | 675 ++++++++++++++++++ .../unittests/data/teststaticnewlinebraces.h | 71 ++ .../data/teststaticnewlinebracesreadded.h | 68 ++ tests/unittests/nim.cfg | 1 + tests/unittests/testfileops.nim | 211 ++++++ 10 files changed, 1783 insertions(+) create mode 100644 tests/rununittests.nim create mode 100644 tests/unittests/common.nim create mode 100644 tests/unittests/data/teststaticexpectedfrontbraces.h create mode 100644 tests/unittests/data/teststaticexpectednewlinebraces.h create mode 100644 tests/unittests/data/teststaticfrontbraces.h create mode 100644 tests/unittests/data/teststaticnewlinebraces.h create mode 100644 tests/unittests/data/teststaticnewlinebracesreadded.h create mode 100644 tests/unittests/nim.cfg create mode 100644 tests/unittests/testfileops.nim diff --git a/nimgen.nimble b/nimgen.nimble index 59de8ed..3e65545 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -14,4 +14,5 @@ skipDirs = @["nimgen", "tests"] requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex >= 0.7.1" task test, "Test nimgen": + exec "nim c -r tests/rununittests.nim" exec "nim e tests/nimgentest.nims" diff --git a/tests/rununittests.nim b/tests/rununittests.nim new file mode 100644 index 0000000..a92ff0a --- /dev/null +++ b/tests/rununittests.nim @@ -0,0 +1,8 @@ +import os, osproc, strutils + +proc main() = + for file in walkFiles(currentSourcePath().splitPath().head / "unittests/*.nim"): + let (path, fname, ext) = file.splitFile() + if fname.startswith("test"): + discard execCmd "nim c -r " & file +main() diff --git a/tests/unittests/common.nim b/tests/unittests/common.nim new file mode 100644 index 0000000..1769b01 --- /dev/null +++ b/tests/unittests/common.nim @@ -0,0 +1,5 @@ +import unittest + +proc checkFile*(filepath, expected: string) = + let result = readFile(filepath) + check result == expected diff --git a/tests/unittests/data/teststaticexpectedfrontbraces.h b/tests/unittests/data/teststaticexpectedfrontbraces.h new file mode 100644 index 0000000..d8c6a0d --- /dev/null +++ b/tests/unittests/data/teststaticexpectedfrontbraces.h @@ -0,0 +1,675 @@ +/** + * @file ipc.h + * @brief Inter-process communication handling + * @author plutoo + * @copyright libnx Authors + */ +#pragma once +#include "../result.h" +#include "../arm/tls.h" +#include "../kernel/svc.h" + +/// IPC input header magic +#define SFCI_MAGIC 0x49434653 +/// IPC output header magic +#define SFCO_MAGIC 0x4f434653 + +/// IPC invalid object ID +#define IPC_INVALID_OBJECT_ID UINT32_MAX + +///@name IPC request building +///@{ + +/// IPC command (request) structure. +#define IPC_MAX_BUFFERS 8 +#define IPC_MAX_OBJECTS 8 + +typedef enum { + BufferType_Normal=0, ///< Regular buffer. + BufferType_Type1=1, ///< Allows ProcessMemory and shared TransferMemory. + BufferType_Invalid=2, + BufferType_Type3=3 ///< Same as Type1 except remote process is not allowed to use device-mapping. +} BufferType; + +typedef enum { + BufferDirection_Send=0, + BufferDirection_Recv=1, + BufferDirection_Exch=2, +} BufferDirection; + +typedef enum { + IpcCommandType_Invalid = 0, + IpcCommandType_LegacyRequest = 1, + IpcCommandType_Close = 2, + IpcCommandType_LegacyControl = 3, + IpcCommandType_Request = 4, + IpcCommandType_Control = 5, + IpcCommandType_RequestWithContext = 6, + IpcCommandType_ControlWithContext = 7, +} IpcCommandType; + +typedef enum { + DomainMessageType_Invalid = 0, + DomainMessageType_SendMessage = 1, + DomainMessageType_Close = 2, +} DomainMessageType; + +/// IPC domain message header. +typedef struct { + u8 Type; + u8 NumObjectIds; + u16 Length; + u32 ThisObjectId; + u32 Pad[2]; +} DomainMessageHeader; + +typedef struct { + size_t NumSend; // A + size_t NumRecv; // B + size_t NumExch; // W + const void* Buffers[IPC_MAX_BUFFERS]; + size_t BufferSizes[IPC_MAX_BUFFERS]; + BufferType BufferTypes[IPC_MAX_BUFFERS]; + + size_t NumStaticIn; // X + size_t NumStaticOut; // C + const void* Statics[IPC_MAX_BUFFERS]; + size_t StaticSizes[IPC_MAX_BUFFERS]; + u8 StaticIndices[IPC_MAX_BUFFERS]; + + bool SendPid; + size_t NumHandlesCopy; + size_t NumHandlesMove; + Handle Handles[IPC_MAX_OBJECTS]; + + size_t NumObjectIds; + u32 ObjectIds[IPC_MAX_OBJECTS]; +} IpcCommand; + +/** + * @brief Initializes an IPC command structure. + * @param cmd IPC command structure. + */ +static inline void ipcInitialize(IpcCommand* cmd);//{ +// *cmd = (IpcCommand){0}; +//} + +/// IPC buffer descriptor. +typedef struct { + u32 Size; ///< Size of the buffer. + u32 Addr; ///< Lower 32-bits of the address of the buffer + u32 Packed; ///< Packed data (including higher bits of the address) +} IpcBufferDescriptor; + +/// IPC static send-buffer descriptor. +typedef struct { + u32 Packed; ///< Packed data (including higher bits of the address) + u32 Addr; ///< Lower 32-bits of the address +} IpcStaticSendDescriptor; + +/// IPC static receive-buffer descriptor. +typedef struct { + u32 Addr; ///< Lower 32-bits of the address of the buffer + u32 Packed; ///< Packed data (including higher bits of the address) +} IpcStaticRecvDescriptor; + +/** + * @brief Adds a buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param type Buffer type. + */ +static inline void ipcAddSendBuffer(IpcCommand* cmd, const void* buffer, size_t size, BufferType type);//{ +// size_t off = cmd->NumSend; +// cmd->Buffers[off] = buffer; +// cmd->BufferSizes[off] = size; +// cmd->BufferTypes[off] = type; +// cmd->NumSend++; +//} + +/** + * @brief Adds a receive-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param type Buffer type. + */ +static inline void ipcAddRecvBuffer(IpcCommand* cmd, void* buffer, size_t size, BufferType type);//{ +// size_t off = cmd->NumSend + cmd->NumRecv; +// cmd->Buffers[off] = buffer; +// cmd->BufferSizes[off] = size; +// cmd->BufferTypes[off] = type; +// cmd->NumRecv++; +//} + +/** + * @brief Adds an exchange-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param type Buffer type. + */ +static inline void ipcAddExchBuffer(IpcCommand* cmd, void* buffer, size_t size, BufferType type);//{ +// size_t off = cmd->NumSend + cmd->NumRecv + cmd->NumExch; +// cmd->Buffers[off] = buffer; +// cmd->BufferSizes[off] = size; +// cmd->BufferTypes[off] = type; +// cmd->NumExch++; +//} + +/** + * @brief Adds a static-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddSendStatic(IpcCommand* cmd, const void* buffer, size_t size, u8 index);//{ +// size_t off = cmd->NumStaticIn; +// cmd->Statics[off] = buffer; +// cmd->StaticSizes[off] = size; +// cmd->StaticIndices[off] = index; +// cmd->NumStaticIn++; +//} + +/** + * @brief Adds a static-receive-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddRecvStatic(IpcCommand* cmd, void* buffer, size_t size, u8 index);//{ +// size_t off = cmd->NumStaticIn + cmd->NumStaticOut; +// cmd->Statics[off] = buffer; +// cmd->StaticSizes[off] = size; +// cmd->StaticIndices[off] = index; +// cmd->NumStaticOut++; +//} + +/** + * @brief Adds a smart-buffer (buffer + static-buffer pair) to an IPC command structure. + * @param cmd IPC command structure. + * @param ipc_buffer_size IPC buffer size. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddSendSmart(IpcCommand* cmd, size_t ipc_buffer_size, const void* buffer, size_t size, u8 index);//{ +// if (ipc_buffer_size != 0 && size <= ipc_buffer_size) { +// ipcAddSendBuffer(cmd, NULL, 0, BufferType_Normal); +// ipcAddSendStatic(cmd, buffer, size, index); +// } else { +// ipcAddSendBuffer(cmd, buffer, size, BufferType_Normal); +// ipcAddSendStatic(cmd, NULL, 0, index); +// } +//} + +/** + * @brief Adds a smart-receive-buffer (buffer + static-receive-buffer pair) to an IPC command structure. + * @param cmd IPC command structure. + * @param ipc_buffer_size IPC buffer size. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddRecvSmart(IpcCommand* cmd, size_t ipc_buffer_size, void* buffer, size_t size, u8 index);//{ +// if (ipc_buffer_size != 0 && size <= ipc_buffer_size) { +// ipcAddRecvBuffer(cmd, NULL, 0, BufferType_Normal); +// ipcAddRecvStatic(cmd, buffer, size, index); +// } else { +// ipcAddRecvBuffer(cmd, buffer, size, BufferType_Normal); +// ipcAddRecvStatic(cmd, NULL, 0, index); +// } +//} + +/** + * @brief Tags an IPC command structure to send the PID. + * @param cmd IPC command structure. + */ +static inline void ipcSendPid(IpcCommand* cmd);//{ +// cmd->SendPid = true; +//} + +/** + * @brief Adds a copy-handle to be sent through an IPC command structure. + * @param cmd IPC command structure. + * @param h Handle to send. + * @remark The receiving process gets a copy of the handle. + */ +static inline void ipcSendHandleCopy(IpcCommand* cmd, Handle h);//{ +// cmd->Handles[cmd->NumHandlesCopy++] = h; +//} + +/** + * @brief Adds a move-handle to be sent through an IPC command structure. + * @param cmd IPC command structure. + * @param h Handle to send. + * @remark The sending process loses ownership of the handle, which is transferred to the receiving process. + */ +static inline void ipcSendHandleMove(IpcCommand* cmd, Handle h);//{ +// cmd->Handles[cmd->NumHandlesCopy + cmd->NumHandlesMove++] = h; +//} + +/** + * @brief Prepares the header of an IPC command structure. + * @param cmd IPC command structure. + * @param sizeof_raw Size in bytes of the raw data structure to embed inside the IPC request + * @return Pointer to the raw embedded data structure in the request, ready to be filled out. + */ +static inline void* ipcPrepareHeader(IpcCommand* cmd, size_t sizeof_raw);//{ +// u32* buf = (u32*)armGetTls(); +// size_t i; +// *buf++ = IpcCommandType_Request | (cmd->NumStaticIn << 16) | (cmd->NumSend << 20) | (cmd->NumRecv << 24) | (cmd->NumExch << 28); +// +// u32* fill_in_size_later = buf; +// +// if (cmd->NumStaticOut > 0) { +// *buf = (cmd->NumStaticOut + 2) << 10; +// } +// else { +// *buf = 0; +// } +// +// if (cmd->SendPid || cmd->NumHandlesCopy > 0 || cmd->NumHandlesMove > 0) { +// *buf++ |= 0x80000000; +// *buf++ = (!!cmd->SendPid) | (cmd->NumHandlesCopy << 1) | (cmd->NumHandlesMove << 5); +// +// if (cmd->SendPid) +// buf += 2; +// +// for (i=0; i<(cmd->NumHandlesCopy + cmd->NumHandlesMove); i++) +// *buf++ = cmd->Handles[i]; +// } +// else { +// buf++; +// } +// +// for (i=0; iNumStaticIn; i++, buf+=2) { +// IpcStaticSendDescriptor* desc = (IpcStaticSendDescriptor*) buf; +// +// uintptr_t ptr = (uintptr_t) cmd->Statics[i]; +// desc->Addr = ptr; +// desc->Packed = cmd->StaticIndices[i] | (cmd->StaticSizes[i] << 16) | +// (((ptr >> 32) & 15) << 12) | (((ptr >> 36) & 15) << 6); +// } +// +// for (i=0; i<(cmd->NumSend + cmd->NumRecv + cmd->NumExch); i++, buf+=3) { +// IpcBufferDescriptor* desc = (IpcBufferDescriptor*) buf; +// desc->Size = cmd->BufferSizes[i]; +// +// uintptr_t ptr = (uintptr_t) cmd->Buffers[i]; +// desc->Addr = ptr; +// desc->Packed = cmd->BufferTypes[i] | +// (((ptr >> 32) & 15) << 28) | ((ptr >> 36) << 2); +// } +// +// u32 padding = ((16 - (((uintptr_t) buf) & 15)) & 15) / 4; +// u32* raw = (u32*) (buf + padding); +// +// size_t raw_size = (sizeof_raw/4) + 4; +// buf += raw_size; +// +// u16* buf_u16 = (u16*) buf; +// +// for (i=0; iNumStaticOut; i++) { +// size_t off = cmd->NumStaticIn + i; +// size_t sz = (uintptr_t) cmd->StaticSizes[off]; +// +// buf_u16[i] = (sz > 0xFFFF) ? 0 : sz; +// } +// +// size_t u16s_size = ((2*cmd->NumStaticOut) + 3)/4; +// buf += u16s_size; +// raw_size += u16s_size; +// +// *fill_in_size_later |= raw_size; +// +// for (i=0; iNumStaticOut; i++, buf+=2) { +// IpcStaticRecvDescriptor* desc = (IpcStaticRecvDescriptor*) buf; +// size_t off = cmd->NumStaticIn + i; +// +// uintptr_t ptr = (uintptr_t) cmd->Statics[off]; +// desc->Addr = ptr; +// desc->Packed = (ptr >> 32) | (cmd->StaticSizes[off] << 16); +// } +// +// return (void*) raw; +//} + +/** + * @brief Dispatches an IPC request. + * @param session IPC session handle. + * @return Result code. + */ +static inline Result ipcDispatch(Handle session);//{ +// return svcSendSyncRequest(session); +//} + +///@} + +///@name IPC response parsing +///@{ + +/// IPC parsed command (response) structure. +typedef struct { + IpcCommandType CommandType; ///< Type of the command + + bool HasPid; ///< true if the 'Pid' field is filled out. + u64 Pid; ///< PID included in the response (only if HasPid is true) + + size_t NumHandles; ///< Number of handles copied. + Handle Handles[IPC_MAX_OBJECTS]; ///< Handles. + bool WasHandleCopied[IPC_MAX_OBJECTS]; ///< true if the handle was moved, false if it was copied. + + bool IsDomainMessage; ///< true if the the message is a Domain message. + DomainMessageType MessageType; ///< Type of the domain message. + u32 MessageLength; ///< Size of rawdata (for domain messages). + u32 ThisObjectId; ///< Object ID to call the command on (for domain messages). + size_t NumObjectIds; ///< Number of object IDs (for domain messages). + u32 ObjectIds[IPC_MAX_OBJECTS]; ///< Object IDs (for domain messages). + + size_t NumBuffers; ///< Number of buffers in the response. + void* Buffers[IPC_MAX_BUFFERS]; ///< Pointers to the buffers. + size_t BufferSizes[IPC_MAX_BUFFERS]; ///< Sizes of the buffers. + BufferType BufferTypes[IPC_MAX_BUFFERS]; ///< Types of the buffers. + BufferDirection BufferDirections[IPC_MAX_BUFFERS]; ///< Direction of each buffer. + + size_t NumStatics; ///< Number of statics in the response. + void* Statics[IPC_MAX_BUFFERS]; ///< Pointers to the statics. + size_t StaticSizes[IPC_MAX_BUFFERS]; ///< Sizes of the statics. + u8 StaticIndices[IPC_MAX_BUFFERS]; ///< Indices of the statics. + + size_t NumStaticsOut; ///< Number of output statics available in the response. + + void* Raw; ///< Pointer to the raw embedded data structure in the response. + void* RawWithoutPadding; ///< Pointer to the raw embedded data structure, without padding. + size_t RawSize; ///< Size of the raw embedded data. +} IpcParsedCommand; + +/** + * @brief Parse an IPC command response into an IPC parsed command structure. + * @param IPC parsed command structure to fill in. + * @return Result code. + */ +static inline Result ipcParse(IpcParsedCommand* r);//{ +// u32* buf = (u32*)armGetTls(); +// u32 ctrl0 = *buf++; +// u32 ctrl1 = *buf++; +// size_t i; +// +// r->IsDomainMessage = false; +// +// r->CommandType = (IpcCommandType) (ctrl0 & 0xffff); +// r->HasPid = false; +// r->RawSize = (ctrl1 & 0x1ff) * 4; +// r->NumHandles = 0; +// +// r->NumStaticsOut = (ctrl1 >> 10) & 15; +// if (r->NumStaticsOut >> 1) r->NumStaticsOut--; // Value 2 -> Single descriptor +// if (r->NumStaticsOut >> 1) r->NumStaticsOut--; // Value 3+ -> (Value - 2) descriptors +// +// if (ctrl1 & 0x80000000) { +// u32 ctrl2 = *buf++; +// +// if (ctrl2 & 1) { +// r->HasPid = true; +// r->Pid = *buf++; +// r->Pid |= ((u64)(*buf++)) << 32; +// } +// +// size_t num_handles_copy = ((ctrl2 >> 1) & 15); +// size_t num_handles_move = ((ctrl2 >> 5) & 15); +// +// size_t num_handles = num_handles_copy + num_handles_move; +// u32* buf_after_handles = buf + num_handles; +// +// if (num_handles > IPC_MAX_OBJECTS) +// num_handles = IPC_MAX_OBJECTS; +// +// for (i=0; iHandles[i] = *(buf+i); +// r->WasHandleCopied[i] = (i < num_handles_copy); +// } +// +// r->NumHandles = num_handles; +// buf = buf_after_handles; +// } +// +// size_t num_statics = (ctrl0 >> 16) & 15; +// u32* buf_after_statics = buf + num_statics*2; +// +// if (num_statics > IPC_MAX_BUFFERS) +// num_statics = IPC_MAX_BUFFERS; +// +// for (i=0; iPacked; +// +// r->Statics[i] = (void*) (desc->Addr | (((packed >> 12) & 15) << 32) | (((packed >> 6) & 15) << 36)); +// r->StaticSizes[i] = packed >> 16; +// r->StaticIndices[i] = packed & 63; +// } +// +// r->NumStatics = num_statics; +// buf = buf_after_statics; +// +// size_t num_bufs_send = (ctrl0 >> 20) & 15; +// size_t num_bufs_recv = (ctrl0 >> 24) & 15; +// size_t num_bufs_exch = (ctrl0 >> 28) & 15; +// +// size_t num_bufs = num_bufs_send + num_bufs_recv + num_bufs_exch; +// r->Raw = (void*)(((uintptr_t)(buf + num_bufs*3) + 15) &~ 15); +// r->RawWithoutPadding = (void*)((uintptr_t)(buf + num_bufs*3)); +// +// if (num_bufs > IPC_MAX_BUFFERS) +// num_bufs = IPC_MAX_BUFFERS; +// +// for (i=0; iPacked; +// +// r->Buffers[i] = (void*) (desc->Addr | ((packed >> 28) << 32) | (((packed >> 2) & 15) << 36)); +// r->BufferSizes[i] = desc->Size; +// r->BufferTypes[i] = (BufferType) (packed & 3); +// +// if (i < num_bufs_send) +// r->BufferDirections[i] = BufferDirection_Send; +// else if (i < (num_bufs_send + num_bufs_recv)) +// r->BufferDirections[i] = BufferDirection_Recv; +// else +// r->BufferDirections[i] = BufferDirection_Exch; +// } +// +// r->NumBuffers = num_bufs; +// return 0; +//} + +/** + * @brief Queries the size of an IPC pointer buffer. + * @param session IPC session handle. + * @param size Output variable in which to store the size. + * @return Result code. + */ +static inline Result ipcQueryPointerBufferSize(Handle session, size_t *size);//{ +// u32* buf = (u32*)armGetTls(); +// +// buf[0] = IpcCommandType_Control; +// buf[1] = 8; +// buf[2] = 0; +// buf[3] = 0; +// buf[4] = SFCI_MAGIC; +// buf[5] = 0; +// buf[6] = 3; +// buf[7] = 0; +// +// Result rc = ipcDispatch(session); +// +// if (R_SUCCEEDED(rc)) { +// IpcParsedCommand r; +// ipcParse(&r); +// +// struct ipcQueryPointerBufferSizeResponse { +// u64 magic; +// u64 result; +// u32 size; +// } *raw = (struct ipcQueryPointerBufferSizeResponse*)r.Raw; +// +// rc = raw->result; +// +// if (R_SUCCEEDED(rc)) { +// *size = raw->size & 0xffff; +// } +// } +// +// return rc; +//} + +/** + * @brief Closes the IPC session with proper clean up. + * @param session IPC session handle. + * @return Result code. + */ +static inline Result ipcCloseSession(Handle session);//{ +// u32* buf = (u32*)armGetTls(); +// buf[0] = IpcCommandType_Close; +// buf[1] = 0; +// return ipcDispatch(session); +//} +///@} + +///@name IPC domain handling +///@{ + +/** + * @brief Converts an IPC session handle into a domain. + * @param session IPC session handle. + * @param object_id_out Output variable in which to store the object ID. + * @return Result code. + */ +static inline Result ipcConvertSessionToDomain(Handle session, u32* object_id_out);//{ +// u32* buf = (u32*)armGetTls(); +// +// buf[0] = IpcCommandType_Control; +// buf[1] = 8; +// buf[4] = SFCI_MAGIC; +// buf[5] = 0; +// buf[6] = 0; +// buf[7] = 0; +// +// Result rc = ipcDispatch(session); +// +// if (R_SUCCEEDED(rc)) { +// IpcParsedCommand r; +// ipcParse(&r); +// +// struct ipcConvertSessionToDomainResponse { +// u64 magic; +// u64 result; +// u32 object_id; +// } *raw = (struct ipcConvertSessionToDomainResponse*)r.Raw; +// +// rc = raw->result; +// +// if (R_SUCCEEDED(rc)) { +// *object_id_out = raw->object_id; +// } +// } +// +// return rc; +//} + +/** + * @brief Adds an object ID to be sent through an IPC domain command structure. + * @param cmd IPC domain command structure. + * @param object_id Object ID to send. + */ +static inline void ipcSendObjectId(IpcCommand* cmd, u32 object_id);//{ +// cmd->ObjectIds[cmd->NumObjectIds++] = object_id; +//} + +/** + * @brief Prepares the header of an IPC command structure (domain version). + * @param cmd IPC command structure. + * @param sizeof_raw Size in bytes of the raw data structure to embed inside the IPC request + * @oaram object_id Domain object ID. + * @return Pointer to the raw embedded data structure in the request, ready to be filled out. + */ +static inline void* ipcPrepareHeaderForDomain(IpcCommand* cmd, size_t sizeof_raw, u32 object_id);//{ +// void* raw = ipcPrepareHeader(cmd, sizeof_raw + sizeof(DomainMessageHeader)); +// DomainMessageHeader* hdr = (DomainMessageHeader*) raw; +// u32 *object_ids = (u32*)(((uintptr_t) raw) + sizeof(DomainMessageHeader) + sizeof_raw); +// +// hdr->Type = DomainMessageType_SendMessage; +// hdr->NumObjectIds = (u8)cmd->NumObjectIds; +// hdr->Length = sizeof_raw; +// hdr->ThisObjectId = object_id; +// hdr->Pad[0] = hdr->Pad[1] = 0; +// +// for(size_t i = 0; i < cmd->NumObjectIds; i++) +// object_ids[i] = cmd->ObjectIds[i]; +// return (void*)(((uintptr_t) raw) + sizeof(DomainMessageHeader)); +//} + +/** + * @brief Parse an IPC command response into an IPC parsed command structure (domain version). + * @param IPC parsed command structure to fill in. + * @return Result code. + */ +static inline Result ipcParseForDomain(IpcParsedCommand* r);//{ +// Result rc = ipcParse(r); +// DomainMessageHeader *hdr; +// u32 *object_ids; +// if(R_FAILED(rc)) +// return rc; +// +// hdr = (DomainMessageHeader*) r->Raw; +// object_ids = (u32*)(((uintptr_t) hdr) + sizeof(DomainMessageHeader) + hdr->Length); +// r->Raw = (void*)(((uintptr_t) r->Raw) + sizeof(DomainMessageHeader)); +// +// r->IsDomainMessage = true; +// r->MessageType = (DomainMessageType)(hdr->Type); +// switch (r->MessageType) { +// case DomainMessageType_SendMessage: +// case DomainMessageType_Close: +// break; +// default: +// return MAKERESULT(Module_Libnx, LibnxError_DomainMessageUnknownType); +// } +// r->ThisObjectId = hdr->ThisObjectId; +// r->NumObjectIds = hdr->NumObjectIds > 8 ? 8 : hdr->NumObjectIds; +// if ((uintptr_t)object_ids + sizeof(u32) * r->NumObjectIds - (uintptr_t)armGetTls() >= 0x100) { +// return MAKERESULT(Module_Libnx, LibnxError_DomainMessageTooManyObjectIds); +// } +// for(size_t i = 0; i < r->NumObjectIds; i++) +// r->ObjectIds[i] = object_ids[i]; +// +// return rc; +//} + +/** + * @brief Closes a domain object by ID. + * @param session IPC session handle. + * @param object_id ID of the object to close. + * @return Result code. + */ +static inline Result ipcCloseObjectById(Handle session, u32 object_id);//{ +// IpcCommand c; +// DomainMessageHeader* hdr; +// +// ipcInitialize(&c); +// hdr = (DomainMessageHeader*)ipcPrepareHeader(&c, sizeof(DomainMessageHeader)); +// +// hdr->Type = 2; +// hdr->NumObjectIds = 0; +// hdr->Length = 0; +// hdr->ThisObjectId = object_id; +// hdr->Pad[0] = hdr->Pad[1] = 0; +// +// return ipcDispatch(session); // this command has no associated response +//} + +///@} + diff --git a/tests/unittests/data/teststaticexpectednewlinebraces.h b/tests/unittests/data/teststaticexpectednewlinebraces.h new file mode 100644 index 0000000..3f90ab6 --- /dev/null +++ b/tests/unittests/data/teststaticexpectednewlinebraces.h @@ -0,0 +1,68 @@ +/** + * @file condvar.h + * @brief Condition variable synchronization primitive. + * @author plutoo + * @copyright libnx Authors + */ +#pragma once +#include "../types.h" +#include "../kernel/mutex.h" + +/// Condition variable structure. +typedef struct { + u32 tag; + Mutex* mutex; +} CondVar; + +/** + * @brief Initializes a condition variable. + * @param[in] c Condition variable object. + * @param[in] m Mutex object to use inside the condition variable. + */ +void condvarInit(CondVar* c, Mutex* m); + +/** + * @brief Waits on a condition variable with a timeout. + * @param[in] c Condition variable object. + * @param[in] timeout Timeout in nanoseconds. + * @return Result code (0xEA01 on timeout). + * @remark On function return, the underlying mutex is acquired. + */ +Result condvarWaitTimeout(CondVar* c, u64 timeout); + +/** + * @brief Waits on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + * @remark On function return, the underlying mutex is acquired. + */ +static inline Result condvarWait(CondVar* c);//{ +// return condvarWaitTimeout(c, -1ull); +//} + +/** + * @brief Wakes up up to the specified number of threads waiting on a condition variable. + * @param[in] c Condition variable object. + * @param[in] num Maximum number of threads to wake up (or -1 to wake them all up). + * @return Result code. + */ +Result condvarWake(CondVar* c, int num); + +/** + * @brief Wakes up a single thread waiting on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + */ +static inline Result condvarWakeOne(CondVar* c);//{ +// return condvarWake(c, 1); +//} + +/** + * @brief Wakes up all thread waiting on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + */ +static inline Result condvarWakeAll(CondVar* c);//{ +// return condvarWake(c, -1); +//} + diff --git a/tests/unittests/data/teststaticfrontbraces.h b/tests/unittests/data/teststaticfrontbraces.h new file mode 100644 index 0000000..d62f694 --- /dev/null +++ b/tests/unittests/data/teststaticfrontbraces.h @@ -0,0 +1,675 @@ +/** + * @file ipc.h + * @brief Inter-process communication handling + * @author plutoo + * @copyright libnx Authors + */ +#pragma once +#include "../result.h" +#include "../arm/tls.h" +#include "../kernel/svc.h" + +/// IPC input header magic +#define SFCI_MAGIC 0x49434653 +/// IPC output header magic +#define SFCO_MAGIC 0x4f434653 + +/// IPC invalid object ID +#define IPC_INVALID_OBJECT_ID UINT32_MAX + +///@name IPC request building +///@{ + +/// IPC command (request) structure. +#define IPC_MAX_BUFFERS 8 +#define IPC_MAX_OBJECTS 8 + +typedef enum { + BufferType_Normal=0, ///< Regular buffer. + BufferType_Type1=1, ///< Allows ProcessMemory and shared TransferMemory. + BufferType_Invalid=2, + BufferType_Type3=3 ///< Same as Type1 except remote process is not allowed to use device-mapping. +} BufferType; + +typedef enum { + BufferDirection_Send=0, + BufferDirection_Recv=1, + BufferDirection_Exch=2, +} BufferDirection; + +typedef enum { + IpcCommandType_Invalid = 0, + IpcCommandType_LegacyRequest = 1, + IpcCommandType_Close = 2, + IpcCommandType_LegacyControl = 3, + IpcCommandType_Request = 4, + IpcCommandType_Control = 5, + IpcCommandType_RequestWithContext = 6, + IpcCommandType_ControlWithContext = 7, +} IpcCommandType; + +typedef enum { + DomainMessageType_Invalid = 0, + DomainMessageType_SendMessage = 1, + DomainMessageType_Close = 2, +} DomainMessageType; + +/// IPC domain message header. +typedef struct { + u8 Type; + u8 NumObjectIds; + u16 Length; + u32 ThisObjectId; + u32 Pad[2]; +} DomainMessageHeader; + +typedef struct { + size_t NumSend; // A + size_t NumRecv; // B + size_t NumExch; // W + const void* Buffers[IPC_MAX_BUFFERS]; + size_t BufferSizes[IPC_MAX_BUFFERS]; + BufferType BufferTypes[IPC_MAX_BUFFERS]; + + size_t NumStaticIn; // X + size_t NumStaticOut; // C + const void* Statics[IPC_MAX_BUFFERS]; + size_t StaticSizes[IPC_MAX_BUFFERS]; + u8 StaticIndices[IPC_MAX_BUFFERS]; + + bool SendPid; + size_t NumHandlesCopy; + size_t NumHandlesMove; + Handle Handles[IPC_MAX_OBJECTS]; + + size_t NumObjectIds; + u32 ObjectIds[IPC_MAX_OBJECTS]; +} IpcCommand; + +/** + * @brief Initializes an IPC command structure. + * @param cmd IPC command structure. + */ +static inline void ipcInitialize(IpcCommand* cmd) { + *cmd = (IpcCommand){0}; +} + +/// IPC buffer descriptor. +typedef struct { + u32 Size; ///< Size of the buffer. + u32 Addr; ///< Lower 32-bits of the address of the buffer + u32 Packed; ///< Packed data (including higher bits of the address) +} IpcBufferDescriptor; + +/// IPC static send-buffer descriptor. +typedef struct { + u32 Packed; ///< Packed data (including higher bits of the address) + u32 Addr; ///< Lower 32-bits of the address +} IpcStaticSendDescriptor; + +/// IPC static receive-buffer descriptor. +typedef struct { + u32 Addr; ///< Lower 32-bits of the address of the buffer + u32 Packed; ///< Packed data (including higher bits of the address) +} IpcStaticRecvDescriptor; + +/** + * @brief Adds a buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param type Buffer type. + */ +static inline void ipcAddSendBuffer(IpcCommand* cmd, const void* buffer, size_t size, BufferType type) { + size_t off = cmd->NumSend; + cmd->Buffers[off] = buffer; + cmd->BufferSizes[off] = size; + cmd->BufferTypes[off] = type; + cmd->NumSend++; +} + +/** + * @brief Adds a receive-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param type Buffer type. + */ +static inline void ipcAddRecvBuffer(IpcCommand* cmd, void* buffer, size_t size, BufferType type) { + size_t off = cmd->NumSend + cmd->NumRecv; + cmd->Buffers[off] = buffer; + cmd->BufferSizes[off] = size; + cmd->BufferTypes[off] = type; + cmd->NumRecv++; +} + +/** + * @brief Adds an exchange-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param type Buffer type. + */ +static inline void ipcAddExchBuffer(IpcCommand* cmd, void* buffer, size_t size, BufferType type) { + size_t off = cmd->NumSend + cmd->NumRecv + cmd->NumExch; + cmd->Buffers[off] = buffer; + cmd->BufferSizes[off] = size; + cmd->BufferTypes[off] = type; + cmd->NumExch++; +} + +/** + * @brief Adds a static-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddSendStatic(IpcCommand* cmd, const void* buffer, size_t size, u8 index) { + size_t off = cmd->NumStaticIn; + cmd->Statics[off] = buffer; + cmd->StaticSizes[off] = size; + cmd->StaticIndices[off] = index; + cmd->NumStaticIn++; +} + +/** + * @brief Adds a static-receive-buffer to an IPC command structure. + * @param cmd IPC command structure. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddRecvStatic(IpcCommand* cmd, void* buffer, size_t size, u8 index) { + size_t off = cmd->NumStaticIn + cmd->NumStaticOut; + cmd->Statics[off] = buffer; + cmd->StaticSizes[off] = size; + cmd->StaticIndices[off] = index; + cmd->NumStaticOut++; +} + +/** + * @brief Adds a smart-buffer (buffer + static-buffer pair) to an IPC command structure. + * @param cmd IPC command structure. + * @param ipc_buffer_size IPC buffer size. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddSendSmart(IpcCommand* cmd, size_t ipc_buffer_size, const void* buffer, size_t size, u8 index) { + if (ipc_buffer_size != 0 && size <= ipc_buffer_size) { + ipcAddSendBuffer(cmd, NULL, 0, BufferType_Normal); + ipcAddSendStatic(cmd, buffer, size, index); + } else { + ipcAddSendBuffer(cmd, buffer, size, BufferType_Normal); + ipcAddSendStatic(cmd, NULL, 0, index); + } +} + +/** + * @brief Adds a smart-receive-buffer (buffer + static-receive-buffer pair) to an IPC command structure. + * @param cmd IPC command structure. + * @param ipc_buffer_size IPC buffer size. + * @param buffer Address of the buffer. + * @param size Size of the buffer. + * @param index Index of buffer. + */ +static inline void ipcAddRecvSmart(IpcCommand* cmd, size_t ipc_buffer_size, void* buffer, size_t size, u8 index) { + if (ipc_buffer_size != 0 && size <= ipc_buffer_size) { + ipcAddRecvBuffer(cmd, NULL, 0, BufferType_Normal); + ipcAddRecvStatic(cmd, buffer, size, index); + } else { + ipcAddRecvBuffer(cmd, buffer, size, BufferType_Normal); + ipcAddRecvStatic(cmd, NULL, 0, index); + } +} + +/** + * @brief Tags an IPC command structure to send the PID. + * @param cmd IPC command structure. + */ +static inline void ipcSendPid(IpcCommand* cmd) { + cmd->SendPid = true; +} + +/** + * @brief Adds a copy-handle to be sent through an IPC command structure. + * @param cmd IPC command structure. + * @param h Handle to send. + * @remark The receiving process gets a copy of the handle. + */ +static inline void ipcSendHandleCopy(IpcCommand* cmd, Handle h) { + cmd->Handles[cmd->NumHandlesCopy++] = h; +} + +/** + * @brief Adds a move-handle to be sent through an IPC command structure. + * @param cmd IPC command structure. + * @param h Handle to send. + * @remark The sending process loses ownership of the handle, which is transferred to the receiving process. + */ +static inline void ipcSendHandleMove(IpcCommand* cmd, Handle h) { + cmd->Handles[cmd->NumHandlesCopy + cmd->NumHandlesMove++] = h; +} + +/** + * @brief Prepares the header of an IPC command structure. + * @param cmd IPC command structure. + * @param sizeof_raw Size in bytes of the raw data structure to embed inside the IPC request + * @return Pointer to the raw embedded data structure in the request, ready to be filled out. + */ +static inline void* ipcPrepareHeader(IpcCommand* cmd, size_t sizeof_raw) { + u32* buf = (u32*)armGetTls(); + size_t i; + *buf++ = IpcCommandType_Request | (cmd->NumStaticIn << 16) | (cmd->NumSend << 20) | (cmd->NumRecv << 24) | (cmd->NumExch << 28); + + u32* fill_in_size_later = buf; + + if (cmd->NumStaticOut > 0) { + *buf = (cmd->NumStaticOut + 2) << 10; + } + else { + *buf = 0; + } + + if (cmd->SendPid || cmd->NumHandlesCopy > 0 || cmd->NumHandlesMove > 0) { + *buf++ |= 0x80000000; + *buf++ = (!!cmd->SendPid) | (cmd->NumHandlesCopy << 1) | (cmd->NumHandlesMove << 5); + + if (cmd->SendPid) + buf += 2; + + for (i=0; i<(cmd->NumHandlesCopy + cmd->NumHandlesMove); i++) + *buf++ = cmd->Handles[i]; + } + else { + buf++; + } + + for (i=0; iNumStaticIn; i++, buf+=2) { + IpcStaticSendDescriptor* desc = (IpcStaticSendDescriptor*) buf; + + uintptr_t ptr = (uintptr_t) cmd->Statics[i]; + desc->Addr = ptr; + desc->Packed = cmd->StaticIndices[i] | (cmd->StaticSizes[i] << 16) | + (((ptr >> 32) & 15) << 12) | (((ptr >> 36) & 15) << 6); + } + + for (i=0; i<(cmd->NumSend + cmd->NumRecv + cmd->NumExch); i++, buf+=3) { + IpcBufferDescriptor* desc = (IpcBufferDescriptor*) buf; + desc->Size = cmd->BufferSizes[i]; + + uintptr_t ptr = (uintptr_t) cmd->Buffers[i]; + desc->Addr = ptr; + desc->Packed = cmd->BufferTypes[i] | + (((ptr >> 32) & 15) << 28) | ((ptr >> 36) << 2); + } + + u32 padding = ((16 - (((uintptr_t) buf) & 15)) & 15) / 4; + u32* raw = (u32*) (buf + padding); + + size_t raw_size = (sizeof_raw/4) + 4; + buf += raw_size; + + u16* buf_u16 = (u16*) buf; + + for (i=0; iNumStaticOut; i++) { + size_t off = cmd->NumStaticIn + i; + size_t sz = (uintptr_t) cmd->StaticSizes[off]; + + buf_u16[i] = (sz > 0xFFFF) ? 0 : sz; + } + + size_t u16s_size = ((2*cmd->NumStaticOut) + 3)/4; + buf += u16s_size; + raw_size += u16s_size; + + *fill_in_size_later |= raw_size; + + for (i=0; iNumStaticOut; i++, buf+=2) { + IpcStaticRecvDescriptor* desc = (IpcStaticRecvDescriptor*) buf; + size_t off = cmd->NumStaticIn + i; + + uintptr_t ptr = (uintptr_t) cmd->Statics[off]; + desc->Addr = ptr; + desc->Packed = (ptr >> 32) | (cmd->StaticSizes[off] << 16); + } + + return (void*) raw; +} + +/** + * @brief Dispatches an IPC request. + * @param session IPC session handle. + * @return Result code. + */ +static inline Result ipcDispatch(Handle session) { + return svcSendSyncRequest(session); +} + +///@} + +///@name IPC response parsing +///@{ + +/// IPC parsed command (response) structure. +typedef struct { + IpcCommandType CommandType; ///< Type of the command + + bool HasPid; ///< true if the 'Pid' field is filled out. + u64 Pid; ///< PID included in the response (only if HasPid is true) + + size_t NumHandles; ///< Number of handles copied. + Handle Handles[IPC_MAX_OBJECTS]; ///< Handles. + bool WasHandleCopied[IPC_MAX_OBJECTS]; ///< true if the handle was moved, false if it was copied. + + bool IsDomainMessage; ///< true if the the message is a Domain message. + DomainMessageType MessageType; ///< Type of the domain message. + u32 MessageLength; ///< Size of rawdata (for domain messages). + u32 ThisObjectId; ///< Object ID to call the command on (for domain messages). + size_t NumObjectIds; ///< Number of object IDs (for domain messages). + u32 ObjectIds[IPC_MAX_OBJECTS]; ///< Object IDs (for domain messages). + + size_t NumBuffers; ///< Number of buffers in the response. + void* Buffers[IPC_MAX_BUFFERS]; ///< Pointers to the buffers. + size_t BufferSizes[IPC_MAX_BUFFERS]; ///< Sizes of the buffers. + BufferType BufferTypes[IPC_MAX_BUFFERS]; ///< Types of the buffers. + BufferDirection BufferDirections[IPC_MAX_BUFFERS]; ///< Direction of each buffer. + + size_t NumStatics; ///< Number of statics in the response. + void* Statics[IPC_MAX_BUFFERS]; ///< Pointers to the statics. + size_t StaticSizes[IPC_MAX_BUFFERS]; ///< Sizes of the statics. + u8 StaticIndices[IPC_MAX_BUFFERS]; ///< Indices of the statics. + + size_t NumStaticsOut; ///< Number of output statics available in the response. + + void* Raw; ///< Pointer to the raw embedded data structure in the response. + void* RawWithoutPadding; ///< Pointer to the raw embedded data structure, without padding. + size_t RawSize; ///< Size of the raw embedded data. +} IpcParsedCommand; + +/** + * @brief Parse an IPC command response into an IPC parsed command structure. + * @param IPC parsed command structure to fill in. + * @return Result code. + */ +static inline Result ipcParse(IpcParsedCommand* r) { + u32* buf = (u32*)armGetTls(); + u32 ctrl0 = *buf++; + u32 ctrl1 = *buf++; + size_t i; + + r->IsDomainMessage = false; + + r->CommandType = (IpcCommandType) (ctrl0 & 0xffff); + r->HasPid = false; + r->RawSize = (ctrl1 & 0x1ff) * 4; + r->NumHandles = 0; + + r->NumStaticsOut = (ctrl1 >> 10) & 15; + if (r->NumStaticsOut >> 1) r->NumStaticsOut--; // Value 2 -> Single descriptor + if (r->NumStaticsOut >> 1) r->NumStaticsOut--; // Value 3+ -> (Value - 2) descriptors + + if (ctrl1 & 0x80000000) { + u32 ctrl2 = *buf++; + + if (ctrl2 & 1) { + r->HasPid = true; + r->Pid = *buf++; + r->Pid |= ((u64)(*buf++)) << 32; + } + + size_t num_handles_copy = ((ctrl2 >> 1) & 15); + size_t num_handles_move = ((ctrl2 >> 5) & 15); + + size_t num_handles = num_handles_copy + num_handles_move; + u32* buf_after_handles = buf + num_handles; + + if (num_handles > IPC_MAX_OBJECTS) + num_handles = IPC_MAX_OBJECTS; + + for (i=0; iHandles[i] = *(buf+i); + r->WasHandleCopied[i] = (i < num_handles_copy); + } + + r->NumHandles = num_handles; + buf = buf_after_handles; + } + + size_t num_statics = (ctrl0 >> 16) & 15; + u32* buf_after_statics = buf + num_statics*2; + + if (num_statics > IPC_MAX_BUFFERS) + num_statics = IPC_MAX_BUFFERS; + + for (i=0; iPacked; + + r->Statics[i] = (void*) (desc->Addr | (((packed >> 12) & 15) << 32) | (((packed >> 6) & 15) << 36)); + r->StaticSizes[i] = packed >> 16; + r->StaticIndices[i] = packed & 63; + } + + r->NumStatics = num_statics; + buf = buf_after_statics; + + size_t num_bufs_send = (ctrl0 >> 20) & 15; + size_t num_bufs_recv = (ctrl0 >> 24) & 15; + size_t num_bufs_exch = (ctrl0 >> 28) & 15; + + size_t num_bufs = num_bufs_send + num_bufs_recv + num_bufs_exch; + r->Raw = (void*)(((uintptr_t)(buf + num_bufs*3) + 15) &~ 15); + r->RawWithoutPadding = (void*)((uintptr_t)(buf + num_bufs*3)); + + if (num_bufs > IPC_MAX_BUFFERS) + num_bufs = IPC_MAX_BUFFERS; + + for (i=0; iPacked; + + r->Buffers[i] = (void*) (desc->Addr | ((packed >> 28) << 32) | (((packed >> 2) & 15) << 36)); + r->BufferSizes[i] = desc->Size; + r->BufferTypes[i] = (BufferType) (packed & 3); + + if (i < num_bufs_send) + r->BufferDirections[i] = BufferDirection_Send; + else if (i < (num_bufs_send + num_bufs_recv)) + r->BufferDirections[i] = BufferDirection_Recv; + else + r->BufferDirections[i] = BufferDirection_Exch; + } + + r->NumBuffers = num_bufs; + return 0; +} + +/** + * @brief Queries the size of an IPC pointer buffer. + * @param session IPC session handle. + * @param size Output variable in which to store the size. + * @return Result code. + */ +static inline Result ipcQueryPointerBufferSize(Handle session, size_t *size) { + u32* buf = (u32*)armGetTls(); + + buf[0] = IpcCommandType_Control; + buf[1] = 8; + buf[2] = 0; + buf[3] = 0; + buf[4] = SFCI_MAGIC; + buf[5] = 0; + buf[6] = 3; + buf[7] = 0; + + Result rc = ipcDispatch(session); + + if (R_SUCCEEDED(rc)) { + IpcParsedCommand r; + ipcParse(&r); + + struct ipcQueryPointerBufferSizeResponse { + u64 magic; + u64 result; + u32 size; + } *raw = (struct ipcQueryPointerBufferSizeResponse*)r.Raw; + + rc = raw->result; + + if (R_SUCCEEDED(rc)) { + *size = raw->size & 0xffff; + } + } + + return rc; +} + +/** + * @brief Closes the IPC session with proper clean up. + * @param session IPC session handle. + * @return Result code. + */ +static inline Result ipcCloseSession(Handle session) { + u32* buf = (u32*)armGetTls(); + buf[0] = IpcCommandType_Close; + buf[1] = 0; + return ipcDispatch(session); +} +///@} + +///@name IPC domain handling +///@{ + +/** + * @brief Converts an IPC session handle into a domain. + * @param session IPC session handle. + * @param object_id_out Output variable in which to store the object ID. + * @return Result code. + */ +static inline Result ipcConvertSessionToDomain(Handle session, u32* object_id_out) { + u32* buf = (u32*)armGetTls(); + + buf[0] = IpcCommandType_Control; + buf[1] = 8; + buf[4] = SFCI_MAGIC; + buf[5] = 0; + buf[6] = 0; + buf[7] = 0; + + Result rc = ipcDispatch(session); + + if (R_SUCCEEDED(rc)) { + IpcParsedCommand r; + ipcParse(&r); + + struct ipcConvertSessionToDomainResponse { + u64 magic; + u64 result; + u32 object_id; + } *raw = (struct ipcConvertSessionToDomainResponse*)r.Raw; + + rc = raw->result; + + if (R_SUCCEEDED(rc)) { + *object_id_out = raw->object_id; + } + } + + return rc; +} + +/** + * @brief Adds an object ID to be sent through an IPC domain command structure. + * @param cmd IPC domain command structure. + * @param object_id Object ID to send. + */ +static inline void ipcSendObjectId(IpcCommand* cmd, u32 object_id) { + cmd->ObjectIds[cmd->NumObjectIds++] = object_id; +} + +/** + * @brief Prepares the header of an IPC command structure (domain version). + * @param cmd IPC command structure. + * @param sizeof_raw Size in bytes of the raw data structure to embed inside the IPC request + * @oaram object_id Domain object ID. + * @return Pointer to the raw embedded data structure in the request, ready to be filled out. + */ +static inline void* ipcPrepareHeaderForDomain(IpcCommand* cmd, size_t sizeof_raw, u32 object_id) { + void* raw = ipcPrepareHeader(cmd, sizeof_raw + sizeof(DomainMessageHeader)); + DomainMessageHeader* hdr = (DomainMessageHeader*) raw; + u32 *object_ids = (u32*)(((uintptr_t) raw) + sizeof(DomainMessageHeader) + sizeof_raw); + + hdr->Type = DomainMessageType_SendMessage; + hdr->NumObjectIds = (u8)cmd->NumObjectIds; + hdr->Length = sizeof_raw; + hdr->ThisObjectId = object_id; + hdr->Pad[0] = hdr->Pad[1] = 0; + + for(size_t i = 0; i < cmd->NumObjectIds; i++) + object_ids[i] = cmd->ObjectIds[i]; + return (void*)(((uintptr_t) raw) + sizeof(DomainMessageHeader)); +} + +/** + * @brief Parse an IPC command response into an IPC parsed command structure (domain version). + * @param IPC parsed command structure to fill in. + * @return Result code. + */ +static inline Result ipcParseForDomain(IpcParsedCommand* r) { + Result rc = ipcParse(r); + DomainMessageHeader *hdr; + u32 *object_ids; + if(R_FAILED(rc)) + return rc; + + hdr = (DomainMessageHeader*) r->Raw; + object_ids = (u32*)(((uintptr_t) hdr) + sizeof(DomainMessageHeader) + hdr->Length); + r->Raw = (void*)(((uintptr_t) r->Raw) + sizeof(DomainMessageHeader)); + + r->IsDomainMessage = true; + r->MessageType = (DomainMessageType)(hdr->Type); + switch (r->MessageType) { + case DomainMessageType_SendMessage: + case DomainMessageType_Close: + break; + default: + return MAKERESULT(Module_Libnx, LibnxError_DomainMessageUnknownType); + } + r->ThisObjectId = hdr->ThisObjectId; + r->NumObjectIds = hdr->NumObjectIds > 8 ? 8 : hdr->NumObjectIds; + if ((uintptr_t)object_ids + sizeof(u32) * r->NumObjectIds - (uintptr_t)armGetTls() >= 0x100) { + return MAKERESULT(Module_Libnx, LibnxError_DomainMessageTooManyObjectIds); + } + for(size_t i = 0; i < r->NumObjectIds; i++) + r->ObjectIds[i] = object_ids[i]; + + return rc; +} + +/** + * @brief Closes a domain object by ID. + * @param session IPC session handle. + * @param object_id ID of the object to close. + * @return Result code. + */ +static inline Result ipcCloseObjectById(Handle session, u32 object_id) { + IpcCommand c; + DomainMessageHeader* hdr; + + ipcInitialize(&c); + hdr = (DomainMessageHeader*)ipcPrepareHeader(&c, sizeof(DomainMessageHeader)); + + hdr->Type = 2; + hdr->NumObjectIds = 0; + hdr->Length = 0; + hdr->ThisObjectId = object_id; + hdr->Pad[0] = hdr->Pad[1] = 0; + + return ipcDispatch(session); // this command has no associated response +} + +///@} + diff --git a/tests/unittests/data/teststaticnewlinebraces.h b/tests/unittests/data/teststaticnewlinebraces.h new file mode 100644 index 0000000..c72c11a --- /dev/null +++ b/tests/unittests/data/teststaticnewlinebraces.h @@ -0,0 +1,71 @@ +/** + * @file condvar.h + * @brief Condition variable synchronization primitive. + * @author plutoo + * @copyright libnx Authors + */ +#pragma once +#include "../types.h" +#include "../kernel/mutex.h" + +/// Condition variable structure. +typedef struct { + u32 tag; + Mutex* mutex; +} CondVar; + +/** + * @brief Initializes a condition variable. + * @param[in] c Condition variable object. + * @param[in] m Mutex object to use inside the condition variable. + */ +void condvarInit(CondVar* c, Mutex* m); + +/** + * @brief Waits on a condition variable with a timeout. + * @param[in] c Condition variable object. + * @param[in] timeout Timeout in nanoseconds. + * @return Result code (0xEA01 on timeout). + * @remark On function return, the underlying mutex is acquired. + */ +Result condvarWaitTimeout(CondVar* c, u64 timeout); + +/** + * @brief Waits on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + * @remark On function return, the underlying mutex is acquired. + */ +static inline Result condvarWait(CondVar* c) +{ + return condvarWaitTimeout(c, -1ull); +} + +/** + * @brief Wakes up up to the specified number of threads waiting on a condition variable. + * @param[in] c Condition variable object. + * @param[in] num Maximum number of threads to wake up (or -1 to wake them all up). + * @return Result code. + */ +Result condvarWake(CondVar* c, int num); + +/** + * @brief Wakes up a single thread waiting on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + */ +static inline Result condvarWakeOne(CondVar* c) +{ + return condvarWake(c, 1); +} + +/** + * @brief Wakes up all thread waiting on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + */ +static inline Result condvarWakeAll(CondVar* c) +{ + return condvarWake(c, -1); +} + diff --git a/tests/unittests/data/teststaticnewlinebracesreadded.h b/tests/unittests/data/teststaticnewlinebracesreadded.h new file mode 100644 index 0000000..176f4fc --- /dev/null +++ b/tests/unittests/data/teststaticnewlinebracesreadded.h @@ -0,0 +1,68 @@ +/** + * @file condvar.h + * @brief Condition variable synchronization primitive. + * @author plutoo + * @copyright libnx Authors + */ +#pragma once +#include "../types.h" +#include "../kernel/mutex.h" + +/// Condition variable structure. +typedef struct { + u32 tag; + Mutex* mutex; +} CondVar; + +/** + * @brief Initializes a condition variable. + * @param[in] c Condition variable object. + * @param[in] m Mutex object to use inside the condition variable. + */ +void condvarInit(CondVar* c, Mutex* m); + +/** + * @brief Waits on a condition variable with a timeout. + * @param[in] c Condition variable object. + * @param[in] timeout Timeout in nanoseconds. + * @return Result code (0xEA01 on timeout). + * @remark On function return, the underlying mutex is acquired. + */ +Result condvarWaitTimeout(CondVar* c, u64 timeout); + +/** + * @brief Waits on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + * @remark On function return, the underlying mutex is acquired. + */ +static inline Result condvarWait(CondVar* c) { + return condvarWaitTimeout(c, -1ull); +} + +/** + * @brief Wakes up up to the specified number of threads waiting on a condition variable. + * @param[in] c Condition variable object. + * @param[in] num Maximum number of threads to wake up (or -1 to wake them all up). + * @return Result code. + */ +Result condvarWake(CondVar* c, int num); + +/** + * @brief Wakes up a single thread waiting on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + */ +static inline Result condvarWakeOne(CondVar* c) { + return condvarWake(c, 1); +} + +/** + * @brief Wakes up all thread waiting on a condition variable. + * @param[in] c Condition variable object. + * @return Result code. + */ +static inline Result condvarWakeAll(CondVar* c) { + return condvarWake(c, -1); +} + diff --git a/tests/unittests/nim.cfg b/tests/unittests/nim.cfg new file mode 100644 index 0000000..e9b9a2e --- /dev/null +++ b/tests/unittests/nim.cfg @@ -0,0 +1 @@ +--path="../../src" diff --git a/tests/unittests/testfileops.nim b/tests/unittests/testfileops.nim new file mode 100644 index 0000000..2fff387 --- /dev/null +++ b/tests/unittests/testfileops.nim @@ -0,0 +1,211 @@ +import nimgen/fileops, common, regex, os + +import unittest + +let testFileContent = """ +this is text +this is text +replace me +prepend me +end +""" + +let prependMiddleExpected = """ +this is text +this is text +replace me +prepended data +prepend me +end +""" + +let prependEndExpected = """ +this is text +this is text +replace me +prepend me +data +end +""" + +let appendEndExpected = """ +this is text +this is text +replace me +prepend me +end +data +""" + +let appendMiddleExpected = """ +this is data + text +this is text +replace me +prepend me +end +""" + +let freplaceDefaultExpected = """ + + +replace me +prepend me +end +""" + +let freplaceWithExpected = """ +this is text +this is text +foobar +prepend me +end +""" + +let freplaceRegexExpected = """ +foobar +foobar +replace me +prepend me +end +""" + +let commentExpected = """ +this is text +this is text +//replace me +//prepend me +//end +""" + +let commentMiddleExpected = """ +this //is text +//this is text +replace me +prepend me +end +""" + + +let dataDir = currentSourcePath().splitPath().head / "data" + +let testfilename = dataDir / "testing.txt" + + +suite "test file ops": + if not dataDir.dirExists(): + dataDir.createDir() + + setup: + if testfilename.existsFile(): + removeFile(testfilename) + writeFile(testfilename, testFileContent) + + ################### Prepend ####################### + + test "prepend at beginning of file": + prepend(testfilename, "data\n") + let expected = "data\n" & testFileContent + testfilename.checkFile(expected) + + test "prepend at middle of file": + prepend(testfilename, "prepended data\n", "prepend me") + testfilename.checkFile(prependMiddleExpected) + + test "prepend at end of file": + prepend(testfilename, "data\n", "end\n") + testfilename.checkFile(prependEndExpected) + + ################### Pipe ######################### + + test "pipe command into file": + when defined(windows): + pipe(testfilename, "(ECHO foo)>>$file") + testfilename.checkFile("foo") + else: + pipe(testfilename, "cat $file | grep 'this is text'") + testfilename.checkFile("this is text\nthis is text") + + ################# Append ######################### + + test "append file end": + append(testfilename, "data\n") + testfilename.checkFile(appendEndExpected) + + test "append file middle": + append(testfilename, " data\n", "this is") + testfilename.checkFile(appendMiddleExpected) + + ################# FReplace ######################### + + test "freplace default empty": + freplace(testfilename, "this is text") + testfilename.checkFile(freplaceDefaultExpected) + + test "freplace with content": + freplace(testfilename, "replace me", "foobar") + testfilename.checkFile(freplaceWithExpected) + + test "freplace regex": + freplace(testfilename, re"this .*", "foobar") + testfilename.checkFile(freplaceRegexExpected) + + ####################### Comment ###################### + + test "comment": + comment(testfilename, "replace me", "3") + testfilename.checkFile(commentExpected) + + test "comment over length": + comment(testfilename, "replace me", "10") + testfilename.checkFile(commentExpected) + + test "comment negative": + comment(testfilename, "replace me", "-3") + testfilename.checkFile(testFileContent) + + test "comment zero": + comment(testfilename, "replace me", "0") + testfilename.checkFile(testFileContent) + + test "comment middle": + comment(testfilename, "is text", "2") + testfilename.checkFile(commentMiddleExpected) + + ############### Static inline removal ################ + + test "replace static inline with front braces at end of line": + + let + file = dataDir / "teststaticfrontbraces.h" + resFile = dataDir / "teststaticexpectedfrontbraces.h" + + test = readFile(file) + expected = readFile(resFile) + + writeFile(testfilename, test) + + removeStatic(testfilename) + testfilename.checkFile(expected) + + reAddStatic(testfilename) + testfilename.checkFile(test) + + test "replace static inline with newline before brace": + + let + file = dataDir / "teststaticnewlinebraces.h" + resFile = dataDir / "teststaticexpectednewlinebraces.h" + reAddedFile = dataDir / "teststaticnewlinebracesreadded.h" + + test = readFile(file) + expected = readFile(resFile) + reAdded = readFile(reAddedFile) + + writeFile(testfilename, test) + + removeStatic(testfilename) + testfilename.checkFile(expected) + + reAddStatic(testfilename) + testfilename.checkFile(reAdded) From 0473e79c0bbc56c41814f6fb41d91da1bf5ed79a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 14 Jul 2018 22:04:03 +0900 Subject: [PATCH 089/174] Fix miscellaneous windows issues with building --- src/nimgen/c2nim.nim | 5 ++--- src/nimgen/external.nim | 5 ++++- src/nimgen/gencore.nim | 2 +- src/nimgen/runcfg.nim | 3 +++ 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 4d721b4..bca56d1 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -38,7 +38,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = for inc in gIncludes: if inc.isAbsolute(): - passC &= ("""{.passC: "-I\"$#\"".}""" % [inc]) & "\n" + passC &= ("""{.passC: "-I\"$#\"".}""" % [inc.sanitizePath()]) & "\n" else: passC &= ( """{.passC: "-I\"" & currentSourcePath().splitPath().head & "$#\"".}""" % @@ -85,11 +85,10 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = passC &= "const header$# = currentSourcePath().splitPath().head & \"$#\"\n" % [fname, file.relativePath()] extflags = "--header:header$#" % fname - # Run c2nim on generated file var cmd = "c2nim $# $# --out:$# $#" % [c2nimConfig.flags, extflags, outfile, cfile] when defined(windows): - cmd = "cmd /c " & cmd + cmd = "cmd /c " & cmd.quoteShell discard execProc(cmd) if c2nimConfig.preprocess or c2nimConfig.ctags: diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 79426f6..096753c 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -2,6 +2,9 @@ import os, osproc, regex, ropes, streams, strutils import globals +proc sanitizePath*(path: string): string = + path.multiReplace([("\\", "/"), ("//", "/")]) + proc execProc*(cmd: string): string = result = "" var @@ -106,7 +109,7 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = cmd = "$# -E $# $#" % [pproc, ppflags, file] for inc in gIncludes: - cmd &= " -I " & inc + cmd &= " -I " & inc.quoteShell # Run preprocessor var data = execProc(cmd) diff --git a/src/nimgen/gencore.nim b/src/nimgen/gencore.nim index f794991..382fa59 100644 --- a/src/nimgen/gencore.nim +++ b/src/nimgen/gencore.nim @@ -20,7 +20,7 @@ proc addEnv*(str: string): string = discard # if there are still format args, print a warning - if newStr.contains("${"): + if newStr.contains("$") and not newStr.contains("$replace("): echo "WARNING: \"", newStr, "\" still contains an uninterpolated value!" return newStr diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 105cede..7255c52 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -196,6 +196,9 @@ proc runCfg*(cfg: string) = # Reset on a per project basis gCCompiler = getEnv(cCompilerEnv, defaultCCompiler) + gCppCompiler = gCppCompiler.quoteShell + gCCompiler = gCCompiler.quoteShell + if gConfig["n.global"].hasKey("filter"): gFilter = gConfig["n.global"]["filter"] if gConfig["n.global"].hasKey("quotes"): From ffc73c6b6d3d16b4833443a1d2b520cf572c0665 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 15 Jul 2018 10:06:46 +0900 Subject: [PATCH 090/174] Prevent processed file collisions --- src/nimgen/runcfg.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 7255c52..cafb48e 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -25,7 +25,7 @@ proc getKey(ukey: string): tuple[key: string, val: bool] = proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, string]()) = var cfg = cfgin - sfile = search(file) + sfile = search(file).sanitizePath if sfile in gDoneRecursive: return From 2d06bdcccfcc41d42949f22aa7b96624cc7df130 Mon Sep 17 00:00:00 2001 From: data-man Date: Sun, 15 Jul 2018 04:49:24 +0300 Subject: [PATCH 091/174] More multiReplaces --- src/nimgen/external.nim | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 79426f6..39c963e 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -123,18 +123,15 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = if line.strip() != "": if line[0] == '#' and not line.contains("#pragma"): start = false - if sfile in line.replace("\\", "/").replace("//", "/"): + if sfile in line.multiReplace([("\\", "/"), ("//", "/")]): start = true if not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: start = true else: if start: rdata.add( - line.replace("_Noreturn", "") - .replace("(())", "") - .replace("WINAPI", "") - .replace("__attribute__", "") - .replace("extern \"C\"", "") + line.multiReplace([("_Noreturn", ""), ("(())", ""), ("WINAPI", ""), + ("__attribute__", ""), ("extern \"C\"", "")]) .replace(re"\(\([_a-z]+?\)\)", "") .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";") & "\n" ) From f8f918642964f60cee6c8dd5ba9fe8390b8ee369 Mon Sep 17 00:00:00 2001 From: data-man Date: Sun, 15 Jul 2018 07:49:05 +0300 Subject: [PATCH 092/174] Use sanitizePath --- src/nimgen/external.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index c336f0a..350c841 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -126,7 +126,7 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = if line.strip() != "": if line[0] == '#' and not line.contains("#pragma"): start = false - if sfile in line.multiReplace([("\\", "/"), ("//", "/")]): + if sfile in line.sanitizePath: start = true if not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: start = true From 0b2fb6cfbc24041feb92221205623e0d579f0247 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 15 Jul 2018 15:40:00 +0900 Subject: [PATCH 093/174] Fix issue with sourcpath building switch on Windows --- src/nimgen/c2nim.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index bca56d1..f171e8d 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -36,12 +36,14 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = passC = "import ospaths, strutils\n" + passC &= """const sourcePath = currentSourcePath().split({'\\', '/'})[0..^2].join("/")""" & "\n" + for inc in gIncludes: if inc.isAbsolute(): passC &= ("""{.passC: "-I\"$#\"".}""" % [inc.sanitizePath()]) & "\n" else: passC &= ( - """{.passC: "-I\"" & currentSourcePath().splitPath().head & "$#\"".}""" % + """{.passC: "-I\"" & sourcePath & "$#\"".}""" % inc.relativePath() ) & "\n" @@ -82,7 +84,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = if file.isAbsolute(): passC &= "const header$# = \"$#\"\n" % [fname, file] else: - passC &= "const header$# = currentSourcePath().splitPath().head & \"$#\"\n" % + passC &= "const header$# = sourcePath & \"$#\"\n" % [fname, file.relativePath()] extflags = "--header:header$#" % fname # Run c2nim on generated file From 5e3c0999e0f969f7fb18fa14b8587b177e52666e Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 17 Jul 2018 18:11:39 +0900 Subject: [PATCH 094/174] Fix windows not processing headers --- src/nimgen/c2nim.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index f171e8d..fe257f2 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -6,7 +6,7 @@ template relativePath(path: untyped): untyped = path.multiReplace([(gOutput, ""), ("\\", "/"), ("//", "/")]) proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = - var file = search(fl) + var file = search(fl).sanitizePath if file.len() == 0: return From 6343567a9182dccac9edb61c636bd5ddf2732cfe Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 18 Jul 2018 07:52:45 +0900 Subject: [PATCH 095/174] Put sanitizePath in search() --- src/nimgen/c2nim.nim | 2 +- src/nimgen/external.nim | 2 +- src/nimgen/file.nim | 4 ++-- src/nimgen/runcfg.nim | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index fe257f2..f171e8d 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -6,7 +6,7 @@ template relativePath(path: untyped): untyped = path.multiReplace([(gOutput, ""), ("\\", "/"), ("//", "/")]) proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = - var file = search(fl).sanitizePath + var file = search(fl) if file.len() == 0: return diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 350c841..c0d5fdc 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -118,7 +118,7 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = var rdata: Rope start = false - sfile = file.replace("\\", "/") + sfile = file.sanitizePath if inline: sfile = sfile.parentDir() diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index d78c692..708ae4a 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -1,6 +1,6 @@ import os, ospaths, pegs, regex, strutils, tables -import globals +import globals, external # ### # File loction @@ -44,7 +44,7 @@ proc search*(file: string): string = quit(1) # Only keep relative directory - return result.multiReplace([("\\", $DirSep), ("//", $DirSep), (gProjectDir & $DirSep, "")]) + return result.sanitizePath.replace(gProjectDir & "/", "") proc rename*(file: string, renfile: string) = if file.splitFile().ext == ".nim": diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index cafb48e..7255c52 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -25,7 +25,7 @@ proc getKey(ukey: string): tuple[key: string, val: bool] = proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, string]()) = var cfg = cfgin - sfile = search(file).sanitizePath + sfile = search(file) if sfile in gDoneRecursive: return From f3c01e53f985a5f479ee4c58c492cc88e6d7af98 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Wed, 18 Jul 2018 12:25:13 +0900 Subject: [PATCH 096/174] Fix a few more issues with windows paths --- src/nimgen/runcfg.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 7255c52..03c0013 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -161,13 +161,13 @@ proc runCfg*(cfg: string) = echo "Config doesn't exist: " & cfg quit(1) - gProjectDir = parentDir(cfg.expandFilename()) + gProjectDir = parentDir(cfg.expandFilename()).sanitizePath gConfig = loadConfig(cfg) if gConfig.hasKey("n.global"): if gConfig["n.global"].hasKey("output"): - gOutput = gConfig["n.global"]["output"] + gOutput = gConfig["n.global"]["output"].sanitizePath if dirExists(gOutput): if "-f" in commandLineParams(): try: @@ -207,11 +207,11 @@ proc runCfg*(cfg: string) = if gConfig.hasKey("n.include"): for inc in gConfig["n.include"].keys(): - gIncludes.add(inc.addEnv()) + gIncludes.add(inc.addEnv().sanitizePath) if gConfig.hasKey("n.exclude"): for excl in gConfig["n.exclude"].keys(): - gExcludes.add(excl.addEnv()) + gExcludes.add(excl.addEnv().sanitizePath) if gConfig.hasKey("n.prepare"): for prep in gConfig["n.prepare"].keys(): From 2c3cc71540f94783a98f34a82afc5c40bf320dd3 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Aug 2018 20:49:50 -0500 Subject: [PATCH 097/174] Fix permission denied and file flushing issues --- src/nimgen/c2nim.nim | 4 ++-- src/nimgen/external.nim | 26 +++++++++----------------- src/nimgen/file.nim | 10 +++++++--- src/nimgen/runcfg.nim | 2 +- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index f171e8d..428c767 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -20,10 +20,10 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = var cfile = file if c2nimConfig.preprocess: cfile = "temp-$#.c" % [outfile.extractFilename()] - writeFile(cfile, runPreprocess(file, c2nimConfig.ppflags, c2nimConfig.flags, c2nimConfig.inline)) + writeFileFlush(cfile, runPreprocess(file, c2nimConfig.ppflags, c2nimConfig.flags, c2nimConfig.inline)) elif c2nimConfig.ctags: cfile = "temp-$#.c" % [outfile.extractFilename()] - writeFile(cfile, runCtags(file)) + writeFileFlush(cfile, runCtags(file)) if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): prepend(cfile, getDefines(file, c2nimConfig.inline)) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index c0d5fdc..9575c38 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -6,22 +6,11 @@ proc sanitizePath*(path: string): string = path.multiReplace([("\\", "/"), ("//", "/")]) proc execProc*(cmd: string): string = - result = "" - var - p = startProcess(cmd, options = {poStdErrToStdOut, poUsePath, poEvalCommand}) + var ret: int - outp = outputStream(p) - line = newStringOfCap(120).TaintedString - - while true: - if outp.readLine(line): - result.add(line) - result.add("\n") - elif not running(p): break - - var x = p.peekExitCode() - if x != 0: - echo "Command failed: " & $x + (result, ret) = execCmdEx(cmd) + if ret != 0: + echo "Command failed: " & $ret echo cmd echo result quit(1) @@ -64,12 +53,15 @@ proc gitReset*() = discard execProc("git reset --hard HEAD") proc gitCheckout*(file: string) = - echo " Resetting " & file + echo "Resetting " & file setCurrentDir(gOutput) defer: setCurrentDir(gProjectDir) - discard execProc("git checkout $#" % file.replace(gOutput & "/", "")) + let cmd = "git checkout $#" % file.replace(gOutput & "/", "") + if execCmdEx(cmd)[0].contains("Permission denied"): + sleep(500) + discard execProc(cmd) proc gitRemotePull*(url: string, pull=true) = if dirExists(gOutput/".git"): diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index 708ae4a..edf39ea 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -82,6 +82,12 @@ proc openRetry*(file: string, mode: FileMode = fmRead): File = except IOError: sleep(100) +template writeFileFlush*(file, content: string): untyped = + let f = openRetry(file, fmWrite) + f.write(content) + f.flushFile() + f.close() + template withFile*(file: string, body: untyped): untyped = if fileExists(file): var f = openRetry(file) @@ -93,9 +99,7 @@ template withFile*(file: string, body: untyped): untyped = body if content != contentOrig: - f = openRetry(file, fmWrite) - write(f, content) - f.close() + writeFileFlush(file, content) else: echo "Missing file " & file diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 03c0013..95cb1c4 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -57,7 +57,7 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str if action == "create": echo "Creating " & file createDir(file.splitPath().head) - writeFile(file, cfg[act]) + writeFileFlush(file, cfg[act]) if file in gExcludes: gExcludes.delete(gExcludes.find(file)) sfile = search(file) From 87e08bfb666789942462739e90afa3e4bd23a1de Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 3 Aug 2018 13:02:56 +0900 Subject: [PATCH 098/174] Fix unit test for Windows and quit with failure code --- tests/rununittests.nim | 4 +++- tests/unittests/testfileops.nim | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/rununittests.nim b/tests/rununittests.nim index a92ff0a..865f77d 100644 --- a/tests/rununittests.nim +++ b/tests/rununittests.nim @@ -1,8 +1,10 @@ import os, osproc, strutils proc main() = + var failures = 0 for file in walkFiles(currentSourcePath().splitPath().head / "unittests/*.nim"): let (path, fname, ext) = file.splitFile() if fname.startswith("test"): - discard execCmd "nim c -r " & file + failures += execCmd "nim c -r " & file + quit(failures) main() diff --git a/tests/unittests/testfileops.nim b/tests/unittests/testfileops.nim index 2fff387..5d2119f 100644 --- a/tests/unittests/testfileops.nim +++ b/tests/unittests/testfileops.nim @@ -120,7 +120,7 @@ suite "test file ops": test "pipe command into file": when defined(windows): - pipe(testfilename, "(ECHO foo)>>$file") + pipe(testfilename, "ECHO foo > $file") testfilename.checkFile("foo") else: pipe(testfilename, "cat $file | grep 'this is text'") From c6148571c10d073b2c4c4a6392c5a43e747315b4 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 3 Aug 2018 15:08:03 +0900 Subject: [PATCH 099/174] Fix windows echo --- tests/unittests/testfileops.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/testfileops.nim b/tests/unittests/testfileops.nim index 5d2119f..076c586 100644 --- a/tests/unittests/testfileops.nim +++ b/tests/unittests/testfileops.nim @@ -120,7 +120,7 @@ suite "test file ops": test "pipe command into file": when defined(windows): - pipe(testfilename, "ECHO foo > $file") + pipe(testfilename, "ECHO foo") testfilename.checkFile("foo") else: pipe(testfilename, "cat $file | grep 'this is text'") From 1dbff5cbd14f69529869720e72e153f6f9029957 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 3 Aug 2018 23:27:26 -0500 Subject: [PATCH 100/174] Inline fixes --- nimgen.nimble | 2 +- src/nimgen/c2nim.nim | 12 ++--------- src/nimgen/external.nim | 9 +++++--- src/nimgen/gencore.nim | 1 - src/nimgen/runcfg.nim | 47 +++++++++++++++++++++++------------------ 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/nimgen.nimble b/nimgen.nimble index 3e65545..841cb09 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.3.0" +version = "0.3.1" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 428c767..73e4ebb 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -10,12 +10,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = if file.len() == 0: return - echo " Generating " & outfile - - # Remove static inline function bodies - removeStatic(file) - - fixFuncProtos(file) + echo "Generating " & outfile var cfile = file if c2nimConfig.preprocess: @@ -121,7 +116,4 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = # Add dynamic library if outlib != "": - prepend(outfile, outlib) - - # Add back static functions for compilation - reAddStatic(file) + prepend(outfile, outlib) \ No newline at end of file diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 9575c38..571c7e0 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -50,7 +50,10 @@ proc gitReset*() = setCurrentDir(gOutput) defer: setCurrentDir(gProjectDir) - discard execProc("git reset --hard HEAD") + let cmd = "git reset --hard HEAD" + while execCmdEx(cmd)[0].contains("Permission denied"): + sleep(1000) + echo " Retrying ..." proc gitCheckout*(file: string) = echo "Resetting " & file @@ -59,9 +62,9 @@ proc gitCheckout*(file: string) = defer: setCurrentDir(gProjectDir) let cmd = "git checkout $#" % file.replace(gOutput & "/", "") - if execCmdEx(cmd)[0].contains("Permission denied"): + while execCmdEx(cmd)[0].contains("Permission denied"): sleep(500) - discard execProc(cmd) + echo " Retrying ..." proc gitRemotePull*(url: string, pull=true) = if dirExists(gOutput/".git"): diff --git a/src/nimgen/gencore.nim b/src/nimgen/gencore.nim index 382fa59..ff070df 100644 --- a/src/nimgen/gencore.nim +++ b/src/nimgen/gencore.nim @@ -87,7 +87,6 @@ proc getDefines*(file: string, inline=false): string = for incl in incls: let sincl = search(incl) if sincl != "": - echo "Inlining " & sincl result &= getDefines(sincl) withFile(file): for def in content.findAll(re"(?m)^(\s*#\s*define\s+[\w\d_]+\s+[\d\-.xf]+)(?:\r|//|/*).*?$"): diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 95cb1c4..fd65708 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -60,7 +60,7 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str writeFileFlush(file, cfg[act]) if file in gExcludes: gExcludes.delete(gExcludes.find(file)) - sfile = search(file) + sfile = file gDoneRecursive.add(sfile) elif action in @["prepend", "append", "replace", "comment", "rename", "compile", "dynlib", "pragma", @@ -127,31 +127,38 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str echo "Cannot use recurse and inline simultaneously" quit(1) - if not noprocess: - let outfile = getNimout(sfile) - c2nim(file, outfile, c2nimConfig) + removeStatic(sfile) + fixFuncProtos(sfile) - if c2nimConfig.recurse: - var - cfg = newOrderedTable[string, string]() - incls = getIncls(sfile) - incout = "" + let outfile = getNimout(sfile) + if c2nimConfig.recurse or c2nimConfig.inline: + var + cfg = newOrderedTable[string, string]() + incls = getIncls(sfile) + incout = "" - for name, value in c2nimConfig.fieldPairs: - when value is string: - cfg[name] = value - when value is bool: - cfg[name] = $value + for name, value in c2nimConfig.fieldPairs: + when value is string: + cfg[name] = value + when value is bool: + cfg[name] = $value - for i in c2nimConfig.dynlib: - cfg["dynlib." & i] = i + for i in c2nimConfig.dynlib: + cfg["dynlib." & i] = i - for inc in incls: - runFile(inc, cfg) + if c2nimConfig.inline: + cfg["noprocess"] = "true" + + for inc in incls: + runFile(inc, cfg) + if c2nimConfig.recurse: incout &= "import $#\n" % inc.search().getNimout()[0 .. ^5] - if incout.len() != 0: - prepend(outfile, incout) + if c2nimConfig.recurse and incout.len() != 0: + prepend(outfile, incout) + + if not noprocess: + c2nim(file, outfile, c2nimConfig) if reset: gitCheckout(sfile) From 32debc5534bc4a82e5f944adb2e4b42043bf5fab Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 5 Aug 2018 01:10:47 -0500 Subject: [PATCH 101/174] Regex search/replace support - #2 --- README.md | 6 ++++-- src/nimgen/c2nim.nim | 2 +- src/nimgen/fileops.nim | 8 +------- src/nimgen/gencore.nim | 22 +++++++--------------- src/nimgen/runcfg.nim | 14 ++++++++++---- 5 files changed, 23 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 5a1945d..c88af16 100644 --- a/README.md +++ b/README.md @@ -174,15 +174,17 @@ The following keys apply to library source code (before processing) and generate ```create``` = create a file at exact location with contents specified. File needs to be in the _[n.exclude]_ list in order to be created. +```pipe``` = execute a command on a file and store the output of the command as the new file contents. E.g. pipe = "cat $file | grep 'static inline'" + ```search``` = search string providing context for following prepend/append/replace directives -```pipe``` = execute a command on a file and store the output of the command as the new file contents. Ex: pipe = "cat $file | grep 'static inline'" +```regex``` = regex search string providing context for the following replace directive. Specify using """ to avoid regex parsing issues ```prepend``` = string value to prepend into file at beginning or before search ```append``` = string value to append into file at the end or after search -```replace``` = string value to replace search string in file +```replace``` = string value to replace search string in file. Regex captures can be referred to using $1, $2, etc. ```comment``` = number of lines to comment from search location diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 73e4ebb..05954e6 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -95,7 +95,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = discard # Nim doesn't like {.cdecl.} for type proc() - freplace(outfile, re"(?m)(.*? = proc.*?)\{.cdecl.\}", "$#") + freplace(outfile, re"(?m)(.*? = proc.*?)\{.cdecl.\}", "$1") freplace(outfile, " {.cdecl.})", ")") # Include {.compile.} directives diff --git a/src/nimgen/fileops.nim b/src/nimgen/fileops.nim index c4e67a0..4e8f858 100644 --- a/src/nimgen/fileops.nim +++ b/src/nimgen/fileops.nim @@ -38,13 +38,7 @@ proc freplace*(file: string, pattern: string, repl="") = proc freplace*(file: string, pattern: Regex, repl="") = withFile(file): - var m: RegexMatch - if content.find(pattern, m): - if "$#" in repl: - content = content.replace(pattern, - proc (m: RegexMatch, s: string): string = repl % s[m.group(0)[0]]) - else: - content = content.replace(pattern, repl) + content = content.replace(pattern, repl) proc comment*(file: string, pattern: string, numlines: string) = let diff --git a/src/nimgen/gencore.nim b/src/nimgen/gencore.nim index ff070df..4e79d04 100644 --- a/src/nimgen/gencore.nim +++ b/src/nimgen/gencore.nim @@ -4,24 +4,16 @@ import file, globals proc addEnv*(str: string): string = var newStr = str - for pair in envPairs(): - try: - newStr = newStr % [pair.key, pair.value.string] - except ValueError: - # Ignore if there are no values to replace. We - # want to continue anyway - discard - try: + if "$output" in newStr or "${output}" in newStr: newStr = newStr % ["output", gOutput] - except ValueError: - # Ignore if there are no values to replace. We - # want to continue anyway - discard - # if there are still format args, print a warning - if newStr.contains("$") and not newStr.contains("$replace("): - echo "WARNING: \"", newStr, "\" still contains an uninterpolated value!" + for pair in envPairs(): + if pair.key.len() == 0: + continue + + if ("$" & pair.key) in newStr or ("${" & pair.key & "}") in newStr: + newStr = newStr % [pair.key, pair.value.string] return newStr diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index fd65708..a0e51c5 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -44,6 +44,7 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str var srch = "" + rgx = "" c2nimConfig = c2nimConfigObj( flags: "--stdcall", ppflags: "", @@ -78,6 +79,8 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str elif action == "replace": if srch != "": freplace(sfile, cfg[srch], cfg[act]) + elif rgx != "": + freplace(sfile, toPattern(cfg[rgx]), cfg[act]) elif action == "comment": if srch != "": comment(sfile, cfg[srch], cfg[act]) @@ -92,8 +95,11 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str elif action == "pipe": pipe(sfile, cfg[act]) srch = "" + rgx = "" elif action == "search": srch = act + elif action == "regex": + rgx = act if file.splitFile().ext != ".nim": var @@ -131,11 +137,11 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str fixFuncProtos(sfile) let outfile = getNimout(sfile) + var incout = "" if c2nimConfig.recurse or c2nimConfig.inline: var cfg = newOrderedTable[string, string]() incls = getIncls(sfile) - incout = "" for name, value in c2nimConfig.fieldPairs: when value is string: @@ -154,12 +160,12 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str if c2nimConfig.recurse: incout &= "import $#\n" % inc.search().getNimout()[0 .. ^5] - if c2nimConfig.recurse and incout.len() != 0: - prepend(outfile, incout) - if not noprocess: c2nim(file, outfile, c2nimConfig) + if c2nimConfig.recurse and incout.len() != 0: + prepend(outfile, incout) + if reset: gitCheckout(sfile) From 667d6caf844133452f0a9145813817b08b0be896 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 8 Aug 2018 11:06:21 -0500 Subject: [PATCH 102/174] CLI support - #20 --- README.md | 35 ++++++++ src/nimgen.nim | 4 +- src/nimgen/runcfg.nim | 138 ++++++++++++++++++++++++++++---- tests/unittests/testfileops.nim | 2 - 4 files changed, 158 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index c88af16..c023d6b 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,41 @@ The following key only applies before processing and allows renaming the generat `$replace(srch1=repl1, srch2=reply2)` = rename specific portions in `$nimout` +__Command Line__ + +A subset of capabilities are available through the command line to enable quick tests using nimgen. Command line flags only apply to source files specified on the command line and do not influence any ```cfg``` files which are expected to be self-sufficient. + +``` +Usage: + nimgen [options] file.cfg|file.h ... + +Params: + -C add compile entry * + -E add n.exclude entry * + -F set c2nim flags * + -I add n.include dir * + -O set output directory + -P set preprocessor flags * + +Options: + -c set ctags = true + -d set defines = true + -i set inline = true + -n set noprocess = true + -p set preprocess = true + -r set recurse = true + +Editing: + -a append string * + -e prepend string * + -l replace string * + -o#lines comment X lines * + -s search string * + -x regex search string * + +* supports multiple instances +``` + __Feedback__ Nimgen is a work in progress and any feedback or suggestions are welcome. It is hosted on [GitHub](https://github.com/genotrance/nimgen) with an MIT license so issues, forks and PRs are most appreciated. Also join us at https://gitter.im/nimgen/Lobby to chat about nimgen and the future of Nim wrappers. diff --git a/src/nimgen.nim b/src/nimgen.nim index 5777398..69f994e 100644 --- a/src/nimgen.nim +++ b/src/nimgen.nim @@ -2,6 +2,4 @@ import os import nimgen/runcfg -for i in commandLineParams(): - if i != "-f": - runCfg(i) +runCli() \ No newline at end of file diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index a0e51c5..91e7078 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -169,6 +169,24 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str if reset: gitCheckout(sfile) +proc setOutputDir(dir: string) = + gOutput = dir.sanitizePath + if dirExists(gOutput): + if "-f" in commandLineParams(): + try: + removeDir(gOutput) + except OSError: + echo "Directory in use: " & gOutput + quit(1) + else: + for f in walkFiles(gOutput/"*.nim"): + try: + removeFile(f) + except OSError: + echo "Unable to delete: " & f + quit(1) + createDir(gOutput) + proc runCfg*(cfg: string) = if not fileExists(cfg): echo "Config doesn't exist: " & cfg @@ -180,22 +198,7 @@ proc runCfg*(cfg: string) = if gConfig.hasKey("n.global"): if gConfig["n.global"].hasKey("output"): - gOutput = gConfig["n.global"]["output"].sanitizePath - if dirExists(gOutput): - if "-f" in commandLineParams(): - try: - removeDir(gOutput) - except OSError: - echo "Directory in use: " & gOutput - quit(1) - else: - for f in walkFiles(gOutput/"*.nim"): - try: - removeFile(f) - except OSError: - echo "Unable to delete: " & f - quit(1) - createDir(gOutput) + setOutputDir(gConfig["n.global"]["output"]) if gConfig["n.global"].hasKey("cpp_compiler"): gCppCompiler = gConfig["n.global"]["cpp_compiler"] @@ -278,3 +281,106 @@ proc runCfg*(cfg: string) = gitReset() elif key == "execute": discard execProc(postVal) + +let gHelp = """ +Nimgen is a helper for c2nim to simplify and automate the wrapping of C libraries + +Usage: + nimgen [options] file.cfg|file.h ... + +Params: + -C add compile entry * + -E add n.exclude entry * + -F set c2nim flags * + -I add n.include dir * + -O set output directory + -P set preprocessor flags * + +Options: + -c set ctags = true + -d set defines = true + -i set inline = true + -n set noprocess = true + -p set preprocess = true + -r set recurse = true + +Editing: + -a append string * + -e prepend string * + -l replace string * + -o#lines comment X lines * + -s search string * + -x regex search string * + +* supports multiple instances +""" + +proc runCli*() = + var + cfg = newOrderedTable[string, string]() + files: seq[string] + uniq = 1 + + gProjectDir = getCurrentDir().sanitizePath + for param in commandLineParams(): + let flag = if param.len() <= 2: param else: param[0..<2] + + if fileExists(param): + if param.splitFile().ext.toLowerAscii() == ".cfg": + runCfg(param) + else: + files.add(param) + + elif flag == "-C": + cfg["compile." & $uniq] = param[2..^1] + elif flag == "-E": + gExcludes.add(param[2..^1].addEnv().sanitizePath) + elif flag == "-F": + if cfg.hasKey("flags"): + cfg["flags"] = cfg["flags"] & " " & param[2..^1] + else: + cfg["flags"] = param[2..^1] + elif flag == "-I": + gIncludes.add(param[2..^1].addEnv().sanitizePath) + elif flag == "-O": + setOutputDir(param[2..^1]) + elif flag == "-P": + if cfg.hasKey("ppflags"): + cfg["ppflags"] = cfg["ppflags"] & " " & param[2..^1] + else: + cfg["ppflags"] = param[2..^1] + + elif flag == "-c": + cfg["ctags"] = "true" + elif flag == "-d": + cfg["defines"] = "true" + elif flag == "-i": + cfg["inline"] = "true" + elif flag == "-n": + cfg["noprocess"] = "true" + elif flag == "-p": + cfg["preprocess"] = "true" + elif flag == "-r": + cfg["recurse"] = "true" + + elif flag == "-a": + cfg["append." & $uniq] = param[2..^1] + elif flag == "-e": + cfg["prepend." & $uniq] = param[2..^1] + elif flag == "-l": + cfg["replace." & $uniq] = param[2..^1] + elif flag == "-o": + cfg["comment." & $uniq] = param[2..^1] + elif flag == "-s": + cfg["search." & $uniq] = param[2..^1] + elif flag == "-x": + cfg["regex." & $uniq] = param[2..^1] + + elif param == "-h" or param == "-?" or param == "--help": + echo gHelp + quit(0) + + uniq += 1 + + for file in files: + runFile(file, cfg) \ No newline at end of file diff --git a/tests/unittests/testfileops.nim b/tests/unittests/testfileops.nim index 076c586..2d24d4e 100644 --- a/tests/unittests/testfileops.nim +++ b/tests/unittests/testfileops.nim @@ -97,8 +97,6 @@ suite "test file ops": dataDir.createDir() setup: - if testfilename.existsFile(): - removeFile(testfilename) writeFile(testfilename, testFileContent) ################### Prepend ####################### From bb2c9474facddbefa5b1bc4c4fa176e127aa70be Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 14 Aug 2018 11:18:31 -0500 Subject: [PATCH 103/174] Add badges --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index c023d6b..d8465cb 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +[![Chat on Gitter](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/nimgen/Lobby) +[![Build status](https://ci.appveyor.com/api/projects/status/05t5ja88lmv1rt3r/branch/master?svg=true)](https://ci.appveyor.com/project/genotrance/nimgen/branch/master) + Nimgen is a helper for [c2nim](https://github.com/nim-lang/c2nim/) to simplify and automate the wrapping of C libraries. Nimgen can be used to automate the process of manipulating C files so that c2nim can be run on them without issues. This includes adding/removing code snippets, removal of complex preprocessor definitions that c2nim doesn't yet comprehend and recursively running on #include files. From 1fb8674d07220868d9fa3015e7d7c31c593a226f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 14 Aug 2018 11:19:32 -0500 Subject: [PATCH 104/174] Add badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d8465cb..ca76311 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Chat on Gitter](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/nimgen/Lobby) -[![Build status](https://ci.appveyor.com/api/projects/status/05t5ja88lmv1rt3r/branch/master?svg=true)](https://ci.appveyor.com/project/genotrance/nimgen/branch/master) +[![Build status](https://ci.appveyor.com/api/projects/status/05t5ja88lmv1rt3r?svg=true)](https://ci.appveyor.com/project/genotrance/nimgen) Nimgen is a helper for [c2nim](https://github.com/nim-lang/c2nim/) to simplify and automate the wrapping of C libraries. From 1a29b9f0f8fc08e2d6eba283026320d1aef04cc2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 14 Aug 2018 11:21:51 -0500 Subject: [PATCH 105/174] Add badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ca76311..d8465cb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Chat on Gitter](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/nimgen/Lobby) -[![Build status](https://ci.appveyor.com/api/projects/status/05t5ja88lmv1rt3r?svg=true)](https://ci.appveyor.com/project/genotrance/nimgen) +[![Build status](https://ci.appveyor.com/api/projects/status/05t5ja88lmv1rt3r/branch/master?svg=true)](https://ci.appveyor.com/project/genotrance/nimgen/branch/master) Nimgen is a helper for [c2nim](https://github.com/nim-lang/c2nim/) to simplify and automate the wrapping of C libraries. From 1e26d3e6eb9ab1036d4992476b71ac02616af4ed Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Aug 2018 14:41:41 -0500 Subject: [PATCH 106/174] More flexibility in compile flag --- README.md | 2 +- src/nimgen/c2nim.nim | 6 +++--- src/nimgen/gencore.nim | 19 +++++++++++++++---- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d8465cb..66db546 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ The following keys apply to library source code and help with generating the .ni Multiple entries for the all following keys are possible by appending any .string to the key. E.g. dynlib.win, compile.dir -```compile``` = file or dir of files of source code to {.compile.} into generated .nim +```compile``` = file or dir of files of source code to {.compile.} into generated .nim. If directory, picks *.c if C mode and *.cxx, *.cpp, *.cc, *.c++ and *.C for cpp mode. Dir can also include wildcards. e.g. compile = """dir/A*.cxx""" ```pragma``` = pragmas to define in generated .nim file. E.g. pragma = "passL: \"-lssl\"" => {.passL: "-lssl".} diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 05954e6..e887b29 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -29,7 +29,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = outlib = "" outpragma = "" - passC = "import ospaths, strutils\n" + passC = "import strutils\n" passC &= """const sourcePath = currentSourcePath().split({'\\', '/'})[0..^2].join("/")""" & "\n" @@ -102,9 +102,9 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = for cpl in c2nimConfig.compile: let fcpl = search(cpl) if getFileInfo(fcpl).kind == pcFile: - prepend(outfile, compile(file=fcpl)) + prepend(outfile, compile(c2nimConfig.flags, file=fcpl)) else: - prepend(outfile, compile(dir=fcpl)) + prepend(outfile, compile(c2nimConfig.flags, dir=fcpl)) # Add any pragmas if outpragma != "": diff --git a/src/nimgen/gencore.nim b/src/nimgen/gencore.nim index 4e79d04..42b82c0 100644 --- a/src/nimgen/gencore.nim +++ b/src/nimgen/gencore.nim @@ -17,15 +17,26 @@ proc addEnv*(str: string): string = return newStr -proc compile*(dir="", file=""): string = +proc compile*(flags: string, dir="", file=""): string = + var data = "" + proc fcompile(file: string): string = return "{.compile: \"$#\".}" % file.replace("\\", "/") - var data = "" - if dir != "" and dirExists(dir): - for f in walkFiles(dir / "*.c"): + proc dcompile(dir: string) = + for f in walkFiles(dir): data &= fcompile(f) & "\n" + if dir != "": + if dir.contains("*") or dir.contains("?"): + dcompile(dir) + elif dirExists(dir): + if flags.contains("cpp"): + for i in @["*.C", "*.cpp", "*.c++", "*.cc", "*.cxx"]: + dcompile(dir / i) + else: + dcompile(dir / "*.c") + if file != "" and fileExists(file): data &= fcompile(file) & "\n" From 7e21b2799eafaad9662bf2d754e94d25eea25100 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Aug 2018 15:19:59 -0500 Subject: [PATCH 107/174] Improve compile --- src/nimgen/c2nim.nim | 6 +----- src/nimgen/gencore.nim | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index e887b29..30f23f0 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -100,11 +100,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = # Include {.compile.} directives for cpl in c2nimConfig.compile: - let fcpl = search(cpl) - if getFileInfo(fcpl).kind == pcFile: - prepend(outfile, compile(c2nimConfig.flags, file=fcpl)) - else: - prepend(outfile, compile(c2nimConfig.flags, dir=fcpl)) + prepend(outfile, compile(cpl, c2nimConfig.flags)) # Add any pragmas if outpragma != "": diff --git a/src/nimgen/gencore.nim b/src/nimgen/gencore.nim index 42b82c0..000651e 100644 --- a/src/nimgen/gencore.nim +++ b/src/nimgen/gencore.nim @@ -17,7 +17,7 @@ proc addEnv*(str: string): string = return newStr -proc compile*(flags: string, dir="", file=""): string = +proc compile*(cpl, flags: string): string = var data = "" proc fcompile(file: string): string = @@ -27,18 +27,18 @@ proc compile*(flags: string, dir="", file=""): string = for f in walkFiles(dir): data &= fcompile(f) & "\n" - if dir != "": - if dir.contains("*") or dir.contains("?"): - dcompile(dir) - elif dirExists(dir): + if cpl.contains("*") or cpl.contains("?"): + dcompile(cpl) + else: + let fcpl = search(cpl) + if getFileInfo(fcpl).kind == pcFile: + data &= fcompile(fcpl) & "\n" + elif getFileInfo(fcpl).kind == pcDir: if flags.contains("cpp"): for i in @["*.C", "*.cpp", "*.c++", "*.cc", "*.cxx"]: - dcompile(dir / i) + dcompile(fcpl / i) else: - dcompile(dir / "*.c") - - if file != "" and fileExists(file): - data &= fcompile(file) & "\n" + dcompile(fcpl / "*.c") return data From 1711052eae985e042ab22108f0975d9bb0906abf Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 19 Aug 2018 12:31:21 -0500 Subject: [PATCH 108/174] Add move capability, handle compile duplicate names --- README.md | 2 ++ nimgen.nimble | 2 +- src/nimgen/fileops.nim | 23 +++++++++++++++++------ src/nimgen/gencore.nim | 14 +++++++++++++- src/nimgen/globals.nim | 1 + src/nimgen/runcfg.nim | 11 ++++++++--- 6 files changed, 42 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 66db546..7cb0f1c 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,8 @@ The following keys apply to library source code (before processing) and generate ```replace``` = string value to replace search string in file. Regex captures can be referred to using $1, $2, etc. +```move``` = search string providing context for location to move the results of a preceding search or regex match + ```comment``` = number of lines to comment from search location The following key only applies before processing and allows renaming the generated .nim files as required to enable successful wrapping. This may be for organizational purposes or to prevent usage of non-nim supported strings in module names (E.g. first letter is a number). Destination is relative to output directory if defined. diff --git a/nimgen.nimble b/nimgen.nimble index 841cb09..6ca5b9e 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.3.1" +version = "0.4.0" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" diff --git a/src/nimgen/fileops.nim b/src/nimgen/fileops.nim index 4e8f858..e416cf1 100644 --- a/src/nimgen/fileops.nim +++ b/src/nimgen/fileops.nim @@ -31,15 +31,26 @@ proc append*(file: string, data: string, search="") = if idx != -1: content = content[0.. Date: Sun, 19 Aug 2018 17:05:19 -0500 Subject: [PATCH 109/174] Fix nil rope --- src/nimgen/external.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 571c7e0..5abc1c6 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -111,7 +111,7 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = # Include content only from file var - rdata: Rope + rdata = rope("") start = false sfile = file.sanitizePath From 864506afbc431ddb62fad36bda38361cfcfe9801 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 19 Aug 2018 19:25:04 -0500 Subject: [PATCH 110/174] Remove ropes dependency --- src/nimgen/external.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 5abc1c6..651ccf3 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -1,4 +1,4 @@ -import os, osproc, regex, ropes, streams, strutils +import os, osproc, regex, streams, strutils import globals @@ -111,7 +111,7 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = # Include content only from file var - rdata = rope("") + rdata: seq[string] = @[] start = false sfile = file.sanitizePath @@ -131,9 +131,9 @@ proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = line.multiReplace([("_Noreturn", ""), ("(())", ""), ("WINAPI", ""), ("__attribute__", ""), ("extern \"C\"", "")]) .replace(re"\(\([_a-z]+?\)\)", "") - .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";") & "\n" + .replace(re"\(\(__format__[\s]*\(__[gnu_]*printf__, [\d]+, [\d]+\)\)\);", ";") ) - return $rdata + return rdata.join("\n") proc runCtags*(file: string): string = var From a2a2c41704beb49e69e28188b1f58283aab70bc7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 5 Sep 2018 13:18:45 -0500 Subject: [PATCH 111/174] Add nimtess2 to test suite --- README.md | 2 ++ tests/nimgentest.nims | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7cb0f1c..a8e751c 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ To see examples of nimgen in action check out the following wrappers:- * git checkout * [nimssl](https://github.com/genotrance/nimssl) - OpenSSL wrapper: [docs](http://nimgen.genotrance.com/nimssl) * git sparse checkout + * [nimtess2](https://github.com/genotrance/nimtess2) - libtess2 wrapper: [docs](http://nimgen.genotrance.com/nimtess2) + * git checkout * [libsvm](https://github.com/genotrance/libsvm) - libsvm wrapper: [docs](http://nimgen.genotrance.com/libsvm) * git sparse checkout diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 9c3e2d7..44c23c6 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -3,7 +3,7 @@ import distros, ospaths, strutils var full = true comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", - "nimfuzz", "nimpcre", "nimrax", "nimssl", "nimssh2"] + "nimfuzz", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2"] if detectOs(Windows): comps.add("nimkerberos") From 7082f897812634362a410dbe2d3d3790fd61fc89 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 5 Sep 2018 14:32:19 -0500 Subject: [PATCH 112/174] Add duktape-nim --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a8e751c..8cc8b04 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,8 @@ To see examples of nimgen in action check out the following wrappers:- * git checkout * [libsvm](https://github.com/genotrance/libsvm) - libsvm wrapper: [docs](http://nimgen.genotrance.com/libsvm) * git sparse checkout + * [duktape-nim](https://github.com/manguluka/duktape-nim) - Duktape wrapper + * git checkout * Compile in as static binary * [nimssh2](https://github.com/genotrance/nimssh2) - libssh2 wrapper: [docs](http://nimgen.genotrance.com/nimssh2) From 23bb86803db4a4b9d0481a7804a945ce591002a4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 5 Sep 2018 14:33:35 -0500 Subject: [PATCH 113/174] Add duktape-nim --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8cc8b04..be88f9e 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ To see examples of nimgen in action check out the following wrappers:- * [libsvm](https://github.com/genotrance/libsvm) - libsvm wrapper: [docs](http://nimgen.genotrance.com/libsvm) * git sparse checkout * [duktape-nim](https://github.com/manguluka/duktape-nim) - Duktape wrapper - * git checkout + * static * Compile in as static binary * [nimssh2](https://github.com/genotrance/nimssh2) - libssh2 wrapper: [docs](http://nimgen.genotrance.com/nimssh2) From 8eaa76560191901dc64e4d8aa4c7d7f3779e3e71 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 26 Sep 2018 16:44:28 -0500 Subject: [PATCH 114/174] Add nimnuklear and nimfastText --- README.md | 6 +++++- src/nimgen.nim | 2 +- tests/nimgentest.nims | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index be88f9e..121c766 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ To see examples of nimgen in action check out the following wrappers:- * [nim-clblast](https://github.com/numforge/nim-clblast) - OpenCL BLAS wrapper * static -* Compile C code into binary +* Compile C/C++ code into binary * [nim7z](https://github.com/genotrance/nim7z) - 7z decoder wrapper: [docs](http://nimgen.genotrance.com/nim7z) * git sparse checkout * [nimarchive](https://github.com/genotrance/nimarchive) - libarchive wrapper: [docs](http://nimgen.genotrance.com/nimarchive) @@ -51,10 +51,14 @@ To see examples of nimgen in action check out the following wrappers:- * git checkout * [nimclipboard](https://github.com/genotrance/nimclipboard) - libclipboard wrapper: [docs](http://nimgen.genotrance.com/nimclipboard) * git checkout + * [nimfastText](https://github.com/genotrance/nimfastText) - fastText wrapper: [docs](http://nimgen.genotrance.com/nimfastText) + * git sparse checkout * [nimfuzz](https://github.com/genotrance/nimfuzz) - fts_fuzzy_match wrapper: [docs](http://nimgen.genotrance.com/nimfuzz) * download header file * [nimkerberos](https://github.com/genotrance/nimkerberos) - WinKerberos wrapper: [docs](http://nimgen.genotrance.com/nimkerberos) * git sparse checkout + * [nimnuklear](https://github.com/genotrance/nimnuklear) - nuklear wrapper: [docs](https://nimgen.genotrance.com/nimnuklear) + * git sparse checkout * [nimpcre](https://github.com/genotrance/nimpcre) - PCRE wrapper: [docs](http://nimgen.genotrance.com/nimpcre) * git checkout * [nimrax](https://github.com/genotrance/nimrax) - Radix tree wrapper: [docs](http://nimgen.genotrance.com/nimrax) diff --git a/src/nimgen.nim b/src/nimgen.nim index 69f994e..dc98ae4 100644 --- a/src/nimgen.nim +++ b/src/nimgen.nim @@ -2,4 +2,4 @@ import os import nimgen/runcfg -runCli() \ No newline at end of file +runCli() diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 44c23c6..03c8dc9 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,8 +2,8 @@ import distros, ospaths, strutils var full = true - comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", - "nimfuzz", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2"] + comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfastText", + "nimfuzz", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2"] if detectOs(Windows): comps.add("nimkerberos") From 598214c1d2fe9aa68e584de05966173ada1edf78 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 27 Sep 2018 18:35:11 -0500 Subject: [PATCH 115/174] Save project artifacts --- appveyor.yml | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 6b5f682..0b7c921 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -31,7 +31,7 @@ for: BASE_DIR: c:\projects install: - - CD %BASE_DIR% + - CD c:\ - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - 7z x -y "%MINGW_ARCHIVE%"> nul - IF not exist "%LIBSSH2_ARCHIVE%" appveyor DownloadFile "%LIBSSH2_URL%" -FileName "%LIBSSH2_ARCHIVE%" @@ -42,7 +42,7 @@ for: - 7z x -y "%LIBCRYPTO_ARCHIVE2%"> nul - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - 7z x -y "%NIM_ARCHIVE%"> nul - - SET PATH=%BASE_DIR%\%MINGW_DIR%\bin;%BASE_DIR%\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% + - SET PATH=c:\%MINGW_DIR%\bin;c:\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - CD %BASE_DIR%\nimgen # - git clone --depth 1 https://github.com/nim-lang/nim # - cd nim @@ -59,16 +59,18 @@ for: # - cd .. on_finish: - - 7z a -r buildlogs-win.zip %USERPROFILE%\.nimble\pkgs - - appveyor PushArtifact buildlogs-win.zip + - 7z a -r buildlogs-win-pkgs.zip %USERPROFILE%\.nimble\pkgs + - appveyor PushArtifact buildlogs-win-pkgs.zip + - 7z a -r buildlogs-win-projects.zip %BASE_DIR% + - appveyor PushArtifact buildlogs-win-projects.zip - 7z a -r nimgen-docs.zip %BASE_DIR%\nimgen\web - appveyor PushArtifact nimgen-docs.zip cache: - - c:\projects\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - - c:\projects\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz - - c:\projects\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - - c:\projects\nim-0.18.0_x32.zip + - c:\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z + - c:\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz + - c:\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz + - c:\nim-0.18.0_x32.zip - matrix: @@ -84,7 +86,7 @@ for: install: - sudo apt -qq update - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev - - cd $BASE_DIR + - cd /home/appveyor - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - tar xJf $NIM_ARCHIVE - cd $NIM_VERSION @@ -92,15 +94,17 @@ for: - bin/nim c -d:release koch - ./koch boot -d:release - ./koch nimble -d:release - - export PATH=$BASE_DIR/$NIM_VERSION/bin:~/.nimble/bin:$PATH + - export PATH=/home/appveyor/$NIM_VERSION/bin:~/.nimble/bin:$PATH - cd $BASE_DIR/nimgen on_finish: - - zip -r -q buildlogs-lin.zip ~/.nimble/pkgs - - appveyor PushArtifact buildlogs-lin.zip + - zip -r -q buildlogs-lin-pkgs.zip ~/.nimble/pkgs + - appveyor PushArtifact buildlogs-lin-pkgs.zip + - zip -r -q buildlogs-lin-projects.zip $BASE_DIR + - appveyor PushArtifact buildlogs-lin-projects.zip cache: - - /home/appveyor/projects/nim-0.18.0.tar.xz + - /home/appveyor/nim-0.18.0.tar.xz build_script: - nimble install -y From 6c8bf5e8429e2d29eb909b1087a527480e413282 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 27 Sep 2018 19:06:16 -0500 Subject: [PATCH 116/174] Save artifacts --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 0b7c921..ce74b7f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -61,7 +61,7 @@ for: on_finish: - 7z a -r buildlogs-win-pkgs.zip %USERPROFILE%\.nimble\pkgs - appveyor PushArtifact buildlogs-win-pkgs.zip - - 7z a -r buildlogs-win-projects.zip %BASE_DIR% + - 7z a -r buildlogs-win-projects.zip c:\projects - appveyor PushArtifact buildlogs-win-projects.zip - 7z a -r nimgen-docs.zip %BASE_DIR%\nimgen\web - appveyor PushArtifact nimgen-docs.zip From 2f27ad0d0525a7d30ebf7fa067673e0e8c47a818 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 27 Sep 2018 19:27:22 -0500 Subject: [PATCH 117/174] Save artifacts --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index ce74b7f..c4f1d69 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -61,7 +61,7 @@ for: on_finish: - 7z a -r buildlogs-win-pkgs.zip %USERPROFILE%\.nimble\pkgs - appveyor PushArtifact buildlogs-win-pkgs.zip - - 7z a -r buildlogs-win-projects.zip c:\projects + - 7z a -r buildlogs-win-projects.zip c:\projects\* - appveyor PushArtifact buildlogs-win-projects.zip - 7z a -r nimgen-docs.zip %BASE_DIR%\nimgen\web - appveyor PushArtifact nimgen-docs.zip From 0a32e7fa2ccea82eed74d33a4e4bdc90605193be Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 2 Oct 2018 12:24:28 -0500 Subject: [PATCH 118/174] Add nimmonocrypt, remove nimfastText --- README.md | 2 ++ tests/nimgentest.nims | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 121c766..0ee6c9f 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ To see examples of nimgen in action check out the following wrappers:- * download header file * [nimkerberos](https://github.com/genotrance/nimkerberos) - WinKerberos wrapper: [docs](http://nimgen.genotrance.com/nimkerberos) * git sparse checkout + * [nimmonocypher](https://github.com/genotrance/nimmonocypher) - monocypher wrapper: [docs](http://nimgen.genotrance.com/nimmonocypher) + * git sparse checkout * [nimnuklear](https://github.com/genotrance/nimnuklear) - nuklear wrapper: [docs](https://nimgen.genotrance.com/nimnuklear) * git sparse checkout * [nimpcre](https://github.com/genotrance/nimpcre) - PCRE wrapper: [docs](http://nimgen.genotrance.com/nimpcre) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 03c8dc9..03b3666 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,8 +2,8 @@ import distros, ospaths, strutils var full = true - comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfastText", - "nimfuzz", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2"] + comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", # "nimfastText", + "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2"] if detectOs(Windows): comps.add("nimkerberos") From 6b16a83572a5a3fc4528cce630c7a4dbe9cd934a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 4 Oct 2018 17:10:43 -0500 Subject: [PATCH 119/174] Edit comps --- tests/nimgentest.nims | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 03b3666..f7e5aee 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,11 +2,15 @@ import distros, ospaths, strutils var full = true - comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", # "nimfastText", - "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2"] + comps = @["libsvm", "nim7z", "nimarchive", "nimbass", #"nimbigwig", + "nimclipboard", "nimfastText", "nimfuzz", "nimmonocypher", + "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", + "nimtess2" + ] if detectOs(Windows): comps.add("nimkerberos") + comps.add("nimzbar") if paramCount() > 2: for i in 3 .. paramCount(): From 4df59320fd43e111a952ae5be4f193bef5252ff8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 4 Oct 2018 21:17:02 -0500 Subject: [PATCH 120/174] Comp fixes --- tests/nimgentest.nims | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index f7e5aee..25eeaaf 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,15 +2,14 @@ import distros, ospaths, strutils var full = true - comps = @["libsvm", "nim7z", "nimarchive", "nimbass", #"nimbigwig", + comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfastText", "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", - "nimtess2" + "nimtess2", "nimzbar" ] if detectOs(Windows): comps.add("nimkerberos") - comps.add("nimzbar") if paramCount() > 2: for i in 3 .. paramCount(): From e04b40d6a88339f60a7099e1f9c46edc70446ea1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 5 Oct 2018 10:47:36 -0500 Subject: [PATCH 121/174] Remove nimfastText --- tests/nimgentest.nims | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 25eeaaf..364cbdd 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -3,7 +3,8 @@ import distros, ospaths, strutils var full = true comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", - "nimclipboard", "nimfastText", "nimfuzz", "nimmonocypher", + "nimclipboard", #"nimfastText", + "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2", "nimzbar" ] From efc04c38cc0b2decff014e084b7854cecc84cb0c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 12:08:38 -0500 Subject: [PATCH 122/174] Multi-version CI --- appveyor.yml | 93 ++++++++++++++++++++++--------------------- tests/nimgentest.nims | 7 +++- 2 files changed, 52 insertions(+), 48 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index c4f1d69..1673432 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,36 +14,39 @@ for: - image: Visual Studio 2017 environment: - MINGW_DIR: mingw32 - MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf/i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z/download + ARCH: 32 + MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf MINGW_ARCHIVE: i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - NIM_URL: https://nim-lang.org/download/nim-0.18.0_x32.zip - NIM_ARCHIVE: nim-0.18.0_x32.zip - NIM_VERSION: nim-0.18.0 - LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz - LIBSSH2_ARCHIVE2: mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar - LIBSSH2_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz/download - LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - LIBCRYPTO_ARCHIVE2: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar - LIBCRYPTO_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686/mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz/download - DLLS_URL: http://nim-lang.org/download/dlls.zip - DLLS_ARCHIVE: dlls.zip - BASE_DIR: c:\projects + SFNET_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686 + LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg + LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg + matrix: + - NIM_VERSION: 0.18.0 + - NIM_VERSION: 0.19.0 install: - CD c:\ - - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%" - - 7z x -y "%MINGW_ARCHIVE%"> nul - - IF not exist "%LIBSSH2_ARCHIVE%" appveyor DownloadFile "%LIBSSH2_URL%" -FileName "%LIBSSH2_ARCHIVE%" - - 7z x -y "%LIBSSH2_ARCHIVE%"> nul - - 7z x -y "%LIBSSH2_ARCHIVE2%"> nul - - IF not exist "%LIBCRYPTO_ARCHIVE%" appveyor DownloadFile "%LIBCRYPTO_URL%" -FileName "%LIBCRYPTO_ARCHIVE%" - - 7z x -y "%LIBCRYPTO_ARCHIVE%"> nul - - 7z x -y "%LIBCRYPTO_ARCHIVE2%"> nul - - IF not exist "%NIM_ARCHIVE%" appveyor DownloadFile "%NIM_URL%" -FileName "%NIM_ARCHIVE%" - - 7z x -y "%NIM_ARCHIVE%"> nul - - SET PATH=c:\%MINGW_DIR%\bin;c:\%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - - CD %BASE_DIR%\nimgen + - IF not exist "binaries" ( + MKDIR binaries && + CD binaries && + appveyor DownloadFile "%MINGW_URL%/%MINGW_ARCHIVE%/download" -FileName "%MINGW_ARCHIVE%" && + 7z x -y "%MINGW_ARCHIVE%"> nul && + del "%MINGW_ARCHIVE%" && + appveyor DownloadFile "%SFNET_URL%/%LIBSSH2_ARCHIVE%.tar.xz/download" -FileName "%LIBSSH2_ARCHIVE%.tar.xz" && + 7z x -y "%LIBSSH2_ARCHIVE%.tar.xz"> nul && + del "%LIBSSH2_ARCHIVE%.tar.xz" && + 7z x -y "%LIBSSH2_ARCHIVE%.tar"> nul && + del "%LIBSSH2_ARCHIVE%.tar" && + appveyor DownloadFile "%SFNET_URL%/%LIBCRYPTO_ARCHIVE%.tar.xz/download" -FileName "%LIBCRYPTO_ARCHIVE%.tar.xz" && + 7z x -y "%LIBCRYPTO_ARCHIVE%.tar.xz"> nul && + del "%LIBCRYPTO_ARCHIVE%.tar.xz" + 7z x -y "%LIBCRYPTO_ARCHIVE2%.tar"> nul && + del "%LIBCRYPTO_ARCHIVE%.tar" + appveyor DownloadFile "https://nim-lang.org/download/nim-%NIM_VERSION%_x%ARCH%.zip" -FileName "nim-%NIM_VERSION%_x%ARCH%.zip" && + 7z x -y "nim-%NIM_VERSION%_x%ARCH%.zip"> nul && + del "nim-%NIM_VERSION%_x%ARCH%.zip") + - SET PATH=c:\binaries\mingw%ARCH%\bin;c:\nim-%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% + - CD c:\projects\nimgen # - git clone --depth 1 https://github.com/nim-lang/nim # - cd nim # - git clone --depth 1 https://github.com/nim-lang/csources @@ -67,10 +70,7 @@ for: - appveyor PushArtifact nimgen-docs.zip cache: - - c:\i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z - - c:\mingw-w64-i686-libssh2-1.8.0-1-any.pkg.tar.xz - - c:\mingw-w64-i686-openssl-1.0.2.o-1-any.pkg.tar.xz - - c:\nim-0.18.0_x32.zip + - c:\binaries - matrix: @@ -78,33 +78,34 @@ for: - image: Ubuntu environment: - NIM_URL: https://nim-lang.org/download/nim-0.18.0.tar.xz - NIM_ARCHIVE: nim-0.18.0.tar.xz - NIM_VERSION: nim-0.18.0 - BASE_DIR: /home/appveyor/projects + matrix: + - NIM_VERSION: 0.18.0 + - NIM_VERSION: 0.19.0 install: - sudo apt -qq update - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev - - cd /home/appveyor - - if [ ! -e $NIM_ARCHIVE ]; then curl -s -o $NIM_ARCHIVE $NIM_URL; fi - - tar xJf $NIM_ARCHIVE - - cd $NIM_VERSION - - sh build.sh - - bin/nim c -d:release koch - - ./koch boot -d:release - - ./koch nimble -d:release - - export PATH=/home/appveyor/$NIM_VERSION/bin:~/.nimble/bin:$PATH - - cd $BASE_DIR/nimgen + - cd /home/appveyor/binaries + - if [ ! -e nim-$NIM_VERSION ]; then + curl -s -o nim-$nim_version.tar.xz https://nim-lang.org/download/nim-$nim_version.tar.xz && + tar xjf nim-$nim_version.tar.xz && + cd nim-$NIM_VERSION && + sh build.sh && + bin/nim c -d:release koch && + ./koch boot -d:release && + ./koch nimble -d:release; + fi + - export PATH=/home/appveyor/binaries/nim-$NIM_VERSION/bin:~/.nimble/bin:$PATH + - cd /home/appveyor/projects/nimgen on_finish: - zip -r -q buildlogs-lin-pkgs.zip ~/.nimble/pkgs - appveyor PushArtifact buildlogs-lin-pkgs.zip - - zip -r -q buildlogs-lin-projects.zip $BASE_DIR + - zip -r -q buildlogs-lin-projects.zip /home/appveyor/projects - appveyor PushArtifact buildlogs-lin-projects.zip cache: - - /home/appveyor/nim-0.18.0.tar.xz + - /home/appveyor/binaries build_script: - nimble install -y diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 364cbdd..134e94b 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -3,12 +3,15 @@ import distros, ospaths, strutils var full = true comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", - "nimclipboard", #"nimfastText", - "nimfuzz", "nimmonocypher", + "nimclipboard", "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2", "nimzbar" ] +var nimver = staticExec("nim -v").split("\n")[0].split(" ")[3] +if nimver >= "0.19.0": + comps.add("nimfastText") + if detectOs(Windows): comps.add("nimkerberos") From 83465b25de63f0a2bb5982c9aa17652c6071b1a6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 12:15:01 -0500 Subject: [PATCH 123/174] Multi-version CI --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 1673432..e9b7001 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -85,6 +85,7 @@ for: install: - sudo apt -qq update - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev + - mkdir /home/appveyor/binaries - cd /home/appveyor/binaries - if [ ! -e nim-$NIM_VERSION ]; then curl -s -o nim-$nim_version.tar.xz https://nim-lang.org/download/nim-$nim_version.tar.xz && From db26f9580d84758cc7bd71b5de95bfd7551e7a67 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 12:21:49 -0500 Subject: [PATCH 124/174] Multi-version CI --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e9b7001..af39ed8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -88,8 +88,8 @@ for: - mkdir /home/appveyor/binaries - cd /home/appveyor/binaries - if [ ! -e nim-$NIM_VERSION ]; then - curl -s -o nim-$nim_version.tar.xz https://nim-lang.org/download/nim-$nim_version.tar.xz && - tar xjf nim-$nim_version.tar.xz && + curl -s -o nim-$NIM_VERSION.tar.xz https://nim-lang.org/download/nim-$NIM_VERSION.tar.xz && + tar xJf nim-$nim_version.tar.xz && cd nim-$NIM_VERSION && sh build.sh && bin/nim c -d:release koch && From 35765f046b1a0b1785067d45fc128057cb7a2725 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 12:28:12 -0500 Subject: [PATCH 125/174] Multi-version CI --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index af39ed8..8daa767 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -89,7 +89,7 @@ for: - cd /home/appveyor/binaries - if [ ! -e nim-$NIM_VERSION ]; then curl -s -o nim-$NIM_VERSION.tar.xz https://nim-lang.org/download/nim-$NIM_VERSION.tar.xz && - tar xJf nim-$nim_version.tar.xz && + tar xJf nim-$NIM_VERSION.tar.xz && cd nim-$NIM_VERSION && sh build.sh && bin/nim c -d:release koch && From c4d45b318bbb9049cb856101fdb5a5f76b4b1779 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 17:17:33 -0500 Subject: [PATCH 126/174] Multi-version CI --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 8daa767..d80a59c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -85,10 +85,12 @@ for: install: - sudo apt -qq update - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev + - echo $NIM_VERSION - mkdir /home/appveyor/binaries - cd /home/appveyor/binaries - if [ ! -e nim-$NIM_VERSION ]; then curl -s -o nim-$NIM_VERSION.tar.xz https://nim-lang.org/download/nim-$NIM_VERSION.tar.xz && + ls -l && tar xJf nim-$NIM_VERSION.tar.xz && cd nim-$NIM_VERSION && sh build.sh && From b5030a8bd54198e9973385b98477c861f30c765c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 17:27:07 -0500 Subject: [PATCH 127/174] Multi-version CI --- appveyor.yml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d80a59c..52dee07 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,6 +7,11 @@ image: matrix: fast_finish: true +environment: + matrix: + - NIM_VERSION: 0.18.0 + - NIM_VERSION: 0.19.0 + for: - matrix: @@ -20,13 +25,11 @@ for: SFNET_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686 LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg - matrix: - - NIM_VERSION: 0.18.0 - - NIM_VERSION: 0.19.0 - install: + install: - CD c:\ - IF not exist "binaries" ( + echo $NIM_VERSION && MKDIR binaries && CD binaries && appveyor DownloadFile "%MINGW_URL%/%MINGW_ARCHIVE%/download" -FileName "%MINGW_ARCHIVE%" && @@ -77,20 +80,14 @@ for: only: - image: Ubuntu - environment: - matrix: - - NIM_VERSION: 0.18.0 - - NIM_VERSION: 0.19.0 - install: - sudo apt -qq update - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev - - echo $NIM_VERSION - - mkdir /home/appveyor/binaries - - cd /home/appveyor/binaries - - if [ ! -e nim-$NIM_VERSION ]; then + - if [ ! -e /home/appveyor/binaries ]; then + echo $NIM_VERSION && + mkdir /home/appveyor/binaries && + cd /home/appveyor/binaries && curl -s -o nim-$NIM_VERSION.tar.xz https://nim-lang.org/download/nim-$NIM_VERSION.tar.xz && - ls -l && tar xJf nim-$NIM_VERSION.tar.xz && cd nim-$NIM_VERSION && sh build.sh && From da865618be271f0cbf4fd8a66c03a74986f23b74 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 17:28:58 -0500 Subject: [PATCH 128/174] Multi-version CI --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 52dee07..a7337a1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -26,7 +26,7 @@ for: LIBSSH2_ARCHIVE: mingw-w64-i686-libssh2-1.8.0-1-any.pkg LIBCRYPTO_ARCHIVE: mingw-w64-i686-openssl-1.0.2.o-1-any.pkg - install: + install: - CD c:\ - IF not exist "binaries" ( echo $NIM_VERSION && From caba6fcc50629456ac06dbe306f765d617ced14c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 8 Oct 2018 17:54:43 -0500 Subject: [PATCH 129/174] Fix regex crash --- nimgen.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimgen.nimble b/nimgen.nimble index 6ca5b9e..cb85eec 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -11,7 +11,7 @@ skipDirs = @["nimgen", "tests"] # Dependencies -requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex >= 0.7.1" +requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex <= 0.7.4" task test, "Test nimgen": exec "nim c -r tests/rununittests.nim" From 1c057f2520dd912176d2559e0761b060029fd217 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Oct 2018 21:55:17 -0500 Subject: [PATCH 130/174] OS specific sections, execAction, nowildcard, multi git repos, branch/tag/commit --- README.md | 20 +++- appveyor.yml | 2 +- nimgen.nimble | 4 +- src/nimgen/c2nim.nim | 9 +- src/nimgen/external.nim | 57 ++++++++---- src/nimgen/file.nim | 3 +- src/nimgen/globals.nim | 26 ++++-- src/nimgen/runcfg.nim | 202 ++++++++++++++++++++++------------------ 8 files changed, 189 insertions(+), 134 deletions(-) diff --git a/README.md b/README.md index 0ee6c9f..153fe98 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,8 @@ In all sections below, environment variables are supported via Nim's string inte "${output}/library/include" "${MY_INCLUDE_PATH}/include" +Append -win, -lin and -mac/osx for OS specific sections and tasks. E.g. n.global-win, n.post-lin, download-win, execute-lin-mac.unique1. -unix can be used as a shortcut for -lin-mac. + _[n.global]_ ```output``` = name of the Nimble project once installed, also location to place generated .nim files @@ -119,17 +121,21 @@ List of all directories or files to exclude from all parsing. If an entry here m _[n.prepare]_ -The following keys can be used to prepare dependencies such as downloading ZIP files, cloning Git repositories, etc. Multiple entries are possible by appending any .string to the key. E.g. download.file1. -win, -lin and -mac/osx can be used for OS specific tasks. E.g. download-win, execute-lin,mac.unique1 +The following keys can be used to prepare dependencies such as downloading ZIP files, cloning Git repositories, etc. Multiple entries are possible by appending any .string to the key. E.g. download.file1. ```download``` = url to download to the output directory. ZIP files are automatically extracted. Files are not redownloaded if already present but re-extracted ```extract``` = ZIP file to extract in case they are local and don't need to be downloaded. Path is relative to output directory. -```git``` = url of Git repository to clone. Full repo is pulled so gitremote + gitsparse is preferable. Resets to HEAD if already present +```gitcheckout``` = branch, commit or tag of repository to checkout in following Git command, resets after each use. Use "-b name" for branches + +```gitoutput`` = directory for all following Git commands relative to `n.global:output` [default: `n.global:output` directory] + +```git``` = url of Git repository to clone. Full repo is pulled so gitremote + gitsparse is preferable. Resets if already present ```gitremote``` = url of Git repository to partially checkout. Use with gitsparse to pull only files and dirs of interest -```gitsparse``` = list of files and/or dirs to include in partial checkout, one per line. Resets to HEAD if already present +```gitsparse``` = list of files and/or dirs to include in partial checkout, one per line. Resets if already present ```execute``` = command to run during preparation @@ -139,7 +145,9 @@ _[n.post]_ This section is the same as the prepare section, but for performing actions after the project has been processed. -```reset``` = whether or not to perform a git reset on all files after processing [default: false] +```gitoutput`` = output directory for Git reset [default: `n.global:output` directory] + +```reset``` = perform a Git reset on all files after processing [default: false] ```execute``` = command to run after processing @@ -155,7 +163,7 @@ This section allows selection of multiple sourcefiles without requiring a detail _[sourcefile]_ -The following keys apply to library source code and help with generating the .nim files. -win, -lin and -osx can be used for OS specific tasks. E.g. dynlib-win, pragma-win +The following keys apply to library source code and help with generating the .nim files. -win, -lin and -osx can be used for OS specific tasks. E.g. dynlib-win, pragma-win. ```recurse``` = find #include files and process them [default: false] @@ -173,6 +181,8 @@ The following keys apply to library source code and help with generating the .ni ```noprocess``` = do not process this source file with c2nim [default: false] - this is useful if a file only needs to be manipulated +```nowildcard``` = ignore any wildcard definitions for this sourcefile + ```reset``` = reset the file back to original state after all processing [default: false] Multiple entries for the all following keys are possible by appending any .string to the key. E.g. dynlib.win, compile.dir diff --git a/appveyor.yml b/appveyor.yml index a7337a1..531866f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,7 +10,7 @@ matrix: environment: matrix: - NIM_VERSION: 0.18.0 - - NIM_VERSION: 0.19.0 + # - NIM_VERSION: 0.19.0 for: - diff --git a/nimgen.nimble b/nimgen.nimble index cb85eec..c99b7a6 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.4.0" +version = "0.5.0" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" @@ -11,7 +11,7 @@ skipDirs = @["nimgen", "tests"] # Dependencies -requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex <= 0.7.4" +requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex >= 0.10.0" task test, "Test nimgen": exec "nim c -r tests/rununittests.nim" diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 30f23f0..0464c6a 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -2,8 +2,13 @@ import os, ospaths, regex, strutils import external, file, fileops, gencore, globals -template relativePath(path: untyped): untyped = - path.multiReplace([(gOutput, ""), ("\\", "/"), ("//", "/")]) +proc relativePath(path: string): string = + if gOutput.len() == 0: + result = path + else: + # multiReplace() bug - #9557 + result = path.replace(gOutput, "") + return result.multiReplace([("\\", "/"), ("//", "/")]) proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = var file = search(fl) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 651ccf3..a34ea69 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -15,6 +15,16 @@ proc execProc*(cmd: string): string = echo result quit(1) +proc execAction*(cmd: string): string = + var ccmd = "" + when defined(Windows): + ccmd = "cmd /c " & cmd.replace("/", "\\") + when defined(Linux) or defined(MacOSX): + ccmd = "bash -c '" & cmd & "'" + + echo "Running '" & ccmd[0..min(64, len(ccmd))] & "'" + return execProc(ccmd) + proc extractZip*(zipfile: string) = var cmd = "unzip -o $#" if defined(Windows): @@ -44,13 +54,16 @@ proc downloadUrl*(url: string) = if ext == ".zip": extractZip(file) -proc gitReset*() = - echo "Resetting Git repo" - - setCurrentDir(gOutput) +template setGitDir() = + setCurrentDir(gGitOutput) defer: setCurrentDir(gProjectDir) - let cmd = "git reset --hard HEAD" +proc gitReset*() = + echo "Resetting " & gGitOutput + + setGitDir() + + let cmd = "git reset --hard" while execCmdEx(cmd)[0].contains("Permission denied"): sleep(1000) echo " Retrying ..." @@ -58,45 +71,51 @@ proc gitReset*() = proc gitCheckout*(file: string) = echo "Resetting " & file - setCurrentDir(gOutput) - defer: setCurrentDir(gProjectDir) + setGitDir() - let cmd = "git checkout $#" % file.replace(gOutput & "/", "") + let cmd = "git checkout $#" % file.replace(gGitOutput & "/", "") while execCmdEx(cmd)[0].contains("Permission denied"): sleep(500) echo " Retrying ..." +proc gitPull() = + if gGitCheckout.len() != 0: + echo "Checking out " & gGitCheckout + #discard execProc("git fetch --depth=1 origin " & gGitCheckout) + discard execProc("git pull --tags origin master") + discard execProc("git checkout " & gGitCheckout) + gGitCheckout = "" + else: + echo "Pulling repository" + discard execProc("git pull --depth=1 origin master") + proc gitRemotePull*(url: string, pull=true) = - if dirExists(gOutput/".git"): + if dirExists(gGitOutput/".git"): if pull: gitReset() return - setCurrentDir(gOutput) - defer: setCurrentDir(gProjectDir) + setGitDir() - echo "Setting up Git repo" + echo "Setting up Git repo: " & url discard execProc("git init .") discard execProc("git remote add origin " & url) if pull: - echo "Checking out artifacts" - discard execProc("git pull --depth=1 origin master") + gitPull() proc gitSparseCheckout*(plist: string) = let sparsefile = ".git/info/sparse-checkout" - if fileExists(gOutput/sparsefile): + if fileExists(gGitOutput/sparsefile): gitReset() return - setCurrentDir(gOutput) - defer: setCurrentDir(gProjectDir) + setGitDir() discard execProc("git config core.sparsecheckout true") writeFile(sparsefile, plist) - echo "Checking out artifacts" - discard execProc("git pull --depth=1 origin master") + gitPull() proc runPreprocess*(file, ppflags, flags: string, inline: bool): string = var diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index edf39ea..769c914 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -16,8 +16,7 @@ proc getNimout*(file: string, rename=true): string = if gRenames.hasKey(file): result = gRenames[file] - if not dirExists(parentDir(result)): - createDir(parentDir(result)) + createDir(parentDir(result)) proc exclude*(file: string): bool = for excl in gExcludes: diff --git a/src/nimgen/globals.nim b/src/nimgen/globals.nim index a83460f..9892b99 100644 --- a/src/nimgen/globals.nim +++ b/src/nimgen/globals.nim @@ -7,21 +7,27 @@ const defaultCppCompiler* = "g++" var - gDoneRecursive*: seq[string] = @[] - gDoneInline*: seq[string] = @[] - - gProjectDir* = "" + # Config gConfig*: Config - gFilter* = "" - gQuotes* = true - gCppCompiler* = getEnv(cppCompilerEnv, defaultCCompiler) - gCCompiler* = getEnv(cCompilerEnv, defaultCppCompiler) - gOutput* = "" - gIncludes*: seq[string] = @[] gExcludes*: seq[string] = @[] + gIncludes*: seq[string] = @[] gRenames* = initTable[string, string]() gWildcards* = newConfig() + + # n.global + gOutput* = "." + gQuotes* = true + gFilter* = "" + gCppCompiler* = getEnv(cppCompilerEnv, defaultCCompiler) + gCCompiler* = getEnv(cCompilerEnv, defaultCppCompiler) + + # State tracking + gGitCheckout* = "" + gGitOutput* = "" + gProjectDir* = "" gCompile*: seq[string] = @[] + gDoneInline*: seq[string] = @[] + gDoneRecursive*: seq[string] = @[] type c2nimConfigObj* = object diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index 5f04b04..f2310ff 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -6,18 +6,19 @@ proc `[]`*(table: OrderedTableRef[string, string], key: string): string = ## Gets table values with env vars inserted tables.`[]`(table, key).addEnv -proc getKey(ukey: string): tuple[key: string, val: bool] = - var kv = ukey.replace(re"\..*", "").split("-", 1) +proc getKey(ukey: string, section = false): tuple[key: string, val: bool] = + var kv = if not section: ukey.replace(re"\..*", "").split("-", 1) else: ukey.split("-", 1) if kv.len() == 1: kv.add("") if kv[1] == "": return (kv[0], true) - for ostyp in kv[1].split(","): + for ostyp in kv[1].split("-"): if (ostyp == "win" and defined(Windows)) or (ostyp == "lin" and defined(Linux)) or - ((ostyp == "osx" or ostyp == "mac") and defined(MacOSX)): + ((ostyp == "osx" or ostyp == "mac") and defined(MacOSX)) or + (ostyp == "unix" and (defined(Linux) or defined(MacOSX))): return (kv[0], true) return (kv[0], false) @@ -26,6 +27,7 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str var cfg = cfgin sfile = search(file) + nowildcard = false if sfile in gDoneRecursive: return @@ -34,13 +36,20 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str echo "Processing " & sfile gDoneRecursive.add(sfile) - for pattern in gWildcards.keys(): - var m: RegexMatch - let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") - if file.find(toPattern(pat), m): - echo " Appending keys for wildcard " & pattern - for key in gWildcards[pattern].keys(): - cfg[key & "." & pattern] = gWildcards[pattern][key] + for act in cfg.keys(): + let (action, val) = getKey(act) + if val == true and action == "nowildcard" and cfg[act] == "true": + nowildcard = true + break + + if not nowildcard: + for pattern in gWildcards.keys(): + var m: RegexMatch + let pat = pattern.replace(".", "\\.").replace("*", ".*").replace("?", ".?") + if file.find(toPattern(pat), m): + echo " Appending keys for wildcard " & pattern + for key in gWildcards[pattern].keys(): + cfg[key & "." & pattern] = gWildcards[pattern][key] var srch = "" @@ -200,92 +209,99 @@ proc runCfg*(cfg: string) = gProjectDir = parentDir(cfg.expandFilename()).sanitizePath gConfig = loadConfig(cfg) + gCppCompiler = getEnv(cppCompilerEnv, defaultCppCompiler).quoteShell + gCCompiler = getEnv(cCompilerEnv, defaultCCompiler).quoteShell + gGitOutput = gOutput - if gConfig.hasKey("n.global"): - if gConfig["n.global"].hasKey("output"): - setOutputDir(gConfig["n.global"]["output"]) - - if gConfig["n.global"].hasKey("cpp_compiler"): - gCppCompiler = gConfig["n.global"]["cpp_compiler"] - else: - # Reset on a per project basis - gCppCompiler = getEnv(cppCompilerEnv, defaultCppCompiler) - - if gConfig["n.global"].hasKey("c_compiler"): - gCCompiler = gConfig["n.global"]["c_compiler"] - else: - # Reset on a per project basis - gCCompiler = getEnv(cCompilerEnv, defaultCCompiler) - - gCppCompiler = gCppCompiler.quoteShell - gCCompiler = gCCompiler.quoteShell - - if gConfig["n.global"].hasKey("filter"): - gFilter = gConfig["n.global"]["filter"] - if gConfig["n.global"].hasKey("quotes"): - if gConfig["n.global"]["quotes"] == "false": - gQuotes = false - - if gConfig.hasKey("n.include"): - for inc in gConfig["n.include"].keys(): - gIncludes.add(inc.addEnv().sanitizePath) - - if gConfig.hasKey("n.exclude"): - for excl in gConfig["n.exclude"].keys(): - gExcludes.add(excl.addEnv().sanitizePath) - - if gConfig.hasKey("n.prepare"): - for prep in gConfig["n.prepare"].keys(): - let (key, val) = getKey(prep) - if val == true: - let prepVal = gConfig["n.prepare"][prep] - if key == "download": - downloadUrl(prepVal) - elif key == "extract": - extractZip(prepVal) - elif key == "git": - gitRemotePull(prepVal) - elif key == "gitremote": - gitRemotePull(prepVal, false) - elif key == "gitsparse": - gitSparseCheckout(prepVal) - elif key == "execute": - discard execProc(prepVal) - elif key == "copy": - doCopy(prepVal) - - if gConfig.hasKey("n.wildcard"): - var wildcard = "" - for wild in gConfig["n.wildcard"].keys(): - let (key, val) = getKey(wild) - if val == true: - if key == "wildcard": - wildcard = gConfig["n.wildcard"][wild] - else: - gWildcards.setSectionKey(wildcard, wild, - gConfig["n.wildcard"][wild]) - - for file in gConfig.keys(): - if file in @["n.global", "n.include", "n.exclude", - "n.prepare", "n.wildcard", "n.post"]: + for section in gConfig.keys(): + let (sname, sval) = getKey(section, true) + if not sval: continue - if file == "n.sourcefile": - for pattern in gConfig["n.sourcefile"].keys(): - for file in walkFiles(pattern.addEnv): - runFile(file) - else: - runFile(file, gConfig[file]) + case sname: + of "n.global": + for glob in gConfig[section].keys(): + let (key, val) = getKey(glob) + if val == true: + let globVal = gConfig[section][glob] + case key: + of "output": + setOutputDir(globVal) + of "cpp_compiler": + gCppCompiler = globVal.quoteShell + of "c_compiler": + gCCompiler = globVal.quoteShell + of "filter": + gFilter = globVal + of "quotes": + if globVal == "false": + gQuotes = false - if gConfig.hasKey("n.post"): - for post in gConfig["n.post"].keys(): - let (key, val) = getKey(post) - if val == true: - let postVal = gConfig["n.post"][post] - if key == "reset": - gitReset() - elif key == "execute": - discard execProc(postVal) + of "n.include": + for inc in gConfig[section].keys(): + gIncludes.add(inc.addEnv().sanitizePath) + + of "n.exclude": + for excl in gConfig[section].keys(): + gExcludes.add(excl.addEnv().sanitizePath) + + of "n.prepare": + for prep in gConfig[section].keys(): + let (key, val) = getKey(prep) + if val == true: + let prepVal = gConfig[section][prep] + case key: + of "download": + downloadUrl(prepVal) + of "extract": + extractZip(prepVal) + of "gitcheckout": + gGitCheckout = prepVal + of "gitoutput": + gGitOutput = gOutput/prepVal + createDir(gGitOutput) + of "git": + gitRemotePull(prepVal) + of "gitremote": + gitRemotePull(prepVal, false) + of "gitsparse": + gitSparseCheckout(prepVal) + of "execute": + discard execAction(prepVal) + of "copy": + doCopy(prepVal) + + of "n.wildcard": + var wildcard = "" + for wild in gConfig[section].keys(): + let (key, val) = getKey(wild) + if val == true: + if key == "wildcard": + wildcard = gConfig[section][wild] + else: + gWildcards.setSectionKey(wildcard, wild, + gConfig[section][wild]) + + of "n.sourcefile": + for pattern in gConfig[section].keys(): + for file in walkFiles(pattern.addEnv): + runFile(file) + + of "n.post": + for post in gConfig[section].keys(): + let (key, val) = getKey(post) + if val == true: + let postVal = gConfig[section][post] + case key: + of "gitoutput": + gGitOutput = gOutput/postVal + of "reset": + gitReset() + of "execute": + discard execAction(postVal) + + else: + runFile(section, gConfig[section]) let gHelp = """ Nimgen is a helper for c2nim to simplify and automate the wrapping of C libraries From 1f3ae34dc0fd9959cdd187a57cc4854cb9ca9b67 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Oct 2018 22:16:29 -0500 Subject: [PATCH 131/174] Fix gGitOutput set --- src/nimgen/external.nim | 1 - src/nimgen/runcfg.nim | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index a34ea69..8656147 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -81,7 +81,6 @@ proc gitCheckout*(file: string) = proc gitPull() = if gGitCheckout.len() != 0: echo "Checking out " & gGitCheckout - #discard execProc("git fetch --depth=1 origin " & gGitCheckout) discard execProc("git pull --tags origin master") discard execProc("git checkout " & gGitCheckout) gGitCheckout = "" diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index f2310ff..a924734 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -201,6 +201,8 @@ proc setOutputDir(dir: string) = quit(1) createDir(gOutput) + gGitOutput = gOutput + proc runCfg*(cfg: string) = if not fileExists(cfg): echo "Config doesn't exist: " & cfg From 9ec27678ea8acccce9089f4736defad97631713d Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Oct 2018 22:43:55 -0500 Subject: [PATCH 132/174] Appveyor bugfix --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 531866f..4df9fe9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -42,9 +42,9 @@ for: del "%LIBSSH2_ARCHIVE%.tar" && appveyor DownloadFile "%SFNET_URL%/%LIBCRYPTO_ARCHIVE%.tar.xz/download" -FileName "%LIBCRYPTO_ARCHIVE%.tar.xz" && 7z x -y "%LIBCRYPTO_ARCHIVE%.tar.xz"> nul && - del "%LIBCRYPTO_ARCHIVE%.tar.xz" + del "%LIBCRYPTO_ARCHIVE%.tar.xz" && 7z x -y "%LIBCRYPTO_ARCHIVE2%.tar"> nul && - del "%LIBCRYPTO_ARCHIVE%.tar" + del "%LIBCRYPTO_ARCHIVE%.tar" && appveyor DownloadFile "https://nim-lang.org/download/nim-%NIM_VERSION%_x%ARCH%.zip" -FileName "nim-%NIM_VERSION%_x%ARCH%.zip" && 7z x -y "nim-%NIM_VERSION%_x%ARCH%.zip"> nul && del "nim-%NIM_VERSION%_x%ARCH%.zip") From e8faaf8d0cf0619dca0d9ab0a53563cc66d572b1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Oct 2018 22:54:08 -0500 Subject: [PATCH 133/174] Appveyor bugfix --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 4df9fe9..2cfa477 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -43,7 +43,7 @@ for: appveyor DownloadFile "%SFNET_URL%/%LIBCRYPTO_ARCHIVE%.tar.xz/download" -FileName "%LIBCRYPTO_ARCHIVE%.tar.xz" && 7z x -y "%LIBCRYPTO_ARCHIVE%.tar.xz"> nul && del "%LIBCRYPTO_ARCHIVE%.tar.xz" && - 7z x -y "%LIBCRYPTO_ARCHIVE2%.tar"> nul && + 7z x -y "%LIBCRYPTO_ARCHIVE%.tar"> nul && del "%LIBCRYPTO_ARCHIVE%.tar" && appveyor DownloadFile "https://nim-lang.org/download/nim-%NIM_VERSION%_x%ARCH%.zip" -FileName "nim-%NIM_VERSION%_x%ARCH%.zip" && 7z x -y "nim-%NIM_VERSION%_x%ARCH%.zip"> nul && From ebd362d934af435bb6c329a08afd8ddbdb53e553 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Oct 2018 23:04:28 -0500 Subject: [PATCH 134/174] Appveyor bugfix --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 2cfa477..a9b8462 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -48,7 +48,7 @@ for: appveyor DownloadFile "https://nim-lang.org/download/nim-%NIM_VERSION%_x%ARCH%.zip" -FileName "nim-%NIM_VERSION%_x%ARCH%.zip" && 7z x -y "nim-%NIM_VERSION%_x%ARCH%.zip"> nul && del "nim-%NIM_VERSION%_x%ARCH%.zip") - - SET PATH=c:\binaries\mingw%ARCH%\bin;c:\nim-%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% + - SET PATH=c:\binaries\mingw%ARCH%\bin;c:\binaries\nim-%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - CD c:\projects\nimgen # - git clone --depth 1 https://github.com/nim-lang/nim # - cd nim From ad499177337fc8b3f682f5ea91648191300cfc56 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Oct 2018 23:25:43 -0500 Subject: [PATCH 135/174] Appveyor bugfix --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index a9b8462..5d3cd62 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -69,7 +69,7 @@ for: - appveyor PushArtifact buildlogs-win-pkgs.zip - 7z a -r buildlogs-win-projects.zip c:\projects\* - appveyor PushArtifact buildlogs-win-projects.zip - - 7z a -r nimgen-docs.zip %BASE_DIR%\nimgen\web + - 7z a -r nimgen-docs.zip c:\projects\nimgen\web - appveyor PushArtifact nimgen-docs.zip cache: From 8188c7511858c28259f0244056f886c25a8f0cc5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 1 Nov 2018 14:31:09 -0500 Subject: [PATCH 136/174] Enable 0.19.0 --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 5d3cd62..35c49fb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,8 @@ version: '{build}' image: - - Ubuntu - Visual Studio 2017 + - Ubuntu matrix: fast_finish: true @@ -10,7 +10,7 @@ matrix: environment: matrix: - NIM_VERSION: 0.18.0 - # - NIM_VERSION: 0.19.0 + - NIM_VERSION: 0.19.0 for: - From f1a8d178910e150746e6cc630cc0f08eb6e86a34 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 1 Nov 2018 15:37:56 -0500 Subject: [PATCH 137/174] Fix version print --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 35c49fb..a4d7c74 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -29,7 +29,7 @@ for: install: - CD c:\ - IF not exist "binaries" ( - echo $NIM_VERSION && + echo %NIM_VERSION% && MKDIR binaries && CD binaries && appveyor DownloadFile "%MINGW_URL%/%MINGW_ARCHIVE%/download" -FileName "%MINGW_ARCHIVE%" && From ee7a55f2ecf801d411f34e284ef531dffe22de6e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 10:38:38 -0500 Subject: [PATCH 138/174] Add Travis for devel/OSX --- .travis.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..9c56b1f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,22 @@ +sudo: false +language: c +os: + - linux + - osx +dist: trusty +before_script: + - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` + - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` + - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` + - curl -L --silent -o $TXZ https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ + - tar xf $TXZ + - cd nim-$VERSION + - sh build.sh + - bin/nim c koch + - ./koch boot -d:release + - ./koch nimble + - export PATH=$(pwd)/bin:$PATH + - cd .. +script: + - nimble install -y + - nimble tests From 713703caae174888a10b27144a967a936de02b21 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 10:55:05 -0500 Subject: [PATCH 139/174] Travis fixes --- .travis.yml | 12 ++++++++++-- README.md | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c56b1f..720d143 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,19 @@ os: - linux - osx dist: trusty +addons: + apt: + packages: + - python-pygments + - libssh2-1-dev + - libgcrypt20-dev + - libgpg-error-dev before_script: - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - - curl -L --silent -o $TXZ https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ + - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" + - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - tar xf $TXZ - cd nim-$VERSION - sh build.sh @@ -19,4 +27,4 @@ before_script: - cd .. script: - nimble install -y - - nimble tests + - nimble test diff --git a/README.md b/README.md index 153fe98..88bed4d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Chat on Gitter](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/nimgen/Lobby) [![Build status](https://ci.appveyor.com/api/projects/status/05t5ja88lmv1rt3r/branch/master?svg=true)](https://ci.appveyor.com/project/genotrance/nimgen/branch/master) +[![Build Status](https://travis-ci.org/genotrance/nimgen.svg?branch=master)](https://travis-ci.org/genotrance/nimgen) Nimgen is a helper for [c2nim](https://github.com/nim-lang/c2nim/) to simplify and automate the wrapping of C libraries. From c40919b87cf16c1066ea028914a828f6d1c04398 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 10:58:38 -0500 Subject: [PATCH 140/174] Fix deps --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 720d143..ad9de5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,6 @@ addons: packages: - python-pygments - libssh2-1-dev - - libgcrypt20-dev - - libgpg-error-dev before_script: - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` From 715b502274da13adb810891d7a493e4ee73ba8dc Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 11:25:48 -0500 Subject: [PATCH 141/174] Fix path --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ad9de5c..340e9a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ before_script: - bin/nim c koch - ./koch boot -d:release - ./koch nimble - - export PATH=$(pwd)/bin:$PATH + - export PATH=$(pwd)/bin:~/.nimble/bin:$PATH - cd .. script: - nimble install -y From 34efd6619ffdb4d52047d13ca05000a75941f605 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 14:14:15 -0500 Subject: [PATCH 142/174] Docgen only on Windows --- .travis.yml | 2 +- appveyor.yml | 2 +- tests/nimgentest.nims | 29 +++++++++++++++-------------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 340e9a3..9e2b6dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,12 @@ dist: trusty addons: apt: packages: - - python-pygments - libssh2-1-dev before_script: - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` + - gcc -v - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - tar xf $TXZ diff --git a/appveyor.yml b/appveyor.yml index a4d7c74..ff7f260 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -82,7 +82,7 @@ for: install: - sudo apt -qq update - - sudo apt -qq install --yes python-pygments libssh2-1-dev libgcrypt20-dev libgpg-error-dev + - sudo apt -qq install --yes libssh2-1-dev libgcrypt20-dev libgpg-error-dev - if [ ! -e /home/appveyor/binaries ]; then echo $NIM_VERSION && mkdir /home/appveyor/binaries && diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 134e94b..f3ab526 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -40,20 +40,21 @@ for comp in comps: exec "nimble install -y" exec "nimble test" - if dirExists("web"/comp): - rmDir("web"/comp) + when defined(windows): + if dirExists("web"/comp): + rmDir("web"/comp) - mkDir("web"/comp) - for file in listFiles(".."/comp/comp) & listFiles(".."/comp): - if file.splitFile().ext == ".nim": - cpFile(file, "web"/comp/extractFilename(file)) - - cpFile("web"/"nimdoc.cfg", "web"/comp/"nimdoc.cfg") - withDir("web"/comp): - for file in listFiles("."): + mkDir("web"/comp) + for file in listFiles(".."/comp/comp) & listFiles(".."/comp): if file.splitFile().ext == ".nim": - exec "nim doc --git.url:. --index:on -o:" & file.changeFileExt("html") & " " & file - exec "pygmentize -f html -O full,linenos=1,anchorlinenos=True,lineanchors=L,style=vs -o " & file & ".html " & file + cpFile(file, "web"/comp/extractFilename(file)) - exec "nim buildIndex -o:index.html ." - rmFile("web"/comp/"nimdoc.cfg") + cpFile("web"/"nimdoc.cfg", "web"/comp/"nimdoc.cfg") + withDir("web"/comp): + for file in listFiles("."): + if file.splitFile().ext == ".nim": + exec "nim doc --git.url:. --index:on -o:" & file.changeFileExt("html") & " " & file + exec "pygmentize -f html -O full,linenos=1,anchorlinenos=True,lineanchors=L,style=vs -o " & file & ".html " & file + + exec "nim buildIndex -o:index.html ." + rmFile("web"/comp/"nimdoc.cfg") From 5549505f61dc5f2a6b1da5cd3b12130dafe5bae8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 15:09:28 -0500 Subject: [PATCH 143/174] Cleanup --- .travis.yml | 6 +++--- appveyor.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e2b6dd..fc24973 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,10 +9,10 @@ addons: packages: - libssh2-1-dev before_script: - - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` + - export RELEASE=`curl --silent "https://api.github.com/repos/nim-lang/nightlies/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` + - export TXZ=`curl --silent "https://api.github.com/repos/nim-lang/nightlies/releases/latest" | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - - gcc -v + - curl --silent "https://api.github.com/repos/nim-lang/nightlies/releases/latest" - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - tar xf $TXZ diff --git a/appveyor.yml b/appveyor.yml index ff7f260..e4e062e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,8 @@ version: '{build}' image: - - Visual Studio 2017 - Ubuntu + - Visual Studio 2017 matrix: fast_finish: true From c7c419e6fad121dee205146f60dd451789aa6a5a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 17:12:37 -0500 Subject: [PATCH 144/174] Backrev c2nim --- nimgen.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimgen.nimble b/nimgen.nimble index c99b7a6..c6a2625 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -11,7 +11,7 @@ skipDirs = @["nimgen", "tests"] # Dependencies -requires "nim >= 0.17.0", "c2nim >= 0.9.13", "regex >= 0.10.0" +requires "nim >= 0.17.0", "c2nim#3ec45c24585ebaed", "regex >= 0.10.0" task test, "Test nimgen": exec "nim c -r tests/rununittests.nim" From d27d28f0d94bbd495cc912fae78300712c71a970 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 17:15:35 -0500 Subject: [PATCH 145/174] Fix travis again --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index fc24973..79889fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,10 +9,10 @@ addons: packages: - libssh2-1-dev before_script: - - export RELEASE=`curl --silent "https://api.github.com/repos/nim-lang/nightlies/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - - export TXZ=`curl --silent "https://api.github.com/repos/nim-lang/nightlies/releases/latest" | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` + - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` + - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - - curl --silent "https://api.github.com/repos/nim-lang/nightlies/releases/latest" + - curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - tar xf $TXZ From d080be55a548049b170c88a356724d505401d545 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 21:56:14 -0500 Subject: [PATCH 146/174] Github download --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 79889fb..ebf0bd4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,12 +9,11 @@ addons: packages: - libssh2-1-dev before_script: - - export RELEASE=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - - export TXZ=`curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` + - curl -u $TOKEN --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest -o latest.json + - export RELEASE=`echo latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` + - export TXZ=`echo latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - - curl --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - tar xf $TXZ - cd nim-$VERSION - sh build.sh From 03c38a244838809fb630431ce20b7183a6d316ca Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 22:04:09 -0500 Subject: [PATCH 147/174] Github download --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ebf0bd4..4687afe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,9 @@ addons: packages: - libssh2-1-dev before_script: - - curl -u $TOKEN --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest -o latest.json - - export RELEASE=`echo latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - - export TXZ=`echo latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` + - curl -u $TOKEN -o latest.json https://api.github.com/repos/nim-lang/nightlies/releases/latest + - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` + - export TXZ=`cat latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - tar xf $TXZ From d6c827bb281f24899a16aa60e0b5cd0a1c3c61dc Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 22:18:05 -0500 Subject: [PATCH 148/174] Github download --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4687afe..7b3241e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,9 @@ addons: packages: - libssh2-1-dev before_script: - - curl -u $TOKEN -o latest.json https://api.github.com/repos/nim-lang/nightlies/releases/latest + - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest + - curl -u $TOKEN https://api.github.com/repos/nim-lang/nightlies/releases/latest + - cat latest.json - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - export TXZ=`cat latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` From 81fa6d15bae2f5529556d917706a50d8df26ea8d Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 22:26:46 -0500 Subject: [PATCH 149/174] Github download --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7b3241e..cddc6a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,12 +10,11 @@ addons: - libssh2-1-dev before_script: - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - - curl -u $TOKEN https://api.github.com/repos/nim-lang/nightlies/releases/latest - - cat latest.json - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - export TXZ=`cat latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" + - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - tar xf $TXZ - cd nim-$VERSION - sh build.sh From a8b836656ef1d3147d16931daa0ddf53e6a925fb Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 23:04:58 -0500 Subject: [PATCH 150/174] libssh2 for OSX --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index cddc6a0..66e3685 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,9 @@ addons: apt: packages: - libssh2-1-dev + brew: + packages: + - libssh2 before_script: - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` From 5be86fcd990162724298b4d3c9692fd3acd9f8ed Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Nov 2018 23:24:45 -0500 Subject: [PATCH 151/174] Brew install --- .travis.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66e3685..27b2cb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,10 @@ addons: apt: packages: - libssh2-1-dev - brew: - packages: - - libssh2 + +before_install: + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libssh2; fi + before_script: - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` @@ -26,6 +27,7 @@ before_script: - ./koch nimble - export PATH=$(pwd)/bin:~/.nimble/bin:$PATH - cd .. + script: - nimble install -y - nimble test From c60a8cd415a3f495242968fb43e6211ff80effd5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 3 Nov 2018 21:14:13 -0500 Subject: [PATCH 152/174] Remove nimzbar from OSX --- tests/nimgentest.nims | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index f3ab526..cb70a63 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -5,7 +5,7 @@ var comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", - "nimtess2", "nimzbar" + "nimtess2" ] var nimver = staticExec("nim -v").split("\n")[0].split(" ")[3] @@ -15,6 +15,9 @@ if nimver >= "0.19.0": if detectOs(Windows): comps.add("nimkerberos") +if not detectOs(MacOSX): + comps.add("nimzbar") + if paramCount() > 2: for i in 3 .. paramCount(): if paramStr(i) == "--full": From 47ca2cffd6fa0f91e390110b4b3b4a7736c5310c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 3 Nov 2018 23:20:28 -0500 Subject: [PATCH 153/174] Brew no autoupdate --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 27b2cb1..988a833 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ addons: - libssh2-1-dev before_install: - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libssh2; fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install libssh2; fi before_script: - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest From ae6d4aff28559b13cec28c878209591ea3b57314 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 3 Nov 2018 23:38:04 -0500 Subject: [PATCH 154/174] Last nightlies release --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 988a833..9b889c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ before_install: before_script: - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - - export TXZ=`cat latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux` + - export TXZ=`cat latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux | tail -n 1` - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" From 44fc30a9cb00f6cb2537cdc2a4fad9a091b7681c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 4 Nov 2018 14:27:27 -0600 Subject: [PATCH 155/174] Add nimdoc.cfg --- nimgen.nimble | 2 +- web/CNAME | 1 + web/nimdoc.cfg | 1401 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1403 insertions(+), 1 deletion(-) create mode 100644 web/CNAME create mode 100644 web/nimdoc.cfg diff --git a/nimgen.nimble b/nimgen.nimble index c6a2625..085580a 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -7,7 +7,7 @@ license = "MIT" bin = @["nimgen"] srcDir = "src" -skipDirs = @["nimgen", "tests"] +skipDirs = @["tests", "web"] # Dependencies diff --git a/web/CNAME b/web/CNAME new file mode 100644 index 0000000..2c6879b --- /dev/null +++ b/web/CNAME @@ -0,0 +1 @@ +nimgen.genotrance.com \ No newline at end of file diff --git a/web/nimdoc.cfg b/web/nimdoc.cfg new file mode 100644 index 0000000..55c2827 --- /dev/null +++ b/web/nimdoc.cfg @@ -0,0 +1,1401 @@ +# This is the config file for the documentation generator. +# (c) 2016 Andreas Rumpf +# Feel free to edit the templates as you need. If you modify this file, it +# might be worth updating the hardcoded values in packages/docutils/rstgen.nim + +split.item.toc = "20" +# too long entries in the table of contents wrap around +# after this number of characters + +doc.section = """ +
+

$sectionTitle

+
+$content +
+""" + +doc.section.toc = """ +
  • + $sectionTitle +
      + $content +
    +
    • +""" + +# Chunk of HTML emitted for each entry in the HTML table of contents. +# Available variables are: +# * $desc: the actual docstring of the item. +# * $header: the full version of name, including types, pragmas, tags, etc. +# * $header_plain: like header but without HTML, for attribute embedding. +# * $itemID: numerical unique entry of the item in the HTML. +# * $itemSym: short symbolic name of the item for easier hyperlinking. +# * $itemSymEnc: quoted version for URLs or attributes. +# * $itemSymOrID: the symbolic name or the ID if that is not unique. +# * $itemSymOrIDEnc: quoted version for URLs or attributes. +# * $name: reduced name of the item. +# * $seeSrc: generated HTML from doc.item.seesrc (if some switches are used). + +doc.item = """ +
    $header
    +
    +$desc +$seeSrc +
    +""" + +# Chunk of HTML emitted for each entry in the HTML table of contents. +# See doc.item for available substitution variables. +doc.item.toc = """ +
  • $name
    • +""" + +# HTML rendered for doc.item's seeSrc variable. Note that this will render to +# the empty string if you don't pass anything through --docSeeSrcURL. Available +# substitutaion variables here are: +# * $path: relative path to the file being processed. +# * $line: line of the item in the original source file. +# * $url: whatever you did pass through the --docSeeSrcUrl switch (which also +# gets variables path/line replaced!) +doc.item.seesrc = """  Source +""" + +doc.toc = """ +
      +$content +
    +""" + +doc.body_toc = """ +
    +
    +
    + Search: +
    + $tableofcontents +
    +
    +
    +

    $moduledesc

    + $content +
    +
    +""" + +@if boot: +# This is enabled with the "boot" directive to generate +# the compiler documentation. +# As a user, tweak the block below instead. +# You can add your own global-links entries +doc.body_toc_group = """ +
    +
    + +
    + Search: +
    +
    + Group by: + +
    + $tableofcontents +
    +
    +
    +

    $moduledesc

    + $content +
    +
    +""" + +@else + +doc.body_toc_group = """ +
    +
    + +
    + Search: +
    +
    + Group by: + +
    + $tableofcontents +
    +
    +
    +

    $moduledesc

    + $content +
    +
    +""" +@end + +doc.body_no_toc = """ +$moduledesc +$content +""" + +doc.listing_start = "
    "
+doc.listing_end = "
    " + +# * $analytics: Google analytics location, includes + + + + + +
    +
    +

    $title

    + $content +
    +
    + +
    + Made with Nim. Generated: $date $time UTC +
    +
    +
    +
    +$analytics + + +""" From b78f6d38fdfabe7fddc5aa1abe7cae41d77e46ad Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 4 Nov 2018 14:34:50 -0600 Subject: [PATCH 156/174] Fix nimgen dir regression --- nimgen.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimgen.nimble b/nimgen.nimble index 085580a..8c44026 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -7,7 +7,7 @@ license = "MIT" bin = @["nimgen"] srcDir = "src" -skipDirs = @["tests", "web"] +skipDirs = @["nimgen", "tests", "web"] # Dependencies From 02459f3d82eda6a79f58eed8b50526089a9cb47c Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 4 Nov 2018 15:06:37 -0600 Subject: [PATCH 157/174] Remove nimfastText from gcc < 5.x --- tests/nimgentest.nims | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index cb70a63..e544f43 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -8,8 +8,11 @@ var "nimtess2" ] -var nimver = staticExec("nim -v").split("\n")[0].split(" ")[3] -if nimver >= "0.19.0": +let + gccver = staticExec("gcc --version").split("\n")[0].split(" ")[^1] + nimver = staticExec("nim -v").split("\n")[0].split(" ")[3] + +if nimver >= "0.19.0" and gccver >= "5.0.0": comps.add("nimfastText") if detectOs(Windows): @@ -18,6 +21,10 @@ if detectOs(Windows): if not detectOs(MacOSX): comps.add("nimzbar") +echo "Testing comps:" +for comp in comps: + echo " " & comp + if paramCount() > 2: for i in 3 .. paramCount(): if paramStr(i) == "--full": From d6241b1da4571c846e321b008826400fefe75112 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 4 Nov 2018 16:19:54 -0600 Subject: [PATCH 158/174] Pygmentize only --- tests/nimgentest.nims | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index e544f43..0f1810c 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -1,7 +1,7 @@ import distros, ospaths, strutils var - full = true + pygonly = false comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfuzz", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", @@ -27,44 +27,47 @@ for comp in comps: if paramCount() > 2: for i in 3 .. paramCount(): - if paramStr(i) == "--full": - full = true + if paramStr(i) == "--pygonly": + pygonly = true elif paramStr(i).len() > 10 and "--comps=" in paramStr(i)[0 ..< 8]: comps = paramStr(i)[8 .. ^1].split(",") for comp in comps: - if not dirExists(".."/comp): - withDir(".."): - exec "git clone --depth=1 https://github.com/genotrance/" & comp + if not pygonly: + if not dirExists(".."/comp): + withDir(".."): + exec "git clone --depth=1 https://github.com/genotrance/" & comp - exec "nimble uninstall -y " & comp, "", "" - withDir(".."/comp): - exec "git pull" + exec "nimble uninstall -y " & comp, "", "" + withDir(".."/comp): + exec "git pull" - if full: rmDir(comp) exec "nimble install -y" exec "nimble test" - exec "nimble install -y" - exec "nimble test" + exec "nimble install -y" + exec "nimble test" when defined(windows): - if dirExists("web"/comp): - rmDir("web"/comp) + if not pygonly: + if dirExists("web"/comp): + rmDir("web"/comp) - mkDir("web"/comp) - for file in listFiles(".."/comp/comp) & listFiles(".."/comp): - if file.splitFile().ext == ".nim": - cpFile(file, "web"/comp/extractFilename(file)) + mkDir("web"/comp) + for file in listFiles(".."/comp/comp) & listFiles(".."/comp): + if file.splitFile().ext == ".nim": + cpFile(file, "web"/comp/extractFilename(file)) cpFile("web"/"nimdoc.cfg", "web"/comp/"nimdoc.cfg") withDir("web"/comp): for file in listFiles("."): if file.splitFile().ext == ".nim": - exec "nim doc --git.url:. --index:on -o:" & file.changeFileExt("html") & " " & file + if not pygonly: + exec "nim doc --git.url:. --index:on -o:" & file.changeFileExt("html") & " " & file exec "pygmentize -f html -O full,linenos=1,anchorlinenos=True,lineanchors=L,style=vs -o " & file & ".html " & file - exec "nim buildIndex -o:index.html ." + if not pygonly: + exec "nim buildIndex -o:index.html ." rmFile("web"/comp/"nimdoc.cfg") From 86df7b19f1ddc1939dfdc0d2851b064fd66712c2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 8 Nov 2018 21:39:20 -0600 Subject: [PATCH 159/174] Fix README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 88bed4d..4df389c 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ The following keys can be used to prepare dependencies such as downloading ZIP f ```gitcheckout``` = branch, commit or tag of repository to checkout in following Git command, resets after each use. Use "-b name" for branches -```gitoutput`` = directory for all following Git commands relative to `n.global:output` [default: `n.global:output` directory] +```gitoutput``` = directory for all following Git commands relative to `n.global:output` [default: `n.global:output` directory] ```git``` = url of Git repository to clone. Full repo is pulled so gitremote + gitsparse is preferable. Resets if already present @@ -146,7 +146,7 @@ _[n.post]_ This section is the same as the prepare section, but for performing actions after the project has been processed. -```gitoutput`` = output directory for Git reset [default: `n.global:output` directory] +```gitoutput``` = output directory for Git reset [default: `n.global:output` directory] ```reset``` = perform a Git reset on all files after processing [default: false] From 24c62dbc188fe638f7640b50a3767e5c09c9f9d7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 7 Dec 2018 17:58:46 -0600 Subject: [PATCH 160/174] Enable nimfastText on OSX --- tests/nimgentest.nims | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 0f1810c..7df6419 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -12,7 +12,7 @@ let gccver = staticExec("gcc --version").split("\n")[0].split(" ")[^1] nimver = staticExec("nim -v").split("\n")[0].split(" ")[3] -if nimver >= "0.19.0" and gccver >= "5.0.0": +if nimver >= "0.19.0" and (gccver >= "5.0.0" or detectOs(MacOSX)): comps.add("nimfastText") if detectOs(Windows): From 2f1a62a03023981f14a8a3cdfbd1d5238a388cb2 Mon Sep 17 00:00:00 2001 From: data-man Date: Sun, 9 Dec 2018 05:54:36 +0500 Subject: [PATCH 161/174] Don't use ospaths on Nim-devel --- src/nimgen/c2nim.nim | 5 ++++- src/nimgen/file.nim | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 0464c6a..f3b5c33 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -1,4 +1,7 @@ -import os, ospaths, regex, strutils +import os, regex, strutils + +when (NimMajor, NimMinor, NimPatch) < (0, 19, 9): + import ospaths import external, file, fileops, gencore, globals diff --git a/src/nimgen/file.nim b/src/nimgen/file.nim index 769c914..81e1d86 100644 --- a/src/nimgen/file.nim +++ b/src/nimgen/file.nim @@ -1,4 +1,7 @@ -import os, ospaths, pegs, regex, strutils, tables +import os, pegs, regex, strutils, tables + +when (NimMajor, NimMinor, NimPatch) < (0, 19, 9): + import ospaths import globals, external From 46cce39e64a9056d2d6d494047be7dbb31c204f0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 10 Dec 2018 09:48:46 -0600 Subject: [PATCH 162/174] Use choosenim for Travis --- .travis.yml | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9b889c5..1ed925a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,12 @@ -sudo: false -language: c os: - linux - osx -dist: trusty + +language: c + +env: + - BRANCH=devel + addons: apt: packages: @@ -12,21 +15,12 @@ addons: before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install libssh2; fi -before_script: - - curl -u $TOKEN -o latest.json --silent https://api.github.com/repos/nim-lang/nightlies/releases/latest - - export RELEASE=`cat latest.json | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/'` - - export TXZ=`cat latest.json | grep '"name":' | sed -E 's/.*"([^"]+)".*/\1/' | grep linux | tail -n 1` - - export VERSION=`echo $TXZ | cut -d"-" -f 2,2` - - echo "RELEASE = $RELEASE, TXZ = $TXZ, VERSION = $VERSION" - - curl -L --silent -o $TXZ "https://github.com/nim-lang/nightlies/releases/download/$RELEASE/$TXZ" - - tar xf $TXZ - - cd nim-$VERSION - - sh build.sh - - bin/nim c koch - - ./koch boot -d:release - - ./koch nimble - - export PATH=$(pwd)/bin:~/.nimble/bin:$PATH - - cd .. +install: + - export CHOOSENIM_CHOOSE_VERSION=$BRANCH + - | + curl https://nim-lang.org/choosenim/init.sh -sSf > init.sh + sh init.sh -y + - export PATH=$HOME/.nimble/bin:$PATH script: - nimble install -y From 31b63a6ea6d035d23b009d8b18d6f74c004c1ce5 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 10 Dec 2018 12:00:53 -0600 Subject: [PATCH 163/174] Update nimfuzz to nimfuzzy --- README.md | 4 ++-- tests/nimgentest.nims | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4df389c..6a64779 100644 --- a/README.md +++ b/README.md @@ -54,14 +54,14 @@ To see examples of nimgen in action check out the following wrappers:- * git checkout * [nimfastText](https://github.com/genotrance/nimfastText) - fastText wrapper: [docs](http://nimgen.genotrance.com/nimfastText) * git sparse checkout - * [nimfuzz](https://github.com/genotrance/nimfuzz) - fts_fuzzy_match wrapper: [docs](http://nimgen.genotrance.com/nimfuzz) + * [nimfuzzy](https://github.com/genotrance/nimfuzzy) - fts_fuzzy_match wrapper: [docs](http://nimgen.genotrance.com/nimfuzzy) * download header file * [nimkerberos](https://github.com/genotrance/nimkerberos) - WinKerberos wrapper: [docs](http://nimgen.genotrance.com/nimkerberos) * git sparse checkout * [nimmonocypher](https://github.com/genotrance/nimmonocypher) - monocypher wrapper: [docs](http://nimgen.genotrance.com/nimmonocypher) * git sparse checkout * [nimnuklear](https://github.com/genotrance/nimnuklear) - nuklear wrapper: [docs](https://nimgen.genotrance.com/nimnuklear) - * git sparse checkout + * git sparse checkout * [nimpcre](https://github.com/genotrance/nimpcre) - PCRE wrapper: [docs](http://nimgen.genotrance.com/nimpcre) * git checkout * [nimrax](https://github.com/genotrance/nimrax) - Radix tree wrapper: [docs](http://nimgen.genotrance.com/nimrax) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 7df6419..8a5cbd6 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -3,7 +3,7 @@ import distros, ospaths, strutils var pygonly = false comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", - "nimclipboard", "nimfuzz", "nimmonocypher", + "nimclipboard", "nimfuzzy", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2" ] @@ -21,6 +21,9 @@ if detectOs(Windows): if not detectOs(MacOSX): comps.add("nimzbar") +echo "Nim version: " & nimver +echo "GCC version: " & gccver + echo "Testing comps:" for comp in comps: echo " " & comp From cf132951bdc326602532a740791041c139895b5e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 31 Dec 2018 12:33:09 -0600 Subject: [PATCH 164/174] Update appveyor to 0.19.2 --- appveyor.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e4e062e..2e382be 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,7 +10,7 @@ matrix: environment: matrix: - NIM_VERSION: 0.18.0 - - NIM_VERSION: 0.19.0 + - NIM_VERSION: 0.19.2 for: - @@ -50,19 +50,6 @@ for: del "nim-%NIM_VERSION%_x%ARCH%.zip") - SET PATH=c:\binaries\mingw%ARCH%\bin;c:\binaries\nim-%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH% - CD c:\projects\nimgen - # - git clone --depth 1 https://github.com/nim-lang/nim - # - cd nim - # - git clone --depth 1 https://github.com/nim-lang/csources - # - cd csources - # - IF "%PLATFORM%" == "x64" ( build64.bat ) else ( build.bat ) - # - cd .. - # - IF not exist "%DLLS_ARCHIVE%" appveyor DownloadFile "%DLLS_URL%" -FileName "%DLLS_ARCHIVE%" - # - 7z x -y "%DLLS_ARCHIVE%" -o"%CD%\bin"> nul - # - bin\nim c -d:release koch - # - koch boot -d:release - # - koch nimble -d:release - # - SET PATH=%CD%\bin;%PATH% - # - cd .. on_finish: - 7z a -r buildlogs-win-pkgs.zip %USERPROFILE%\.nimble\pkgs From 293476c0a967e07b9b277ced2437a821a5604921 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 6 Mar 2019 00:34:11 -0600 Subject: [PATCH 165/174] Remove nimarchive since it is currently broken --- tests/nimgentest.nims | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index 8a5cbd6..d8e5737 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,7 +2,8 @@ import distros, ospaths, strutils var pygonly = false - comps = @["libsvm", "nim7z", "nimarchive", "nimbass", "nimbigwig", + comps = @["libsvm", "nim7z", #"nimarchive", + "nimbass", "nimbigwig", "nimclipboard", "nimfuzzy", "nimmonocypher", "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2" From 69b9ee9083d1ef81764141407b6b114e43abdc37 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 22 Mar 2019 09:15:49 -0600 Subject: [PATCH 166/174] Re-enable remove static --- src/nimgen/c2nim.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index f3b5c33..3d15243 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -31,6 +31,7 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): prepend(cfile, getDefines(file, c2nimConfig.inline)) + removeStatic(cfile) var extflags = "" passC = "" @@ -101,6 +102,8 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = removeFile(cfile) except: discard + else: + reAddStatic(cfile) # Nim doesn't like {.cdecl.} for type proc() freplace(outfile, re"(?m)(.*? = proc.*?)\{.cdecl.\}", "$1") @@ -120,4 +123,4 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = # Add dynamic library if outlib != "": - prepend(outfile, outlib) \ No newline at end of file + prepend(outfile, outlib) From d6f8d140ccd25567996e018d3cebe4f698b63eaa Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Fri, 22 Mar 2019 09:53:23 -0600 Subject: [PATCH 167/174] Add function to remove function bodies --- src/nimgen/c2nim.nim | 5 +++++ src/nimgen/fileops.nim | 30 ++++++++++++++++++++++++++++++ src/nimgen/globals.nim | 2 +- src/nimgen/runcfg.nim | 4 +++- 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/nimgen/c2nim.nim b/src/nimgen/c2nim.nim index 3d15243..c9ccff7 100644 --- a/src/nimgen/c2nim.nim +++ b/src/nimgen/c2nim.nim @@ -28,6 +28,9 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = cfile = "temp-$#.c" % [outfile.extractFilename()] writeFileFlush(cfile, runCtags(file)) + if c2nimConfig.removeBodies: + removeBodies(cfile) + if c2nimConfig.defines and (c2nimConfig.preprocess or c2nimConfig.ctags): prepend(cfile, getDefines(file, c2nimConfig.inline)) @@ -103,6 +106,8 @@ proc c2nim*(fl, outfile: string, c2nimConfig: c2nimConfigObj) = except: discard else: + if c2nimConfig.removeBodies: + reAddBodies(cfile) reAddStatic(cfile) # Nim doesn't like {.cdecl.} for type proc() diff --git a/src/nimgen/fileops.nim b/src/nimgen/fileops.nim index e416cf1..7f16139 100644 --- a/src/nimgen/fileops.nim +++ b/src/nimgen/fileops.nim @@ -91,6 +91,36 @@ proc removeStatic*(filename: string) = result.add(body.replace(re"(?m)^(.*\n?)", "//$1")) ) +proc removeBodies*(filename: string) = + ## Replace function bodies with a semicolon and commented + ## out body + withFile(filename): + content = content.replace( + re"(?m)(.*?\))(\s*\{(\s*?.*?$)*?[\n\r]\})", + proc (m: RegexMatch, s: string): string = + let funcDecl = s[m.group(0)[0]] + let body = s[m.group(1)[0]].strip() + result = "" + + result.add("$#;" % [funcDecl]) + result.add(body.replace(re"(?m)^(.*\n?)", "//$1")) + ) + +proc reAddBodies*(filename: string) = + ## Uncomment out the body and remove the semicolon. Undoes + ## removeBodies + withFile(filename): + content = content.replace( + re"(?m)(.*?\))(\s*\{(\s*?.*?$)*?[\n\r]\})", + proc (m: RegexMatch, s: string): string = + let funcDecl = s[m.group(0)[0]] + let body = s[m.group(1)[0]].strip() + result = "" + + result.add("$# " % [funcDecl]) + result.add(body.replace(re"(?m)^\/\/(.*\n?)", "$1")) + ) + proc reAddStatic*(filename: string) = ## Uncomment out the body and remove the semicolon. Undoes ## removeStatic diff --git a/src/nimgen/globals.nim b/src/nimgen/globals.nim index 9892b99..da90635 100644 --- a/src/nimgen/globals.nim +++ b/src/nimgen/globals.nim @@ -32,7 +32,7 @@ var type c2nimConfigObj* = object flags*, ppflags*: string - recurse*, inline*, preprocess*, ctags*, defines*: bool + recurse*, inline*, preprocess*, removeBodies*, ctags*, defines*: bool dynlib*, compile*, pragma*: seq[string] const gDoc* = """ diff --git a/src/nimgen/runcfg.nim b/src/nimgen/runcfg.nim index a924734..c582c2c 100644 --- a/src/nimgen/runcfg.nim +++ b/src/nimgen/runcfg.nim @@ -130,6 +130,8 @@ proc runFile*(file: string, cfgin: OrderedTableRef = newOrderedTable[string, str c2nimConfig.inline = true elif action == "preprocess": c2nimConfig.preprocess = true + elif action == "removeBodies": + c2nimConfig.removeBodies = true elif action == "ctags": c2nimConfig.ctags = true elif action == "defines": @@ -406,4 +408,4 @@ proc runCli*() = uniq += 1 for file in files: - runFile(file, cfg) \ No newline at end of file + runFile(file, cfg) From 97141f523fbdf5d5b0b4594ea615f05faf7efcfb Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 9 Jul 2019 23:58:01 -0500 Subject: [PATCH 168/174] Bump to latest --- appveyor.yml | 4 ++-- nimgen.nimble | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 2e382be..7d8dabf 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,8 +9,8 @@ matrix: environment: matrix: - - NIM_VERSION: 0.18.0 - - NIM_VERSION: 0.19.2 + - NIM_VERSION: 0.20.0 + - NIM_VERSION: 0.19.6 for: - diff --git a/nimgen.nimble b/nimgen.nimble index 8c44026..0427d5a 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -11,7 +11,7 @@ skipDirs = @["nimgen", "tests", "web"] # Dependencies -requires "nim >= 0.17.0", "c2nim#3ec45c24585ebaed", "regex >= 0.10.0" +requires "nim >= 0.17.0", "c2nim >= 0.9.14", "regex >= 0.10.0" task test, "Test nimgen": exec "nim c -r tests/rununittests.nim" From 4ab5ff93b8c18ea63deefa5a947950756d215a24 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 15 Jul 2019 22:49:04 -0500 Subject: [PATCH 169/174] Fix c2nim dependency --- nimgen.nimble | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nimgen.nimble b/nimgen.nimble index 0427d5a..10bddf5 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.0" +version = "0.5.1" author = "genotrance" description = "c2nim helper to simplify and automate the wrapping of C libraries" license = "MIT" @@ -11,7 +11,12 @@ skipDirs = @["nimgen", "tests", "web"] # Dependencies -requires "nim >= 0.17.0", "c2nim >= 0.9.14", "regex >= 0.10.0" +requires "nim >= 0.19.0", "regex >= 0.10.0" + +when NimVersion < "0.20.0": + requires "c2nim#8f1705509084ae47319f6fcfb131e515b134e0f1" +else: + requires "c2nim >= 0.9.14" task test, "Test nimgen": exec "nim c -r tests/rununittests.nim" From b54feea3fcdae07fb0462327807e0632a5892123 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jul 2019 13:04:05 -0500 Subject: [PATCH 170/174] Remove nimnuklear --- tests/nimgentest.nims | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index d8e5737..b1816cd 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -5,7 +5,8 @@ var comps = @["libsvm", "nim7z", #"nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfuzzy", "nimmonocypher", - "nimnuklear", "nimpcre", "nimrax", "nimssl", "nimssh2", + #"nimnuklear", + "nimpcre", "nimrax", "nimssl", "nimssh2", "nimtess2" ] From fabd61537939b0e1a30d784a4761714a93d1abb6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jul 2019 14:06:02 -0500 Subject: [PATCH 171/174] C2nim is now backwards compatible --- nimgen.nimble | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nimgen.nimble b/nimgen.nimble index 10bddf5..fc7409e 100644 --- a/nimgen.nimble +++ b/nimgen.nimble @@ -11,12 +11,7 @@ skipDirs = @["nimgen", "tests", "web"] # Dependencies -requires "nim >= 0.19.0", "regex >= 0.10.0" - -when NimVersion < "0.20.0": - requires "c2nim#8f1705509084ae47319f6fcfb131e515b134e0f1" -else: - requires "c2nim >= 0.9.14" +requires "nim >= 0.19.0", "c2nim >= 0.9.14", "regex >= 0.10.0" task test, "Test nimgen": exec "nim c -r tests/rununittests.nim" From 39f5f320a627a895d3000673a0fb29603354061f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jul 2019 22:06:06 -0500 Subject: [PATCH 172/174] Remove libsvm --- README.md | 2 -- tests/nimgentest.nims | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 6a64779..818b8cd 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,6 @@ To see examples of nimgen in action check out the following wrappers:- * git sparse checkout * [nimtess2](https://github.com/genotrance/nimtess2) - libtess2 wrapper: [docs](http://nimgen.genotrance.com/nimtess2) * git checkout - * [libsvm](https://github.com/genotrance/libsvm) - libsvm wrapper: [docs](http://nimgen.genotrance.com/libsvm) - * git sparse checkout * [duktape-nim](https://github.com/manguluka/duktape-nim) - Duktape wrapper * static diff --git a/tests/nimgentest.nims b/tests/nimgentest.nims index b1816cd..0de436a 100644 --- a/tests/nimgentest.nims +++ b/tests/nimgentest.nims @@ -2,7 +2,7 @@ import distros, ospaths, strutils var pygonly = false - comps = @["libsvm", "nim7z", #"nimarchive", + comps = @["nim7z", #"nimarchive", "nimbass", "nimbigwig", "nimclipboard", "nimfuzzy", "nimmonocypher", #"nimnuklear", From 2be81614a9851ec13547bffcfe73c20fec6f7708 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jul 2019 22:17:32 -0500 Subject: [PATCH 173/174] Fix #42 - curl should follow redirects, powershell wget backport --- src/nimgen/external.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index 8656147..cc7a5e8 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -43,9 +43,10 @@ proc downloadUrl*(url: string) = file = url.extractFilename() ext = file.splitFile().ext.toLowerAscii() - var cmd = "curl $# -o $#" - if defined(Windows): - cmd = "powershell wget $# -OutFile $#" + var cmd = if defined(Windows): + "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" + else: + "curl -L $# -o $#" if not (ext == ".zip" and fileExists(gOutput/file)): echo "Downloading " & file From 638c9b4a5ae4d09c9451feaca363840e130ffbb7 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 7 Sep 2019 09:12:32 -0600 Subject: [PATCH 174/174] Fix bug with length --- src/nimgen/external.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nimgen/external.nim b/src/nimgen/external.nim index cc7a5e8..a887e20 100644 --- a/src/nimgen/external.nim +++ b/src/nimgen/external.nim @@ -22,7 +22,7 @@ proc execAction*(cmd: string): string = when defined(Linux) or defined(MacOSX): ccmd = "bash -c '" & cmd & "'" - echo "Running '" & ccmd[0..min(64, len(ccmd))] & "'" + echo "Running '" & ccmd[0..