From a0ecf4d21667a10f7ae9c4871dd0026871a7df72 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 18 May 2020 16:58:07 -0600 Subject: [PATCH 001/106] Remove regex from comment processing --- nimterop/tshelp.nim | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index 762cf7c..9565a1f 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -1,7 +1,5 @@ import sets, strformat, strutils -import regex - import "."/[getters, globals] import "."/treesitter/[api, c, cpp] @@ -273,8 +271,14 @@ proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = for commentNode in commentNodes: result &= "\n " & gState.getNodeVal(commentNode).strip() - result = result.replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "") - result = result.multiReplace([("\n", "\n "), ("`", "")]).strip() + result = result.multiReplace( + { + "/**": "", "**/": "", "/*": "", + "*/": "", "/*": "", "//": "", + "\n": "\n ", "`": "" + } + # need to replace this last otherwise it supercedes other replacements + ).replace(" *", "").strip() proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch`` From 6dcd74d510a82f688715fd8be9ddcceca01dc6e7 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 18 May 2020 17:52:21 -0600 Subject: [PATCH 002/106] Remove more regex from exprparser --- nimterop/exprparser.nim | 61 ++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index b089fd2..b2b8c2c 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -1,4 +1,4 @@ -import strformat, strutils, macros, sets +import strformat, strutils, macros, sets, sequtils import regex @@ -32,6 +32,10 @@ import "."/[globals, getters, comphelp, tshelp] type ExprParseError* = object of CatchableError +const + CharRegStr = "(\\\\x[[:xdigit:]]{2}|\\\\\\d{3}|\\\\0|\\\\a|\\\\b|\\\\e|\\\\f|\\\\n|\\\\r|\\\\t|\\\\v|\\\\\\\\|\\\\'|\\\\\"|[[:ascii:]])" + CharRegex = re(CharRegStr) + template val(node: TSNode): string = gState.currentExpr.getNodeVal(node) @@ -138,13 +142,13 @@ proc getIntNode(number, suffix: string): PNode {.inline.} = flags: TNodeFlags # I realize these regex are wasteful on performance, but # couldn't come up with a better idea. - if number.contains(re"0[xX]"): + if number.startsWith("0X") or number.startsWith("0x"): val = parseHexInt(number) flags = {nfBase16} - elif number.contains(re"0[bB]"): + elif number.startsWith("0B") or number.startsWith("0b"): val = parseBinInt(number) flags = {nfBase2} - elif number.contains(re"0[oO]"): + elif number.startsWith("0O") or number.startsWith("0o"): val = parseOctInt(number) flags = {nfBase8} else: @@ -186,25 +190,36 @@ proc processNumberLiteral(gState: State, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val + var + prefix: string + number = nodeVal + suffix: string - var match: RegexMatch - const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.\d*[fFlL]?|\d*\.\d+[fFlL]?|\d+)([ulUL]*)" - let found = nodeVal.find(reg, match) - if found: - let - prefix = if match.group(0).len > 0: nodeVal[match.group(0)[0]] else: "" - number = nodeVal[match.group(1)[0]] - suffix = nodeVal[match.group(2)[0]] + const + singleEndings = ["u", "l", "U", "L"] + doubleEndings = ["ul", "UL", "ll", "LL"] + tripleEndings = ["ull", "ULL"] - result = getNumNode(number, suffix) + if number.startsWith("-"): + number = number[1 ..< number.len] + prefix = "-" + if tripleEndings.any(proc (s: string): bool = number.endsWith(s)): + suffix = number[^3 .. ^1] + number = number[0 ..< ^3] + elif doubleEndings.any(proc (s: string): bool = number.endsWith(s)): + suffix = number[^2 .. ^1] + number = number[0 ..< ^2] + elif singleEndings.any(proc (s: string): bool = number.endsWith(s)): + suffix = $number[number.len - 1] + number = number[0 ..< ^1] - if result.kind != nkNone and prefix == "-": - result = nkPrefix.newTree( - gState.getIdent("-"), - result - ) - else: - raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") + result = getNumNode(number, suffix) + + if result.kind != nkNone and prefix == "-": + result = nkPrefix.newTree( + gState.getIdent("-"), + result + ) proc processCharacterLiteral(gState: State, node: TSNode): PNode = # Input => 'G' @@ -234,13 +249,9 @@ proc processStringLiteral(gState: State, node: TSNode): PNode = nodeVal = node.val strVal = nodeVal[1 ..< nodeVal.len - 1] - const - str = "(\\\\x[[:xdigit:]]{2}|\\\\\\d{3}|\\\\0|\\\\a|\\\\b|\\\\e|\\\\f|\\\\n|\\\\r|\\\\t|\\\\v|\\\\\\\\|\\\\'|\\\\\"|[[:ascii:]])" - reg = re(str) - # Convert the c string escape sequences/etc to Nim chars var nimStr = newStringOfCap(nodeVal.len) - for m in strVal.findAll(reg): + for m in strVal.findAll(CharRegex): nimStr.add(parseChar(strVal[m.group(0)[0]]).chr) result = newStrNode(nkStrLit, nimStr) From 5895db1d18e1d3c9f1e502f0a68f671d4e53e02b Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 18 May 2020 16:58:07 -0600 Subject: [PATCH 003/106] Remove regex from comment processing --- nimterop/tshelp.nim | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index 762cf7c..9565a1f 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -1,7 +1,5 @@ import sets, strformat, strutils -import regex - import "."/[getters, globals] import "."/treesitter/[api, c, cpp] @@ -273,8 +271,14 @@ proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = for commentNode in commentNodes: result &= "\n " & gState.getNodeVal(commentNode).strip() - result = result.replace(re" *(//|/\*\*|\*\*/|/\*|\*/|\*)", "") - result = result.multiReplace([("\n", "\n "), ("`", "")]).strip() + result = result.multiReplace( + { + "/**": "", "**/": "", "/*": "", + "*/": "", "/*": "", "//": "", + "\n": "\n ", "`": "" + } + # need to replace this last otherwise it supercedes other replacements + ).replace(" *", "").strip() proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch`` From 985741cc8de35ea36933ddb845f93316288c30b8 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Mon, 18 May 2020 17:52:21 -0600 Subject: [PATCH 004/106] Remove more regex from exprparser --- nimterop/exprparser.nim | 61 ++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index b089fd2..b2b8c2c 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -1,4 +1,4 @@ -import strformat, strutils, macros, sets +import strformat, strutils, macros, sets, sequtils import regex @@ -32,6 +32,10 @@ import "."/[globals, getters, comphelp, tshelp] type ExprParseError* = object of CatchableError +const + CharRegStr = "(\\\\x[[:xdigit:]]{2}|\\\\\\d{3}|\\\\0|\\\\a|\\\\b|\\\\e|\\\\f|\\\\n|\\\\r|\\\\t|\\\\v|\\\\\\\\|\\\\'|\\\\\"|[[:ascii:]])" + CharRegex = re(CharRegStr) + template val(node: TSNode): string = gState.currentExpr.getNodeVal(node) @@ -138,13 +142,13 @@ proc getIntNode(number, suffix: string): PNode {.inline.} = flags: TNodeFlags # I realize these regex are wasteful on performance, but # couldn't come up with a better idea. - if number.contains(re"0[xX]"): + if number.startsWith("0X") or number.startsWith("0x"): val = parseHexInt(number) flags = {nfBase16} - elif number.contains(re"0[bB]"): + elif number.startsWith("0B") or number.startsWith("0b"): val = parseBinInt(number) flags = {nfBase2} - elif number.contains(re"0[oO]"): + elif number.startsWith("0O") or number.startsWith("0o"): val = parseOctInt(number) flags = {nfBase8} else: @@ -186,25 +190,36 @@ proc processNumberLiteral(gState: State, node: TSNode): PNode = ## Parse a number literal from a TSNode. Can be a float, hex, long, etc result = newNode(nkNone) let nodeVal = node.val + var + prefix: string + number = nodeVal + suffix: string - var match: RegexMatch - const reg = re"(\-)?(0\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|\d+\.\d*[fFlL]?|\d*\.\d+[fFlL]?|\d+)([ulUL]*)" - let found = nodeVal.find(reg, match) - if found: - let - prefix = if match.group(0).len > 0: nodeVal[match.group(0)[0]] else: "" - number = nodeVal[match.group(1)[0]] - suffix = nodeVal[match.group(2)[0]] + const + singleEndings = ["u", "l", "U", "L"] + doubleEndings = ["ul", "UL", "ll", "LL"] + tripleEndings = ["ull", "ULL"] - result = getNumNode(number, suffix) + if number.startsWith("-"): + number = number[1 ..< number.len] + prefix = "-" + if tripleEndings.any(proc (s: string): bool = number.endsWith(s)): + suffix = number[^3 .. ^1] + number = number[0 ..< ^3] + elif doubleEndings.any(proc (s: string): bool = number.endsWith(s)): + suffix = number[^2 .. ^1] + number = number[0 ..< ^2] + elif singleEndings.any(proc (s: string): bool = number.endsWith(s)): + suffix = $number[number.len - 1] + number = number[0 ..< ^1] - if result.kind != nkNone and prefix == "-": - result = nkPrefix.newTree( - gState.getIdent("-"), - result - ) - else: - raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"") + result = getNumNode(number, suffix) + + if result.kind != nkNone and prefix == "-": + result = nkPrefix.newTree( + gState.getIdent("-"), + result + ) proc processCharacterLiteral(gState: State, node: TSNode): PNode = # Input => 'G' @@ -234,13 +249,9 @@ proc processStringLiteral(gState: State, node: TSNode): PNode = nodeVal = node.val strVal = nodeVal[1 ..< nodeVal.len - 1] - const - str = "(\\\\x[[:xdigit:]]{2}|\\\\\\d{3}|\\\\0|\\\\a|\\\\b|\\\\e|\\\\f|\\\\n|\\\\r|\\\\t|\\\\v|\\\\\\\\|\\\\'|\\\\\"|[[:ascii:]])" - reg = re(str) - # Convert the c string escape sequences/etc to Nim chars var nimStr = newStringOfCap(nodeVal.len) - for m in strVal.findAll(reg): + for m in strVal.findAll(CharRegex): nimStr.add(parseChar(strVal[m.group(0)[0]]).chr) result = newStrNode(nkStrLit, nimStr) From 11f84d4d8075d96aed623d09972a79f2702486a8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 18 May 2020 11:22:54 -0500 Subject: [PATCH 005/106] var support, interleaved #defines, fix ast1 default, misc performance --- nimterop/ast2.nim | 501 ++++++++++++++++++++++------------------ nimterop/exprparser.nim | 3 +- nimterop/getters.nim | 35 +-- nimterop/globals.nim | 2 +- nimterop/toast.nim | 8 +- nimterop/tshelp.nim | 2 +- tests/rsa.nim | 3 +- 7 files changed, 314 insertions(+), 240 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index dff355e..3d52e02 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1,7 +1,5 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times -import options as opts - import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] import "."/treesitter/api @@ -136,6 +134,8 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = result.add valident[0] else: result.add valident + # In case symbol was skipped earlier + gState.skippedSyms.excl origname else: gecho &"# const '{origname}' is duplicate, skipped" else: @@ -251,7 +251,7 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: (tname, torigname, info) = if not atom.isNil: - gState.getNameInfo(node.getAtom(), kind) + gState.getNameInfo(atom, kind) else: ("", "", gState.getLineInfo(node)) @@ -342,46 +342,42 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: result.add prident result.add newNode(nkEmpty) elif kind == nskVar: - # var name* {.importc: "abc".} + # Used by `addVar()` for regular vars and proc vars # - # nkIdentDefs( - # nkPragmaExpr( - # nkPostfix( - # nkIdent("*"), - # nkIdent(name) - # ), - # nkPragma( - # nkExprColonExpr( - # nkIdent("importc"), - # nkStrLit("abc") - # ) + # name* {.importc: "abc".} + # + # nkPragmaExpr( + # nkPostfix( + # nkIdent("*"), + # nkIdent(name) + # ), + # nkPragma( + # nkExprColonExpr( + # nkIdent("importc"), + # nkStrLit("abc") # ) # ) # ) - let - prident = block: - var - prident: PNode - # Add {.importc.} pragma - if name != origname: - # Name changed - prident = gState.newPragmaExpr(node, ident, "importc", newStrNode(nkStrLit, &"{origname}")) - else: - prident = gState.newPragmaExpr(node, ident, "importc") + result = block: + var + prident: PNode + # Add {.importc.} pragma + if name != origname: + # Name changed + prident = gState.newPragmaExpr(node, ident, "importc", newStrNode(nkStrLit, &"{origname}")) + else: + prident = gState.newPragmaExpr(node, ident, "importc") - if gState.dynlib.nBl: - # Add {.dynlib.} - gState.addPragma(node, prident[1], gState.impShort & "Dyn") - elif not gState.noHeader: - # Add {.header.} - gState.addPragma(node, prident[1], gState.impShort & "Hdr") + if gState.dynlib.nBl: + # Add {.dynlib.} + gState.addPragma(node, prident[1], gState.impShort & "Dyn") + elif not gState.noHeader: + # Add {.header.} + gState.addPragma(node, prident[1], gState.impShort & "Hdr") - if pragmas.nBl: - gState.addPragma(node, prident[1], pragmas) - prident - - result = newNode(nkIdentDefs) - result.add prident + if pragmas.nBl: + gState.addPragma(node, prident[1], pragmas) + prident elif kind == nskProc: # name* # @@ -426,12 +422,39 @@ proc newArrayTree(gState: State, node: TSNode, typ, size: PNode = nil): PNode = proc getTypeArray(gState: State, node: TSNode, tident: PNode, name: string): PNode proc getTypeProc(gState: State, name: string, node, rnode: TSNode): PNode -iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInteger, ftname = "", exported = false): PNode = - # Create nkIdentDefs tree for specified proc parameter or object field +proc getTypeAndStart(gState: State, name: string, node: TSNode, ftname = ""): + tuple[tname: string, tinfo: TLineInfo, tident: PNode, start: int] = + # Shortcut to get start node and type info from first node # - # For proc, param should not be `exported` + # `name` is the parent type or proc # - # If `ftname` is set, use it as the type name + # If `ftname` is set, use it as the type name instead + result.start = getStartAtom(node) + + let + # node[start] - param type + (tname0, _, tinfo) = gState.getNameInfo(node[result.start].getAtom(), nskType, parent = name) + + # Override type name + result.tname = + if ftname.nBl: + ftname + else: + tname0 + + result.tinfo = tinfo + result.tident = gState.getIdent(result.tname, tinfo, exported = false) + +proc newIdentDef(gState: State, name: string, node: TSNode, tname: string, tinfo: TLineInfo, tident: PNode, + start, offset, poffset: SomeInteger, exported = false): PNode = + # Create nkIdentDefs tree for specified proc parameter, object field or var/proc var + # + # `name` is the parent type or proc - if blank, this is a var so add pragmas + # `tname`, `tinfo` and `tident` are the type info for the node + # `start` is the `node` child where type is located + # `offset` is the nth `node` child where specified param/field/var is located - from `addVar()` + # `poffset` is the param number and used for unnamed params in procs - from `newIdentDefs()` iterator + # `isvar` when true adds pragmas # # pname: [ptr ..] typ # @@ -441,7 +464,7 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt # nkEmpty() # ) # - # For object, field should be exported + # For objects and vars/proc vars, field should be exported # # pname*: [ptr ..] typ # @@ -453,6 +476,125 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt # typ, # nkEmpty() # ) + result = newNode(nkIdentDefs) + + let + fdecl = node[offset].firstChildInTree("function_declarator") + afdecl = node[offset].firstChildInTree("abstract_function_declarator") + adecl = node[offset].firstChildInTree("array_declarator") + abst = node[offset].getName() == "abstract_pointer_declarator" + if fdecl.isNil and afdecl.isNil and adecl.isNil: + if abst: + # Only for proc with no named param with pointer type + # Create a param name based on poffset + # + # int func(char *, int **); + let + pname = "a" & $(poffset+1) + pident = gState.getIdent(pname, tinfo, exported) + acount = node[offset].getXCount("abstract_pointer_declarator") + result.add pident + result.add gState.newPtrTree(acount, tident) + result.add newNode(nkEmpty) + else: + # Named param, simple type + if name.nBl: + # Types and procs - `newIdentDefs()` iterator + let + (pname, _, pinfo) = gState.getNameInfo(node[offset].getAtom(), nskField, parent = name) + pident = gState.getIdent(pname, pinfo, exported) + + # Bitfield support - typedef struct { int field: 1; }; + prident = + if node.len > offset and node[offset + 1].getName() == "bitfield_clause": + gState.newPragmaExpr(node, pident, "bitsize", + newIntNode(nkIntLit, parseInt(gState.getNodeVal(node[offset + 1].getAtom())))) + else: + pident + + result.add prident + else: + # Vars - `addVar()` + let + pident = gState.newXIdent(node[offset], nskVar) + if pident.isNil: + return nil + + result.add pident + + let + count = node[offset].getPtrCount() + if count > 0: + result.add gState.newPtrTree(count, tident) + else: + result.add tident + result.add newNode(nkEmpty) + elif not fdecl.isNil: + # Named param, function pointer + var + name = name + pident: PNode + if name.nBl: + # Types and procs - `newIdentDefs()` iterator + let + (pname, _, pinfo) = gState.getNameInfo(node[offset].getAtom(), nskField, parent = name) + pident = gState.getIdent(pname, pinfo, exported) + else: + # Vars - `addVar()` + pident = gState.newXIdent(node[offset], nskVar) + if pident.isNil: + return nil + # The var is the parent + name = pident.getIdentName() + + result.add pident + result.add gState.getTypeProc(name, node[offset], node[start]) + result.add newNode(nkEmpty) + elif not afdecl.isNil: + # Only for proc with no named param with function pointer type + # Create a param name based on poffset + # + # int func(int (*)(int *)); + let + pname = "a" & $(poffset+1) + pident = gState.getIdent(pname, tinfo, exported) + procTy = gState.getTypeProc(name, node[offset], node[start]) + result.add pident + result.add procTy + result.add newNode(nkEmpty) + elif not adecl.isNil: + # Named param, array type + var + name = name + pident: PNode + if name.nBl: + # Types and procs - `newIdentDefs()` iterator + let + (pname, _, pinfo) = gState.getNameInfo(node[offset].getAtom(), nskField, parent = name) + pident = gState.getIdent(pname, pinfo, exported) + else: + # Vars - `addVar()` + pident = gState.newXIdent(node[offset], nskVar) + if pident.isNil: + return nil + # The var is the parent + name = pident.getIdentName() + + result.add pident + result.add gState.getTypeArray(node[offset], tident, name) + result.add newNode(nkEmpty) + else: + result = nil + +iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInteger, ftname = "", exported = false): PNode = + # Create nkIdentDefs tree for specified proc parameter or object field + # + # `name` is the parent type or proc + # `offset` is the param number and used for unnamed params in procs + # + # For proc, param should not be `exported` + # + # If `ftname` is set, use it as the type name # # Iterator since structs can have multiple comma separated fields for the # same type so can yield multiple results. @@ -460,23 +602,12 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt # struct ABC { int w, h; }; # # This is not applicable for procs. - var - start = getStartAtom(node) - let - # node[start] - param type - (tname0, _, tinfo) = gState.getNameInfo(node[start].getAtom(), nskType, parent = name) - - # Override type name - tname = - if ftname.nBl: - ftname - else: - tname0 - - tident = gState.getIdent(tname, tinfo, exported = false) + (tname, tinfo, tident, start0) = gState.getTypeAndStart(name, node, ftname) # Skip qualifiers after type + var + start = start0 while start < node.len - 1 and node[start+1].getName() == "type_qualifier": start += 1 @@ -503,82 +634,7 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt for i in start+1 ..< node.len: if node[i].getName() == "bitfield_clause": continue - - var - result = newNode(nkIdentDefs) - - let - fdecl = node[i].firstChildInTree("function_declarator") - afdecl = node[i].firstChildInTree("abstract_function_declarator") - adecl = node[i].firstChildInTree("array_declarator") - abst = node[i].getName() == "abstract_pointer_declarator" - if fdecl.isNil and afdecl.isNil and adecl.isNil: - if abst: - # Only for proc with no named param with pointer type - # Create a param name based on offset - # - # int func(char *, int **); - let - pname = "a" & $(offset+1) - pident = gState.getIdent(pname, tinfo, exported) - acount = node[i].getXCount("abstract_pointer_declarator") - result.add pident - result.add gState.newPtrTree(acount, tident) - result.add newNode(nkEmpty) - else: - # Named param, simple type - let - (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) - pident = gState.getIdent(pname, pinfo, exported) - - # Bitfield support - typedef struct { int field: 1; }; - prident = - if node.len > i and node[i + 1].getName() == "bitfield_clause": - gState.newPragmaExpr(node, pident, "bitsize", - newIntNode(nkIntLit, parseInt(gState.getNodeVal(node[i + 1].getAtom())))) - else: - pident - - count = node[i].getPtrCount() - - result.add prident - if count > 0: - result.add gState.newPtrTree(count, tident) - else: - result.add tident - result.add newNode(nkEmpty) - elif not fdecl.isNil: - # Named param, function pointer - let - (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) - pident = gState.getIdent(pname, pinfo, exported) - result.add pident - result.add gState.getTypeProc(name, node[i], node[start]) - result.add newNode(nkEmpty) - elif not afdecl.isNil: - # Only for proc with no named param with function pointer type - # Create a param name based on offset - # - # int func(int (*)(int *)); - let - pname = "a" & $(offset+1) - pident = gState.getIdent(pname, tinfo, exported) - procTy = gState.getTypeProc(name, node[i], node[start]) - result.add pident - result.add procTy - result.add newNode(nkEmpty) - elif not adecl.isNil: - # Named param, array type - let - (pname, _, pinfo) = gState.getNameInfo(node[i].getAtom(), nskField, parent = name) - pident = gState.getIdent(pname, pinfo, exported) - result.add pident - result.add gState.getTypeArray(node[i], tident, name) - result.add newNode(nkEmpty) - else: - result = nil - - yield result + yield gState.newIdentDef(name, node, tname, tinfo, tident, start, i, offset, exported) proc newFormalParams(gState: State, name: string, node: TSNode, rtyp: PNode): PNode = # Create nkFormalParams tree for specified params and return type @@ -967,11 +1023,8 @@ proc addTypeArray(gState: State, node: TSNode) = # Add a type of array type decho("addTypeArray()") let - start = getStartAtom(node) + (_, _, tident, start) = gState.getTypeAndStart("addTypeArray", node) - # node[start] = identifier = type name - (tname, _, info) = gState.getNameInfo(node[start].getAtom(), nskType, parent = "addTypeArray") - tident = gState.getIdent(tname, info, exported = false) commentNodes = gState.getCommentNodes(node) # Could have multiple types, comma separated @@ -1015,6 +1068,7 @@ proc addTypeArray(gState: State, node: TSNode) = proc getTypeProc(gState: State, name: string, node, rnode: TSNode): PNode = # Create proc type tree # + # `name` is the parent type or proc # `rnode` is the return type let # rnode = identifier = return type name @@ -1368,7 +1422,7 @@ proc addEnum(gState: State, node: TSNode) = fnames: HashSet[string] # Hold all of field information so that we can add all of them # after the const identifiers has been updated - fieldDeclarations: seq[tuple[fname: string, fval: string, cexpr: Option[TSNode], comment: seq[TSNode]]] + fieldDeclarations: seq[tuple[fname, forigname, fval, cexpr: string, comment: seq[TSNode]]] for i in 0 .. enumlist.len - 1: let en = enumlist[i] @@ -1378,7 +1432,8 @@ proc addEnum(gState: State, node: TSNode) = let atom = en.getAtom() commentNodes = gState.getCommentNodes(en) - fname = gState.getIdentifier(gState.getNodeVal(atom), nskEnumField) + forigname = gState.getNodeVal(atom) + fname = gState.getIdentifier(forigname, nskEnumField) if fname.nBl and gState.addNewIdentifer(fname): var @@ -1391,9 +1446,9 @@ proc addEnum(gState: State, node: TSNode) = fval = &"({prev} + 1).{name}" if en.len > 1 and en[1].getName() in gEnumVals: - fieldDeclarations.add((fname, "", some(en[1]), commentNodes)) + fieldDeclarations.add((fname, forigname, "", gState.getNodeVal(en[1]), commentNodes)) else: - fieldDeclarations.add((fname, fval, none(TSNode), commentNodes)) + fieldDeclarations.add((fname, forigname, fval, "", commentNodes)) fnames.incl fname prev = fname @@ -1403,78 +1458,25 @@ proc addEnum(gState: State, node: TSNode) = gState.constIdentifiers.incl fnames # parseCExpression requires all const identifiers to be present for the enum - for (fname, fval, cexprNode, commentNodes) in fieldDeclarations: - var fval = fval - if cexprNode.isSome: - fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name - # Cannot use newConstDef() since parseString(fval) adds backticks to and/or - let constNode = gState.parseString(&"const {fname}* = {fval}")[0][0] + for (fname, forigname, fval, cexpr, commentNodes) in fieldDeclarations: + let + fval = + if fval.Bl: + "(" & $gState.parseCExpression(cexpr, name) & ")." & name + else: fval + + # Cannot use newConstDef() since parseString(fval) adds backticks to and/or + constNode = gState.parseString(&"const {fname}* = {fval}")[0][0] + constNode.comment = gState.getCommentsStr(commentNodes) gState.constSection.add constNode + # In case symbol was skipped earlier + gState.skippedSyms.excl forigname # Add other names if node.getName() == "type_definition" and node.len > 1: gState.addTypeTyped(node, ftname = name, offset = offset) -proc addProcVar(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = - # Add a proc variable - decho("addProcVar()") - let - # node = identifier = name - identDefs = gState.newXIdent(node, kind = nskVar, istype = true) - - if not identDefs.isNil: - let - name = identDefs.getIdentName() - # origname = gState.getNodeVal(node.getAtom()) - - procTy = gState.getTypeProc(name, node, rnode) - - identDefs.add procTy - identDefs.add newNode(nkEmpty) - - # var X* {.importc: "_X": proc(a1: Y, a2: Z): P {.cdecl.} - # - # nkIdentDefs( - # nkPragmaExpr( - # nkPostfix( - # nkIdent("*"), - # nkIdent("X") - # ), - # nkPragma( - # nkExprColonExpr( - # nkIdent("importc"), - # nkStrLit("_X") - # ) - # ) - # ), - # nkProcTy( - # nkFormalParams( - # nkIdent("P"), - # nkIdentDefs( - # nkIdent("a1"), - # nkIdent("Y"), - # nkEmpty() - # ), - # nkIdentDefs( - # nkIdent("a2"), - # nkIdent("Z"), - # nkEmpty() - # ) - # ), - # nkPragma( - # nkIdent("cdecl") - # ) - # ), - # nkEmpty() - # ) - - identDefs.comment = gState.getCommentsStr(commentNodes) - # nkVarSection.add - gState.varSection.add identDefs - - gState.printDebug(identDefs) - proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = # Add a proc # @@ -1585,6 +1587,65 @@ proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = gState.printDebug(procDef) +proc addVar(gState: State, node: TSNode, offset: SomeInteger, commentNodes: seq[TSNode]) = + # Add a regular variable + # + # `node` is the `nth` child of (declaration) + # `tnode` is the type node, the first child of (declaration) + decho("addVar()") + let + (tname, tinfo, tident, start) = gState.getTypeAndStart("", node) + + identDefs = gState.newIdentDef("", node, tname, tinfo, tident, start, offset, 0, exported = true) + + if not identDefs.isNil: + # proc var + # + # P (*_X)(Y a1, Z a2); + # + # var X* {.importc: "_X": proc(a1: Y, a2: Z): P {.cdecl.} + # + # nkIdentDefs( + # nkPragmaExpr( + # nkPostfix( + # nkIdent("*"), + # nkIdent("X") + # ), + # nkPragma( + # nkExprColonExpr( + # nkIdent("importc"), + # nkStrLit("_X") + # ) + # ) + # ), + # nkProcTy( + # nkFormalParams( + # nkIdent("P"), + # nkIdentDefs( + # nkIdent("a1"), + # nkIdent("Y"), + # nkEmpty() + # ), + # nkIdentDefs( + # nkIdent("a2"), + # nkIdent("Z"), + # nkEmpty() + # ) + # ), + # nkPragma( + # nkIdent("cdecl") + # ) + # ), + # nkEmpty() + # ) + + identDefs.comment = gState.getCommentsStr(commentNodes) + + # nkVarSection.add + gState.varSection.add identDefs + + gState.printDebug(identDefs) + proc addDecl(gState: State, node: TSNode) = # Add a declaration decho("addDecl()") @@ -1598,31 +1659,29 @@ proc addDecl(gState: State, node: TSNode) = commentNodes: seq[TSNode] for i in start+1 ..< node.len: + if node[i].getName() == "comment": + continue + + if firstDecl: + # If it's the first declaration, use the whole node + # to get the comment above/below + commentNodes = gState.getCommentNodes(node) + firstDecl = false + else: + commentNodes = gState.getCommentNodes(node[i]) + if not node[i].firstChildInTree("function_declarator").isNil: # Proc declaration - var or actual proc if node[i].getAtom().getPxName(1) == "pointer_declarator": # proc var - if firstDecl: - # If it's the first declaration, use the whole node - # to get the comment above/below - commentNodes = gState.getCommentNodes(node) - firstDecl = false - else: - commentNodes = gState.getCommentNodes(node[i]) - gState.addProcVar(node[i], node[start], commentNodes) + #gState.addProcVar(node[i], node[start], commentNodes) + gState.addVar(node, i, commentNodes) else: # proc - if firstDecl: - # If it's the first declaration, use the whole node - # to get the comment above/below - commentNodes = gState.getCommentNodes(node) - firstDecl = false - else: - commentNodes = gState.getCommentNodes(node[i]) gState.addProc(node[i], node[start], commentNodes) else: # Regular var - discard + gState.addVar(node, i, commentNodes) proc addDef(gState: State, node: TSNode) = # Wrap static inline definition if {.header.} mode is specified diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index b2b8c2c..0fea184 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -600,7 +600,8 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = else: raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"") - decho "NODE RESULT: ", result + if result.kind != nkNone: + decho "NODE RESULT: ", result proc parseCExpression*(gState: State, codeRoot: TSNode): PNode = ## Parse a c expression from a root ts node diff --git a/nimterop/getters.nim b/nimterop/getters.nim index e600b18..825b1c4 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -116,7 +116,7 @@ proc getType*(str: string): string = if str == "void": return "object" - result = str.strip(chars={'_'}).replace(re"\s+", " ") + result = str.strip(chars={'_'}).splitWhitespace().join(" ") if gTypeMap.hasKey(result): result = gTypeMap[result] @@ -246,17 +246,23 @@ proc getKeyword*(kind: NimSymKind): string = proc getCurrentHeader*(fullpath: string): string = ("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")])) -proc getPreprocessor*(gState: State, fullpath: string): string = +proc getPreprocessor*(gState: State, fullpath: string) = var cmts = if gState.noComments: "" else: "-CC" cmd = &"""{getCompiler()} -E {cmts} -dD {getGccModeArg(gState.mode)} -w """ - rdata: seq[string] = @[] + ddata: seq[string] + rdata: seq[string] start = false sfile = fullpath.sanitizePath(noQuote = true) + sfileName = sfile.extractFilename() + pDir = sfile.expandFilename().parentDir() + includeDirs: seq[string] + for inc in gState.includeDirs: cmd &= &"-I{inc.sanitizePath} " + includeDirs.add inc.absolutePath().sanitizePath(noQuote = true) for def in gState.defines: cmd &= &"-D{def} " @@ -267,7 +273,10 @@ proc getPreprocessor*(gState: State, fullpath: string): string = else: cmd &= "-D__attribute__(x)= " - cmd &= "-D__restrict= -D__extension__= " + cmd &= "-D__restrict= -D__extension__= -D__inline__=inline -D__inline=inline " + + # https://github.com/tree-sitter/tree-sitter-c/issues/43 + cmd &= "-D_Noreturn= " cmd &= &"{fullpath.sanitizePath}" @@ -278,26 +287,26 @@ proc getPreprocessor*(gState: State, fullpath: string): string = start = false let saniLine = line.sanitizePath(noQuote = true) - if sfile in saniLine: - start = true - elif not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line: + if sfile in saniLine or + (DirSep notin saniLine and sfileName in saniLine): start = true elif gState.recurse: - let - pDir = sfile.expandFilename().parentDir().sanitizePath(noQuote = true) if pDir.Bl or pDir in saniLine: start = true else: - for inc in gState.includeDirs: - if inc.absolutePath().sanitizePath(noQuote = true) in saniLine: + for inc in includeDirs: + if inc in saniLine: start = true break else: if start: if "#undef" in line: continue - rdata.add line - return rdata.join("\n") + elif line.startsWith("#define"): + ddata.add line + else: + rdata.add line + gState.code = ddata.join("\n") & "\n" & rdata.join("\n") converter toString*(kind: Kind): string = return case kind: diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 05284a9..79b1637 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -139,7 +139,7 @@ template Bl*(s: typed): untyped {.used.} = # Redirect output to file when required template gecho*(args: string) = if gState.outputHandle.isNil: - stdout.writeLine(args) + echo args else: gState.outputHandle.writeLine(args) diff --git a/nimterop/toast.nim b/nimterop/toast.nim index f54aa67..02d6494 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -13,7 +13,7 @@ proc process(gState: State, path: string, astTable: AstTable) = gState.mode = getCompilerMode(path) if gState.preprocess: - gState.code = gState.getPreprocessor(path) + gState.getPreprocessor(path) else: gState.code = readFile(path) @@ -35,7 +35,7 @@ proc main( debug = false, defines: seq[string] = @[], dynlib: string = "", - feature: seq[Feature] = @[Feature.ast1], + feature: seq[Feature] = @[], includeDirs: seq[string] = @[], mode = "", nim: string = "nim", @@ -83,6 +83,10 @@ proc main( # Set gDebug in build.nim build.gDebug = debug + # Default `ast` mode + if gState.feature.Bl: + gState.feature.add Feature.ast1 + # Split some arguments with , gState.symOverride = gState.symOverride.getSplitComma() gState.prefix = gState.prefix.getSplitComma() diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index 9565a1f..6760aa9 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -50,7 +50,7 @@ proc getName*(node: TSNode): string {.inline.} = proc getNodeVal*(code: var string, node: TSNode): string = if not node.isNil: - return code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1].strip() + return code[node.tsNodeStartByte() .. node.tsNodeEndByte()-1] proc getNodeVal*(gState: State, node: TSNode): string = gState.code.getNodeVal(node) diff --git a/tests/rsa.nim b/tests/rsa.nim index e7bde89..a4aa64b 100644 --- a/tests/rsa.nim +++ b/tests/rsa.nim @@ -35,10 +35,11 @@ cPlugin: cOverride: proc OPENSSL_die*(assertion: cstring; file: cstring; line: cint) {.importc.} +# Skip comments for https://github.com/tree-sitter/tree-sitter-c/issues/44 cImport(@[ basePath / "rsa.h", basePath / "err.h", -], recurse = true, flags = "-f:ast2 -s " & FLAGS) +], recurse = true, flags = "-f:ast2 -s -c " & FLAGS) {.passL: cryptoLPath.} From 6f96437f39572f3b01f4e61b684f2a74cb524bba Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 18 May 2020 16:39:18 -0500 Subject: [PATCH 006/106] Reduce regex for perf, reduce globals for CT --- nimterop.nimble | 8 +- nimterop/build.nim | 4 +- nimterop/cimport.nim | 8 +- nimterop/getters.nim | 2 +- nimterop/globals.nim | 200 ++++++++++++++++++++++--------------------- nimterop/toast.nim | 2 - nimterop/tshelp.nim | 17 ++++ tests/tast2.nim | 2 + 8 files changed, 128 insertions(+), 115 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index 187b0ce..1eb605d 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -30,15 +30,15 @@ proc execTest(test: string, flags = "", runDocs = true) = mkDir docPath buildDocs(@[test], docPath, nimArgs = "--hints:off " & flags) -task buildToast, "build toast": - execCmd("nim c --hints:off nimterop/toast.nim") - task buildTimeit, "build timer": exec "nim c --hints:off -d:danger tests/timeit" -task bt, "build toast": +task buildToast, "build toast": execCmd("nim c --hints:off -d:danger nimterop/toast.nim") +task bt, "build toast": + buildToastTask() + task btd, "build toast": execCmd("nim c -g nimterop/toast.nim") diff --git a/nimterop/build.nim b/nimterop/build.nim index b02132d..6f13ac7 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -2,8 +2,6 @@ import hashes, macros, osproc, sets, strformat, strutils, tables import os except findExe, sleep -import regex - type BuildType* = enum btAutoconf, btCmake @@ -995,7 +993,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta ## prior to the build process. var origname = header.extractFilename().split(".")[0] - name = origname.replace(re"[[:^alnum:]]", "") + name = origname.split(seps = AllChars-Letters-Digits).join() # -d:xxx for this header stdStr = name & "Std" diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 99be1f5..0559db7 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -17,8 +17,6 @@ All `{.compileTime.}` procs must be used in a compile time context, e.g. using: import hashes, macros, os, strformat, strutils -import regex - import "."/[build, globals, paths, types] export types @@ -684,7 +682,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: hash = output.hash().abs() hpath = getProjectCacheDir("c2nimCache", forceClean = false) / "nimterop_" & $hash & ".h" npath = hpath[0 .. hpath.rfind('.')] & "nim" - header = ("header" & fullpath.splitFile().name.replace(re"[-.]+", "")) + header = "header" & fullpath.splitFile().name.split(seps = {'-', '.'}).join() if not fileExists(hpath) or gStateCT.nocache or compileOption("forceBuild"): mkDir(hpath.parentDir()) @@ -715,10 +713,6 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: var nimout = &"const {header} = \"{fullpath}\"\n\n" & readFile(npath) - nimout = nimout. - replace(re"([u]?int[\d]+)_t", "$1"). - replace(re"([u]?int)ptr_t", "ptr $1") - if gStateCT.debug: echo nimout diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 825b1c4..716c577 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -227,7 +227,7 @@ proc getOverride*(gState: State, name: string, kind: NimSymKind): string = result = sym.override if kind != nskProc: - result = result.replace(re"(?m)^(.*?)$", " $1") + result = " " & result.replace("\n", "\n ") proc getOverrideFinal*(gState: State, kind: NimSymKind): string = # Get all unused cOverride symbols of `kind` diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 79b1637..ac25a40 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -1,58 +1,19 @@ -import sequtils, sets, tables, strutils - -import regex - -import "."/plugin +import tables when defined(TOAST): + import sets, sequtils, strutils + + import regex + + import "."/plugin + import compiler/[ast, idents, modulegraphs, options] import "."/treesitter/api -const - gAtoms* {.used.} = @[ - "field_identifier", - "identifier", - "number_literal", - "char_literal", - "preproc_arg", - "primitive_type", - "sized_type_specifier", - "type_identifier" - ].toHashSet() - - gExpressions* {.used.} = @[ - "parenthesized_expression", - "bitwise_expression", - "shift_expression", - "math_expression", - "escape_sequence" - ].toHashSet() - - gEnumVals* {.used.} = @[ - "identifier", - "number_literal", - "char_literal" - ].concat(toSeq(gExpressions.items)) - type - Kind* = enum - exactlyOne - oneOrMore # + - zeroOrMore # * - zeroOrOne # ? - orWithNext # ! - - Ast* = object - name*: string - kind*: Kind - recursive*: bool - children*: seq[ref Ast] - when defined(TOAST): - tonim*: proc (ast: ref Ast, node: TSNode, gState: State) - regex*: Regex - - AstTable* {.used.} = TableRef[string, seq[ref Ast]] + Feature* = enum + ast1, ast2 State* = ref object # Command line arguments to toast - some forwarded from cimport.nim @@ -79,31 +40,25 @@ type typeMap*: TableRef[string, string] # `--typeMap | -T` to map instances of type X to Y - e.g. ABC=cint - # cimport.nim specific - compile*: seq[string] # `cCompile()` list of files already processed - nocache*: bool # `cDisableCaching()` to disable caching of artifacts - overrides*: string # `cOverride()` code which gets added to `cPlugin()` output - pluginSource*: string # `cPlugin()` generated code to write to plugin file from - searchDirs*: seq[string] # `cSearchPath()` added directories for header search - - # Data fields - code*: string # Contents of header file currently being processed - currentHeader*: string # Const name of header being currently processed - impShort*: string # Short base name for pragma in output - outputHandle*: File # `--output | -o` open file handle - sourceFile*: string # Full path of header being currently processed - - # Plugin callbacks - onSymbol*, onSymbolOverride*: OnSymbol - onSymbolOverrideFinal*: OnSymbolOverrideFinal - - # Symbol tables - constIdentifiers*: HashSet[string] # Const names for enum casting - identifiers*: TableRef[string, string] # Symbols that have been declared so far indexed by nimName - skippedSyms*: HashSet[string] # Symbols that have been skipped due to being unwrappable or - # the user provided override is blank - # Nim compiler objects when defined(TOAST): + # Data fields + code*: string # Contents of header file currently being processed + currentHeader*: string # Const name of header being currently processed + impShort*: string # Short base name for pragma in output + outputHandle*: File # `--output | -o` open file handle + sourceFile*: string # Full path of header being currently processed + + # Plugin callbacks + onSymbol*, onSymbolOverride*: OnSymbol + onSymbolOverrideFinal*: OnSymbolOverrideFinal + + # Symbol tables + constIdentifiers*: HashSet[string] # Const names for enum casting + identifiers*: TableRef[string, string] # Symbols that have been declared so far indexed by nimName + skippedSyms*: HashSet[string] # Symbols that have been skipped due to being unwrappable or + # the user provided override is blank + + # Nim compiler objects constSection*, enumSection*, pragmaSection*, procSection*, typeSection*, varSection*: PNode identCache*: IdentCache config*: ConfigRef @@ -112,37 +67,86 @@ type # Table of symbols to generated AST PNode - used to implement forward declarations identifierNodes*: TableRef[string, PNode] - # Used for the exprparser.nim module - currentExpr*, currentTyCastName*: string - # Controls whether or not the current expression - # should validate idents against currently defined idents - skipIdentValidation*: bool + # Used for the exprparser.nim module + currentExpr*, currentTyCastName*: string + # Controls whether or not the current expression + # should validate idents against currently defined idents + skipIdentValidation*: bool - # Legacy AST fields, remove when ast2 becomes default - constStr*, enumStr*, procStr*, typeStr*: string - commentStr*, debugStr*, skipStr*: string - data*: seq[tuple[name, val: string]] - nodeBranch*: seq[string] + # Legacy AST fields, remove when ast2 becomes default + constStr*, enumStr*, procStr*, typeStr*: string + commentStr*, debugStr*, skipStr*: string + data*: seq[tuple[name, val: string]] + nodeBranch*: seq[string] + else: + # cimport.nim specific + compile*: seq[string] # `cCompile()` list of files already processed + nocache*: bool # `cDisableCaching()` to disable caching of artifacts + overrides*: string # `cOverride()` code which gets added to `cPlugin()` output + pluginSource*: string # `cPlugin()` generated code to write to plugin file from + searchDirs*: seq[string] # `cSearchPath()` added directories for header search - Feature* = enum - ast1, ast2 +when defined(TOAST): + const + gAtoms* {.used.} = @[ + "field_identifier", + "identifier", + "number_literal", + "char_literal", + "preproc_arg", + "primitive_type", + "sized_type_specifier", + "type_identifier" + ].toHashSet() -var - gStateCT* {.compiletime, used.} = new(State) + gExpressions* {.used.} = @[ + "parenthesized_expression", + "bitwise_expression", + "shift_expression", + "math_expression", + "escape_sequence" + ].toHashSet() + + gEnumVals* {.used.} = @[ + "identifier", + "number_literal", + "char_literal" + ].concat(toSeq(gExpressions.items)) + + type + Kind* = enum + exactlyOne + oneOrMore # + + zeroOrMore # * + zeroOrOne # ? + orWithNext # ! + + Ast* = object + name*: string + kind*: Kind + recursive*: bool + children*: seq[ref Ast] + tonim*: proc (ast: ref Ast, node: TSNode, gState: State) + regex*: Regex + + AstTable* {.used.} = TableRef[string, seq[ref Ast]] + + # Redirect output to file when required + template gecho*(args: string) = + if gState.outputHandle.isNil: + echo args + else: + gState.outputHandle.writeLine(args) + + template decho*(args: varargs[string, `$`]): untyped = + if gState.debug: + gecho join(args, "").getCommented() +else: + var + gStateCT* {.compiletime, used.} = new(State) template nBl*(s: typed): untyped {.used.} = (s.len != 0) template Bl*(s: typed): untyped {.used.} = - (s.len == 0) - -# Redirect output to file when required -template gecho*(args: string) = - if gState.outputHandle.isNil: - echo args - else: - gState.outputHandle.writeLine(args) - -template decho*(args: varargs[string, `$`]): untyped = - if gState.debug: - gecho join(args, "").getCommented() \ No newline at end of file + (s.len == 0) \ No newline at end of file diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 02d6494..84835a3 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -4,8 +4,6 @@ import "."/treesitter/[api, c, cpp] import "."/[ast, ast2, build, globals, getters, grammar, tshelp] -{.passC: "-DNIMTEROP".} - proc process(gState: State, path: string, astTable: AstTable) = doAssert existsFile(path), &"Invalid path {path}" diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index 6760aa9..fa2a750 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -79,6 +79,23 @@ proc getStartAtom*(node: TSNode): int = else: break +proc getConstQualifier*(gState: State, node: TSNode): bool = + # Check if node siblings have type_qualifier = `const` + var + curr = node.tsNodePrevNamedSibling() + while not curr.isNil: + # Check previous siblings + if curr.getName() == "type_qualifier" and + gState.getNodeVal(curr) == "const": + return true + curr = curr.tsNodePrevNamedSibling() + + # Check immediate next sibling + curr = node.tsNodePrevNamedSibling() + if curr.getName() == "type_qualifier" and + gState.getNodeVal(curr) == "const": + return true + proc getXCount*(node: TSNode, ntype: string, reverse = false): int = if not node.isNil: # Get number of ntype nodes nested in tree diff --git a/tests/tast2.nim b/tests/tast2.nim index 8deb99e..fd0175a 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -2,6 +2,8 @@ import macros, os, sets, strutils import nimterop/[cimport] +{.passC: "-DNIMTEROP".} + static: # Skip casting on lower nim compilers because # the VM does not support it From f079d851db1db0dd64d7538a8b417b1e5c0985a1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 19 May 2020 10:27:31 -0500 Subject: [PATCH 007/106] Changelog for variable support --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index ea3f427..e47974d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -35,6 +35,7 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 - Nested function pointers - [#155][i155] [#156][i156] - Various enum fixes - [#159][i159] [#171][i171] - Map `int arr[]` to `arr: UncheckedArray[cint]` - [#174][i174] + - Global variables including arrays and procs (since v0.5.4) - `ast2` also includes an advanced expression parser that can reliably handle constructs typically seen with `#define` statements and enumeration values: - Integers + integer like expressions (hex, octal, suffixes) From 7d3f52f1dc0829bc8540a413925147b96b5298fe Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 19 May 2020 10:27:36 -0500 Subject: [PATCH 008/106] v0.5.4 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 1eb605d..d1ad7ce 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.3" +version = "0.5.4" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From e32fd4d1d2542a6ffaeb8221a3da7ef136669651 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 20 May 2020 15:16:26 -0500 Subject: [PATCH 009/106] ast2 handle tree-sitter errors, multi-reorder --- nimterop/ast2.nim | 100 +++++++++++++++++++++++++------------------ nimterop/getters.nim | 8 +--- nimterop/globals.nim | 3 ++ 3 files changed, 64 insertions(+), 47 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 3d52e02..e2cdf8f 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -27,7 +27,7 @@ proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimS if not pnode.isNil: result = pnode[0][0] else: - gecho &"\n# $1'{origname}' skipped" % skind + gecho &"# {skind}'{origname}' skipped" gState.skippedSyms.incl origname proc addOverrideFinal(gState: State, kind: NimSymKind) = @@ -391,7 +391,7 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: gState.identifierNodes[name] = result else: - gecho &"# $1 '{origname}' is duplicate, skipped" % getKeyword(kind) + gecho &"# {getKeyword(kind)} '{origname}' is duplicate, skipped" proc newArrayTree(gState: State, node: TSNode, typ, size: PNode = nil): PNode = # Create nkBracketExpr tree depending on input @@ -682,7 +682,7 @@ proc newProcTy(gState: State, name: string, node: TSNode, rtyp: PNode): PNode = if node.getVarargs(): gState.addPragma(node, result[^1], "varargs") -proc processNode(gState: State, node: TSNode): bool +proc processNode(gState: State, node: TSNode): Status proc newRecListTree(gState: State, name: string, node: TSNode): PNode = # Create nkRecList tree for specified object if not node.isNil: @@ -720,9 +720,9 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = $gState.enumSection[^1][0][1] ) else: - (true, "") + (success, "") - if not processed: + if processed != success: return nil # Add nkIdentDefs for each field @@ -1699,35 +1699,52 @@ proc addDef(gState: State, node: TSNode) = if not gState.noHeader: gState.addProc(node[start+1], node[start], commentNodes) else: - gecho &"\n# proc '$1' skipped - static inline procs cannot work with '--noHeader | -H'" % + gecho "# proc '$1' skipped - static inline procs cannot work with '--noHeader | -H'" % gState.getNodeVal(node[start+1].getAtom()) -proc processNode(gState: State, node: TSNode): bool = - result = true +proc processNode(gState: State, node: TSNode): Status = + const + known = ["preproc_def", "type_definition", + "struct_specifier", "union_specifier", "enum_specifier", + "declaration", "function_definition"].toHashSet() - case node.getName() - of "preproc_def": - gState.addConst(node) - of "type_definition": - if node.len > 0 and node[0].getName() == "enum_specifier": - gState.addEnum(node) - elif node.len > 0 and node[0].getName() == "union_specifier": - gState.addType(node, union = true) + result = success + let + name = node.getName() + if name in known: + # Recognized top-level nodes + let + err = node.anyChildInTree("ERROR") + if not err.isNil: + # Bail on errors + gState.printDebug(node) + gecho &"# tree-sitter parse error: '{gState.getNodeVal(node).splitLines()[0]}', skipped" + result = Status.error else: - gState.addType(node) - of "struct_specifier": - gState.addType(node) - of "union_specifier": - gState.addType(node, union = true) - of "enum_specifier": - gState.addEnum(node) - of "declaration": - gState.addDecl(node) - of "function_definition": - gState.addDef(node) + # Process nodes + case name + of "preproc_def": + gState.addConst(node) + of "type_definition": + if node.len > 0 and node[0].getName() == "enum_specifier": + gState.addEnum(node) + elif node.len > 0 and node[0].getName() == "union_specifier": + gState.addType(node, union = true) + else: + gState.addType(node) + of "struct_specifier": + gState.addType(node) + of "union_specifier": + gState.addType(node, union = true) + of "enum_specifier": + gState.addEnum(node) + of "declaration": + gState.addDecl(node) + of "function_definition": + gState.addDef(node) else: - # Unknown - result = false + # Unknown, will check child nodes + result = unknown proc searchTree(gState: State, root: TSNode) = # Search AST generated by tree-sitter for recognized elements @@ -1735,7 +1752,7 @@ proc searchTree(gState: State, root: TSNode) = node = root nextnode: TSNode depth = 0 - processed = false + processed = success while true: if not node.isNil and depth > -1: @@ -1743,7 +1760,7 @@ proc searchTree(gState: State, root: TSNode) = else: break - if not processed and node.len != 0: + if processed == unknown and node.len != 0: nextnode = node[0] depth += 1 else: @@ -1773,27 +1790,28 @@ proc searchTree(gState: State, root: TSNode) = proc setupPragmas(gState: State, root: TSNode, fullpath: string) = # Create shortcut pragmas to reduce clutter var - hdrPragma: PNode - dynPragma: PNode + count = 0 if not gState.noHeader: # {.pragma: impnameHdr, header: "xxx".} - hdrPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Hdr")) + let + hdrPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Hdr")) gState.addPragma(root, hdrPragma, "header", newStrNode(nkStrLit, fullpath)) + gState.pragmaSection.add hdrPragma + count += 1 if gState.dynlib.nBl: # {.pragma: impnameDyn, dynlib: libname.} - dynPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Dyn")) + let + dynPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Dyn")) gState.addPragma(root, dynPragma, "dynlib", gState.getIdent(gState.dynlib)) - - # Add pragma shortcuts to output - if not hdrPragma.isNil: - gState.pragmaSection.add hdrPragma - if not dynPragma.isNil: gState.pragmaSection.add dynPragma + count += 1 # Add `{.experimental: "codeReordering".} for #206 - gState.pragmaSection.add gState.newPragma(root, "experimental", newStrNode(nkStrLit, "codeReordering")) + if gState.pragmaSection.len == count: + # Only if not already done + gState.pragmaSection.add gState.newPragma(root, "experimental", newStrNode(nkStrLit, "codeReordering")) proc initNim*(gState: State) = # Initialize for parseNim() one time diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 716c577..0906d56 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -251,7 +251,6 @@ proc getPreprocessor*(gState: State, fullpath: string) = cmts = if gState.noComments: "" else: "-CC" cmd = &"""{getCompiler()} -E {cmts} -dD {getGccModeArg(gState.mode)} -w """ - ddata: seq[string] rdata: seq[string] start = false sfile = fullpath.sanitizePath(noQuote = true) @@ -302,11 +301,8 @@ proc getPreprocessor*(gState: State, fullpath: string) = if start: if "#undef" in line: continue - elif line.startsWith("#define"): - ddata.add line - else: - rdata.add line - gState.code = ddata.join("\n") & "\n" & rdata.join("\n") + rdata.add line + gState.code = rdata.join("\n") converter toString*(kind: Kind): string = return case kind: diff --git a/nimterop/globals.nim b/nimterop/globals.nim index ac25a40..8062ac8 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -131,6 +131,9 @@ when defined(TOAST): AstTable* {.used.} = TableRef[string, seq[ref Ast]] + Status* = enum + success, unknown, error + # Redirect output to file when required template gecho*(args: string) = if gState.outputHandle.isNil: From 80cfc99dcd08a145569f28ad6b9999dbd9868bda Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 20 May 2020 16:58:52 -0500 Subject: [PATCH 010/106] Fix nimconf path to nim --- nimterop/ast2.nim | 2 +- nimterop/build.nim | 14 +++++++++----- nimterop/getters.nim | 2 +- nimterop/nimconf.nim | 2 +- nimterop/toast.nim | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index e2cdf8f..b604f1b 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -139,7 +139,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = else: gecho &"# const '{origname}' is duplicate, skipped" else: - gecho &"# const '{origname}' has unsupported value '{val}'" + gecho &"# const '{origname}' has unsupported value '{val.strip()}'" gState.skippedSyms.incl origname proc addConst(gState: State, node: TSNode) = diff --git a/nimterop/build.nim b/nimterop/build.nim index 6f13ac7..58ecbd1 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -15,6 +15,7 @@ type var gDebug* = false gDebugCT* {.compileTime.} = false + gNimExe* = "" proc echoDebug(str: string) = let str = "\n# " & str.strip().replace("\n", "\n# ") @@ -46,6 +47,14 @@ proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = if not noQuote: result = result.quoteShell +proc getCurrentNimCompiler*(): string = + when nimvm: + result = getCurrentCompilerExe() + when defined(nimsuggest): + result = result.replace("nimsuggest", "nim") + else: + result = gNimExe + # Nim cfg file related functionality include "."/nimconf @@ -64,11 +73,6 @@ proc getNimteropCacheDir(): string = # Get location to cache all nimterop artifacts result = getNimcacheDir() / "nimterop" -proc getCurrentNimCompiler*(): string = - result = getCurrentCompilerExe() - when defined(nimsuggest): - result = result.replace("nimsuggest", "nim") - proc execAction*(cmd: string, retry = 0, die = true, cache = false, cacheKey = ""): tuple[output: string, ret: int] = ## Execute an external command - supported at compile time diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 0906d56..b092663 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -454,7 +454,7 @@ proc loadPlugin*(gState: State, sourcePath: string) = outflags = &"--out:\"{pdll}\"" # Compile plugin as library with `markAndSweep` GC - cmd = &"{gState.nim.sanitizePath} c --app:lib --gc:markAndSweep {flags} {outflags} {sourcePath.sanitizePath}" + cmd = &"{gState.nim} c --app:lib --gc:markAndSweep {flags} {outflags} {sourcePath.sanitizePath}" discard execAction(cmd) doAssert fileExists(pdll), "No plugin binary generated for " & sourcePath diff --git a/nimterop/nimconf.nim b/nimterop/nimconf.nim index 2a54d7e..33215ef 100644 --- a/nimterop/nimconf.nim +++ b/nimterop/nimconf.nim @@ -20,7 +20,7 @@ type proc getJson(projectDir: string): JsonNode = # Get `nim dump` json value for `projectDir` var - cmd = "nim --hints:off --dump.format:json dump dummy" + cmd = &"{getCurrentNimCompiler()} --hints:off --dump.format:json dump dummy" dump = "" ret = 0 diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 84835a3..4a9553b 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -64,7 +64,7 @@ proc main( feature: feature, includeDirs: includeDirs, mode: mode, - nim: nim, + nim: nim.sanitizePath, noComments: noComments, noHeader: noHeader, past: past, @@ -79,7 +79,8 @@ proc main( ) # Set gDebug in build.nim - build.gDebug = debug + build.gDebug = gState.debug + build.gNimExe = gState.nim # Default `ast` mode if gState.feature.Bl: From 6c01f5504771d2a3b013097007f16a5414baa5b7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 21 May 2020 13:51:15 -0500 Subject: [PATCH 011/106] No pointless hints --- nimterop/ast.nim | 2 ++ nimterop/ast2.nim | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/nimterop/ast.nim b/nimterop/ast.nim index 96116c7..78501fd 100644 --- a/nimterop/ast.nim +++ b/nimterop/ast.nim @@ -238,6 +238,8 @@ proc printNim*(gState: State) = if gState.procStr.nBl: gecho &"{gState.procStr}\n" + gecho "{.pop.}" + if gState.debug: if gState.debugStr.nBl: gecho gState.debugStr diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index e2cdf8f..f4e15db 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -1815,7 +1815,10 @@ proc setupPragmas(gState: State, root: TSNode, fullpath: string) = proc initNim*(gState: State) = # Initialize for parseNim() one time - gecho "import nimterop/types\n" + gecho """import nimterop/types + +{.push hint[ConvFromXtoItselfNotNeeded]: off.} +""" # Track identifiers already rendered and corresponding PNodes gState.identifiers = newTable[string, string]() @@ -1867,3 +1870,5 @@ proc printNim*(gState: State) = tree.add gState.procSection gecho tree.renderTree() + + gecho "{.pop.}" \ No newline at end of file From a54991122de7f586f3671316eaf079b25d1eae16 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 20 May 2020 16:58:52 -0500 Subject: [PATCH 012/106] Fix nimconf path to nim --- nimterop/ast2.nim | 2 +- nimterop/build.nim | 14 +++++++++----- nimterop/getters.nim | 2 +- nimterop/nimconf.nim | 2 +- nimterop/toast.nim | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index f4e15db..759ddfd 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -139,7 +139,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = else: gecho &"# const '{origname}' is duplicate, skipped" else: - gecho &"# const '{origname}' has unsupported value '{val}'" + gecho &"# const '{origname}' has unsupported value '{val.strip()}'" gState.skippedSyms.incl origname proc addConst(gState: State, node: TSNode) = diff --git a/nimterop/build.nim b/nimterop/build.nim index 6f13ac7..58ecbd1 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -15,6 +15,7 @@ type var gDebug* = false gDebugCT* {.compileTime.} = false + gNimExe* = "" proc echoDebug(str: string) = let str = "\n# " & str.strip().replace("\n", "\n# ") @@ -46,6 +47,14 @@ proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = if not noQuote: result = result.quoteShell +proc getCurrentNimCompiler*(): string = + when nimvm: + result = getCurrentCompilerExe() + when defined(nimsuggest): + result = result.replace("nimsuggest", "nim") + else: + result = gNimExe + # Nim cfg file related functionality include "."/nimconf @@ -64,11 +73,6 @@ proc getNimteropCacheDir(): string = # Get location to cache all nimterop artifacts result = getNimcacheDir() / "nimterop" -proc getCurrentNimCompiler*(): string = - result = getCurrentCompilerExe() - when defined(nimsuggest): - result = result.replace("nimsuggest", "nim") - proc execAction*(cmd: string, retry = 0, die = true, cache = false, cacheKey = ""): tuple[output: string, ret: int] = ## Execute an external command - supported at compile time diff --git a/nimterop/getters.nim b/nimterop/getters.nim index 0906d56..b092663 100644 --- a/nimterop/getters.nim +++ b/nimterop/getters.nim @@ -454,7 +454,7 @@ proc loadPlugin*(gState: State, sourcePath: string) = outflags = &"--out:\"{pdll}\"" # Compile plugin as library with `markAndSweep` GC - cmd = &"{gState.nim.sanitizePath} c --app:lib --gc:markAndSweep {flags} {outflags} {sourcePath.sanitizePath}" + cmd = &"{gState.nim} c --app:lib --gc:markAndSweep {flags} {outflags} {sourcePath.sanitizePath}" discard execAction(cmd) doAssert fileExists(pdll), "No plugin binary generated for " & sourcePath diff --git a/nimterop/nimconf.nim b/nimterop/nimconf.nim index 2a54d7e..33215ef 100644 --- a/nimterop/nimconf.nim +++ b/nimterop/nimconf.nim @@ -20,7 +20,7 @@ type proc getJson(projectDir: string): JsonNode = # Get `nim dump` json value for `projectDir` var - cmd = "nim --hints:off --dump.format:json dump dummy" + cmd = &"{getCurrentNimCompiler()} --hints:off --dump.format:json dump dummy" dump = "" ret = 0 diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 84835a3..4a9553b 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -64,7 +64,7 @@ proc main( feature: feature, includeDirs: includeDirs, mode: mode, - nim: nim, + nim: nim.sanitizePath, noComments: noComments, noHeader: noHeader, past: past, @@ -79,7 +79,8 @@ proc main( ) # Set gDebug in build.nim - build.gDebug = debug + build.gDebug = gState.debug + build.gNimExe = gState.nim # Default `ast` mode if gState.feature.Bl: From f1b5f5b8905a015c4e220c7689b7a05ac9375eee Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 21 May 2020 14:57:58 -0500 Subject: [PATCH 013/106] v0.5.5 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index d1ad7ce..76f05ea 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.4" +version = "0.5.5" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From ebf9018df85aa3385b19aed6273d80390753fdeb Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 23 May 2020 11:29:59 -0600 Subject: [PATCH 014/106] Try markdown code block --- nimterop/tshelp.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nimterop/tshelp.nim b/nimterop/tshelp.nim index fa2a750..ad3e89e 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/tshelp.nim @@ -284,18 +284,17 @@ proc getCommentsStr*(gState: State, commentNodes: seq[TSNode]): string = ## Generate a comment from a set of comment nodes. Comment is guaranteed ## to be able to be rendered using nim doc if commentNodes.len > 0: - result = "::" for commentNode in commentNodes: result &= "\n " & gState.getNodeVal(commentNode).strip() - result = result.multiReplace( + result = "```\n " & result.multiReplace( { "/**": "", "**/": "", "/*": "", "*/": "", "/*": "", "//": "", "\n": "\n ", "`": "" } # need to replace this last otherwise it supercedes other replacements - ).replace(" *", "").strip() + ).replace(" *", "").strip() & "\n```" proc getCommentNodes*(gState: State, node: TSNode, maxSearch=1): seq[TSNode] = ## Get a set of comment nodes in order of priority. Will search up to ``maxSearch`` From 0e3c2d02f9566786d106687239fc39331e72cf41 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 25 May 2020 22:56:32 -0500 Subject: [PATCH 015/106] v0.5.6 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 76f05ea..3c8e74c 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.5" +version = "0.5.6" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From e8baeb92294328d9750469f852816c5a47375954 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 27 May 2020 10:25:48 -0500 Subject: [PATCH 016/106] Fix #222 --- nimterop/ast2.nim | 2 +- nimterop/build.nim | 3 +++ tests/include/tast2.h | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nimterop/ast2.nim b/nimterop/ast2.nim index 759ddfd..de3b452 100644 --- a/nimterop/ast2.nim +++ b/nimterop/ast2.nim @@ -608,7 +608,7 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt # Skip qualifiers after type var start = start0 - while start < node.len - 1 and node[start+1].getName() == "type_qualifier": + while start < node.len - 1 and node[start+1].getName() in ["type_qualifier", "comment"]: start += 1 if start == node.len - 1: diff --git a/nimterop/build.nim b/nimterop/build.nim index 58ecbd1..e0d630a 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -365,6 +365,9 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = discard execAction(&"cd {outdirQ} && git config core.sparsecheckout true") writeFile(sparsefile, plist) + # In case directory has old files from another run + discard execAction(&"cd {outdirQ} && git clean -fxd") + if checkout.len != 0: echo "# Checking out " & checkout discard execAction(&"cd {outdirQ} && git fetch", retry = 1) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index b8433f9..019f7c4 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -90,7 +90,7 @@ typedef char *(*A11)[3]; typedef struct A0 *A111[12]; typedef int - **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)), + **(*A12)(int, int b, int *c, int *, int /*out*/ *count[4], int (*func)(int, int)), **(*A121)(float, float b, float *c, float *, float *count[4], float (*func)(float, float)), **(*A122)(char, char b, char *c, char *, char *count[4], char (*func)(char, char)); typedef int (*A13)(int, int, void (*func)(void)); From d875252238ceb6a8134b2c9429f8a6c383aff48a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 27 May 2020 11:54:50 -0500 Subject: [PATCH 017/106] v0.5.7 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 3c8e74c..eed70ba 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.6" +version = "0.5.7" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From fa02a6e0bc955042ac867c5a3a462a00d9cfe092 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jun 2020 17:42:31 -0500 Subject: [PATCH 018/106] Fix #227 - paramCount/Str revert --- nimterop/docs.nim | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nimterop/docs.nim b/nimterop/docs.nim index 6c2bbfd..5fe87a1 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -2,11 +2,8 @@ import macros, strformat from os import parentDir, getCurrentCompilerExe, DirSep -when defined(nimdoc) or (NimMajor, NimMinor) >= (1, 3): - from os import paramCount, paramStr - when defined(nimdoc): - from os import getCurrentDir + from os import getCurrentDir, paramCount, paramStr proc getNimRootDir(): string = #[ From 2885753dca1acacd3bcebb299fb7fa2aa7a88fe4 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jun 2020 20:42:42 -0500 Subject: [PATCH 019/106] v0.5.8 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index eed70ba..b5fab89 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.7" +version = "0.5.8" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From eb2fd1c1dd40ad5ecb0ee6389f67c7704375ac85 Mon Sep 17 00:00:00 2001 From: Euan Date: Thu, 18 Jun 2020 19:08:29 +0100 Subject: [PATCH 020/106] Fix #229 - FreeBSD build error --- nimterop/toast.nims | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/nimterop/toast.nims b/nimterop/toast.nims index 83d5cf4..e9a3c99 100644 --- a/nimterop/toast.nims +++ b/nimterop/toast.nims @@ -1,10 +1,8 @@ import os # Workaround for C++ scanner.cc causing link error with other C obj files -when defined(MacOSX): - switch("clang.linkerexe", "g++") -else: - switch("gcc.linkerexe", "g++") +switch("clang.linkerexe", "clang++") +switch("gcc.linkerexe", "g++") # Workaround for NilAccessError crash on Windows #98 # Could also help for OSX/Linux crash @@ -25,4 +23,4 @@ when not defined(danger): switch("out", currentSourcePath.parentDir() / "toast".addFileExt(ExeExt)) # Define TOAST for globals.nim -switch("define", "TOAST") \ No newline at end of file +switch("define", "TOAST") From 68cb3ab0b449c6155d5a797bec4d715a80423f05 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 18 Jun 2020 15:10:49 -0500 Subject: [PATCH 021/106] v0.5.9 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index b5fab89..1657aed 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.8" +version = "0.5.9" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 290ad2b96733170ad653abd9f3705b5f50c4dcca Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 9 Jun 2020 23:21:11 -0500 Subject: [PATCH 022/106] Initial conan.io support --- nimterop/build.nim | 111 ++++++++++++-- nimterop/conan.nim | 373 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 468 insertions(+), 16 deletions(-) create mode 100644 nimterop/conan.nim diff --git a/nimterop/build.nim b/nimterop/build.nim index e0d630a..d5ce59c 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -206,6 +206,14 @@ proc rmDir*(dir: string) = ## Remove a directory or pattern at compile time rmFile(dir, dir = true) +proc cleanDir*(dir: string) = + ## Remove all contents of a directory at compile time + for kind, path in walkDir(dir): + if kind == pcDir: + rmDir(path) + else: + rmFile(path) + proc getProjectCacheDir*(name: string, forceClean = true): string = ## Get a cache directory where all nimterop artifacts can be stored ## @@ -281,7 +289,7 @@ proc extractTar*(tarfile, outdir: string) = if name.len != 0: rmFile(outdir / name) -proc downloadUrl*(url, outdir: string) = +proc downloadUrl*(url, outdir: string, quiet = false) = ## Download a file using `curl` or `wget` (or `powershell` on Windows) to the specified directory ## ## If an archive file, it is automatically extracted after download. @@ -291,7 +299,8 @@ proc downloadUrl*(url, outdir: string) = archives = @[".zip", ".xz", ".gz", ".bz2", ".tgz", ".tar"] if not (ext in archives and fileExists(outdir/file)): - echo "# Downloading " & file + if not quiet: + echo "# Downloading " & file mkDir(outdir) var cmd = findExe("curl") if cmd.len != 0: @@ -304,7 +313,7 @@ proc downloadUrl*(url, outdir: string) = cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" else: doAssert false, "No download tool available - curl, wget" - discard execAction(cmd % [url, (outdir/file).sanitizePath], retry = 1) + discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 1) if ext == ".zip": extractZip(file, outdir) @@ -732,6 +741,36 @@ proc getGccLibPaths*(mode: string): seq[string] = when defined(osx): result.add "/usr/lib" +proc getGccInfo*(): tuple[arch, os, compiler, version: string] = + let + (outp, _) = execAction(&"{getCompiler()} -v") + for line in outp.splitLines(): + if line.startsWith("Target: "): + result.arch = line.split(' ')[1].split('-')[0] + result.os = + if "linux" in line: + "linux" + elif "android" in line: + "android" + elif "darwin" in line: + "macos" + elif "w64" in line or "mingw" in line: + "windows" + else: + "unknown" + elif " version " in line: + result.version = line.split(" version ")[1].split(' ')[0] + if "clang" in outp: + if result.os == "macos": + result.compiler = "apple-clang" + else: + result.compiler = "clang" + else: + result.compiler = "gcc" + +# Conan support +include conan + proc getStdPath(header, mode: string): string = for inc in getGccPaths(mode): result = findFile(header, inc, recurse = false, first = true) @@ -784,6 +823,28 @@ proc getDlPath(header, url, outdir, version: string): string = result = findFile(header, outdir) +proc getConanPath(header, uri, outdir, version: string, shared: bool): string = + var + uri = uri + + if "$#" in uri or "$1" in uri: + doAssert version.len != 0, "Need version for Conan uri" + uri = uri % version + else: + doAssert version.len == 0, "Conan uri does not contain version" + + let + pkg = newConanPackageFromUri(uri, shared) + downloadConan(pkg, outdir) + + result = findFile(header, outdir) + +proc getConanLPath(outdir: string): string = + let + pkg = loadConanInfo(outdir) + + result = pkg.getConanLibs(outdir).join(" ") + proc getLocalPath(header, outdir: string): string = if outdir.len != 0: result = findFile(header, outdir) @@ -937,7 +998,9 @@ macro isDefined*(def: untyped): untyped = false ) -macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: static[string] = "", outdir: static[string] = "", +macro getHeader*( + header: static[string], giturl: static[string] = "", dlurl: static[string] = "", + conanuri: static[string] = "", outdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = ## Get the path to a header file for wrapping with @@ -950,16 +1013,18 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta ## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux ## `-d:xxxGit` - clone source from a git repo specified in `giturl` ## `-d:xxxDL` - download source from `dlurl` and extract if required + ## `-d:xxxConan` - download headers and binary from conan.io using `conanuri` + ## typically formatted as `name/version[@user/channel][:bhash]` ## ## This allows a single wrapper to be used in different ways depending on the user's needs. ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. ## - ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git` or `DL`. - ## This allows using a system installed library if available before falling back to manual - ## building. + ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL` or + ## `Conan`. This allows using a system installed library if available before falling back + ## to manual building. ## ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag - ## name for Git whereas for DL, it replaces `$1` in the URL defined. + ## name for Git whereas for DL and Conan, it replaces `$1` in the URL defined. ## ## All defines can also be set in code using `setDefines()`. ## @@ -1006,6 +1071,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta stdStr = name & "Std" gitStr = name & "Git" dlStr = name & "DL" + conanStr = name & "Conan" staticStr = name & "Static" verStr = name & "SetVer" @@ -1014,6 +1080,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta nameStd = newIdentNode(stdStr) nameGit = newIdentNode(gitStr) nameDL = newIdentNode(dlStr) + nameConan = newIdentNode(conanStr) nameStatic = newIdentNode(staticStr) @@ -1031,6 +1098,7 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta stdVal = gDefines.hasKey(stdStr) gitVal = gDefines.hasKey(gitStr) dlVal = gDefines.hasKey(dlStr) + conanVal = gDefines.hasKey(conanStr) staticVal = gDefines.hasKey(staticStr) verVal = if gDefines.hasKey(verStr): @@ -1052,14 +1120,17 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta `nameStd`* = when defined(`nameStd`): true else: `stdVal` == 1 `nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1 `nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1 + `nameConan`* = when defined(`nameConan`): true else: `conanVal` == 1 `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, outdir, version: string): string = + proc getPath(header, giturl, dlurl, conanuri, outdir, version: string, shared: bool): string = when `nameGit`: getGitPath(header, giturl, outdir, version) elif `nameDL`: getDlPath(header, dlurl, outdir, version) + elif `nameConan`: + getConanPath(header, conanuri, outdir, version, shared) else: getLocalPath(header, outdir) @@ -1077,23 +1148,30 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta stdLPath = when `nameStd`: getStdLibPath(`lname`, `mode`) else: "" + useStd = stdPath.len != 0 and stdLPath.len != 0 + # Look elsewhere if requested while prioritizing standard paths prePath = - when stdPath.len != 0 and stdLPath.len != 0: + when useStd: stdPath else: - getPath(`header`, `giturl`, `dlurl`, `outdir`, `version`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`) - # Run preBuild hook before building library if not standard - when (prePath != stdPath or prePath.len == 0) and declared(`preBuild`): + # Run preBuild hook before building library if not Std or Conan + when not (useStd or `nameConan`) and declared(`preBuild`): static: `preBuild`(`outdir`, prePath) const # Library binary path - build if not standard `lpath`* = - when stdPath.len != 0 and stdLPath.len != 0: + when useStd: stdLPath + elif `nameConan`: + when `nameStatic`: + getConanLPath(`outdir`) + else: + findFile(`lname`, `outdir`, regex = true) else: buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) @@ -1102,10 +1180,11 @@ macro getHeader*(header: static[string], giturl: static[string] = "", dlurl: sta if prePath.len != 0: prePath else: - getPath(`header`, `giturl`, `dlurl`, `outdir`, `version`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`) static: - doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & "missing/empty outdir or -d:$1Std -d:$1Git or -d:$1DL not specified" % `name` + doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & + "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL or -d:$1Conan not specified" % `name` doAssert `lpath`.len != 0, "\nLibrary " & `lname` & " not found" echo "# Including library " & `lpath` diff --git a/nimterop/conan.nim b/nimterop/conan.nim new file mode 100644 index 0000000..0707ecc --- /dev/null +++ b/nimterop/conan.nim @@ -0,0 +1,373 @@ +import json, marshal, os, strformat, strutils, tables + +type + ConanPackage* = ref object + ## ConanPackage type that stores conan uri and recipes/builds/revisions + name*: string + version*: string + user*: string + channel*: string + recipes*: OrderedTableRef[string, seq[ConanBuild]] + + bhash*: string + shared*: bool + headers*: seq[string] + libs*: seq[string] + requires*: seq[ConanPackage] + + ConanBuild* = ref object + ## Build type that stores build specific info and revisions + bhash*: string + settings*: TableRef[string, string] + options*: TableRef[string, string] + requires*: seq[string] + recipe_hash*: string + revisions*: seq[string] + +const + # Conan API urls + baseUrl = "https://conan.bintray.com/v2/conans" + searchUrl = baseUrl & "/search?q=$query" + pkgUrl = baseUrl & "/$name/$version/$user/$channel/search$query" + cfgUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions" + dlUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions/$revision/files/$file" + + # Bintray download sub-URL for explicit `user/channel` (not _/_) + dlAltUrl = "/download_file?file_path=$user%2F$name%2F$version%2F$channel%2F0%2Fpackage%2F$build%2F0%2F$file" + + # Strings + conanInfo = "conaninfo.json" + conanPackage = "conan_package.tgz" + conanManifest = "conanmanifest.txt" + +# https://github.com/kdheepak/binary-builder-downloader/blob/master/src/bbd.nim +# https://bintray.com/conan/conan-center +# https://bintray.com/bincrafters/public-conan/ + +# https://docs.conan.io/en/latest/howtos/manage_shared_libraries.html#manage-shared + +var + # Bintray download URL for explicit `user/channel` + baseAltUrl {.compiletime.} = { + "bincrafters": "https://bintray.com/bincrafters/public-conan", + "conan": "https://bintray.com/conan-community/conan" + }.toTable() + +template fixOutDir() {.dirty.} = + let + outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir + +proc addAltBaseUrl*(name, url: string) = + # Add an alternate base URL for a custom conan repo on bintray + baseAltUrl[name] = url + +proc jsonGet(url: string): JsonNode = + # Make HTTP call and return content as JSON + let + temp = getTempDir() + file = temp / url.extractFilename() + + downloadUrl(url, temp, quiet = true) + result = readFile(file).parseJson() + rmFile(file) + +proc `==`(pkg1, pkg2: ConanPackage): bool = + (not pkg1.isNil and not pkg2.isNil and + pkg1.name == pkg2.name and + pkg1.version == pkg2.version and + pkg1.user == pkg2.user and + pkg1.channel == pkg2.channel and + + pkg1.bhash == pkg2.bhash and + pkg1.shared == pkg2.shared) + +proc newConanPackage*(name, version, user = "_", channel = "_", bhash = "", shared = true): ConanPackage = + ## Create a new ConanPackage with specified name and version + result = new(ConanPackage) + result.name = name + result.version = version + result.user = user + result.channel = channel + result.recipes = newOrderedTable[string, seq[ConanBuild]](2) + + result.bhash = bhash + result.shared = shared + +proc newConanPackageFromUri*(uri: string, shared = true): ConanPackage = + ## Create a new ConanPackage from a conan uri + ## + ## name/version[@user/channel][:bhash] + var + name, version, user, channel, bhash: string + + spl = uri.split(":") + + if spl.len > 1: + bhash = spl[1] + + spl = spl[0].split('/') + + name = spl[0] + user = "_" + channel = "_" + + if spl.len > 2: + channel = spl[2] + if spl.len > 1: + spl = spl[1].split('@') + + version = spl[0] + if spl.len > 1: + user = spl[1] + + result = newConanPackage(name, version, user, channel, bhash, shared) + +proc getUriFromConanPackage*(pkg: ConanPackage): string = + ## Convert a ConanPackage to a conan uri + result = pkg.name + if pkg.version.len != 0: + result &= "/" & pkg.version + if pkg.user.len != 0: + result &= "@" & pkg.user + if pkg.channel.len != 0: + result &= "/" & pkg.channel + if pkg.bhash.len != 0: + result &= ":" & pkg.bhash + +proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPackage = + ## Search for package by `name` and optional `version`, `user` and `channel` + var + query = name + if version.len != 0: + query &= "/" & version + if user.len != 0: + query &= "@" & user + if channel.len != 0: + query &= "/" & channel + + let + j1 = jsonGet(searchUrl % ["query", query]) + res = j1.getOrDefault("results").getElems() + + if res.len != 0: + # Return last entry - latest + result = newConanPackageFromUri(res[^1].getStr()) + +proc searchConan*(pkg: ConanPackage): ConanPackage = + ## Search for latest package based on incomplete package info + result = searchConan(pkg.name, pkg.version, pkg.user, pkg.channel) + +proc getConanBuilds*(pkg: ConanPackage, filter = "") = + ## Get all builds for a package based on the C compiler's target OS/arch info + ## + ## `filter` can be used to tweak search terms + ## e.g. build_type=Debug&compiler=clang + let + (arch, os, compiler, _) = getGccInfo() + + query = + if pkg.bhash.len == 0: + block: + var + query = &"?q=arch={arch}&os={os.capitalizeAscii()}" + if "build_type" notin filter: + query &= "&build_type=Release" + if "shared=" notin filter: + query &= &"&options.shared={($pkg.shared).capitalizeAscii()}" + if filter.len != 0: + query &= &"&{filter}" + if "compiler=" notin filter and os != "windows": + query &= &"&compiler={compiler}" + query.replace("&", "%20and%20") + else: "" + + url = pkgUrl % [ + "name", pkg.name, + "version", pkg.version, + "user", pkg.user, + "channel", pkg.channel, + "query", query + ] + + j1 = jsonGet(url) + + if not j1.isNil: + for bhash, bdata in j1.getFields(): + if pkg.bhash.len == 0 or pkg.bhash == bhash: + let + bld = new(ConanBuild) + settings = bdata.getOrDefault("settings") + options = bdata.getOrDefault("options") + requires = bdata.getOrDefault("requires") + bld.bhash = bhash + if not settings.isNil: + bld.settings = newTable[string, string](8) + for key, value in settings.getFields(): + bld.settings[key] = value.getStr() + if not options.isNil: + bld.options = newTable[string, string](8) + for key, value in options.getFields(): + bld.options[key] = value.getStr() + bld.requires = requires.to(seq[string]) + bld.recipe_hash = bdata.getOrDefault("recipe_hash").getStr() + + if pkg.recipes.hasKey(bld.recipe_hash): + pkg.recipes[bld.recipe_hash].add bld + else: + pkg.recipes[bld.recipe_hash] = @[bld] + + # Only need first or matching build + break + +proc getConanRevisions*(pkg: ConanPackage, bld: ConanBuild) = + ## Get all revisions of a build + let + url = cfgUrl % [ + "name", pkg.name, + "version", pkg.version, + "user", pkg.user, + "channel", pkg.channel, + "recipe", bld.recipe_hash, + "build", bld.bhash + ] + + j1 = jsonGet(url) + + if not j1.isNil: + let + revs = j1.getOrDefault("revisions") + for i in revs: + bld.revisions.add i.getOrDefault("revision").getStr() + +proc loadConanInfo*(outdir: string): ConanPackage = + ## Load cached package info from `outdir/conaninfo.txt` + fixOutDir() + let + file = outdir / conanInfo + + if fileExists(file): + result = to[ConanPackage](readFile(file)) + +proc saveConanInfo*(pkg: ConanPackage, outdir: string) = + ## Save downloaded package info to `outdir/conaninfo.txt` + fixOutDir() + let + file = outdir / conanInfo + + writeFile(file, $$pkg) + +proc parseConanManifest(pkg: ConanPackage, outdir: string) = + let + file = outdir / conanManifest + + if fileExists(file): + let + data = readFile(file) + for line in data.splitLines(): + let + line = line.split(':')[0] + if line.startsWith("include/"): + pkg.headers.add line + elif line.startsWith("lib/"): + pkg.libs.add line + elif line.startsWith("bin/") and line.endsWith("dll"): + pkg.libs.add line + +proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision = "") = + ## Download specific `revision` of `bld` to `outdir` + ## + ## If omitted, the latest revision (first) is downloaded + fixOutDir() + let + revision = + if revision.len != 0: + revision + else: + bld.revisions[0] + + url = + if pkg.user == "_": + dlUrl % [ + "name", pkg.name, + "version", pkg.version, + "user", pkg.user, + "channel", pkg.channel, + "recipe", bld.recipe_hash, + "build", bld.bhash, + "revision", revision, + "file", conanPackage + ] + else: + baseAltUrl[pkg.user] & dlAltUrl % [ + "name", pkg.name, + "version", pkg.version, + "user", pkg.user, + "channel", pkg.channel, + "build", bld.bhash, + "file", conanPackage + ] + + downloadUrl(url, outdir, quiet = true) + downloadUrl(url.replace(conanPackage, conanManifest), outdir, quiet = true) + + pkg.parseConanManifest(outdir) + + rmFile(outdir / url.extractFilename()) + rmFile(outdir / conanManifest) + +proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) +proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = + ## Download latest recipe/build/revision of `pkg` to `outdir` + fixOutDir() + let + pkg = + if pkg.version.len == 0: + searchConan(pkg) + else: + pkg + + cpkg = loadConanInfo(outdir) + + if cpkg == pkg: + return + elif clean: + cleanDir(outdir) + + pkg.getConanBuilds() + + for recipe, builds in pkg.recipes: + for build in builds: + if pkg.bhash.len == 0 or pkg.bhash == build.bhash: + pkg.getConanRevisions(build) + pkg.dlConanBuild(build, outdir) + pkg.dlConanRequires(build, outdir) + break + break + + if clean: + pkg.saveConanInfo(outdir) + +proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = + ## Download all required dependencies of this `bld` + ## + ## This is not required for shared libs since conan builds them + ## with all dependencies statically linked in + fixOutDir() + if bld.options["shared"] == "False": + for req in bld.requires: + let + rpkg = newConanPackageFromUri(req) + + downloadConan(rpkg, outdir, clean = false) + pkg.requires.add rpkg + +proc getConanLibs*(pkg: ConanPackage, outdir: string): seq[string] = + ## Get all Conan libs (.so|.a|.lib|.dll) in pkg, including deps + ## in descending order + ## + ## `outdir` is prefixed to each entry + for lib in pkg.libs: + result.add outdir / lib + + for cpkg in pkg.requires: + result.add cpkg.getConanLibs(outdir) \ No newline at end of file From 6d256d127191e546b51c5707ae1bfebac43486c6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 12 Jun 2020 17:45:06 -0500 Subject: [PATCH 023/106] Filter compiler version, Windows .lib detection --- nimterop/build.nim | 2 +- nimterop/conan.nim | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index d5ce59c..fd7ad3a 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1138,7 +1138,7 @@ macro getHeader*( `version`* {.strdefine.} = `verVal` `lname` = when `nameStatic`: - `lre` & ".a" + `lre` & "\\.(a|lib)" else: `lre` & getDynlibExt() diff --git a/nimterop/conan.nim b/nimterop/conan.nim index 0707ecc..1a9237a 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -163,7 +163,15 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = ## `filter` can be used to tweak search terms ## e.g. build_type=Debug&compiler=clang let - (arch, os, compiler, _) = getGccInfo() + (arch, os, compiler, version) = getGccInfo() + + vsplit = version.split('.') + + vfilter = + when defined(OSX): + vsplit[0 .. 1].join(".") + else: + vsplit[0] query = if pkg.bhash.len == 0: @@ -177,7 +185,8 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = if filter.len != 0: query &= &"&{filter}" if "compiler=" notin filter and os != "windows": - query &= &"&compiler={compiler}" + query &= &"&compiler={compiler}&compiler.version=" & vfilter + query.replace("&", "%20and%20") else: "" @@ -370,4 +379,4 @@ proc getConanLibs*(pkg: ConanPackage, outdir: string): seq[string] = result.add outdir / lib for cpkg in pkg.requires: - result.add cpkg.getConanLibs(outdir) \ No newline at end of file + result.add cpkg.getConanLibs(outdir) From c63d82bf77ca6245469586265950f6e2085546f0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 14 Jun 2020 15:16:21 -0500 Subject: [PATCH 024/106] Add tests for conan, recurse implies preprocess --- nimterop/build.nim | 25 +++++++++++++++--------- nimterop/conan.nim | 2 ++ nimterop/toast.nim | 6 +++++- tests/getheader.nims | 8 ++++++++ tests/libssh2.nim | 46 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 tests/libssh2.nim diff --git a/nimterop/build.nim b/nimterop/build.nim index fd7ad3a..d534cf9 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1018,26 +1018,33 @@ macro getHeader*( ## ## This allows a single wrapper to be used in different ways depending on the user's needs. ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. + ## The user can opt to download the sources to `outdir` using any other method such as + ## git sub-modules, vendoring or pointing to a repository that was already cloned. ## ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL` or ## `Conan`. This allows using a system installed library if available before falling back - ## to manual building. + ## to manual building. The user would need to specify both `-d:xxxStd` and one of the other + ## methods. ## ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag ## name for Git whereas for DL and Conan, it replaces `$1` in the URL defined. ## - ## All defines can also be set in code using `setDefines()`. + ## All defines can also be set in code using `setDefines()` and checked for using + ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`. ## ## The library is then configured (with `cmake` or `autotools` if possible) and built ## using `make`, unless using `-d:xxxStd` which presumes that the system package - ## manager was used to install prebuilt headers and binaries. + ## manager was used to install prebuilt headers and binaries, or using `-d:xxxConan` + ## which downloads pre-built binaries. ## ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. ## ## `-d:xxxStatic` can be specified to statically link with the library instead. This - ## will automatically add a `{.passL.}` call to the static library for convenience. + ## will automatically add a `{.passL.}` call to the static library for convenience. Note + ## that `-d:xxxConan` downloads all dependency libs as well and the `xxxLPath` will + ## include all separated by space in the right order for linking. ## ## `conFlags`, `cmakeFlags` and `makeFlags` allow sending custom parameters to `configure`, ## `cmake` and `make` in case additional configuration is required as part of the build process. @@ -1047,19 +1054,19 @@ macro getHeader*( ## with cmake. In this case, `altNames = "z,zlib"`. Comma separate for multiple alternate names without ## spaces. ## - ## `buildTypes` specifies a list of in order build strategies to use when building the downloaded source - ## files. Default is [btCmake, btAutoconf] - ## ## The original header name is not included by default if `altNames` is set since it could cause the ## wrong lib to be selected. E.g. `SDL2/SDL.h` could pick `libSDL.so` even if `altNames = "SDL2"`. ## Explicitly include it in `altNames` like the `zlib` example when required. ## + ## `buildTypes` specifies a list of ordered build strategies to use when building the downloaded source + ## files. Default is [btCmake, btAutoconf] + ## ## `xxxPreBuild` is a hook that is called after the source code is pulled from Git or downloaded but ## before the library is built. This might be needed if some initial prep needs to be done before ## compilation. A few values are provided to the hook to help provide context: ## - ## `outdir` is the same `outdir` passed in and `header` is the discovered header path in the - ## downloaded source code. + ## `outdir` is the same `outdir` passed in and `header` is the discovered header path in the + ## downloaded source code. ## ## Simply define `proc xxxPreBuild(outdir, header: string)` in the wrapper and it will get called ## prior to the build process. diff --git a/nimterop/conan.nim b/nimterop/conan.nim index 1a9237a..e9d23cb 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -342,6 +342,8 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = elif clean: cleanDir(outdir) + echo &"# Downloading {pkg.name} v{pkg.version} from Conan" + pkg.getConanBuilds() for recipe, builds in pkg.recipes: diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 4a9553b..1844a56 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -123,6 +123,10 @@ proc main( if check and outputFile.len == 0: outputFile = getTempDir() / "toast_" & ($getTime().toUnix()).addFileExt("nim") + # Recurse implies preprocess + if gState.recurse: + gState.preprocess = true + # Redirect output to file if outputFile.len != 0: doAssert gState.outputHandle.open(outputFile, fmWrite), @@ -248,7 +252,7 @@ when isMainModule: "pnim": "print Nim output", "prefix": "strip prefix from identifiers", "preprocess": "run preprocessor on header", - "recurse": "process #include files", + "recurse": "process #include files - implies --preprocess", "replace": "replace X with Y in identifiers, X1=Y1,X2=Y2, @X for regex", "source" : "C/C++ source/header(s) and command line file(s)", "stub": "stub out undefined type references as objects", diff --git a/tests/getheader.nims b/tests/getheader.nims index 5d3b427..55d00f1 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -17,10 +17,12 @@ var cmd = "nim c -f --hints:off -d:FLAGS=\"-f:ast2\" -d:checkAbi" lrcmd = " -r lzma.nim" zrcmd = " -r zlib.nim" + sshcmd = " -r libssh2.nim" lexp = "liblzma version = " zexp = "zlib version = " testCall(cmd & lrcmd, "No build files found", 1) +testCall(cmd & " -d:libssh2Conan" & sshcmd, "Need version for Conan uri", 1) when defined(posix): # stdlib @@ -35,6 +37,9 @@ when defined(posix): testCall(cmd & " -d:lzmaGit -d:lzmaSetVer=v5.2.0" & lrcmd, lexp & "5.2.0", 0) testCall(cmd & " -d:lzmaGit -d:lzmaStatic -d:lzmaSetVer=v5.2.0" & lrcmd, lexp & "5.2.0", 0, delete = false) + # conan static + testCall(cmd & " -d:libssh2Conan -d:libssh2SetVer=1.9.0 -d:libssh2Static" & sshcmd, zexp, 0) + # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) testCall(cmd & " -d:envTestStatic" & zrcmd, zexp, 0, delete = false) @@ -51,3 +56,6 @@ testCall(cmd & " -d:lzmaDL -d:lzmaStatic -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5 # dl testCall(cmd & " -d:zlibDL -d:zlibSetVer=1.2.11" & zrcmd, zexp & "1.2.11", 0) testCall(cmd & " -d:zlibDL -d:zlibStatic -d:zlibSetVer=1.2.11" & zrcmd, zexp & "1.2.11", 0, delete = false) + +# conan +testCall(cmd & " -d:libssh2Conan -d:libssh2SetVer=1.9.0" & sshcmd, zexp, 0) diff --git a/tests/libssh2.nim b/tests/libssh2.nim new file mode 100644 index 0000000..b7e591c --- /dev/null +++ b/tests/libssh2.nim @@ -0,0 +1,46 @@ +import nimterop/[build, cimport] + +const + outdir = getProjectCacheDir("libssh2") + +getHeader( + header = "libssh2.h", + conanuri = "libssh2/$1", + outdir = outdir +) + +cOverride: + type + stat = object + stat64 = object + SOCKET = object + +when not libssh2Static: + cImport(libssh2Path, recurse = true, dynlib = "libssh2LPath", flags = "-f:ast2 -c -E_ -F_") + + when not defined(Windows): + proc zlibVersion(): cstring {.importc, dynlib: libssh2LPath.} +else: + cImport(libssh2Path, recurse = true, flags = "-f:ast2 -c -E_ -F_") + + when not defined(Windows): + proc zlibVersion(): cstring {.importc.} + + {.passL: "-lpthread".} + +assert libssh2_init(0) == 0 + +let + session = libssh2_session_init_ex(nil, nil, nil, nil) + +if session == nil: + quit(1) + +libssh2_session_set_blocking(session, 0.cint) + +echo "zlib version = " & (block: + when not defined(Windows): + $zlibVersion() + else: + "" +) \ No newline at end of file From b6278280dcc5018f043a1112c003e31c8305edab Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 14 Jun 2020 16:10:34 -0500 Subject: [PATCH 025/106] Fix for Windows, README --- README.md | 11 ++++++----- nimterop/build.nim | 45 +++++++++++++++++++++++++++------------------ nimterop/conan.nim | 24 +++++++++++++++++------- 3 files changed, 50 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 48822ac..3fd48e2 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ getHeader( "header.h", # The header file to wrap, full path is returned in `headerPath` giturl = "https://github.com/username/repo", # Git repo URL dlurl = "https://website.org/download/repo-$1.tar.gz", # Download URL for archive or raw file + conanuri = "repo/$1", # Conan.io URI outdir = baseDir, # Where to download/build/search conFlags = "--disable-comp --enable-feature", # Flags to pass configure script cmakeFlags = "-DENABLE_STATIC_LIB=ON" # Flags to pass to Cmake @@ -61,7 +62,7 @@ getHeader( # Wrap headerPath as returned from getHeader() and link statically # or dynamically depending on user input -when not defined(headerStatic): +when not isDefined(headerStatic): cImport(headerPath, recurse = true, dynlib = "headerLPath") # Pass dynlib if not static link else: cImport(headerPath, recurse = true) @@ -74,8 +75,8 @@ Module documentation for the build API can be found [here](https://nimterop.gith The above wrapper is generic and allows the end user to control how it works. Note that `headerPath` is derived from `header.h` so if you have `SDL.h` as the argument to `getHeader()`, it generates `SDLPath` and `SDLLPath` and is controlled by `-d:SDLStatic`, `-d:SDLGit` and so forth. - If the library is already installed in `/usr/include` then the `-d:headerStd` define to Nim can be used to instruct `getHeader()` to search for `header.h` in the standard system path. -- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL or `-d:headerDL` to get the source from download URL. - - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git whereas for DL, it replaces `$1` in the URL if defined. +- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL, `-d:headerDL` to get the source from download URL or `-d:headerConan` to download from https://conan.io/center. + - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git and for DL and Conan, it replaces `$1` in the URL if specified. - If no flag is provided, `getHeader()` simply looks for the library in `outdir`. The user could use Git submodules or manually download or check-in the library to that directory and `getHeader()` will use it directly. #### Pre build @@ -92,7 +93,7 @@ Flags can be specified to these tools via `getHeader()` or directly via the unde #### Linking -- If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If it is omitted, the dynamic library is returned in `headerLPath`. +- If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`. - `getHeader()` searches for libraries based on the header name by default: - `libheader.so` or `libheader.a` on Linux - `libheader.dylib` on OSX @@ -224,7 +225,7 @@ Options: -n, --pnim bool false print Nim output -E=, --prefix= strings {} strip prefix from identifiers -p, --preprocess bool false run preprocessor on header - -r, --recurse bool false process #include files + -r, --recurse bool false process #include files, implies --preprocess -G=, --replace= strings {} replace X with Y in identifiers, X1=Y1,X2=Y2, @X for regex -s, --stub bool false stub out undefined type references as objects -F=, --suffix= strings {} strip suffix from identifiers diff --git a/nimterop/build.nim b/nimterop/build.nim index d534cf9..74e6a61 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1044,32 +1044,41 @@ macro getHeader*( ## `-d:xxxStatic` can be specified to statically link with the library instead. This ## will automatically add a `{.passL.}` call to the static library for convenience. Note ## that `-d:xxxConan` downloads all dependency libs as well and the `xxxLPath` will - ## include all separated by space in the right order for linking. + ## include paths to all of them separated by space in the right order for linking. + ## + ## Note also that Conan currently builds all OSX binaries on 10.14 so older versions of + ## OSX will complain if statically linking to these binaries. Further, all Conan binaries + ## for Windows are built with Visual Studio so static linking the `.lib` files with gcc + ## or clang might lead to incompatibility issues if the library uses Visual Studio + ## specific compiler features. ## ## `conFlags`, `cmakeFlags` and `makeFlags` allow sending custom parameters to `configure`, - ## `cmake` and `make` in case additional configuration is required as part of the build process. + ## `cmake` and `make` in case additional configuration is required as part of the build + ## process. ## - ## `altNames` is a list of alternate names for the library - e.g. zlib uses `zlib.h` for the header but - ## the typical lib name is `libz.so` and not `libzlib.so`. However, it is libzlib.dll on Windows if built - ## with cmake. In this case, `altNames = "z,zlib"`. Comma separate for multiple alternate names without - ## spaces. + ## `altNames` is a list of alternate names for the library - e.g. zlib uses `zlib.h` for + ## the header but the typical lib name is `libz.so` and not `libzlib.so`. However, it is + ## libzlib.dll on Windows if built with cmake. In this case, `altNames = "z,zlib"`. Comma + ## separate for multiple alternate names without spaces. ## - ## The original header name is not included by default if `altNames` is set since it could cause the - ## wrong lib to be selected. E.g. `SDL2/SDL.h` could pick `libSDL.so` even if `altNames = "SDL2"`. - ## Explicitly include it in `altNames` like the `zlib` example when required. + ## The original header name is not included by default if `altNames` is set since it could + ## cause the wrong lib to be selected. E.g. `SDL2/SDL.h` could pick `libSDL.so` even if + ## `altNames = "SDL2"`. Explicitly include it in `altNames` like the `zlib` example when + ## required. ## - ## `buildTypes` specifies a list of ordered build strategies to use when building the downloaded source - ## files. Default is [btCmake, btAutoconf] + ## `buildTypes` specifies a list of ordered build strategies to use when building the + ## downloaded source files. Default is [btCmake, btAutoconf] ## - ## `xxxPreBuild` is a hook that is called after the source code is pulled from Git or downloaded but - ## before the library is built. This might be needed if some initial prep needs to be done before - ## compilation. A few values are provided to the hook to help provide context: + ## `xxxPreBuild` is a hook that is called after the source code is pulled from Git or + ## downloaded but before the library is built. This might be needed if some initial prep + ## needs to be done before compilation. A few values are provided to the hook to help + ## provide context: ## - ## `outdir` is the same `outdir` passed in and `header` is the discovered header path in the - ## downloaded source code. + ## `outdir` is the same `outdir` passed in and `header` is the discovered header path + ## in the downloaded source code. ## - ## Simply define `proc xxxPreBuild(outdir, header: string)` in the wrapper and it will get called - ## prior to the build process. + ## Simply define `proc xxxPreBuild(outdir, header: string)` in the wrapper and it will get + ## called prior to the build process. var origname = header.extractFilename().split(".")[0] name = origname.split(seps = AllChars-Letters-Digits).join() diff --git a/nimterop/conan.nim b/nimterop/conan.nim index e9d23cb..872d138 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -65,13 +65,19 @@ proc jsonGet(url: string): JsonNode = # Make HTTP call and return content as JSON let temp = getTempDir() - file = temp / url.extractFilename() + file = block: + var + file = temp / url.extractFilename() + when defined(Windows): + file = file.replace('?', '_') + file downloadUrl(url, temp, quiet = true) result = readFile(file).parseJson() rmFile(file) -proc `==`(pkg1, pkg2: ConanPackage): bool = +proc `==`*(pkg1, pkg2: ConanPackage): bool = + ## Check if two ConanPackage objects are equal (not pkg1.isNil and not pkg2.isNil and pkg1.name == pkg2.name and pkg1.version == pkg2.version and @@ -94,9 +100,7 @@ proc newConanPackage*(name, version, user = "_", channel = "_", bhash = "", shar result.shared = shared proc newConanPackageFromUri*(uri: string, shared = true): ConanPackage = - ## Create a new ConanPackage from a conan uri - ## - ## name/version[@user/channel][:bhash] + ## Create a new ConanPackage from a conan uri typically formatted as name/version[@user/channel][:bhash] var name, version, user, channel, bhash: string @@ -136,6 +140,8 @@ proc getUriFromConanPackage*(pkg: ConanPackage): string = proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPackage = ## Search for package by `name` and optional `version`, `user` and `channel` + ## + ## Search is quite slow so it is preferable to specify a version and use `getConanBuilds()` var query = name if version.len != 0: @@ -249,7 +255,7 @@ proc getConanRevisions*(pkg: ConanPackage, bld: ConanBuild) = bld.revisions.add i.getOrDefault("revision").getStr() proc loadConanInfo*(outdir: string): ConanPackage = - ## Load cached package info from `outdir/conaninfo.txt` + ## Load cached package info from `outdir/conaninfo.json` fixOutDir() let file = outdir / conanInfo @@ -258,7 +264,7 @@ proc loadConanInfo*(outdir: string): ConanPackage = result = to[ConanPackage](readFile(file)) proc saveConanInfo*(pkg: ConanPackage, outdir: string) = - ## Save downloaded package info to `outdir/conaninfo.txt` + ## Save downloaded package info to `outdir/conaninfo.json` fixOutDir() let file = outdir / conanInfo @@ -266,6 +272,7 @@ proc saveConanInfo*(pkg: ConanPackage, outdir: string) = writeFile(file, $$pkg) proc parseConanManifest(pkg: ConanPackage, outdir: string) = + # Get all header and library info from downloaded conan package let file = outdir / conanManifest @@ -327,6 +334,9 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = ## Download latest recipe/build/revision of `pkg` to `outdir` + ## + ## High-level API that handles the end to end Conan process flow to find + ## latest package binary and downloads and extracts it to `outdir`. fixOutDir() let pkg = From f21dab89caab28c20fa7db65969c8089fece55c9 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 14 Jun 2020 16:53:24 -0500 Subject: [PATCH 026/106] Handle bad conan uri, test Windows static --- nimterop/conan.nim | 16 +++++++--------- tests/getheader.nims | 3 +++ tests/zlib.nim | 1 + 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index 872d138..95ab24f 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -40,12 +40,6 @@ const conanPackage = "conan_package.tgz" conanManifest = "conanmanifest.txt" -# https://github.com/kdheepak/binary-builder-downloader/blob/master/src/bbd.nim -# https://bintray.com/conan/conan-center -# https://bintray.com/bincrafters/public-conan/ - -# https://docs.conan.io/en/latest/howtos/manage_shared_libraries.html#manage-shared - var # Bintray download URL for explicit `user/channel` baseAltUrl {.compiletime.} = { @@ -73,7 +67,10 @@ proc jsonGet(url: string): JsonNode = file downloadUrl(url, temp, quiet = true) - result = readFile(file).parseJson() + try: + result = readFile(file).parseJson() + except JsonParsingError: + discard rmFile(file) proc `==`*(pkg1, pkg2: ConanPackage): bool = @@ -352,10 +349,11 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = elif clean: cleanDir(outdir) - echo &"# Downloading {pkg.name} v{pkg.version} from Conan" - pkg.getConanBuilds() + doAssert pkg.recipes.len != 0, &"# Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" + + echo &"# Downloading {pkg.name} v{pkg.version} from Conan" for recipe, builds in pkg.recipes: for build in builds: if pkg.bhash.len == 0 or pkg.bhash == build.bhash: diff --git a/tests/getheader.nims b/tests/getheader.nims index 55d00f1..349b9bb 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -39,6 +39,9 @@ when defined(posix): # conan static testCall(cmd & " -d:libssh2Conan -d:libssh2SetVer=1.9.0 -d:libssh2Static" & sshcmd, zexp, 0) +else: + # conan static for Windows + testCall(cmd & " -d:zlibConan -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) diff --git a/tests/zlib.nim b/tests/zlib.nim index 852dca9..15400d4 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -27,6 +27,7 @@ getHeader( "zlib.h", giturl = "https://github.com/madler/zlib", dlurl = "http://zlib.net/zlib-$1.tar.gz", + conanuri = "zlib/$1", outdir = baseDir, altNames = "z,zlib" ) From 2b3b5e2bec8f7c39540f0da5b4331f5e90c46843 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 15 Jun 2020 21:37:35 -0500 Subject: [PATCH 027/106] Add binarybuilder.org support, conan improvements, headerLDeps --- README.md | 30 ++++--- nimterop/build.nim | 196 ++++++++++++++++++++++++++++++---------- nimterop/conan.nim | 78 ++++++++-------- nimterop/jbb.nim | 210 +++++++++++++++++++++++++++++++++++++++++++ tests/getheader.nims | 6 ++ tests/libssh2.nim | 14 +-- tests/zlib.nim | 4 + 7 files changed, 437 insertions(+), 101 deletions(-) create mode 100644 nimterop/jbb.nim diff --git a/README.md b/README.md index 3fd48e2..ca17b90 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ getHeader( giturl = "https://github.com/username/repo", # Git repo URL dlurl = "https://website.org/download/repo-$1.tar.gz", # Download URL for archive or raw file conanuri = "repo/$1", # Conan.io URI + jbburi = "repo/$1", # BinaryBuilder.org URI outdir = baseDir, # Where to download/build/search conFlags = "--disable-comp --enable-feature", # Flags to pass configure script cmakeFlags = "-DENABLE_STATIC_LIB=ON" # Flags to pass to Cmake @@ -75,8 +76,8 @@ Module documentation for the build API can be found [here](https://nimterop.gith The above wrapper is generic and allows the end user to control how it works. Note that `headerPath` is derived from `header.h` so if you have `SDL.h` as the argument to `getHeader()`, it generates `SDLPath` and `SDLLPath` and is controlled by `-d:SDLStatic`, `-d:SDLGit` and so forth. - If the library is already installed in `/usr/include` then the `-d:headerStd` define to Nim can be used to instruct `getHeader()` to search for `header.h` in the standard system path. -- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL, `-d:headerDL` to get the source from download URL or `-d:headerConan` to download from https://conan.io/center. - - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git and for DL and Conan, it replaces `$1` in the URL if specified. +- If the library needs to be downloaded, the user can use `-d:headerGit` to clone the source from the specified git URL, `-d:headerDL` to get the source from download URL, `-d:headerConan` to download from https://conan.io/center or `-d:headerJBB` to download from https://binarybuilder.org. + - The `-d:headerSetVer=X.Y.Z` flag can be used to specify which version to download. It is used as the tag name for Git and for DL, Conan and JBB, it replaces `$1` in the URL if specified. - If no flag is provided, `getHeader()` simply looks for the library in `outdir`. The user could use Git submodules or manually download or check-in the library to that directory and `getHeader()` will use it directly. #### Pre build @@ -93,16 +94,21 @@ Flags can be specified to these tools via `getHeader()` or directly via the unde #### Linking -- If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`. -- `getHeader()` searches for libraries based on the header name by default: - - `libheader.so` or `libheader.a` on Linux - - `libheader.dylib` on OSX - - `header.dll` or `header.a` on Windows -- If a library has a different header and library binary name, `altNames` can be used to configure an alternate name of library binary. - - For example, Bzip2 has `bzlib.h` but the library is `libbz2.so` so `altNames = "bz2"`. - - In the example above, `altNames = "hdr"` so `getHeader()` will look for `libhdr.so`, `hdr.dll`, etc. - - See [bzlib.nim](https://github.com/genotrance/nimarchive/blob/master/nimarchive/bzlib.nim) for an example. -- [lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library that allows both static and dynamic linking. +If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`. + +All dependency libraries (supported by Conan and JBB) will be returned in `headerLDeps`. Static libraries and dependencies are automatically linked using `{.passL.}`. Conan shared libs include all dependencies whereas JBB shared libs expect the required dependencies to be in the same location or in `LD_LIBRARY_PATH`. + +`getHeader()` searches for libraries based on the header name by default: +- `libheader.so` or `libheader.a` on Linux +- `libheader.dylib` on OSX +- `header.dll`, `header.a` or `header.lib` on Windows + +If a library has a different header and library binary name, `altNames` can be used to configure an alternate name of library binary. +- For example, Bzip2 has `bzlib.h` but the library is `libbz2.so` so `altNames = "bz2"`. +- In the example above, `altNames = "hdr"` so `getHeader()` will look for `libhdr.so`, `hdr.dll`, etc. +- See [bzlib.nim](https://github.com/genotrance/nimarchive/blob/master/nimarchive/bzlib.nim) for an example. + +[lzma.nim](https://github.com/nimterop/nimterop/blob/master/tests/lzma.nim) is an example of a library that allows both static and dynamic linking. #### User control diff --git a/nimterop/build.nim b/nimterop/build.nim index 74e6a61..eda27e7 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -158,7 +158,7 @@ proc mkDir*(dir: string) = flag = when not defined(Windows): "-p" else: "" discard execAction(&"mkdir {flag} {dir.sanitizePath}", retry = 2) -proc cpFile*(source, dest: string, move=false) = +proc cpFile*(source, dest: string, move = false) = ## Copy a file from `source` to `dest` at compile time let source = source.replace("/", $DirSep) @@ -214,6 +214,25 @@ proc cleanDir*(dir: string) = else: rmFile(path) +proc cpTree*(source, dest: string, move = false) = + ## Copy contents of source dir to the destination, not the directory itself + for kind, path in walkDir(source, relative = true): + if kind == pcDir: + cpTree(source / path, dest / path, move) + if move: + rmDir(source / path) + else: + if not dirExists(dest): + mkDir(dest) + if move: + mvFile(source / path, dest / path) + else: + cpFile(source / path, dest / path) + +proc mvTree*(source, dest: string) = + ## Move contents of source dir to the destination, not the directory itself + cpTree(source, dest, move = true) + proc getProjectCacheDir*(name: string, forceClean = true): string = ## Get a cache directory where all nimterop artifacts can be stored ## @@ -242,7 +261,7 @@ proc getProjectCacheDir*(name: string, forceClean = true): string = echo "# Removing " & result rmDir(result) -proc extractZip*(zipfile, outdir: string) = +proc extractZip*(zipfile, outdir: string, quiet = false) = ## Extract a zip file using `powershell` on Windows and `unzip` on other ## systems to the specified output directory var cmd = "unzip -o $#" @@ -251,10 +270,11 @@ proc extractZip*(zipfile, outdir: string) = "'System.IO.Compression.FileSystem'; " & "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" - echo "# Extracting " & zipfile + if not quiet: + echo "# Extracting " & zipfile discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}") -proc extractTar*(tarfile, outdir: string) = +proc extractTar*(tarfile, outdir: string, quiet = false) = ## Extract a tar file using `tar`, `7z` or `7za` to the specified output directory var cmd = "" @@ -284,7 +304,8 @@ proc extractTar*(tarfile, outdir: string) = doAssert cmd.len != 0, "No extraction tool - tar, 7z, 7za - available for " & tarfile.sanitizePath - echo "# Extracting " & tarfile + if not quiet: + echo "# Extracting " & tarfile discard execAction(&"cd {outdir.sanitizePath} && {cmd}") if name.len != 0: rmFile(outdir / name) @@ -316,9 +337,9 @@ proc downloadUrl*(url, outdir: string, quiet = false) = discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 1) if ext == ".zip": - extractZip(file, outdir) + extractZip(file, outdir, quiet) elif ext in archives: - extractTar(file, outdir) + extractTar(file, outdir, quiet) proc gitReset*(outdir: string) = ## Hard reset the git repository at the specified directory @@ -342,7 +363,7 @@ proc gitCheckout*(file, outdir: string) = sleep(500) echo "# Retrying ..." -proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = +proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = ## Pull the specified git repository to the output directory ## ## `plist` is the list of specific files and directories or wildcards @@ -362,7 +383,8 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = mkDir(outdir) - echo "# Setting up Git repo: " & url + if not quiet: + echo "# Setting up Git repo: " & url discard execAction(&"cd {outdirQ} && git init .") discard execAction(&"cd {outdirQ} && git remote add origin {url}") @@ -378,20 +400,32 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") = discard execAction(&"cd {outdirQ} && git clean -fxd") if checkout.len != 0: - echo "# Checking out " & checkout + if not quiet: + echo "# Checking out " & checkout discard execAction(&"cd {outdirQ} && git fetch", retry = 1) discard execAction(&"cd {outdirQ} && git checkout {checkout}") else: - echo "# Pulling repository" + if not quiet: + echo "# Pulling repository" discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 1) -proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string = - ## Find the file in the specified directory +proc gitTags*(outdir: string): seq[string] = + ## Get all the git tags in the specified directory + let + cmd = &"cd {outdir.sanitizePath} && git tag" + tags = execAction(cmd).output.splitLines() + for tag in tags: + let + tag = tag.strip() + if tag.len != 0: + result.add tag + +proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[string] = + ## Find all matching files in the specified directory ## ## `file` is a regular expression if `regex` is true ## - ## Turn off recursive search with `recurse` and stop on first match with - ## `first`. Without it, the shortest match is returned. + ## Turn off recursive search with `recurse` var cmd = when defined(Windows): @@ -435,10 +469,22 @@ proc findFile*(file: string, dir: string, recurse = true, first = false, regex = "" else: line + if f.len != 0: + result.add f - if (f.len != 0 and (result.len == 0 or result.len > f.len)): - result = f - if first: break +proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string = + ## Find the file in the specified directory + ## + ## `file` is a regular expression if `regex` is true + ## + ## Turn off recursive search with `recurse` and stop on first match with + ## `first`. Without it, the shortest match is returned. + let + matches = findFiles(file, dir, recurse, regex) + for match in matches: + if (result.len == 0 or result.len > match.len): + result = match + if first: break proc flagBuild*(base: string, flags: openArray[string]): string = ## Simple helper proc to generate flags for `configure`, `cmake`, etc. @@ -533,7 +579,7 @@ proc configure*(path, check: string, flags = "") = echoDebug execAction(cmd).output - doAssert (path / check).fileExists(), "# Configure failed" + doAssert (path / check).fileExists(), "Configure failed" proc getCmakePropertyStr(name, property, value: string): string = &"\nset_target_properties({name} PROPERTIES {property} \"{value}\")\n" @@ -630,7 +676,7 @@ proc cmake*(path, check, flags: string) = echoDebug execAction(cmd).output - doAssert (path / check).fileExists(), "# cmake failed" + doAssert (path / check).fileExists(), "cmake failed" proc make*(path, check: string, flags = "", regex = false) = ## Run the `make` command to build all binaries in the specified path @@ -666,7 +712,7 @@ proc make*(path, check: string, flags = "", regex = false) = echoDebug execAction(cmd).output - doAssert findFile(check, path, regex = regex).len != 0, "# make failed" + doAssert findFile(check, path, regex = regex).len != 0, "make failed" proc getCompilerMode*(path: string): string = ## Determines a target language mode from an input filename, if one is not already specified. @@ -768,9 +814,16 @@ proc getGccInfo*(): tuple[arch, os, compiler, version: string] = else: result.compiler = "gcc" +template fixOutDir() {.dirty.} = + let + outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir + # Conan support include conan +# Julia Binary Builder support +include jbb + proc getStdPath(header, mode: string): string = for inc in getGccPaths(mode): result = findFile(header, inc, recurse = false, first = true) @@ -839,11 +892,41 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string = result = findFile(header, outdir) -proc getConanLPath(outdir: string): string = +proc getConanLDeps(outdir: string): seq[string] = let pkg = loadConanInfo(outdir) - - result = pkg.getConanLibs(outdir).join(" ") + + result = pkg.getConanLDeps(outdir) + +proc getJBBPath(header, uri, outdir, version: string): string = + let + spl = uri.split('/', 1) + name = spl[0] + + var + ver = + if spl.len == 2: + spl[1] + else: + "" + + if "$#" in ver or "$1" in ver: + doAssert version.len != 0, "Need version for BinaryBuilder.org uri" + ver = ver % version + else: + doAssert version.len == 0, "BinaryBuilder.org uri does not contain version" + + let + pkg = newJBBPackage(name, ver) + downloadJBB(pkg, outdir) + + result = findFile(header, outdir) + +proc getJBBLDeps(outdir: string, shared: bool): seq[string] = + let + pkg = loadJBBInfo(outdir) + + result = pkg.getJBBLDeps(outdir, shared) proc getLocalPath(header, outdir: string): string = if outdir.len != 0: @@ -932,7 +1015,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT buildStatus.built = true let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir - doAssert buildStatus.built, &"\n# Build configuration failed - {error}\n" + doAssert buildStatus.built, &"\nBuild configuration failed - {error}\n" result = findFile(lname, outdir, regex = true) @@ -1000,7 +1083,7 @@ macro isDefined*(def: untyped): untyped = macro getHeader*( header: static[string], giturl: static[string] = "", dlurl: static[string] = "", - conanuri: static[string] = "", outdir: static[string] = "", + conanuri: static[string] = "", jbburi: static[string] = "", outdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = ## Get the path to a header file for wrapping with @@ -1015,19 +1098,22 @@ macro getHeader*( ## `-d:xxxDL` - download source from `dlurl` and extract if required ## `-d:xxxConan` - download headers and binary from conan.io using `conanuri` ## typically formatted as `name/version[@user/channel][:bhash]` + ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` + ## typically formatted as `name/version` ## ## This allows a single wrapper to be used in different ways depending on the user's needs. ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. ## The user can opt to download the sources to `outdir` using any other method such as ## git sub-modules, vendoring or pointing to a repository that was already cloned. ## - ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL` or - ## `Conan`. This allows using a system installed library if available before falling back - ## to manual building. The user would need to specify both `-d:xxxStd` and one of the other - ## methods. + ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL`, + ## `Conan` or `JBB`. This allows using a system installed library if available before + ## falling back to manual building. The user would need to specify both `-d:xxxStd` and + ## one of the other methods. ## ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag - ## name for Git whereas for DL and Conan, it replaces `$1` in the URL defined. + ## name for Git whereas for DL, Conan and BinaryBuilder.org, it replaces `$1` in the URL + ## defined. ## ## All defines can also be set in code using `setDefines()` and checked for using ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`. @@ -1035,7 +1121,7 @@ macro getHeader*( ## The library is then configured (with `cmake` or `autotools` if possible) and built ## using `make`, unless using `-d:xxxStd` which presumes that the system package ## manager was used to install prebuilt headers and binaries, or using `-d:xxxConan` - ## which downloads pre-built binaries. + ## or `-d:xxxJBB` which download pre-built binaries. ## ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can @@ -1043,9 +1129,10 @@ macro getHeader*( ## ## `-d:xxxStatic` can be specified to statically link with the library instead. This ## will automatically add a `{.passL.}` call to the static library for convenience. Note - ## that `-d:xxxConan` downloads all dependency libs as well and the `xxxLPath` will - ## include paths to all of them separated by space in the right order for linking. - ## + ## that `-d:xxxConan` and `-d:xxxJBB` download all dependency libs as well and the + ## `xxxLPath` will include paths to all of them separated by space in the right order for + ## linking. + ## ## Note also that Conan currently builds all OSX binaries on 10.14 so older versions of ## OSX will complain if statically linking to these binaries. Further, all Conan binaries ## for Windows are built with Visual Studio so static linking the `.lib` files with gcc @@ -1088,6 +1175,7 @@ macro getHeader*( gitStr = name & "Git" dlStr = name & "DL" conanStr = name & "Conan" + jbbStr = name & "JBB" staticStr = name & "Static" verStr = name & "SetVer" @@ -1097,12 +1185,14 @@ macro getHeader*( nameGit = newIdentNode(gitStr) nameDL = newIdentNode(dlStr) nameConan = newIdentNode(conanStr) + nameJBB = newIdentNode(jbbStr) nameStatic = newIdentNode(staticStr) # Consts to generate path = newIdentNode(name & "Path") lpath = newIdentNode(name & "LPath") + ldeps = newIdentNode(name & "LDeps") version = newIdentNode(verStr) lname = newIdentNode(name & "LName") preBuild = newIdentNode(name & "PreBuild") @@ -1115,6 +1205,7 @@ macro getHeader*( gitVal = gDefines.hasKey(gitStr) dlVal = gDefines.hasKey(dlStr) conanVal = gDefines.hasKey(conanStr) + jbbVal = gDefines.hasKey(jbbStr) staticVal = gDefines.hasKey(staticStr) verVal = if gDefines.hasKey(verStr): @@ -1137,16 +1228,19 @@ macro getHeader*( `nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1 `nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1 `nameConan`* = when defined(`nameConan`): true else: `conanVal` == 1 + `nameJBB`* = when defined(`nameJBB`): true else: `jbbVal` == 1 `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, conanuri, outdir, version: string, shared: bool): string = + proc getPath(header, giturl, dlurl, conanuri, jbburi, outdir, version: string, shared: bool): string = when `nameGit`: getGitPath(header, giturl, outdir, version) elif `nameDL`: getDlPath(header, dlurl, outdir, version) elif `nameConan`: getConanPath(header, conanuri, outdir, version, shared) + elif `nameJBB`: + getJBBPath(header, jbburi, outdir, version) else: getLocalPath(header, outdir) @@ -1171,10 +1265,10 @@ macro getHeader*( when useStd: stdPath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) - # Run preBuild hook before building library if not Std or Conan - when not (useStd or `nameConan`) and declared(`preBuild`): + # Run preBuild hook before building library if not Std, Conan or JBB + when not (useStd or `nameConan` or `nameJBB`) and declared(`preBuild`): static: `preBuild`(`outdir`, prePath) @@ -1183,28 +1277,36 @@ macro getHeader*( `lpath`* = when useStd: stdLPath - elif `nameConan`: - when `nameStatic`: - getConanLPath(`outdir`) - else: - findFile(`lname`, `outdir`, regex = true) + elif `nameConan` or `nameJBB`: + findFile(`lname`, `outdir`, regex = true) else: buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) + # Library dependecy paths + `ldeps`*: seq[string] = + when `nameConan`: + getConanLDeps(`outdir`) + elif `nameJBB`: + getJBBLDeps(`outdir`, not `nameStatic`) + else: + @[] + # Header path - search again in case header is generated in build `path`* = if prePath.len != 0: prePath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `outdir`, `version`, not `nameStatic`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) static: doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & - "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL or -d:$1Conan not specified" % `name` + "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name` doAssert `lpath`.len != 0, "\nLibrary " & `lname` & " not found" echo "# Including library " & `lpath` - # Automatically link with static libary + # Automatically link with static library and dependencies when `nameStatic`: {.passL: `lpath`.} + if `ldeps`.len != 0: + {.passL: `ldeps`.join(" ").} ) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index 95ab24f..bfe2e94 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -11,8 +11,8 @@ type bhash*: string shared*: bool - headers*: seq[string] - libs*: seq[string] + sharedLibs*: seq[string] + staticLibs*: seq[string] requires*: seq[ConanPackage] ConanBuild* = ref object @@ -26,14 +26,14 @@ type const # Conan API urls - baseUrl = "https://conan.bintray.com/v2/conans" - searchUrl = baseUrl & "/search?q=$query" - pkgUrl = baseUrl & "/$name/$version/$user/$channel/search$query" - cfgUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions" - dlUrl = baseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions/$revision/files/$file" + conanBaseUrl = "https://conan.bintray.com/v2/conans" + conanSearchUrl = conanBaseUrl & "/search?q=$query" + conanPkgUrl = conanBaseUrl & "/$name/$version/$user/$channel/search$query" + conanCfgUrl = conanBaseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions" + conanDlUrl = conanBaseUrl & "/$name/$version/$user/$channel/revisions/$recipe/packages/$build/revisions/$revision/files/$file" # Bintray download sub-URL for explicit `user/channel` (not _/_) - dlAltUrl = "/download_file?file_path=$user%2F$name%2F$version%2F$channel%2F0%2Fpackage%2F$build%2F0%2F$file" + conanDlAltUrl = "/download_file?file_path=$user%2F$name%2F$version%2F$channel%2F0%2Fpackage%2F$build%2F0%2F$file" # Strings conanInfo = "conaninfo.json" @@ -42,18 +42,14 @@ const var # Bintray download URL for explicit `user/channel` - baseAltUrl {.compiletime.} = { + conanBaseAltUrl {.compiletime.} = { "bincrafters": "https://bintray.com/bincrafters/public-conan", "conan": "https://bintray.com/conan-community/conan" }.toTable() -template fixOutDir() {.dirty.} = - let - outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir - -proc addAltBaseUrl*(name, url: string) = +proc addAltConanBaseUrl*(name, url: string) = # Add an alternate base URL for a custom conan repo on bintray - baseAltUrl[name] = url + conanBaseAltUrl[name] = url proc jsonGet(url: string): JsonNode = # Make HTTP call and return content as JSON @@ -137,7 +133,7 @@ proc getUriFromConanPackage*(pkg: ConanPackage): string = proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPackage = ## Search for package by `name` and optional `version`, `user` and `channel` - ## + ## ## Search is quite slow so it is preferable to specify a version and use `getConanBuilds()` var query = name @@ -149,7 +145,7 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac query &= "/" & channel let - j1 = jsonGet(searchUrl % ["query", query]) + j1 = jsonGet(conanSearchUrl % ["query", query]) res = j1.getOrDefault("results").getElems() if res.len != 0: @@ -193,7 +189,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = query.replace("&", "%20and%20") else: "" - url = pkgUrl % [ + url = conanPkgUrl % [ "name", pkg.name, "version", pkg.version, "user", pkg.user, @@ -234,7 +230,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = proc getConanRevisions*(pkg: ConanPackage, bld: ConanBuild) = ## Get all revisions of a build let - url = cfgUrl % [ + url = conanCfgUrl % [ "name", pkg.name, "version", pkg.version, "user", pkg.user, @@ -269,22 +265,23 @@ proc saveConanInfo*(pkg: ConanPackage, outdir: string) = writeFile(file, $$pkg) proc parseConanManifest(pkg: ConanPackage, outdir: string) = - # Get all header and library info from downloaded conan package + # Get all library info from downloaded conan package let file = outdir / conanManifest - + if fileExists(file): let data = readFile(file) for line in data.splitLines(): let line = line.split(':')[0] - if line.startsWith("include/"): - pkg.headers.add line - elif line.startsWith("lib/"): - pkg.libs.add line + if line.startsWith("lib/"): + if line.endsWith(".a") or line.endsWith(".lib"): + pkg.staticLibs.add line + elif line.endsWith(".so"): + pkg.sharedLibs.add line elif line.startsWith("bin/") and line.endsWith("dll"): - pkg.libs.add line + pkg.sharedLibs.add line proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision = "") = ## Download specific `revision` of `bld` to `outdir` @@ -300,7 +297,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision url = if pkg.user == "_": - dlUrl % [ + conanDlUrl % [ "name", pkg.name, "version", pkg.version, "user", pkg.user, @@ -311,7 +308,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision "file", conanPackage ] else: - baseAltUrl[pkg.user] & dlAltUrl % [ + conanBaseAltUrl[pkg.user] & conanDlAltUrl % [ "name", pkg.name, "version", pkg.version, "user", pkg.user, @@ -331,7 +328,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = ## Download latest recipe/build/revision of `pkg` to `outdir` - ## + ## ## High-level API that handles the end to end Conan process flow to find ## latest package binary and downloads and extracts it to `outdir`. fixOutDir() @@ -351,7 +348,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = pkg.getConanBuilds() - doAssert pkg.recipes.len != 0, &"# Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" + doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" echo &"# Downloading {pkg.name} v{pkg.version} from Conan" for recipe, builds in pkg.recipes: @@ -375,18 +372,25 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = if bld.options["shared"] == "False": for req in bld.requires: let - rpkg = newConanPackageFromUri(req) + rpkg = newConanPackageFromUri(req, shared = false) downloadConan(rpkg, outdir, clean = false) pkg.requires.add rpkg -proc getConanLibs*(pkg: ConanPackage, outdir: string): seq[string] = - ## Get all Conan libs (.so|.a|.lib|.dll) in pkg, including deps +proc getConanLDeps*(pkg: ConanPackage, outdir: string, main = true): seq[string] = + ## Get all Conan libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps ## in descending order - ## + ## ## `outdir` is prefixed to each entry - for lib in pkg.libs: - result.add outdir / lib + let + libs = if pkg.shared: pkg.sharedLibs else: pkg.staticLibs + str = if pkg.shared: "shared" else: "static" + + doAssert libs.len != 0, &"No {str} libs found for {pkg.name} in {outdir}" + + if not main: + for lib in libs: + result.add outdir / lib for cpkg in pkg.requires: - result.add cpkg.getConanLibs(outdir) + result.add cpkg.getConanLDeps(outdir, main = false) diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim new file mode 100644 index 0000000..9f0d173 --- /dev/null +++ b/nimterop/jbb.nim @@ -0,0 +1,210 @@ +import marshal, os, strutils + +type + JBBPackage* = ref object + ## JBBPackage type that stores package information + name*: string + version*: string + + url*: string + + sharedLibs*: seq[string] + staticLibs*: seq[string] + requires*: seq[JBBPackage] + +const + # JBB URLs + jbbBaseUrl = "https://github.com/JuliaBinaryWrappers/$1_jll.jl" + + jbbInfo = "jbbinfo.json" + jbbProject = "Project.toml" + jbbArtifacts = "Artifacts.toml" + +proc `==`*(pkg1, pkg2: JBBPackage): bool = + ## Check if two JBBPackage objects are equal + (not pkg1.isNil and not pkg2.isNil and + pkg1.name == pkg2.name and + pkg1.version == pkg2.version) + +proc newJBBPackage*(name, version: string): JBBPackage = + ## Create a new JBBPackage with specified name and version + result = new(JBBPackage) + result.name = name + result.version = version + +proc parseJBBProject(pkg: JBBPackage, outdir: string) = + # Get all dependencies from Project.toml + let + file = outdir / jbbProject + + if fileExists(file): + let + data = readFile(file) + var + deps = false + + doAssert pkg.version in data, &"{pkg.name} v{pkg.version} not found" + + for line in data.splitLines(): + let + line = line.strip() + if line.len != 0: + if line.startsWith('['): + if line == "[deps]": + deps = true + else: + deps = false + elif deps: + let + name = line.split()[0] + if name.endsWith("_jll"): + pkg.requires.add newJBBPackage(name[0 .. ^5], "") + +proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = + # Get build information from Artifacts.toml + let + file = outdir / jbbArtifacts + + (arch, os, _, _) = getGccInfo() + + if fileExists(file): + let + data = readFile(file) + + doAssert pkg.version in data, &"{pkg.name} v{pkg.version} not found" + + var + found = false + for line in data.splitLines(): + let + line = line.strip() + if line.len != 0: + if line.startsWith("arch = ") and not found: + let + barch = line.split(" = ")[1].strip(chars = {'"'}) + if barch == arch: + found = true + elif line.startsWith("os = ") and found: + let + bos = line.split(" = ")[1].strip(chars = {'"'}) + if bos != os: + found = false + elif line.startsWith("url = ") and found: + pkg.url = line.split(" = ")[1].strip(chars = {'"'}) + break + +proc findJBBLibs(pkg: JBBPackage, outdir: string) = + pkg.sharedLibs = findFiles("lib[\\\\/].*\\.(so|dylib)", outdir) + pkg.sharedLibs.add findFiles("bin[\\\\/].*\\.(dll)", outdir) + for i in 0 ..< pkg.sharedLibs.len: + if pkg.sharedLibs[i].isAbsolute: + pkg.sharedLibs[i] = pkg.sharedLibs[i][outdir.len+1 .. ^1] + + for lib in findFiles("lib[\\\\/].*\\.(a|lib)$", outdir): + if not lib.endsWith(".dll.a"): + if lib.isAbsolute: + pkg.staticLibs.add lib[outdir.len+1 .. ^1] + else: + pkg.staticLibs.add lib + +proc getJBBRepo*(pkg: JBBPackage, outdir: string) = + ## Clone JBB package repo and checkout version tag if version is + ## specified in package + let + path = outdir / "repos" / pkg.name + + gitPull( + jbbBaseUrl % pkg.name, + outdir = path, + plist = "*.toml", + "master", + quiet = true + ) + + if pkg.version.len != 0: + # Checkout correct tag + let + tags = gitTags(path) + for i in tags.len - 1 .. 0: + if pkg.version in tags[i] and i != tags.len - 1: + gitCheckout(path, tags[i-1]) + + pkg.parseJBBProject(path) + pkg.parseJBBArtifacts(path) + +proc loadJBBInfo*(outdir: string): JBBPackage = + ## Load cached package info from `outdir/jbbinfo.json` + fixOutDir() + let + file = outdir / jbbInfo + + if fileExists(file): + result = to[JBBPackage](readFile(file)) + +proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = + ## Save downloaded package info to `outdir/jbbinfo.json` + fixOutDir() + let + file = outdir / jbbInfo + + writeFile(file, $$pkg) + +proc dlJBBRequires*(pkg: JBBPackage, outdir: string) +proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) = + ## Download `pkg` from BinaryBuilder.org to `outdir` + ## + ## High-level API that handles the end to end JBB process flow to find + ## latest package binary and downloads and extracts it to `outdir`. + fixOutDir() + let + cpkg = loadJBBInfo(outdir) + + if cpkg == pkg: + return + elif clean: + cleanDir(outdir) + + pkg.getJBBRepo(outdir) + + doAssert pkg.url.len != 0, &"Failed to download {pkg.name} info from BinaryBuilder.org" + + let + vstr = + if pkg.version.len != 0: + &" v{pkg.version}" + else: + "" + path = outdir / "downloads" / pkg.name + echo &"# Downloading {pkg.name}{vstr} from BinaryBuilder.org" + downloadUrl(pkg.url, path, quiet = true) + pkg.findJBBLibs(path) + mvTree(path, outdir) + + pkg.dlJBBRequires(outdir) + + if clean: + pkg.saveJBBInfo(outdir) + +proc dlJBBRequires*(pkg: JBBPackage, outdir: string) = + ## Download all required dependencies of this `pkg` + fixOutDir() + for rpkg in pkg.requires: + downloadJBB(rpkg, outdir, clean = false) + +proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): seq[string] = + ## Get all BinaryBuilder.org libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps + ## in descending order + ## + ## `outdir` is prefixed to each entry + let + libs = if shared: pkg.sharedLibs else: pkg.staticLibs + str = if shared: "shared" else: "static" + + doAssert libs.len != 0, &"No {str} libs found for {pkg.name} in {outdir}" + + if not main: + for lib in libs: + result.add outdir / lib + + for cpkg in pkg.requires: + result.add cpkg.getJBBLDeps(outdir, shared, main = false) diff --git a/tests/getheader.nims b/tests/getheader.nims index 349b9bb..1d0ba1d 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -23,6 +23,7 @@ var testCall(cmd & lrcmd, "No build files found", 1) testCall(cmd & " -d:libssh2Conan" & sshcmd, "Need version for Conan uri", 1) +testCall(cmd & " -d:libssh2JBB" & sshcmd, "Need version for BinaryBuilder.org uri", 1) when defined(posix): # stdlib @@ -43,6 +44,11 @@ else: # conan static for Windows testCall(cmd & " -d:zlibConan -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) +# JBB +testCall(cmd & " -d:libssh2JBB -d:libssh2SetVer=1.9.0" & sshcmd, zexp, 0) +testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11" & zrcmd, zexp, 0) +testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) + # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) testCall(cmd & " -d:envTestStatic" & zrcmd, zexp, 0, delete = false) diff --git a/tests/libssh2.nim b/tests/libssh2.nim index b7e591c..48b8ba9 100644 --- a/tests/libssh2.nim +++ b/tests/libssh2.nim @@ -6,6 +6,7 @@ const getHeader( header = "libssh2.h", conanuri = "libssh2/$1", + jbburi = "libssh2/$1", outdir = outdir ) @@ -14,16 +15,16 @@ cOverride: stat = object stat64 = object SOCKET = object - + when not libssh2Static: cImport(libssh2Path, recurse = true, dynlib = "libssh2LPath", flags = "-f:ast2 -c -E_ -F_") - when not defined(Windows): + when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc, dynlib: libssh2LPath.} else: cImport(libssh2Path, recurse = true, flags = "-f:ast2 -c -E_ -F_") - when not defined(Windows): + when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc.} {.passL: "-lpthread".} @@ -39,8 +40,11 @@ if session == nil: libssh2_session_set_blocking(session, 0.cint) echo "zlib version = " & (block: - when not defined(Windows): + when not defined(Windows) and not isDefined(libssh2JBB): $zlibVersion() else: "" -) \ No newline at end of file +) + +static: + echo libssh2LDeps \ No newline at end of file diff --git a/tests/zlib.nim b/tests/zlib.nim index 15400d4..2bd9395 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -28,6 +28,7 @@ getHeader( giturl = "https://github.com/madler/zlib", dlurl = "http://zlib.net/zlib-$1.tar.gz", conanuri = "zlib/$1", + jbburi = "zlib/$1", outdir = baseDir, altNames = "z,zlib" ) @@ -72,3 +73,6 @@ else: cImport(zlibPath, recurse = true, flags = FLAGS) echo "zlib version = " & $zlibVersion() + +when isDefined(zlibJBB) and isDefined(zlibStatic): + {.passL: "-no-pie".} From 9d8fee19c01881619b68164a51aaf51dd17e8be3 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jun 2020 17:41:18 -0500 Subject: [PATCH 028/106] libdir support --- nimterop/build.nim | 97 +++++++++++++++++++++++++++++++++++++++----- nimterop/cimport.nim | 13 ------ nimterop/conan.nim | 2 +- nimterop/jbb.nim | 16 ++------ nimterop/nimconf.nim | 61 +++++++++++++--------------- 5 files changed, 120 insertions(+), 69 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index eda27e7..6cb497d 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -158,8 +158,10 @@ proc mkDir*(dir: string) = flag = when not defined(Windows): "-p" else: "" discard execAction(&"mkdir {flag} {dir.sanitizePath}", retry = 2) -proc cpFile*(source, dest: string, move = false) = +proc cpFile*(source, dest: string, psymlink = false, move = false) = ## Copy a file from `source` to `dest` at compile time + ## + ## `psymlink = true` preserves symlinks instead of dereferencing on posix let source = source.replace("/", $DirSep) dest = dest.replace("/", $DirSep) @@ -173,7 +175,10 @@ proc cpFile*(source, dest: string, move = false) = if move: "mv -f" else: - "cp -f" + if psymlink: + "cp -fd" + else: + "cp -f" discard execAction(&"{cmd} {source.sanitizePath} {dest.sanitizePath}", retry = 2) @@ -233,6 +238,20 @@ proc mvTree*(source, dest: string) = ## Move contents of source dir to the destination, not the directory itself cpTree(source, dest, move = true) +proc getFileDate*(fullpath: string): string = + ## Get file date for `fullpath` + var + ret = 0 + cmd = + when defined(Windows): + &"cmd /c for %a in ({fullpath.sanitizePath}) do echo %~ta" + elif defined(Linux): + &"stat -c %y {fullpath.sanitizePath}" + elif defined(OSX) or defined(FreeBSD): + &"stat -f %m {fullpath.sanitizePath}" + + (result, ret) = execAction(cmd) + proc getProjectCacheDir*(name: string, forceClean = true): string = ## Get a cache directory where all nimterop artifacts can be stored ## @@ -1083,7 +1102,8 @@ macro isDefined*(def: untyped): untyped = macro getHeader*( header: static[string], giturl: static[string] = "", dlurl: static[string] = "", - conanuri: static[string] = "", jbburi: static[string] = "", outdir: static[string] = "", + conanuri: static[string] = "", jbburi: static[string] = "", + outdir: static[string] = "", libdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = ## Get the path to a header file for wrapping with @@ -1125,7 +1145,14 @@ macro getHeader*( ## ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can - ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. + ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. Any dependency + ## libraries downloaded by `Conan` or `JBB` are returned in `const xxxLDeps` as a seq[string]. + ## + ## `libdir` can be used to instruct `getHeader()` to copy shared libraries and their + ## dependencies to that directory. This prevents any runtime failures if `outdir` gets + ## removed or its contents changed. By default, `libdir` is set to the output directory + ## where the program binary will be created. The values of `xxxLPath` and `xxxLDeps` will + ## reflect this new location. ## ## `-d:xxxStatic` can be specified to statically link with the library instead. This ## will automatically add a `{.passL.}` call to the static library for convenience. Note @@ -1214,6 +1241,8 @@ macro getHeader*( "" mode = getCompilerMode(header) + libdir = if libdir.len != 0: libdir else: getOutDir() + # Use alternate library names if specified for regex search if altNames.len != 0: lre = lre % ("(" & altNames.replace(",", "|") & ")") @@ -1272,9 +1301,9 @@ macro getHeader*( static: `preBuild`(`outdir`, prePath) - const - # Library binary path - build if not standard - `lpath`* = + let + # Library binary path - build if not standard / conan / jbb + lpath {.compiletime.} = when useStd: stdLPath elif `nameConan` or `nameJBB`: @@ -1283,7 +1312,7 @@ macro getHeader*( buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) # Library dependecy paths - `ldeps`*: seq[string] = + ldeps {.compiletime.}: seq[string] = when `nameConan`: getConanLDeps(`outdir`) elif `nameJBB`: @@ -1291,6 +1320,7 @@ macro getHeader*( else: @[] + const # Header path - search again in case header is generated in build `path`* = if prePath.len != 0: @@ -1301,12 +1331,57 @@ macro getHeader*( static: doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name` - doAssert `lpath`.len != 0, "\nLibrary " & `lname` & " not found" - echo "# Including library " & `lpath` + doAssert lpath.len != 0, "\nLibrary " & `lname` & " not found" + + proc extractFilenameStatic(str: string): string {.compiletime.} = + var + pos = -1 + for i in countdown(str.len - 1, 0): + if str[i] == '/' or str[i] == '\\': + pos = i + 1 + break + result = str[pos .. ^1] + + proc joinPathStatic(str1, str2: string): string {.compiletime.} = + let + sep = when defined(Windows): "\\" else: "/" + result = str1 & sep & str2 - # Automatically link with static library and dependencies when `nameStatic`: + const + `lpath`* = lpath + `ldeps`* = ldeps + + # Automatically link with static library and dependencies {.passL: `lpath`.} if `ldeps`.len != 0: {.passL: `ldeps`.join(" ").} + + static: + echo "# Including library " & lpath + if `ldeps`.len != 0: + echo "# Including dependencies " & `ldeps`.join(" ") + else: + const + `lpath`* = joinPathStatic(`libdir`, lpath.extractFilenameStatic()) + `ldeps`* = block: + var + ldeps = ldeps + for i in 0 ..< ldeps.len: + let + lname = ldeps[i].extractFilenameStatic() + ldeptgt = joinPathStatic(`libdir`, lname) + if not fileExists(ldeptgt) or getFileDate(ldeps[i]) > getFileDate(ldeptgt): + echo "# Copying " & lname & " to " & `libdir` + cpFile(ldeps[i], ldeptgt, psymlink = true) + ldeps[i] = ldeptgt + ldeps + + static: + # Copy shared libraries and dependencies to `libdir` + if not fileExists(`lpath`) or getFileDate(lpath) > getFileDate(`lpath`): + echo "# Copying " & `lpath`.extractFilenameStatic() & " to " & `libdir` + cpFile(lpath, `lpath`) + + echo "# Including library " & `lpath` ) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 0559db7..c81c35b 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -70,19 +70,6 @@ proc walkDirImpl(indir, inext: string, file=true): seq[string] = if ret == 0: result = output.splitLines() -proc getFileDate(fullpath: string): string = - var - ret = 0 - cmd = - when defined(Windows): - &"cmd /c for %a in ({fullpath.sanitizePath}) do echo %~ta" - elif defined(Linux): - &"stat -c %y {fullpath.sanitizePath}" - elif defined(OSX) or defined(FreeBSD): - &"stat -f %m {fullpath.sanitizePath}" - - (result, ret) = execAction(cmd) - proc getCacheValue(fullpath: string): string = if not gStateCT.nocache: result = fullpath.getFileDate() diff --git a/nimterop/conan.nim b/nimterop/conan.nim index bfe2e94..f8740d0 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -278,7 +278,7 @@ proc parseConanManifest(pkg: ConanPackage, outdir: string) = if line.startsWith("lib/"): if line.endsWith(".a") or line.endsWith(".lib"): pkg.staticLibs.add line - elif line.endsWith(".so"): + elif line.endsWith(".so") or line.endsWith(".dylib"): pkg.sharedLibs.add line elif line.startsWith("bin/") and line.endsWith("dll"): pkg.sharedLibs.add line diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim index 9f0d173..3e7e623 100644 --- a/nimterop/jbb.nim +++ b/nimterop/jbb.nim @@ -94,18 +94,11 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = break proc findJBBLibs(pkg: JBBPackage, outdir: string) = - pkg.sharedLibs = findFiles("lib[\\\\/].*\\.(so|dylib)", outdir) - pkg.sharedLibs.add findFiles("bin[\\\\/].*\\.(dll)", outdir) - for i in 0 ..< pkg.sharedLibs.len: - if pkg.sharedLibs[i].isAbsolute: - pkg.sharedLibs[i] = pkg.sharedLibs[i][outdir.len+1 .. ^1] + pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dynlib)[0-9.]*", outdir) for lib in findFiles("lib[\\\\/].*\\.(a|lib)$", outdir): if not lib.endsWith(".dll.a"): - if lib.isAbsolute: - pkg.staticLibs.add lib[outdir.len+1 .. ^1] - else: - pkg.staticLibs.add lib + pkg.staticLibs.add lib proc getJBBRepo*(pkg: JBBPackage, outdir: string) = ## Clone JBB package repo and checkout version tag if version is @@ -174,11 +167,10 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) = &" v{pkg.version}" else: "" - path = outdir / "downloads" / pkg.name + path = outdir / pkg.name echo &"# Downloading {pkg.name}{vstr} from BinaryBuilder.org" downloadUrl(pkg.url, path, quiet = true) pkg.findJBBLibs(path) - mvTree(path, outdir) pkg.dlJBBRequires(outdir) @@ -204,7 +196,7 @@ proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): s if not main: for lib in libs: - result.add outdir / lib + result.add lib for cpkg in pkg.requires: result.add cpkg.getJBBLDeps(outdir, shared, main = false) diff --git a/nimterop/nimconf.nim b/nimterop/nimconf.nim index 33215ef..98fc8fe 100644 --- a/nimterop/nimconf.nim +++ b/nimterop/nimconf.nim @@ -16,6 +16,7 @@ type paths*: OrderedSet[string] nimblePaths*: OrderedSet[string] nimcacheDir*: string + outDir*: string proc getJson(projectDir: string): JsonNode = # Get `nim dump` json value for `projectDir` @@ -67,35 +68,6 @@ proc stripName(path, projectName: string): string = else: result = path -proc getNimcacheDir*(projectDir = ""): string = - ## Get nimcache directory for current compilation or specified `projectDir` - when nimvm: - when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): - # Get value at compile time from `std/compilesettings` - result = stripName( - querySetting(SingleValueSetting.nimcacheDir), - querySetting(SingleValueSetting.projectName) - ) - else: - discard - - # Not Nim v1.2.0+ or runtime - if result.len == 0: - let - # Get project directory for < v1.2.0 at compile time - projectDir = if projectDir.len != 0: projectDir else: getProjectDir() - - # Use `nim dump` to figure out nimcache for `projectDir` - let - dumpJson = getJson(projectDir) - - if dumpJson != nil and dumpJson.hasKey("nimcache"): - result = stripName(dumpJson["nimcache"].getStr(), "dummy") - - # Set to OS defaults if not detectable - if result.len == 0: - result = getOsCacheDir() - proc jsonToSeq(node: JsonNode, key: string): seq[string] = # Convert JsonArray to seq[string] for specified `key` if node.hasKey(key): @@ -126,7 +98,11 @@ proc getNimConfig*(projectDir = ""): Config = libPath = getCurrentCompilerExe().parentDir().parentDir() / "lib" lazyPaths = querySettingSeq(MultipleValueSetting.lazyPaths) searchPaths = querySettingSeq(MultipleValueSetting.searchPaths) - result.nimcacheDir = querySetting(SingleValueSetting.nimcacheDir) + result.nimcacheDir = stripName( + querySetting(SingleValueSetting.nimcacheDir), + querySetting(SingleValueSetting.projectName) + ) + result.outDir = querySetting(SingleValueSetting.outDir) else: discard @@ -150,6 +126,11 @@ proc getNimConfig*(projectDir = ""): Config = # Usually `libPath` is last entry in `searchPaths` libPath = searchPaths[^1] + if dumpJson.hasKey("nimcache"): + result.nimcacheDir = stripName(dumpJson["nimcache"].getStr(), "dummy") + if dumpJson.hasKey("outdir"): + result.outDir = dumpJson["outdir"].getStr() + # Parse version if version.len != 0: let @@ -188,7 +169,11 @@ proc getNimConfig*(projectDir = ""): Config = if not skip: result.paths.incl path - result.nimcacheDir = getNimcacheDir(projectDir) + if result.nimcacheDir.len == 0: + result.nimcacheDir = getOsCacheDir() + + if result.outDir.len == 0: + result.outDir = projectDir proc getNimConfigFlags(cfg: Config): string = # Convert configuration into Nim flags for cfg file or command line @@ -227,4 +212,16 @@ proc writeNimConfig*(cfgFile: string, projectDir = "") = let cfg = getNimConfig(projectDir) cfgOut = getNimConfigFlags(cfg) - writeFile(cfgFile, cfgOut) \ No newline at end of file + writeFile(cfgFile, cfgOut) + +proc getNimcacheDir*(projectDir = ""): string = + ## Get nimcache directory for current compilation or specified `projectDir` + let + cfg = getNimConfig(projectDir) + result = cfg.nimcacheDir + +proc getOutDir*(projectDir = ""): string = + ## Get output directory for current compilation or specified `projectDir` + let + cfg = getNimConfig(projectDir) + result = cfg.outDir From c359cc0226effd3b3dc47af74defb6b6b7bc7bca Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jun 2020 20:41:32 -0500 Subject: [PATCH 029/106] Fix lib copy issue --- nimterop/build.nim | 20 +++++++++++++++++--- nimterop/conan.nim | 2 ++ tests/getheader.nims | 3 +++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 6cb497d..b5cdc59 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -252,6 +252,17 @@ proc getFileDate*(fullpath: string): string = (result, ret) = execAction(cmd) +proc touchFile*(fullpath: string) = + ## Touch file to update modified date + var + cmd = + when defined(Windows): + &"cmd /c copy /b {fullpath.sanitizePath}+" + else: + &"touch {fullpath.sanitizePath}" + + discard execAction(cmd) + proc getProjectCacheDir*(name: string, forceClean = true): string = ## Get a cache directory where all nimterop artifacts can be stored ## @@ -1367,19 +1378,22 @@ macro getHeader*( `ldeps`* = block: var ldeps = ldeps + copied: seq[string] for i in 0 ..< ldeps.len: let lname = ldeps[i].extractFilenameStatic() ldeptgt = joinPathStatic(`libdir`, lname) - if not fileExists(ldeptgt) or getFileDate(ldeps[i]) > getFileDate(ldeptgt): - echo "# Copying " & lname & " to " & `libdir` + if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt): cpFile(ldeps[i], ldeptgt, psymlink = true) + copied.add lname ldeps[i] = ldeptgt + if copied.len != 0: + echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` ldeps static: # Copy shared libraries and dependencies to `libdir` - if not fileExists(`lpath`) or getFileDate(lpath) > getFileDate(`lpath`): + if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): echo "# Copying " & `lpath`.extractFilenameStatic() & " to " & `libdir` cpFile(lpath, `lpath`) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index f8740d0..fcb0495 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -346,6 +346,8 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = elif clean: cleanDir(outdir) + echo &"# Downloading {pkg.name} v{pkg.version} from Conan" + pkg.getConanBuilds() doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" diff --git a/tests/getheader.nims b/tests/getheader.nims index 1d0ba1d..7a3c176 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -40,6 +40,7 @@ when defined(posix): # conan static testCall(cmd & " -d:libssh2Conan -d:libssh2SetVer=1.9.0 -d:libssh2Static" & sshcmd, zexp, 0) +<<<<<<< HEAD else: # conan static for Windows testCall(cmd & " -d:zlibConan -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) @@ -48,6 +49,8 @@ else: testCall(cmd & " -d:libssh2JBB -d:libssh2SetVer=1.9.0" & sshcmd, zexp, 0) testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11" & zrcmd, zexp, 0) testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) +======= +>>>>>>> c35eb74... Add tests for conan, recurse implies preprocess # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) From 0a014e084f3afef204b7539fa688183d9b2f62d7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jun 2020 22:57:28 -0500 Subject: [PATCH 030/106] Fix OSX --- nimterop/build.nim | 4 ++-- nimterop/jbb.nim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index b5cdc59..28f2b3c 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -176,7 +176,7 @@ proc cpFile*(source, dest: string, psymlink = false, move = false) = "mv -f" else: if psymlink: - "cp -fd" + "cp -fa" else: "cp -f" @@ -1055,7 +1055,7 @@ proc getDynlibExt(): string = elif defined(linux) or defined(FreeBSD): result = "\\.so[0-9.]*" elif defined(macosx): - result = "\\.dylib[0-9.]*" + result = "[0-9.\\-]*\\.dylib" var gDefines {.compileTime.} = initTable[string, string]() diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim index 3e7e623..de59a87 100644 --- a/nimterop/jbb.nim +++ b/nimterop/jbb.nim @@ -94,7 +94,7 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = break proc findJBBLibs(pkg: JBBPackage, outdir: string) = - pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dynlib)[0-9.]*", outdir) + pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dylib)[0-9.]*", outdir) for lib in findFiles("lib[\\\\/].*\\.(a|lib)$", outdir): if not lib.endsWith(".dll.a"): From 7870840d2d6992f53d1bfc319e2b52422b234f36 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 16 Jun 2020 23:37:03 -0500 Subject: [PATCH 031/106] Fix Windows file date --- nimterop/build.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 28f2b3c..dbdcd4c 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -244,7 +244,9 @@ proc getFileDate*(fullpath: string): string = ret = 0 cmd = when defined(Windows): - &"cmd /c for %a in ({fullpath.sanitizePath}) do echo %~ta" + let + (head, tail) = fullpath.splitPath() + &"cmd /c forfiles /P {head.sanitizePath()} /M {tail.sanitizePath} /C \"cmd /c echo @fdate @ftime @fsize\"" elif defined(Linux): &"stat -c %y {fullpath.sanitizePath}" elif defined(OSX) or defined(FreeBSD): From 1820fdffcf8af4b1c5ecca8bdc548c206e24b439 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 18 Jun 2020 10:22:22 -0500 Subject: [PATCH 032/106] Optional version and uri, no Std copy, reuse deps, jbb libc check --- .travis.yml | 2 +- appveyor.yml | 1 + nimterop/build.nim | 150 +++++++++++++++++++++++++++---------------- nimterop/conan.nim | 58 +++++++++++++---- nimterop/globals.nim | 2 +- nimterop/jbb.nim | 63 +++++++++++------- tests/libssh2.nim | 5 +- tests/lzma.nim | 2 + tests/zlib.nim | 2 - 9 files changed, 185 insertions(+), 100 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4e9bde..173da68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ language: c env: - BRANCH=0.20.2 - BRANCH=1.0.6 - - BRANCH=1.2.0 + - BRANCH=1.2.2 - BRANCH=devel cache: diff --git a/appveyor.yml b/appveyor.yml index d67b01d..b572ea1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,6 +11,7 @@ environment: matrix: - NIM_VERSION: 0.20.2 - NIM_VERSION: 1.0.6 + - NIM_VERSION: 1.2.2 for: - diff --git a/nimterop/build.nim b/nimterop/build.nim index dbdcd4c..b391f7a 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -2,6 +2,8 @@ import hashes, macros, osproc, sets, strformat, strutils, tables import os except findExe, sleep +export extractFilename, `/` + type BuildType* = enum btAutoconf, btCmake @@ -850,6 +852,31 @@ template fixOutDir() {.dirty.} = let outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir +proc compareVersions*(ver1, ver2: string): int = + ## Compare two version strings x.y.z and return -1, 0, 1 + ## + ## ver1 < ver2 = -1 + ## ver1 = ver2 = 0 + ## ver1 > ver2 = 1 + let + ver1seq = ver1.replace("-", "").split('.') + ver2seq = ver2.replace("-", "").split('.') + for i in 0 ..< ver1seq.len: + let + p1 = ver1seq[i] + p2 = if i < ver2seq.len: ver2seq[i] else: "0" + + try: + let + h1 = p1.parseHexInt() + h2 = p2.parseHexInt() + + if h1 < h2: return -1 + elif h1 > h2: return 1 + except ValueError: + if p1 < p2: return -1 + elif p1 > p2: return 1 + # Conan support include conan @@ -913,10 +940,10 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string = uri = uri if "$#" in uri or "$1" in uri: - doAssert version.len != 0, "Need version for Conan uri" + doAssert version.len != 0, "Need version for Conan.io uri: " & uri uri = uri % version - else: - doAssert version.len == 0, "Conan uri does not contain version" + elif version.len != 0: + uri = uri & "/" & version let pkg = newConanPackageFromUri(uri, shared) @@ -934,6 +961,7 @@ proc getJBBPath(header, uri, outdir, version: string): string = let spl = uri.split('/', 1) name = spl[0] + hasVersion = version.len != 0 var ver = @@ -942,11 +970,15 @@ proc getJBBPath(header, uri, outdir, version: string): string = else: "" - if "$#" in ver or "$1" in ver: - doAssert version.len != 0, "Need version for BinaryBuilder.org uri" - ver = ver % version - else: - doAssert version.len == 0, "BinaryBuilder.org uri does not contain version" + if ver.len != 0: + if "$#" in ver or "$1" in ver: + doAssert hasVersion, "Need version for BinaryBuilder.org uri: " & uri + ver = ver % version + elif hasVersion: + doAssert false, "Version in both uri `" & uri & "` and `-d:xxxSetVer=\"" & + version & "\"` for BinaryBuilder.org" + elif hasVersion: + ver = version let pkg = newJBBPackage(name, ver) @@ -1129,10 +1161,10 @@ macro getHeader*( ## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux ## `-d:xxxGit` - clone source from a git repo specified in `giturl` ## `-d:xxxDL` - download source from `dlurl` and extract if required - ## `-d:xxxConan` - download headers and binary from conan.io using `conanuri` - ## typically formatted as `name/version[@user/channel][:bhash]` - ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` - ## typically formatted as `name/version` + ## `-d:xxxConan` - download headers and binary from Conan.io using `conanuri` with + ## format `pkgname[/version[@user/channel][:bhash]]` + ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` with + ## format `pkgname[/version]` ## ## This allows a single wrapper to be used in different ways depending on the user's needs. ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. @@ -1145,8 +1177,15 @@ macro getHeader*( ## one of the other methods. ## ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag - ## name for Git whereas for DL, Conan and BinaryBuilder.org, it replaces `$1` in the URL - ## defined. + ## name for `Git` whereas for `DL`, `Conan` and `JBB`, it replaces `$1` in the URL + ## if specified. Specifying `-d:xxxSetVer` without a `$1` will download that version for + ## `Conan` and `JBB` if available. If no version is specified, the latest release of the + ## package is downloaded. For `Conan`, `-d:xxxSetVer` can also be used to set additional + ## URI information: + ## `-d:xxxSetVer=1.9.0@bincrafters/stable:bhash` + ## + ## If `conanuri` or `jbburi` are not defined and `Conan` or `JBB` is selected, the `header` + ## filename is used instead. ## ## All defines can also be set in code using `setDefines()` and checked for using ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`. @@ -1165,7 +1204,7 @@ macro getHeader*( ## dependencies to that directory. This prevents any runtime failures if `outdir` gets ## removed or its contents changed. By default, `libdir` is set to the output directory ## where the program binary will be created. The values of `xxxLPath` and `xxxLDeps` will - ## reflect this new location. + ## reflect this new location. `libdir` is ignored for `Std` mode. ## ## `-d:xxxStatic` can be specified to statically link with the library instead. This ## will automatically add a `{.passL.}` call to the static library for convenience. Note @@ -1210,6 +1249,10 @@ macro getHeader*( origname = header.extractFilename().split(".")[0] name = origname.split(seps = AllChars-Letters-Digits).join() + # Default to origname if not specified + conanuri = if conanuri.len != 0: conanuri else: origname + jbburi = if jbburi.len != 0: jbburi else: origname + # -d:xxx for this header stdStr = name & "Std" gitStr = name & "Git" @@ -1316,7 +1359,7 @@ macro getHeader*( let # Library binary path - build if not standard / conan / jbb - lpath {.compiletime.} = + lpath {.compileTime.} = when useStd: stdLPath elif `nameConan` or `nameJBB`: @@ -1325,11 +1368,14 @@ macro getHeader*( buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) # Library dependecy paths - ldeps {.compiletime.}: seq[string] = - when `nameConan`: - getConanLDeps(`outdir`) - elif `nameJBB`: - getJBBLDeps(`outdir`, not `nameStatic`) + ldeps {.compileTime.}: seq[string] = + when not useStd: + when `nameConan`: + getConanLDeps(`outdir`) + elif `nameJBB`: + getJBBLDeps(`outdir`, not `nameStatic`) + else: + @[] else: @[] @@ -1346,20 +1392,6 @@ macro getHeader*( "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name` doAssert lpath.len != 0, "\nLibrary " & `lname` & " not found" - proc extractFilenameStatic(str: string): string {.compiletime.} = - var - pos = -1 - for i in countdown(str.len - 1, 0): - if str[i] == '/' or str[i] == '\\': - pos = i + 1 - break - result = str[pos .. ^1] - - proc joinPathStatic(str1, str2: string): string {.compiletime.} = - let - sep = when defined(Windows): "\\" else: "/" - result = str1 & sep & str2 - when `nameStatic`: const `lpath`* = lpath @@ -1376,28 +1408,34 @@ macro getHeader*( echo "# Including dependencies " & `ldeps`.join(" ") else: const - `lpath`* = joinPathStatic(`libdir`, lpath.extractFilenameStatic()) - `ldeps`* = block: - var - ldeps = ldeps - copied: seq[string] - for i in 0 ..< ldeps.len: - let - lname = ldeps[i].extractFilenameStatic() - ldeptgt = joinPathStatic(`libdir`, lname) - if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt): - cpFile(ldeps[i], ldeptgt, psymlink = true) - copied.add lname - ldeps[i] = ldeptgt - if copied.len != 0: - echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` - ldeps + `lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath + `ldeps`* = + when not useStd: + block: + var + ldeps = ldeps + copied: seq[string] + for i in 0 ..< ldeps.len: + let + lname = ldeps[i].extractFilename() + ldeptgt = `libdir` / lname + if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt): + cpFile(ldeps[i], ldeptgt, psymlink = true) + copied.add lname + ldeps[i] = ldeptgt + # Copy downloaded dependencies to `libdir` + if copied.len != 0: + echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` + ldeps + else: + ldeps static: - # Copy shared libraries and dependencies to `libdir` - if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): - echo "# Copying " & `lpath`.extractFilenameStatic() & " to " & `libdir` - cpFile(lpath, `lpath`) + when not useStd: + # Copy downloaded shared libraries to `libdir` + if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): + echo "# Copying " & `lpath`.extractFilename() & " to " & `libdir` + cpFile(lpath, `lpath`) - echo "# Including library " & `lpath` + echo "# Including library " & `lpath` ) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index fcb0495..6e761df 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -42,11 +42,14 @@ const var # Bintray download URL for explicit `user/channel` - conanBaseAltUrl {.compiletime.} = { + conanBaseAltUrl {.compileTime.} = { "bincrafters": "https://bintray.com/bincrafters/public-conan", "conan": "https://bintray.com/conan-community/conan" }.toTable() + # Reuse dependencies already downloaded + gConanRequires {.compileTime.}: Table[string, ConanPackage] + proc addAltConanBaseUrl*(name, url: string) = # Add an alternate base URL for a custom conan repo on bintray conanBaseAltUrl[name] = url @@ -144,13 +147,28 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac if channel.len != 0: query &= "/" & channel + echo &"# Searching Conan.io for latest version of {name}" + let j1 = jsonGet(conanSearchUrl % ["query", query]) res = j1.getOrDefault("results").getElems() - if res.len != 0: - # Return last entry - latest - result = newConanPackageFromUri(res[^1].getStr()) + # Return latest comparing versions - prefer @_/_ + var + latest = "" + latestv = "" + for i in 0 ..< res.len: + let + str = res[i].getStr() + if "@_/_" in str: + let + ver = str.split('/')[1].split('@')[0] + if latestv.len == 0 or compareVersions(ver, latestv) > 0: + latestv = ver + latest = str + + if latest.len != 0: + result = newConanPackageFromUri(latest) proc searchConan*(pkg: ConanPackage): ConanPackage = ## Search for latest package based on incomplete package info @@ -288,6 +306,9 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision ## ## If omitted, the latest revision (first) is downloaded fixOutDir() + + doAssert bld.revisions.len != 0, "No build revisions found for Conan.io package " & pkg.getUriFromConanPackage() + let revision = if revision.len != 0: @@ -326,7 +347,7 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision rmFile(outdir / conanManifest) proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) -proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = +proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = ## Download latest recipe/build/revision of `pkg` to `outdir` ## ## High-level API that handles the end to end Conan process flow to find @@ -339,11 +360,13 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = else: pkg - cpkg = loadConanInfo(outdir) + if main: + let + cpkg = loadConanInfo(outdir) + + if cpkg == pkg: + return - if cpkg == pkg: - return - elif clean: cleanDir(outdir) echo &"# Downloading {pkg.name} v{pkg.version} from Conan" @@ -352,7 +375,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" - echo &"# Downloading {pkg.name} v{pkg.version} from Conan" + echo &"# Downloading {pkg.name} v{pkg.version} from Conan.io" for recipe, builds in pkg.recipes: for build in builds: if pkg.bhash.len == 0 or pkg.bhash == build.bhash: @@ -362,7 +385,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, clean = true) = break break - if clean: + if main: pkg.saveConanInfo(outdir) proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = @@ -374,10 +397,17 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = if bld.options["shared"] == "False": for req in bld.requires: let - rpkg = newConanPackageFromUri(req, shared = false) + name = req.split('/')[0] + if gConanRequires.hasKey(name): + # Reuse dep already downloaded + pkg.requires.add gConanRequires[name] + else: + let + rpkg = newConanPackageFromUri(req, shared = false) - downloadConan(rpkg, outdir, clean = false) - pkg.requires.add rpkg + downloadConan(rpkg, outdir, main = false) + pkg.requires.add rpkg + gConanRequires[name] = rpkg proc getConanLDeps*(pkg: ConanPackage, outdir: string, main = true): seq[string] = ## Get all Conan libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 8062ac8..c79f4a5 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -146,7 +146,7 @@ when defined(TOAST): gecho join(args, "").getCommented() else: var - gStateCT* {.compiletime, used.} = new(State) + gStateCT* {.compileTime, used.} = new(State) template nBl*(s: typed): untyped {.used.} = (s.len != 0) diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim index de59a87..5d421f0 100644 --- a/nimterop/jbb.nim +++ b/nimterop/jbb.nim @@ -20,6 +20,10 @@ const jbbProject = "Project.toml" jbbArtifacts = "Artifacts.toml" +var + # Reuse dependencies already downloaded + gJBBRequires {.compileTime.}: Table[string, JBBPackage] + proc `==`*(pkg1, pkg2: JBBPackage): bool = ## Check if two JBBPackage objects are equal (not pkg1.isNil and not pkg2.isNil and @@ -79,19 +83,26 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = let line = line.strip() if line.len != 0: - if line.startsWith("arch = ") and not found: - let - barch = line.split(" = ")[1].strip(chars = {'"'}) - if barch == arch: - found = true - elif line.startsWith("os = ") and found: - let - bos = line.split(" = ")[1].strip(chars = {'"'}) - if bos != os: - found = false - elif line.startsWith("url = ") and found: - pkg.url = line.split(" = ")[1].strip(chars = {'"'}) - break + let + spl = line.split(" = ", 1) + name = spl[0] + val = if spl.len == 2: spl[1].strip(chars = {'"', ' '}) else: "" + + # Match arch, os and glibc on Linux to find download URL + case name + of "arch": + if val == arch and not found: found = true + of "os": + if val != os and found: found = false + of "libc": + when defined(Linux): + if val != "glibc" and found: found = false + of "url": + if found: + pkg.url = val + break + else: + discard proc findJBBLibs(pkg: JBBPackage, outdir: string) = pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dylib)[0-9.]*", outdir) @@ -143,18 +154,19 @@ proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = writeFile(file, $$pkg) proc dlJBBRequires*(pkg: JBBPackage, outdir: string) -proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) = +proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = ## Download `pkg` from BinaryBuilder.org to `outdir` ## ## High-level API that handles the end to end JBB process flow to find ## latest package binary and downloads and extracts it to `outdir`. fixOutDir() - let - cpkg = loadJBBInfo(outdir) + if main: + let + cpkg = loadJBBInfo(outdir) + + if cpkg == pkg: + return - if cpkg == pkg: - return - elif clean: cleanDir(outdir) pkg.getJBBRepo(outdir) @@ -174,14 +186,21 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, clean = true) = pkg.dlJBBRequires(outdir) - if clean: + if main: pkg.saveJBBInfo(outdir) proc dlJBBRequires*(pkg: JBBPackage, outdir: string) = ## Download all required dependencies of this `pkg` fixOutDir() - for rpkg in pkg.requires: - downloadJBB(rpkg, outdir, clean = false) + for i in 0 ..< pkg.requires.len: + let + rpkg = pkg.requires[i] + if gJBBRequires.hasKey(rpkg.name): + # Reuse dep already downloaded + pkg.requires[i] = gJBBRequires[rpkg.name] + else: + downloadJBB(rpkg, outdir, main = false) + gJBBRequires[rpkg.name] = rpkg proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): seq[string] = ## Get all BinaryBuilder.org libs - shared (.so|.dll) or static (.a|.lib) in pkg, including deps diff --git a/tests/libssh2.nim b/tests/libssh2.nim index 48b8ba9..172f6ae 100644 --- a/tests/libssh2.nim +++ b/tests/libssh2.nim @@ -6,7 +6,7 @@ const getHeader( header = "libssh2.h", conanuri = "libssh2/$1", - jbburi = "libssh2/$1", + jbburi = "libssh2/1.9.0", outdir = outdir ) @@ -45,6 +45,3 @@ echo "zlib version = " & (block: else: "" ) - -static: - echo libssh2LDeps \ No newline at end of file diff --git a/tests/lzma.nim b/tests/lzma.nim index cff39de..9bda18e 100644 --- a/tests/lzma.nim +++ b/tests/lzma.nim @@ -24,6 +24,8 @@ getHeader( "lzma.h", giturl = "https://github.com/xz-mirror/xz", dlurl = "https://tukaani.org/xz/xz-$1.tar.gz", + conanuri = "xz_utils", + jbburi = "xz", outdir = baseDir, conFlags = "--disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo" ) diff --git a/tests/zlib.nim b/tests/zlib.nim index 2bd9395..53384c3 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -27,8 +27,6 @@ getHeader( "zlib.h", giturl = "https://github.com/madler/zlib", dlurl = "http://zlib.net/zlib-$1.tar.gz", - conanuri = "zlib/$1", - jbburi = "zlib/$1", outdir = baseDir, altNames = "z,zlib" ) From f207911d38537fbadf42069b679259aed2dd929d Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 18 Jun 2020 14:33:32 -0500 Subject: [PATCH 033/106] --gc:arc fixes and testing --- nimterop/conan.nim | 9 ++++++--- nimterop/jbb.nim | 9 ++++++--- tests/getheader.nims | 3 +++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index 6e761df..ee74141 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -1,4 +1,4 @@ -import json, marshal, os, strformat, strutils, tables +import json, os, strformat, strutils, tables type ConanPackage* = ref object @@ -272,7 +272,10 @@ proc loadConanInfo*(outdir: string): ConanPackage = file = outdir / conanInfo if fileExists(file): - result = to[ConanPackage](readFile(file)) + try: + result = to(readFile(file).parseJson(), ConanPackage) + except: + discard proc saveConanInfo*(pkg: ConanPackage, outdir: string) = ## Save downloaded package info to `outdir/conaninfo.json` @@ -280,7 +283,7 @@ proc saveConanInfo*(pkg: ConanPackage, outdir: string) = let file = outdir / conanInfo - writeFile(file, $$pkg) + writeFile(file, $(%pkg)) proc parseConanManifest(pkg: ConanPackage, outdir: string) = # Get all library info from downloaded conan package diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim index 5d421f0..b5e66fb 100644 --- a/nimterop/jbb.nim +++ b/nimterop/jbb.nim @@ -1,4 +1,4 @@ -import marshal, os, strutils +import json, os, strutils type JBBPackage* = ref object @@ -143,7 +143,10 @@ proc loadJBBInfo*(outdir: string): JBBPackage = file = outdir / jbbInfo if fileExists(file): - result = to[JBBPackage](readFile(file)) + try: + result = to(readFile(file).parseJson(), JBBPackage) + except: + discard proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = ## Save downloaded package info to `outdir/jbbinfo.json` @@ -151,7 +154,7 @@ proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = let file = outdir / jbbInfo - writeFile(file, $$pkg) + writeFile(file, $(%pkg)) proc dlJBBRequires*(pkg: JBBPackage, outdir: string) proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = diff --git a/tests/getheader.nims b/tests/getheader.nims index 7a3c176..2fe4d9c 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -21,6 +21,9 @@ var lexp = "liblzma version = " zexp = "zlib version = " +when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): + cmd &= " --gc:arc" + testCall(cmd & lrcmd, "No build files found", 1) testCall(cmd & " -d:libssh2Conan" & sshcmd, "Need version for Conan uri", 1) testCall(cmd & " -d:libssh2JBB" & sshcmd, "Need version for BinaryBuilder.org uri", 1) From 9b2aa7d4b1691db69e76bad1b681eafc5bfb060a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 18 Jun 2020 14:52:27 -0500 Subject: [PATCH 034/106] Use marshal pre 1.2.0 --- nimterop/conan.nim | 23 +++++++++++++++++------ nimterop/jbb.nim | 19 ++++++++++++++----- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/nimterop/conan.nim b/nimterop/conan.nim index ee74141..cc060f8 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -1,4 +1,9 @@ -import json, os, strformat, strutils, tables +import os, strformat, strutils, tables + +when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): + import marshal +else: + import json type ConanPackage* = ref object @@ -272,10 +277,13 @@ proc loadConanInfo*(outdir: string): ConanPackage = file = outdir / conanInfo if fileExists(file): - try: - result = to(readFile(file).parseJson(), ConanPackage) - except: - discard + when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): + result = to[ConanPackage](readFile(file)) + else: + try: + result = to(readFile(file).parseJson(), ConanPackage) + except: + discard proc saveConanInfo*(pkg: ConanPackage, outdir: string) = ## Save downloaded package info to `outdir/conaninfo.json` @@ -283,7 +291,10 @@ proc saveConanInfo*(pkg: ConanPackage, outdir: string) = let file = outdir / conanInfo - writeFile(file, $(%pkg)) + when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): + writeFile(file, $$pkg) + else: + writeFile(file, $(%pkg)) proc parseConanManifest(pkg: ConanPackage, outdir: string) = # Get all library info from downloaded conan package diff --git a/nimterop/jbb.nim b/nimterop/jbb.nim index b5e66fb..c282197 100644 --- a/nimterop/jbb.nim +++ b/nimterop/jbb.nim @@ -1,5 +1,8 @@ import json, os, strutils +when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): + import marshal + type JBBPackage* = ref object ## JBBPackage type that stores package information @@ -143,10 +146,13 @@ proc loadJBBInfo*(outdir: string): JBBPackage = file = outdir / jbbInfo if fileExists(file): - try: - result = to(readFile(file).parseJson(), JBBPackage) - except: - discard + when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): + result = to[JBBPackage](readFile(file)) + else: + try: + result = to(readFile(file).parseJson(), JBBPackage) + except: + discard proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = ## Save downloaded package info to `outdir/jbbinfo.json` @@ -154,7 +160,10 @@ proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = let file = outdir / jbbInfo - writeFile(file, $(%pkg)) + when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): + writeFile(file, $$pkg) + else: + writeFile(file, $(%pkg)) proc dlJBBRequires*(pkg: JBBPackage, outdir: string) proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = From 7e0b0b1a2b69a835a680afce88f7af39b2c944b0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 19 Jun 2020 13:32:01 -0500 Subject: [PATCH 035/106] Retries, conan VS MD/14 --- nimterop/build.nim | 8 ++++---- nimterop/conan.nim | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index b391f7a..edbe439 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -129,7 +129,7 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, # On failure, retry or die as requested if result.ret != 0: if retry > 0: - sleep(500) + sleep(1000) result = execAction(cmd, retry = retry - 1, die, cache, cacheKey) elif die: doAssert false, "Command failed: " & $result.ret & "\ncmd: " & ccmd & @@ -368,7 +368,7 @@ proc downloadUrl*(url, outdir: string, quiet = false) = cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" else: doAssert false, "No download tool available - curl, wget" - discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 1) + discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 3) if ext == ".zip": extractZip(file, outdir, quiet) @@ -436,12 +436,12 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false if checkout.len != 0: if not quiet: echo "# Checking out " & checkout - discard execAction(&"cd {outdirQ} && git fetch", retry = 1) + discard execAction(&"cd {outdirQ} && git fetch", retry = 3) discard execAction(&"cd {outdirQ} && git checkout {checkout}") else: if not quiet: echo "# Pulling repository" - discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 1) + discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 3) proc gitTags*(outdir: string): seq[string] = ## Get all the git tags in the specified directory diff --git a/nimterop/conan.nim b/nimterop/conan.nim index cc060f8..7e1bd8e 100644 --- a/nimterop/conan.nim +++ b/nimterop/conan.nim @@ -208,6 +208,10 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = query &= &"&{filter}" if "compiler=" notin filter and os != "windows": query &= &"&compiler={compiler}&compiler.version=" & vfilter + if "compiler.runtime=" notin filter and os == "windows": + query &= &"&compiler.runtime=MD" + if "compiler.version=" notin filter and os == "windows": + query &= &"&compiler.version=14" query.replace("&", "%20and%20") else: "" From 7c9d826998e7347e613d823ba00ca6160d712a2f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 19 Jun 2020 18:03:37 -0500 Subject: [PATCH 036/106] Handle octal values correctly --- nimterop/exprparser.nim | 5 +++-- tests/include/tast2.h | 1 + tests/tast2.nim | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nimterop/exprparser.nim b/nimterop/exprparser.nim index 0fea184..0d2099a 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/exprparser.nim @@ -140,8 +140,6 @@ proc getIntNode(number, suffix: string): PNode {.inline.} = var val: BiggestInt flags: TNodeFlags - # I realize these regex are wasteful on performance, but - # couldn't come up with a better idea. if number.startsWith("0X") or number.startsWith("0x"): val = parseHexInt(number) flags = {nfBase16} @@ -203,6 +201,9 @@ proc processNumberLiteral(gState: State, node: TSNode): PNode = if number.startsWith("-"): number = number[1 ..< number.len] prefix = "-" + if number.len > 1 and number[0] == '0' and number[1] notin ['x', 'X']: + # Octal 0123 + number = "0o" & number[1 .. ^1] if tripleEndings.any(proc (s: string): bool = number.endsWith(s)): suffix = number[^3 .. ^1] number = number[0 ..< ^3] diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 019f7c4..6f569d5 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -7,6 +7,7 @@ extern "C" { #define C 0x10 #define D "hello" #define E 'c' +#define F 01234 #define UEXPR (1234u << 1) #define ULEXPR (1234ul << 2) diff --git a/tests/tast2.nim b/tests/tast2.nim index fd0175a..124c4e4 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -110,6 +110,7 @@ assert B == 1.0 assert C == 0x10 assert D == "hello" assert E == 'c' +assert F == 0o1234 assert not defined(NOTSUPPORTEDSTR) From 73fec6753fdcade0277c6c529e52ef01096e28e1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 19 Jun 2020 21:31:22 -0500 Subject: [PATCH 037/106] Download retries --- nimterop/build.nim | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index edbe439..62b5e1f 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -76,7 +76,7 @@ proc getNimteropCacheDir(): string = result = getNimcacheDir() / "nimterop" proc execAction*(cmd: string, retry = 0, die = true, cache = false, - cacheKey = ""): tuple[output: string, ret: int] = + cacheKey = "", onRetry: proc() = nil): tuple[output: string, ret: int] = ## Execute an external command - supported at compile time ## ## Checks if command exits successfully before returning. If not, an @@ -129,7 +129,9 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, # On failure, retry or die as requested if result.ret != 0: if retry > 0: - sleep(1000) + if not onRetry.isNil: + onRetry() + sleep(500) result = execAction(cmd, retry = retry - 1, die, cache, cacheKey) elif die: doAssert false, "Command failed: " & $result.ret & "\ncmd: " & ccmd & @@ -344,16 +346,17 @@ proc extractTar*(tarfile, outdir: string, quiet = false) = if name.len != 0: rmFile(outdir / name) -proc downloadUrl*(url, outdir: string, quiet = false) = +proc downloadUrl*(url, outdir: string, quiet = false, retry = 1) = ## Download a file using `curl` or `wget` (or `powershell` on Windows) to the specified directory ## ## If an archive file, it is automatically extracted after download. let file = url.extractFilename() + filePath = outdir / file ext = file.splitFile().ext.toLowerAscii() archives = @[".zip", ".xz", ".gz", ".bz2", ".tgz", ".tar"] - if not (ext in archives and fileExists(outdir/file)): + if not (ext in archives and fileExists(filePath)): if not quiet: echo "# Downloading " & file mkDir(outdir) @@ -368,7 +371,8 @@ proc downloadUrl*(url, outdir: string, quiet = false) = cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" else: doAssert false, "No download tool available - curl, wget" - discard execAction(cmd % [url.quoteShell, (outdir/file).sanitizePath], retry = 3) + discard execAction(cmd % [url.quoteShell, (filePath).sanitizePath], retry = 3, + onRetry = proc() = rmFile(filePath)) if ext == ".zip": extractZip(file, outdir, quiet) From fbc8c2ad9edd413472849500822d929c6578c568 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 20 Jun 2020 00:09:50 -0500 Subject: [PATCH 038/106] Reorganize file structure --- CHANGES.md | 30 + nimterop/build.nim | 1420 +----------------------- nimterop/build/ccompiler.nim | 101 ++ nimterop/{ => build}/conan.nim | 0 nimterop/build/getheader.nim | 504 +++++++++ nimterop/{ => build}/jbb.nim | 2 +- nimterop/{ => build}/nimconf.nim | 0 nimterop/build/shell.nim | 465 ++++++++ nimterop/build/tools.nim | 260 +++++ nimterop/git.nim | 1 - nimterop/toast.nim | 4 +- nimterop/{ => toastlib}/ast.nim | 3 +- nimterop/{ => toastlib}/ast2.nim | 4 +- nimterop/{ => toastlib}/comphelp.nim | 3 +- nimterop/{ => toastlib}/exprparser.nim | 6 +- nimterop/{ => toastlib}/getters.nim | 2 +- nimterop/{ => toastlib}/grammar.nim | 3 +- nimterop/{ => toastlib}/lisp.nim | 3 +- nimterop/{ => toastlib}/tshelp.nim | 5 +- 19 files changed, 1435 insertions(+), 1381 deletions(-) create mode 100644 nimterop/build/ccompiler.nim rename nimterop/{ => build}/conan.nim (100%) create mode 100644 nimterop/build/getheader.nim rename nimterop/{ => build}/jbb.nim (99%) rename nimterop/{ => build}/nimconf.nim (100%) create mode 100644 nimterop/build/shell.nim create mode 100644 nimterop/build/tools.nim delete mode 100644 nimterop/git.nim rename nimterop/{ => toastlib}/ast.nim (99%) rename nimterop/{ => toastlib}/ast2.nim (99%) rename nimterop/{ => toastlib}/comphelp.nim (98%) rename nimterop/{ => toastlib}/exprparser.nim (99%) rename nimterop/{ => toastlib}/getters.nim (99%) rename nimterop/{ => toastlib}/grammar.nim (99%) rename nimterop/{ => toastlib}/lisp.nim (97%) rename nimterop/{ => toastlib}/tshelp.nim (99%) diff --git a/CHANGES.md b/CHANGES.md index e47974d..cedf5a5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,34 @@ # Nimterop Change History +## Version 0.6.0 + +This release adds the ability to download precompiled binaries from [Conan.io](https://conan.io/center) and Julia's [BinaryBuilder.org](https://binarybuilder.org). This alleviates the headache of searching and downloading libraries manually both for wrapper writers as well as end users. There are some known limitations but it should prove to become more useful as these sites expand their capabilities. + +Conan.io shared builds tend to have all dependencies statically linked into the binary so a single so/dll/dylib has everything. For Conan.io static builds and all libraries on BinaryBuilder.org, dependencies are also downloaded and linked as needed. They are returned in the new `const xxxLDeps` in case wrapper writers need it for some reason. + +Known concerns: +- Conan.io only compiles Windows builds with Microsoft's VC++ compiler so static .lib files may not always work with MinGW on Windows. +- Conan.io compiles all Mac builds on OSX 10.14 so older versions of the OS will grumble when statically linking these libraries. +- BinaryBuilder.org does not include static libs for all their projects. + +Refer to the documentation for `getHeader()` for details on how to use this new capability. + +See the full list of changes here: + +https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 + +### Breaking changes + +- All shared libraries installed by `getHeader()` will now get copied into the `libdir` parameter specified. If left blank, `libdir` will default to the directory where the executable binary gets created (outdir). While this is not really a breaking change, it is a change in behavior compared to older versions of nimterop. Note that `Std` libraries are not copied over. [#154](i154) + +- `git.nim` has been removed. This module was an artifact from the early days and was renamed to `build.nim` back in v0.2.0. + +### New functionality + +- `getHeader()` now detects and links against `.lib` files as part of enabling Conan.io. Not all `.lib` files are compatible with MinGW as already stated above but for those that work, this is a required capability. + + + ## Version 0.5.0 This release introduces a new backend for wrapper generation dubbed `ast2` that leverages the Nim compiler AST and renderer. The new design simplifies feature development and already includes all the functionality of the legacy algorithm plus fixes for several open issues. @@ -73,6 +102,7 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 [i148]: https://github.com/nimterop/nimterop/issues/148 [i151]: https://github.com/nimterop/nimterop/issues/151 [i153]: https://github.com/nimterop/nimterop/issues/153 +[i154]: https://github.com/nimterop/nimterop/issues/154 [i155]: https://github.com/nimterop/nimterop/issues/155 [i156]: https://github.com/nimterop/nimterop/issues/156 [i159]: https://github.com/nimterop/nimterop/issues/159 diff --git a/nimterop/build.nim b/nimterop/build.nim index 62b5e1f..99adeca 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -4,21 +4,13 @@ import os except findExe, sleep export extractFilename, `/` -type - BuildType* = enum - btAutoconf, btCmake - - BuildStatus = object - built: bool - buildPath: string - error: string - # build specific debug since we cannot import globals (yet) var gDebug* = false gDebugCT* {.compileTime.} = false gNimExe* = "" +# Misc helpers proc echoDebug(str: string) = let str = "\n# " & str.strip().replace("\n", "\n# ") when nimvm: @@ -26,24 +18,6 @@ proc echoDebug(str: string) = else: if gDebug: echo str -proc fixCmd(cmd: string): string = - when defined(Windows): - # Replace 'cd d:\abc' with 'd: && cd d:\abc` - var filteredCmd = cmd - if cmd.toLower().startsWith("cd"): - var - colonIndex = cmd.find(":") - driveLetter = cmd.substr(colonIndex-1, colonIndex) - if (driveLetter[0].isAlphaAscii() and - driveLetter[1] == ':' and - colonIndex == 4): - filteredCmd = &"{driveLetter} && {cmd}" - result = "cmd /c " & filteredCmd - elif defined(posix): - result = cmd - else: - doAssert false - proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) if not noQuote: @@ -57,801 +31,6 @@ proc getCurrentNimCompiler*(): string = else: result = gNimExe -# Nim cfg file related functionality -include "."/nimconf - -proc sleep*(milsecs: int) = - ## Sleep at compile time - let - cmd = - when defined(Windows): - "cmd /c timeout " - else: - "sleep " - - discard gorgeEx(cmd & $(milsecs / 1000)) - -proc getNimteropCacheDir(): string = - # Get location to cache all nimterop artifacts - result = getNimcacheDir() / "nimterop" - -proc execAction*(cmd: string, retry = 0, die = true, cache = false, - cacheKey = "", onRetry: proc() = nil): tuple[output: string, ret: int] = - ## Execute an external command - supported at compile time - ## - ## Checks if command exits successfully before returning. If not, an - ## error is raised. Always caches results to be used in nimsuggest or nimcheck - ## mode. - ## - ## `retry` - number of times command should be retried before error - ## `die = false` - return on errors - ## `cache = true` - cache results unless cleared with -f - ## `cacheKey` - key to create unique cache entry - let - ccmd = fixCmd(cmd) - - when nimvm: - # Cache results for speedup if cache = true - # Else cache for preserving functionality in nimsuggest and nimcheck - let - hash = (ccmd & cacheKey).hash().abs() - cachePath = getNimteropCacheDir() / "execCache" / "nimterop_" & $hash - cacheFile = cachePath & ".txt" - retFile = cachePath & "_ret.txt" - - when defined(nimsuggest) or defined(nimcheck): - # Load results from cache file if generated in previous run - if fileExists(cacheFile) and fileExists(retFile): - result.output = cacheFile.readFile() - result.ret = retFile.readFile().parseInt() - elif die: - doAssert false, "Results not cached - run nim c/cpp at least once\n" & ccmd - else: - if cache and fileExists(cacheFile) and fileExists(retFile) and not compileOption("forceBuild"): - # Return from cache when requested - result.output = cacheFile.readFile() - result.ret = retFile.readFile().parseInt() - else: - # Execute command and store results in cache - (result.output, result.ret) = gorgeEx(ccmd) - if result.ret == 0 or die == false: - # mkdir for execCache dir (circular dependency) - let dir = cacheFile.parentDir() - if not dirExists(dir): - let flag = when not defined(Windows): "-p" else: "" - discard execAction(&"mkdir {flag} {dir.sanitizePath}") - cacheFile.writeFile(result.output) - retFile.writeFile($result.ret) - else: - # Used by toast - (result.output, result.ret) = execCmdEx(ccmd) - - # On failure, retry or die as requested - if result.ret != 0: - if retry > 0: - if not onRetry.isNil: - onRetry() - sleep(500) - result = execAction(cmd, retry = retry - 1, die, cache, cacheKey) - elif die: - doAssert false, "Command failed: " & $result.ret & "\ncmd: " & ccmd & - "\nresult:\n" & result.output - -proc findExe*(exe: string): string = - ## Find the specified executable using the `which`/`where` command - supported - ## at compile time - var - cmd = - when defined(Windows): - "where " & exe - else: - "which " & exe - - (output, ret) = execAction(cmd, die = false) - - if ret == 0: - return output.splitLines()[0].strip() - -proc mkDir*(dir: string) = - ## Create a directory at compile time - ## - ## The `os` module is not available at compile time so a few - ## crucial helper functions are included with nimterop. - if not dirExists(dir): - let - flag = when not defined(Windows): "-p" else: "" - discard execAction(&"mkdir {flag} {dir.sanitizePath}", retry = 2) - -proc cpFile*(source, dest: string, psymlink = false, move = false) = - ## Copy a file from `source` to `dest` at compile time - ## - ## `psymlink = true` preserves symlinks instead of dereferencing on posix - let - source = source.replace("/", $DirSep) - dest = dest.replace("/", $DirSep) - cmd = - when defined(Windows): - if move: - "move /y" - else: - "copy /y" - else: - if move: - "mv -f" - else: - if psymlink: - "cp -fa" - else: - "cp -f" - - discard execAction(&"{cmd} {source.sanitizePath} {dest.sanitizePath}", retry = 2) - -proc mvFile*(source, dest: string) = - ## Move a file from `source` to `dest` at compile time - cpFile(source, dest, move=true) - -proc rmFile*(source: string, dir = false) = - ## Remove a file or pattern at compile time - let - source = source.replace("/", $DirSep) - cmd = - when defined(Windows): - if dir: - "rd /s/q" - else: - "del /s/q/f" - else: - "rm -rf" - exists = - if dir: - dirExists(source) - else: - fileExists(source) - - if exists: - discard execAction(&"{cmd} {source.sanitizePath}", retry = 2) - -proc rmDir*(dir: string) = - ## Remove a directory or pattern at compile time - rmFile(dir, dir = true) - -proc cleanDir*(dir: string) = - ## Remove all contents of a directory at compile time - for kind, path in walkDir(dir): - if kind == pcDir: - rmDir(path) - else: - rmFile(path) - -proc cpTree*(source, dest: string, move = false) = - ## Copy contents of source dir to the destination, not the directory itself - for kind, path in walkDir(source, relative = true): - if kind == pcDir: - cpTree(source / path, dest / path, move) - if move: - rmDir(source / path) - else: - if not dirExists(dest): - mkDir(dest) - if move: - mvFile(source / path, dest / path) - else: - cpFile(source / path, dest / path) - -proc mvTree*(source, dest: string) = - ## Move contents of source dir to the destination, not the directory itself - cpTree(source, dest, move = true) - -proc getFileDate*(fullpath: string): string = - ## Get file date for `fullpath` - var - ret = 0 - cmd = - when defined(Windows): - let - (head, tail) = fullpath.splitPath() - &"cmd /c forfiles /P {head.sanitizePath()} /M {tail.sanitizePath} /C \"cmd /c echo @fdate @ftime @fsize\"" - elif defined(Linux): - &"stat -c %y {fullpath.sanitizePath}" - elif defined(OSX) or defined(FreeBSD): - &"stat -f %m {fullpath.sanitizePath}" - - (result, ret) = execAction(cmd) - -proc touchFile*(fullpath: string) = - ## Touch file to update modified date - var - cmd = - when defined(Windows): - &"cmd /c copy /b {fullpath.sanitizePath}+" - else: - &"touch {fullpath.sanitizePath}" - - discard execAction(cmd) - -proc getProjectCacheDir*(name: string, forceClean = true): string = - ## Get a cache directory where all nimterop artifacts can be stored - ## - ## Projects can use this location to download source code and build binaries - ## that can be then accessed by multiple apps. This is created under the - ## per-user Nim cache directory. - ## - ## Use `name` to specify the subdirectory name for a project. - ## - ## `forceClean` is enabled by default and effectively deletes the folder - ## if Nim is compiled with the `-f` or `--forceBuild` flag. This allows - ## any project to start out with a clean cache dir on a forced build. - ## - ## NOTE: avoid calling `getProjectCacheDir()` multiple times on the same - ## `name` when `forceClean = true` else checked out source might get deleted - ## at the wrong time during build. - ## - ## E.g. - ## `nimgit2` downloads `libgit2` source so `name = "libgit2"` - ## - ## `nimarchive` downloads `libarchive`, `bzlib`, `liblzma` and `zlib` so - ## `name = "nimarchive" / "libarchive"` for `libarchive`, etc. - result = getNimteropCacheDir() / name - - if forceClean and compileOption("forceBuild"): - echo "# Removing " & result - rmDir(result) - -proc extractZip*(zipfile, outdir: string, quiet = false) = - ## Extract a zip file using `powershell` on Windows and `unzip` on other - ## systems to the specified output directory - var cmd = "unzip -o $#" - if defined(Windows): - cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A " & - "'System.IO.Compression.FileSystem'; " & - "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" - - if not quiet: - echo "# Extracting " & zipfile - discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}") - -proc extractTar*(tarfile, outdir: string, quiet = false) = - ## Extract a tar file using `tar`, `7z` or `7za` to the specified output directory - var - cmd = "" - name = "" - - if findExe("tar").len != 0: - let - ext = tarfile.splitFile().ext.toLowerAscii() - typ = - case ext - of ".gz", ".tgz": "z" - of ".xz": "J" - of ".bz2": "j" - else: "" - - cmd = "tar xvf" & typ & " " & tarfile.sanitizePath - else: - for i in ["7z", "7za"]: - if findExe(i).len != 0: - cmd = i & " x $#" % tarfile.sanitizePath - - name = tarfile.splitFile().name - if ".tar" in name.toLowerAscii(): - cmd &= " && " & i & " x $#" % name.sanitizePath - - break - - doAssert cmd.len != 0, "No extraction tool - tar, 7z, 7za - available for " & tarfile.sanitizePath - - if not quiet: - echo "# Extracting " & tarfile - discard execAction(&"cd {outdir.sanitizePath} && {cmd}") - if name.len != 0: - rmFile(outdir / name) - -proc downloadUrl*(url, outdir: string, quiet = false, retry = 1) = - ## Download a file using `curl` or `wget` (or `powershell` on Windows) to the specified directory - ## - ## If an archive file, it is automatically extracted after download. - let - file = url.extractFilename() - filePath = outdir / file - ext = file.splitFile().ext.toLowerAscii() - archives = @[".zip", ".xz", ".gz", ".bz2", ".tgz", ".tar"] - - if not (ext in archives and fileExists(filePath)): - if not quiet: - echo "# Downloading " & file - mkDir(outdir) - var cmd = findExe("curl") - if cmd.len != 0: - cmd &= " -Lk $# -o $#" - else: - cmd = findExe("wget") - if cmd.len != 0: - cmd &= " $# -O $#" - elif defined(Windows): - cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" - else: - doAssert false, "No download tool available - curl, wget" - discard execAction(cmd % [url.quoteShell, (filePath).sanitizePath], retry = 3, - onRetry = proc() = rmFile(filePath)) - - if ext == ".zip": - extractZip(file, outdir, quiet) - elif ext in archives: - extractTar(file, outdir, quiet) - -proc gitReset*(outdir: string) = - ## Hard reset the git repository at the specified directory - echo "# Resetting " & outdir - - let cmd = &"cd {outdir.sanitizePath} && git reset --hard" - while execAction(cmd).output.contains("Permission denied"): - sleep(1000) - echo "# Retrying ..." - -proc gitCheckout*(file, outdir: string) = - ## Checkout the specified `file` in the git repository at `outdir` - ## - ## This effectively resets all changes in the file and can be - ## used to undo any changes that were made to source files to enable - ## successful wrapping with `cImport()` or `c2nImport()`. - echo "# Resetting " & file - let file2 = file.relativePath outdir - let cmd = &"cd {outdir.sanitizePath} && git checkout {file2.sanitizePath}" - while execAction(cmd).output.contains("Permission denied"): - sleep(500) - echo "# Retrying ..." - -proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = - ## Pull the specified git repository to the output directory - ## - ## `plist` is the list of specific files and directories or wildcards - ## to sparsely checkout. Multiple values can be specified one entry per - ## line. It is optional and if omitted, the entire repository will be - ## checked out. - ## - ## `checkout` is the git tag, branch or commit hash to checkout once - ## the repository is downloaded. This allows for pinning to a specific - ## version of the code. - if dirExists(outdir/".git"): - gitReset(outdir) - return - - let - outdirQ = outdir.sanitizePath - - mkDir(outdir) - - if not quiet: - echo "# Setting up Git repo: " & url - discard execAction(&"cd {outdirQ} && git init .") - discard execAction(&"cd {outdirQ} && git remote add origin {url}") - - if plist.len != 0: - # If a specific list of files is required, create a sparse checkout - # file for git in its config directory - let sparsefile = outdir / ".git/info/sparse-checkout" - - discard execAction(&"cd {outdirQ} && git config core.sparsecheckout true") - writeFile(sparsefile, plist) - - # In case directory has old files from another run - discard execAction(&"cd {outdirQ} && git clean -fxd") - - if checkout.len != 0: - if not quiet: - echo "# Checking out " & checkout - discard execAction(&"cd {outdirQ} && git fetch", retry = 3) - discard execAction(&"cd {outdirQ} && git checkout {checkout}") - else: - if not quiet: - echo "# Pulling repository" - discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 3) - -proc gitTags*(outdir: string): seq[string] = - ## Get all the git tags in the specified directory - let - cmd = &"cd {outdir.sanitizePath} && git tag" - tags = execAction(cmd).output.splitLines() - for tag in tags: - let - tag = tag.strip() - if tag.len != 0: - result.add tag - -proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[string] = - ## Find all matching files in the specified directory - ## - ## `file` is a regular expression if `regex` is true - ## - ## Turn off recursive search with `recurse` - var - cmd = - when defined(Windows): - "nimgrep --filenames --oneline --nocolor $1 \"$2\" $3" - elif defined(linux): - "find $3 $1 -regextype egrep -regex $2" - elif defined(osx) or defined(FreeBSD): - "find -E $3 $1 -regex $2" - - recursive = "" - - if recurse: - when defined(Windows): - recursive = "--recursive" - else: - when not defined(Windows): - recursive = "-maxdepth 1" - - var - dir = dir - file = file - if not recurse: - let - pdir = file.parentDir() - if pdir.len != 0: - dir = dir / pdir - - file = file.extractFilename - - cmd = cmd % [recursive, (".*[\\\\/]" & file & "$").quoteShell, dir.sanitizePath] - - let - (files, ret) = execAction(cmd, die = false) - if ret == 0: - for line in files.splitLines(): - let f = - when defined(Windows): - if ": " in line: - line.split(": ", maxsplit = 1)[1] - else: - "" - else: - line - if f.len != 0: - result.add f - -proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string = - ## Find the file in the specified directory - ## - ## `file` is a regular expression if `regex` is true - ## - ## Turn off recursive search with `recurse` and stop on first match with - ## `first`. Without it, the shortest match is returned. - let - matches = findFiles(file, dir, recurse, regex) - for match in matches: - if (result.len == 0 or result.len > match.len): - result = match - if first: break - -proc flagBuild*(base: string, flags: openArray[string]): string = - ## Simple helper proc to generate flags for `configure`, `cmake`, etc. - ## - ## Every entry in `flags` is replaced into the `base` string and - ## concatenated to the result. - ## - ## E.g. - ## `base = "--disable-$#"` - ## `flags = @["one", "two"]` - ## - ## `flagBuild(base, flags) => " --disable-one --disable-two"` - for i in flags: - result &= " " & base % i - -proc linkLibs*(names: openArray[string], staticLink = true): string = - ## Create linker flags for specified libraries - ## - ## Prepends `lib` to the name so you only need `ssl` for `libssl`. - var - stat = if staticLink: "--static" else: "" - resSet: OrderedSet[string] - resSet.init() - - for name in names: - let - cmd = &"pkg-config --libs --silence-errors {stat} lib{name}" - (libs, _) = execAction(cmd, die = false) - for lib in libs.split(" "): - resSet.incl lib - - if staticLink: - resSet.incl "--static" - - for res in resSet: - result &= " " & res - -proc configure*(path, check: string, flags = "") = - ## Run the GNU `configure` command to generate all Makefiles or other - ## build scripts in the specified path - ## - ## If a `configure` script is not present and an `autogen.sh` script - ## is present, it will be run before attempting `configure`. - ## - ## Next, if `configure.ac` or `configure.in` exist, `autoreconf` will - ## be executed. - ## - ## `check` is a file that will be generated by the `configure` command. - ## This is required to prevent configure from running on every build. It - ## is relative to the `path` and should not be an absolute path. - ## - ## `flags` are any flags that should be passed to the `configure` command. - if (path / check).fileExists(): - return - - echo "# Configuring " & path - - if not fileExists(path / "configure"): - for i in @["autogen.sh", "build" / "autogen.sh"]: - if fileExists(path / i): - echo "# Running autogen.sh" - - when defined(unix): - echoDebug execAction( - &"cd {(path / i).parentDir().sanitizePath} && ./autogen.sh").output - else: - echoDebug execAction( - &"cd {(path / i).parentDir().sanitizePath} && bash ./autogen.sh").output - - break - - if not fileExists(path / "configure"): - for i in @["configure.ac", "configure.in"]: - if fileExists(path / i): - echo "# Running autoreconf" - - echoDebug execAction(&"cd {path.sanitizePath} && autoreconf -fi").output - - break - - if fileExists(path / "configure"): - echo "# Running configure " & flags - - when defined(unix): - var - cmd = &"cd {path.sanitizePath} && ./configure" - else: - var - cmd = &"cd {path.sanitizePath} && bash ./configure" - if flags.len != 0: - cmd &= &" {flags}" - - echoDebug execAction(cmd).output - - doAssert (path / check).fileExists(), "Configure failed" - -proc getCmakePropertyStr(name, property, value: string): string = - &"\nset_target_properties({name} PROPERTIES {property} \"{value}\")\n" - -proc getCmakeIncludePath*(paths: openArray[string]): string = - ## Create a `cmake` flag to specify custom include paths - ## - ## Result can be included in the `flag` parameter for `cmake()` or - ## the `cmakeFlags` parameter for `getHeader()`. - for path in paths: - result &= path & ";" - result = " -DCMAKE_INCLUDE_PATH=" & result[0 .. ^2].sanitizePath(sep = "/") - -proc setCmakeProperty*(outdir, name, property, value: string) = - ## Set a `cmake` property in `outdir / CMakeLists.txt` - usable in the `xxxPreBuild` hook - ## for `getHeader()` - ## - ## `set_target_properties(name PROPERTIES property "value")` - let - cm = outdir / "CMakeLists.txt" - if cm.fileExists(): - cm.writeFile( - cm.readFile() & getCmakePropertyStr(name, property, value) - ) - -proc setCmakeLibName*(outdir, name, prefix = "", oname = "", suffix = "") = - ## Set a `cmake` property in `outdir / CMakeLists.txt` to specify a custom library output - ## name - usable in the `xxxPreBuild` hook for `getHeader()` - ## - ## `prefix` is typically `lib` - ## `oname` is the library name - ## `suffix` is typically `.a` - ## - ## Sometimes, `cmake` generates non-standard library names - e.g. zlib compiles to - ## `libzlibstatic.a` on Windows. This proc can help rename it to `libzlib.a` so that `getHeader()` - ## can find it after the library is compiled. - ## - ## ``` - ## set_target_properties(name PROPERTIES PREFIX "prefix") - ## set_target_properties(name PROPERTIES OUTPUT_NAME "oname") - ## set_target_properties(name PROPERTIES SUFFIX "suffix") - ## ``` - let - cm = outdir / "CMakeLists.txt" - if cm.fileExists(): - var - str = "" - if prefix.len != 0: - str &= getCmakePropertyStr(name, "PREFIX", prefix) - if oname.len != 0: - str &= getCmakePropertyStr(name, "OUTPUT_NAME", oname) - if suffix.len != 0: - str &= getCmakePropertyStr(name, "SUFFIX", suffix) - if str.len != 0: - cm.writeFile(cm.readFile() & str) - -proc setCmakePositionIndependentCode*(outdir: string) = - ## Set a `cmake` directive to create libraries with -fPIC enabled - let - cm = outdir / "CMakeLists.txt" - if cm.fileExists(): - let - pic = "set(CMAKE_POSITION_INDEPENDENT_CODE ON)" - cmd = cm.readFile() - if not cmd.contains(pic): - cm.writeFile( - pic & "\n" & cmd - ) - -proc cmake*(path, check, flags: string) = - ## Run the `cmake` command to generate all Makefiles or other - ## build scripts in the specified path - ## - ## `path` will be created since typically `cmake` is run in an - ## empty directory. - ## - ## `check` is a file that will be generated by the `cmake` command. - ## This is required to prevent `cmake` from running on every build. It - ## is relative to the `path` and should not be an absolute path. - ## - ## `flags` are any flags that should be passed to the `cmake` command. - ## Unlike `configure`, it is required since typically it will be the - ## path to the repository, typically `..` when `path` is a subdir. - if (path / check).fileExists(): - return - - echo "# Running cmake " & flags - echo "# Path: " & path - - mkDir(path) - - let - cmd = &"cd {path.sanitizePath} && cmake {flags}" - - echoDebug execAction(cmd).output - - doAssert (path / check).fileExists(), "cmake failed" - -proc make*(path, check: string, flags = "", regex = false) = - ## Run the `make` command to build all binaries in the specified path - ## - ## `check` is a file that will be generated by the `make` command. - ## This is required to prevent `make` from running on every build. It - ## is relative to the `path` and should not be an absolute path. - ## - ## `flags` are any flags that should be passed to the `make` command. - ## - ## `regex` can be set to true if `check` is a regular expression. - ## - ## If `make.exe` is missing and `mingw32-make.exe` is available, it will - ## be copied over to make.exe in the same location. - if findFile(check, path, regex = regex).len != 0: - return - - echo "# Running make " & flags - echo "# Path: " & path - - var - cmd = findExe("make") - - if cmd.len == 0: - cmd = findExe("mingw32-make") - if cmd.len != 0: - cpFile(cmd, cmd.replace("mingw32-make", "make")) - doAssert cmd.len != 0, "Make not found" - - cmd = &"cd {path.sanitizePath} && make" - if flags.len != 0: - cmd &= &" {flags}" - - echoDebug execAction(cmd).output - - doAssert findFile(check, path, regex = regex).len != 0, "make failed" - -proc getCompilerMode*(path: string): string = - ## Determines a target language mode from an input filename, if one is not already specified. - let file = path.splitFile() - if file.ext in [".hxx", ".hpp", ".hh", ".H", ".h++", ".cpp", ".cxx", ".cc", ".C", ".c++"]: - result = "cpp" - elif file.ext in [".h", ".c"]: - result = "c" - -proc getGccModeArg*(mode: string): string = - ## Produces a GCC argument that explicitly sets the language mode to be used by the compiler. - if mode == "cpp": - result = "-xc++" - elif mode == "c": - result = "-xc" - -proc getCompiler*(): string = - var - compiler = - when defined(gcc): - "gcc" - elif defined(clang): - "clang" - else: - doAssert false, "Nimterop only supports gcc and clang at this time" - - result = getEnv("CC", compiler) - -proc getGccPaths*(mode: string): seq[string] = - var - nul = when defined(Windows): "nul" else: "/dev/null" - inc = false - - (outp, _) = execAction(&"""{getCompiler()} -Wp,-v {getGccModeArg(mode)} {nul}""", die = false) - - for line in outp.splitLines(): - if "#include <...> search starts here" in line: - inc = true - continue - elif "End of search list" in line: - break - if inc: - var - path = line.strip().normalizedPath() - if path notin result: - result.add path - - when defined(osx): - result.add(execAction("xcrun --show-sdk-path").output.strip() & "/usr/include") - -proc getGccLibPaths*(mode: string): seq[string] = - var - nul = when defined(Windows): "nul" else: "/dev/null" - linker = when defined(OSX): "-Xlinker" else: "" - - (outp, _) = execAction(&"""{getCompiler()} {linker} -v {getGccModeArg(mode)} {nul}""", die = false) - - for line in outp.splitLines(): - if "LIBRARY_PATH=" in line: - for path in line[13 .. ^1].split(PathSep): - var - path = path.strip().normalizedPath() - if path notin result: - result.add path - break - elif '\t' in line: - var - path = line.strip().normalizedPath() - if path notin result: - result.add path - - when defined(osx): - result.add "/usr/lib" - -proc getGccInfo*(): tuple[arch, os, compiler, version: string] = - let - (outp, _) = execAction(&"{getCompiler()} -v") - for line in outp.splitLines(): - if line.startsWith("Target: "): - result.arch = line.split(' ')[1].split('-')[0] - result.os = - if "linux" in line: - "linux" - elif "android" in line: - "android" - elif "darwin" in line: - "macos" - elif "w64" in line or "mingw" in line: - "windows" - else: - "unknown" - elif " version " in line: - result.version = line.split(" version ")[1].split(' ')[0] - if "clang" in outp: - if result.os == "macos": - result.compiler = "apple-clang" - else: - result.compiler = "clang" - else: - result.compiler = "gcc" - template fixOutDir() {.dirty.} = let outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir @@ -881,565 +60,74 @@ proc compareVersions*(ver1, ver2: string): int = if p1 < p2: return -1 elif p1 > p2: return 1 -# Conan support -include conan - -# Julia Binary Builder support -include jbb - -proc getStdPath(header, mode: string): string = - for inc in getGccPaths(mode): - result = findFile(header, inc, recurse = false, first = true) - if result.len != 0: - break - -proc getStdLibPath(lname, mode: string): string = - for lib in getGccLibPaths(mode): - result = findFile(lname, lib, recurse = false, first = true, regex = true) - if result.len != 0: - break - -proc getGitPath(header, url, outdir, version: string): string = - doAssert url.len != 0, "No git url setup for " & header - doAssert findExe("git").len != 0, "git executable missing" - - gitPull(url, outdir, checkout = version) - - result = findFile(header, outdir) - -proc getDlPath(header, url, outdir, version: string): string = - doAssert url.len != 0, "No download url setup for " & header - - var - dlurl = url - if "$#" in url or "$1" in url: - doAssert version.len != 0, "Need version for download url" - dlurl = url % version - else: - doAssert version.len == 0, "Download url does not contain version" - - downloadUrl(dlurl, outdir) - - var - dirname = "" - for kind, path in walkDir(outdir, relative = true): - if kind == pcFile and path != dlurl.extractFilename(): - dirname = "" - break - elif kind == pcDir: - if dirname.len == 0: - dirname = path - else: - dirname = "" - break - - if dirname.len != 0: - for kind, path in walkDir(outdir / dirname, relative = true): - mvFile(outdir / dirname / path, outdir / path) - - result = findFile(header, outdir) - -proc getConanPath(header, uri, outdir, version: string, shared: bool): string = - var - uri = uri - - if "$#" in uri or "$1" in uri: - doAssert version.len != 0, "Need version for Conan.io uri: " & uri - uri = uri % version - elif version.len != 0: - uri = uri & "/" & version - - let - pkg = newConanPackageFromUri(uri, shared) - downloadConan(pkg, outdir) - - result = findFile(header, outdir) - -proc getConanLDeps(outdir: string): seq[string] = - let - pkg = loadConanInfo(outdir) - - result = pkg.getConanLDeps(outdir) - -proc getJBBPath(header, uri, outdir, version: string): string = - let - spl = uri.split('/', 1) - name = spl[0] - hasVersion = version.len != 0 - - var - ver = - if spl.len == 2: - spl[1] - else: - "" - - if ver.len != 0: - if "$#" in ver or "$1" in ver: - doAssert hasVersion, "Need version for BinaryBuilder.org uri: " & uri - ver = ver % version - elif hasVersion: - doAssert false, "Version in both uri `" & uri & "` and `-d:xxxSetVer=\"" & - version & "\"` for BinaryBuilder.org" - elif hasVersion: - ver = version - - let - pkg = newJBBPackage(name, ver) - downloadJBB(pkg, outdir) - - result = findFile(header, outdir) - -proc getJBBLDeps(outdir: string, shared: bool): seq[string] = - let - pkg = loadJBBInfo(outdir) - - result = pkg.getJBBLDeps(outdir, shared) - -proc getLocalPath(header, outdir: string): string = - if outdir.len != 0: - result = findFile(header, outdir) - -proc getNumProcs(): string = +proc fixCmd(cmd: string): string = when defined(Windows): - getEnv("NUMBER_OF_PROCESSORS").strip() - elif defined(linux): - execAction("nproc").output.strip() - elif defined(macosx) or defined(FreeBSD): - execAction("sysctl -n hw.ncpu").output.strip() + # Replace 'cd d:\abc' with 'd: && cd d:\abc` + var filteredCmd = cmd + if cmd.toLower().startsWith("cd"): + var + colonIndex = cmd.find(":") + driveLetter = cmd.substr(colonIndex-1, colonIndex) + if (driveLetter[0].isAlphaAscii() and + driveLetter[1] == ':' and + colonIndex == 4): + filteredCmd = &"{driveLetter} && {cmd}" + result = "cmd /c " & filteredCmd + elif defined(posix): + result = cmd else: - "1" + doAssert false -proc buildWithCmake(outdir, flags: string): BuildStatus = - if not fileExists(outdir / "Makefile"): - if fileExists(outdir / "CMakeLists.txt"): - if findExe("cmake").len != 0: - var - gen = "" - when defined(Windows): - if findExe("sh").len != 0: - let - uname = execAction("sh -c uname -a").output.toLowerAscii() - if uname.contains("msys"): - gen = "MSYS Makefiles".quoteShell - elif uname.contains("mingw"): - gen = "MinGW Makefiles".quoteShell & " -DCMAKE_SH=\"CMAKE_SH-NOTFOUND\"" - else: - echo "Unsupported system: " & uname - else: - gen = "MinGW Makefiles".quoteShell - else: - gen = "Unix Makefiles".quoteShell - if findExe("ccache").len != 0: - gen &= " -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" - result.buildPath = outdir / "buildcache" - cmake(result.buildPath, "Makefile", &".. -G {gen} {flags}") - result.built = true - else: - result.error = "cmake capable but cmake executable missing" - else: - result.buildPath = outdir +# Nim cfg file related functionality +include "."/build/nimconf -proc buildWithAutoConf(outdir, flags: string): BuildStatus = - if not fileExists(outdir / "Makefile"): - if findExe("bash").len != 0: - for file in @["configure", "configure.ac", "configure.in", "autogen.sh", "build/autogen.sh"]: - if fileExists(outdir / file): - configure(outdir, "Makefile", flags) - result.buildPath = outdir - result.built = true - break - else: - result.error = "configure capable but bash executable missing" - else: - result.buildPath = outdir +proc getNimteropCacheDir(): string = + # Get location to cache all nimterop artifacts + result = getNimcacheDir() / "nimterop" -proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildTypes: openArray[BuildType]): string = - var - lpath = findFile(lname, outdir, regex = true) - makeFlagsProc = &"-j {getNumProcs()} {makeFlags}" - makePath = outdir +# Functionality shelled out to external executables +include "."/build/shell - if lpath.len != 0: - return lpath - - var buildStatus: BuildStatus - - for buildType in buildTypes: - case buildType - of btCmake: - buildStatus = buildWithCmake(makePath, cmakeFlags) - of btAutoconf: - buildStatus = buildWithAutoConf(makePath, conFlags) - - if buildStatus.built: - break - - if buildStatus.buildPath.len > 0: - let libraryExists = findFile(lname, buildStatus.buildPath, regex = true).len > 0 - - if not libraryExists and fileExists(buildStatus.buildPath / "Makefile"): - make(buildStatus.buildPath, lname, makeFlagsProc, regex = true) - buildStatus.built = true - - let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir - doAssert buildStatus.built, &"\nBuild configuration failed - {error}\n" - - result = findFile(lname, outdir, regex = true) - -proc getDynlibExt(): string = - when defined(Windows): - result = "[0-9.\\-]*\\.dll" - elif defined(linux) or defined(FreeBSD): - result = "\\.so[0-9.]*" - elif defined(macosx): - result = "[0-9.\\-]*\\.dylib" - -var - gDefines {.compileTime.} = initTable[string, string]() - -macro setDefines*(defs: static openArray[string]): untyped = - ## Specify `-d:xxx` values in code instead of having to rely on the command - ## line or `cfg` or `nims` files. +proc getProjectCacheDir*(name: string, forceClean = true): string = + ## Get a cache directory where all nimterop artifacts can be stored ## - ## At this time, Nim does not allow creation of `-d:xxx` defines in code. In - ## addition, Nim only loads config files for the module being compiled but not - ## for imported packages. This becomes a challenge when wanting to ship a wrapper - ## library that wants to control `getHeader()` for an underlying package. + ## Projects can use this location to download source code and build binaries + ## that can be then accessed by multiple apps. This is created under the + ## per-user Nim cache directory. ## - ## E.g. nimarchive wanting to set `-d:lzmaStatic` + ## Use `name` to specify the subdirectory name for a project. ## - ## The consumer of nimarchive would need to set such defines as part of their - ## project, making it inconvenient. + ## `forceClean` is enabled by default and effectively deletes the folder + ## if Nim is compiled with the `-f` or `--forceBuild` flag. This allows + ## any project to start out with a clean cache dir on a forced build. ## - ## By calling this proc with the defines preferred before importing such a module, - ## the caller can set the behavior in code instead. + ## NOTE: avoid calling `getProjectCacheDir()` multiple times on the same + ## `name` when `forceClean = true` else checked out source might get deleted + ## at the wrong time during build. ## - ## .. code-block:: nim + ## E.g. + ## `nimgit2` downloads `libgit2` source so `name = "libgit2"` ## - ## setDefines(@["lzmaStatic", "lzmaDL", "lzmaSetVer=5.2.4"]) - ## - ## import lzma - for def in defs: - let - nv = def.strip().split("=", maxsplit = 1) - if nv.len != 0: - let - n = nv[0] - v = - if nv.len == 2: - nv[1] - else: - "" - gDefines[n] = v + ## `nimarchive` downloads `libarchive`, `bzlib`, `liblzma` and `zlib` so + ## `name = "nimarchive" / "libarchive"` for `libarchive`, etc. + result = getNimteropCacheDir() / name -macro clearDefines*(): untyped = - ## Clear all defines set using `setDefines()`. - gDefines.clear() + if forceClean and compileOption("forceBuild"): + echo "# Removing " & result + rmDir(result) -macro isDefined*(def: untyped): untyped = - ## Check if `-d:xxx` is set globally or via `setDefines()` - let - sdef = gDefines.hasKey(def.strVal()) - result = newNimNode(nnkStmtList) - result.add(quote do: - when defined(`def`) or `sdef` != 0: - true - else: - false - ) +# C compiler support +include "."/build/ccompiler -macro getHeader*( - header: static[string], giturl: static[string] = "", dlurl: static[string] = "", - conanuri: static[string] = "", jbburi: static[string] = "", - outdir: static[string] = "", libdir: static[string] = "", - conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", - altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = - ## Get the path to a header file for wrapping with - ## `cImport() `_ or - ## `c2nImport() `_. - ## - ## This proc checks `-d:xxx` defines based on the header name (e.g. lzma from lzma.h), - ## and accordingly employs different ways to obtain the source. - ## - ## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux - ## `-d:xxxGit` - clone source from a git repo specified in `giturl` - ## `-d:xxxDL` - download source from `dlurl` and extract if required - ## `-d:xxxConan` - download headers and binary from Conan.io using `conanuri` with - ## format `pkgname[/version[@user/channel][:bhash]]` - ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` with - ## format `pkgname[/version]` - ## - ## This allows a single wrapper to be used in different ways depending on the user's needs. - ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. - ## The user can opt to download the sources to `outdir` using any other method such as - ## git sub-modules, vendoring or pointing to a repository that was already cloned. - ## - ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL`, - ## `Conan` or `JBB`. This allows using a system installed library if available before - ## falling back to manual building. The user would need to specify both `-d:xxxStd` and - ## one of the other methods. - ## - ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag - ## name for `Git` whereas for `DL`, `Conan` and `JBB`, it replaces `$1` in the URL - ## if specified. Specifying `-d:xxxSetVer` without a `$1` will download that version for - ## `Conan` and `JBB` if available. If no version is specified, the latest release of the - ## package is downloaded. For `Conan`, `-d:xxxSetVer` can also be used to set additional - ## URI information: - ## `-d:xxxSetVer=1.9.0@bincrafters/stable:bhash` - ## - ## If `conanuri` or `jbburi` are not defined and `Conan` or `JBB` is selected, the `header` - ## filename is used instead. - ## - ## All defines can also be set in code using `setDefines()` and checked for using - ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`. - ## - ## The library is then configured (with `cmake` or `autotools` if possible) and built - ## using `make`, unless using `-d:xxxStd` which presumes that the system package - ## manager was used to install prebuilt headers and binaries, or using `-d:xxxConan` - ## or `-d:xxxJBB` which download pre-built binaries. - ## - ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call - ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can - ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. Any dependency - ## libraries downloaded by `Conan` or `JBB` are returned in `const xxxLDeps` as a seq[string]. - ## - ## `libdir` can be used to instruct `getHeader()` to copy shared libraries and their - ## dependencies to that directory. This prevents any runtime failures if `outdir` gets - ## removed or its contents changed. By default, `libdir` is set to the output directory - ## where the program binary will be created. The values of `xxxLPath` and `xxxLDeps` will - ## reflect this new location. `libdir` is ignored for `Std` mode. - ## - ## `-d:xxxStatic` can be specified to statically link with the library instead. This - ## will automatically add a `{.passL.}` call to the static library for convenience. Note - ## that `-d:xxxConan` and `-d:xxxJBB` download all dependency libs as well and the - ## `xxxLPath` will include paths to all of them separated by space in the right order for - ## linking. - ## - ## Note also that Conan currently builds all OSX binaries on 10.14 so older versions of - ## OSX will complain if statically linking to these binaries. Further, all Conan binaries - ## for Windows are built with Visual Studio so static linking the `.lib` files with gcc - ## or clang might lead to incompatibility issues if the library uses Visual Studio - ## specific compiler features. - ## - ## `conFlags`, `cmakeFlags` and `makeFlags` allow sending custom parameters to `configure`, - ## `cmake` and `make` in case additional configuration is required as part of the build - ## process. - ## - ## `altNames` is a list of alternate names for the library - e.g. zlib uses `zlib.h` for - ## the header but the typical lib name is `libz.so` and not `libzlib.so`. However, it is - ## libzlib.dll on Windows if built with cmake. In this case, `altNames = "z,zlib"`. Comma - ## separate for multiple alternate names without spaces. - ## - ## The original header name is not included by default if `altNames` is set since it could - ## cause the wrong lib to be selected. E.g. `SDL2/SDL.h` could pick `libSDL.so` even if - ## `altNames = "SDL2"`. Explicitly include it in `altNames` like the `zlib` example when - ## required. - ## - ## `buildTypes` specifies a list of ordered build strategies to use when building the - ## downloaded source files. Default is [btCmake, btAutoconf] - ## - ## `xxxPreBuild` is a hook that is called after the source code is pulled from Git or - ## downloaded but before the library is built. This might be needed if some initial prep - ## needs to be done before compilation. A few values are provided to the hook to help - ## provide context: - ## - ## `outdir` is the same `outdir` passed in and `header` is the discovered header path - ## in the downloaded source code. - ## - ## Simply define `proc xxxPreBuild(outdir, header: string)` in the wrapper and it will get - ## called prior to the build process. - var - origname = header.extractFilename().split(".")[0] - name = origname.split(seps = AllChars-Letters-Digits).join() +when not defined(TOAST): + # configure, cmake, make support + include "."/build/tools - # Default to origname if not specified - conanuri = if conanuri.len != 0: conanuri else: origname - jbburi = if jbburi.len != 0: jbburi else: origname + # Conan.io support + include "."/build/conan - # -d:xxx for this header - stdStr = name & "Std" - gitStr = name & "Git" - dlStr = name & "DL" - conanStr = name & "Conan" - jbbStr = name & "JBB" + # Julia BinaryBuilder.org support + include "."/build/jbb - staticStr = name & "Static" - verStr = name & "SetVer" - - # Ident nodes of the -d:xxx to check in when statements - nameStd = newIdentNode(stdStr) - nameGit = newIdentNode(gitStr) - nameDL = newIdentNode(dlStr) - nameConan = newIdentNode(conanStr) - nameJBB = newIdentNode(jbbStr) - - nameStatic = newIdentNode(staticStr) - - # Consts to generate - path = newIdentNode(name & "Path") - lpath = newIdentNode(name & "LPath") - ldeps = newIdentNode(name & "LDeps") - version = newIdentNode(verStr) - lname = newIdentNode(name & "LName") - preBuild = newIdentNode(name & "PreBuild") - - # Regex for library search - lre = "(lib)?$1[_-]?(static)?" - - # If -d:xxx set with setDefines() - stdVal = gDefines.hasKey(stdStr) - gitVal = gDefines.hasKey(gitStr) - dlVal = gDefines.hasKey(dlStr) - conanVal = gDefines.hasKey(conanStr) - jbbVal = gDefines.hasKey(jbbStr) - staticVal = gDefines.hasKey(staticStr) - verVal = - if gDefines.hasKey(verStr): - gDefines[verStr] - else: - "" - mode = getCompilerMode(header) - - libdir = if libdir.len != 0: libdir else: getOutDir() - - # Use alternate library names if specified for regex search - if altNames.len != 0: - lre = lre % ("(" & altNames.replace(",", "|") & ")") - else: - lre = lre % origname - - result = newNimNode(nnkStmtList) - result.add(quote do: - # Need to check -d:xxx or setDefines() - const - `nameStd`* = when defined(`nameStd`): true else: `stdVal` == 1 - `nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1 - `nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1 - `nameConan`* = when defined(`nameConan`): true else: `conanVal` == 1 - `nameJBB`* = when defined(`nameJBB`): true else: `jbbVal` == 1 - `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 - - # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, conanuri, jbburi, outdir, version: string, shared: bool): string = - when `nameGit`: - getGitPath(header, giturl, outdir, version) - elif `nameDL`: - getDlPath(header, dlurl, outdir, version) - elif `nameConan`: - getConanPath(header, conanuri, outdir, version, shared) - elif `nameJBB`: - getJBBPath(header, jbburi, outdir, version) - else: - getLocalPath(header, outdir) - - const - `version`* {.strdefine.} = `verVal` - `lname` = - when `nameStatic`: - `lre` & "\\.(a|lib)" - else: - `lre` & getDynlibExt() - - # Look in standard path if requested by user - stdPath = - when `nameStd`: getStdPath(`header`, `mode`) else: "" - stdLPath = - when `nameStd`: getStdLibPath(`lname`, `mode`) else: "" - - useStd = stdPath.len != 0 and stdLPath.len != 0 - - # Look elsewhere if requested while prioritizing standard paths - prePath = - when useStd: - stdPath - else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) - - # Run preBuild hook before building library if not Std, Conan or JBB - when not (useStd or `nameConan` or `nameJBB`) and declared(`preBuild`): - static: - `preBuild`(`outdir`, prePath) - - let - # Library binary path - build if not standard / conan / jbb - lpath {.compileTime.} = - when useStd: - stdLPath - elif `nameConan` or `nameJBB`: - findFile(`lname`, `outdir`, regex = true) - else: - buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) - - # Library dependecy paths - ldeps {.compileTime.}: seq[string] = - when not useStd: - when `nameConan`: - getConanLDeps(`outdir`) - elif `nameJBB`: - getJBBLDeps(`outdir`, not `nameStatic`) - else: - @[] - else: - @[] - - const - # Header path - search again in case header is generated in build - `path`* = - if prePath.len != 0: - prePath - else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) - - static: - doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & - "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name` - doAssert lpath.len != 0, "\nLibrary " & `lname` & " not found" - - when `nameStatic`: - const - `lpath`* = lpath - `ldeps`* = ldeps - - # Automatically link with static library and dependencies - {.passL: `lpath`.} - if `ldeps`.len != 0: - {.passL: `ldeps`.join(" ").} - - static: - echo "# Including library " & lpath - if `ldeps`.len != 0: - echo "# Including dependencies " & `ldeps`.join(" ") - else: - const - `lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath - `ldeps`* = - when not useStd: - block: - var - ldeps = ldeps - copied: seq[string] - for i in 0 ..< ldeps.len: - let - lname = ldeps[i].extractFilename() - ldeptgt = `libdir` / lname - if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt): - cpFile(ldeps[i], ldeptgt, psymlink = true) - copied.add lname - ldeps[i] = ldeptgt - # Copy downloaded dependencies to `libdir` - if copied.len != 0: - echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` - ldeps - else: - ldeps - - static: - when not useStd: - # Copy downloaded shared libraries to `libdir` - if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): - echo "# Copying " & `lpath`.extractFilename() & " to " & `libdir` - cpFile(lpath, `lpath`) - - echo "# Including library " & `lpath` - ) + # getHeader support + include "."/build/getheader diff --git a/nimterop/build/ccompiler.nim b/nimterop/build/ccompiler.nim new file mode 100644 index 0000000..c33c94a --- /dev/null +++ b/nimterop/build/ccompiler.nim @@ -0,0 +1,101 @@ +import os, strformat, strutils + +proc getCompilerMode*(path: string): string = + ## Determines a target language mode from an input filename, if one is not already specified. + let file = path.splitFile() + if file.ext in [".hxx", ".hpp", ".hh", ".H", ".h++", ".cpp", ".cxx", ".cc", ".C", ".c++"]: + result = "cpp" + elif file.ext in [".h", ".c"]: + result = "c" + +proc getGccModeArg*(mode: string): string = + ## Produces a GCC argument that explicitly sets the language mode to be used by the compiler. + if mode == "cpp": + result = "-xc++" + elif mode == "c": + result = "-xc" + +proc getCompiler*(): string = + var + compiler = + when defined(gcc): + "gcc" + elif defined(clang): + "clang" + else: + doAssert false, "Nimterop only supports gcc and clang at this time" + + result = getEnv("CC", compiler) + +proc getGccPaths*(mode: string): seq[string] = + var + nul = when defined(Windows): "nul" else: "/dev/null" + inc = false + + (outp, _) = execAction(&"""{getCompiler()} -Wp,-v {getGccModeArg(mode)} {nul}""", die = false) + + for line in outp.splitLines(): + if "#include <...> search starts here" in line: + inc = true + continue + elif "End of search list" in line: + break + if inc: + var + path = line.strip().normalizedPath() + if path notin result: + result.add path + + when defined(osx): + result.add(execAction("xcrun --show-sdk-path").output.strip() & "/usr/include") + +proc getGccLibPaths*(mode: string): seq[string] = + var + nul = when defined(Windows): "nul" else: "/dev/null" + linker = when defined(OSX): "-Xlinker" else: "" + + (outp, _) = execAction(&"""{getCompiler()} {linker} -v {getGccModeArg(mode)} {nul}""", die = false) + + for line in outp.splitLines(): + if "LIBRARY_PATH=" in line: + for path in line[13 .. ^1].split(PathSep): + var + path = path.strip().normalizedPath() + if path notin result: + result.add path + break + elif '\t' in line: + var + path = line.strip().normalizedPath() + if path notin result: + result.add path + + when defined(osx): + result.add "/usr/lib" + +proc getGccInfo*(): tuple[arch, os, compiler, version: string] = + let + (outp, _) = execAction(&"{getCompiler()} -v") + for line in outp.splitLines(): + if line.startsWith("Target: "): + result.arch = line.split(' ')[1].split('-')[0] + result.os = + if "linux" in line: + "linux" + elif "android" in line: + "android" + elif "darwin" in line: + "macos" + elif "w64" in line or "mingw" in line: + "windows" + else: + "unknown" + elif " version " in line: + result.version = line.split(" version ")[1].split(' ')[0] + if "clang" in outp: + if result.os == "macos": + result.compiler = "apple-clang" + else: + result.compiler = "clang" + else: + result.compiler = "gcc" diff --git a/nimterop/conan.nim b/nimterop/build/conan.nim similarity index 100% rename from nimterop/conan.nim rename to nimterop/build/conan.nim diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim new file mode 100644 index 0000000..3852fbf --- /dev/null +++ b/nimterop/build/getheader.nim @@ -0,0 +1,504 @@ +import macros, os, strutils, tables + +var + gDefines {.compileTime.} = initTable[string, string]() + +macro setDefines*(defs: static openArray[string]): untyped = + ## Specify `-d:xxx` values in code instead of having to rely on the command + ## line or `cfg` or `nims` files. + ## + ## At this time, Nim does not allow creation of `-d:xxx` defines in code. In + ## addition, Nim only loads config files for the module being compiled but not + ## for imported packages. This becomes a challenge when wanting to ship a wrapper + ## library that wants to control `getHeader()` for an underlying package. + ## + ## E.g. nimarchive wanting to set `-d:lzmaStatic` + ## + ## The consumer of nimarchive would need to set such defines as part of their + ## project, making it inconvenient. + ## + ## By calling this proc with the defines preferred before importing such a module, + ## the caller can set the behavior in code instead. + ## + ## .. code-block:: nim + ## + ## setDefines(@["lzmaStatic", "lzmaDL", "lzmaSetVer=5.2.4"]) + ## + ## import lzma + for def in defs: + let + nv = def.strip().split("=", maxsplit = 1) + if nv.len != 0: + let + n = nv[0] + v = + if nv.len == 2: + nv[1] + else: + "" + gDefines[n] = v + +macro clearDefines*(): untyped = + ## Clear all defines set using `setDefines()`. + gDefines.clear() + +macro isDefined*(def: untyped): untyped = + ## Check if `-d:xxx` is set globally or via `setDefines()` + let + sdef = gDefines.hasKey(def.strVal()) + result = newNimNode(nnkStmtList) + result.add(quote do: + when defined(`def`) or `sdef` != 0: + true + else: + false + ) + +proc getDynlibExt(): string = + when defined(Windows): + result = "[0-9.\\-]*\\.dll" + elif defined(linux) or defined(FreeBSD): + result = "\\.so[0-9.]*" + elif defined(macosx): + result = "[0-9.\\-]*\\.dylib" + +proc getStdPath(header, mode: string): string = + for inc in getGccPaths(mode): + result = findFile(header, inc, recurse = false, first = true) + if result.len != 0: + break + +proc getStdLibPath(lname, mode: string): string = + for lib in getGccLibPaths(mode): + result = findFile(lname, lib, recurse = false, first = true, regex = true) + if result.len != 0: + break + +proc getGitPath(header, url, outdir, version: string): string = + doAssert url.len != 0, "No git url setup for " & header + doAssert findExe("git").len != 0, "git executable missing" + + gitPull(url, outdir, checkout = version) + + result = findFile(header, outdir) + +proc getDlPath(header, url, outdir, version: string): string = + doAssert url.len != 0, "No download url setup for " & header + + var + dlurl = url + if "$#" in url or "$1" in url: + doAssert version.len != 0, "Need version for download url" + dlurl = url % version + else: + doAssert version.len == 0, "Download url does not contain version" + + downloadUrl(dlurl, outdir) + + var + dirname = "" + for kind, path in walkDir(outdir, relative = true): + if kind == pcFile and path != dlurl.extractFilename(): + dirname = "" + break + elif kind == pcDir: + if dirname.len == 0: + dirname = path + else: + dirname = "" + break + + if dirname.len != 0: + for kind, path in walkDir(outdir / dirname, relative = true): + mvFile(outdir / dirname / path, outdir / path) + + result = findFile(header, outdir) + +proc getConanPath(header, uri, outdir, version: string, shared: bool): string = + var + uri = uri + + if "$#" in uri or "$1" in uri: + doAssert version.len != 0, "Need version for Conan.io uri: " & uri + uri = uri % version + elif version.len != 0: + uri = uri & "/" & version + + let + pkg = newConanPackageFromUri(uri, shared) + downloadConan(pkg, outdir) + + result = findFile(header, outdir) + +proc getConanLDeps(outdir: string): seq[string] = + let + pkg = loadConanInfo(outdir) + + result = pkg.getConanLDeps(outdir) + +proc getJBBPath(header, uri, outdir, version: string): string = + let + spl = uri.split('/', 1) + name = spl[0] + hasVersion = version.len != 0 + + var + ver = + if spl.len == 2: + spl[1] + else: + "" + + if ver.len != 0: + if "$#" in ver or "$1" in ver: + doAssert hasVersion, "Need version for BinaryBuilder.org uri: " & uri + ver = ver % version + elif hasVersion: + doAssert false, "Version in both uri `" & uri & "` and `-d:xxxSetVer=\"" & + version & "\"` for BinaryBuilder.org" + elif hasVersion: + ver = version + + let + pkg = newJBBPackage(name, ver) + downloadJBB(pkg, outdir) + + result = findFile(header, outdir) + +proc getJBBLDeps(outdir: string, shared: bool): seq[string] = + let + pkg = loadJBBInfo(outdir) + + result = pkg.getJBBLDeps(outdir, shared) + +proc getLocalPath(header, outdir: string): string = + if outdir.len != 0: + result = findFile(header, outdir) + +proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildTypes: openArray[BuildType]): string = + var + lpath = findFile(lname, outdir, regex = true) + makeFlagsProc = &"-j {getNumProcs()} {makeFlags}" + makePath = outdir + + if lpath.len != 0: + return lpath + + var buildStatus: BuildStatus + + for buildType in buildTypes: + case buildType + of btCmake: + buildStatus = buildWithCmake(makePath, cmakeFlags) + of btAutoconf: + buildStatus = buildWithAutoConf(makePath, conFlags) + + if buildStatus.built: + break + + if buildStatus.buildPath.len > 0: + let libraryExists = findFile(lname, buildStatus.buildPath, regex = true).len > 0 + + if not libraryExists and fileExists(buildStatus.buildPath / "Makefile"): + make(buildStatus.buildPath, lname, makeFlagsProc, regex = true) + buildStatus.built = true + + let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir + doAssert buildStatus.built, &"\nBuild configuration failed - {error}\n" + + result = findFile(lname, outdir, regex = true) + +macro getHeader*( + header: static[string], giturl: static[string] = "", dlurl: static[string] = "", + conanuri: static[string] = "", jbburi: static[string] = "", + outdir: static[string] = "", libdir: static[string] = "", + conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", + altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = + ## Get the path to a header file for wrapping with + ## `cImport() `_ or + ## `c2nImport() `_. + ## + ## This proc checks `-d:xxx` defines based on the header name (e.g. lzma from lzma.h), + ## and accordingly employs different ways to obtain the source. + ## + ## `-d:xxxStd` - search standard system paths. E.g. `/usr/include` and `/usr/lib` on Linux + ## `-d:xxxGit` - clone source from a git repo specified in `giturl` + ## `-d:xxxDL` - download source from `dlurl` and extract if required + ## `-d:xxxConan` - download headers and binary from Conan.io using `conanuri` with + ## format `pkgname[/version[@user/channel][:bhash]]` + ## `-d:xxxJBB` - download headers and binary from BinaryBuilder.org using `jbburi` with + ## format `pkgname[/version]` + ## + ## This allows a single wrapper to be used in different ways depending on the user's needs. + ## If no `-d:xxx` defines are specified, `outdir` will be searched for the header as is. + ## The user can opt to download the sources to `outdir` using any other method such as + ## git sub-modules, vendoring or pointing to a repository that was already cloned. + ## + ## If multiple `-d:xxx` defines are specified, precedence is `Std` and then `Git`, `DL`, + ## `Conan` or `JBB`. This allows using a system installed library if available before + ## falling back to manual building. The user would need to specify both `-d:xxxStd` and + ## one of the other methods. + ## + ## `-d:xxxSetVer=x.y.z` can be used to specify which version to use. It is used as a tag + ## name for `Git` whereas for `DL`, `Conan` and `JBB`, it replaces `$1` in the URL + ## if specified. Specifying `-d:xxxSetVer` without a `$1` will download that version for + ## `Conan` and `JBB` if available. If no version is specified, the latest release of the + ## package is downloaded. For `Conan`, `-d:xxxSetVer` can also be used to set additional + ## URI information: + ## `-d:xxxSetVer=1.9.0@bincrafters/stable:bhash` + ## + ## If `conanuri` or `jbburi` are not defined and `Conan` or `JBB` is selected, the `header` + ## filename is used instead. + ## + ## All defines can also be set in code using `setDefines()` and checked for using + ## `isDefined()` which checks for defines set from both `-d` and `setDefines()`. + ## + ## The library is then configured (with `cmake` or `autotools` if possible) and built + ## using `make`, unless using `-d:xxxStd` which presumes that the system package + ## manager was used to install prebuilt headers and binaries, or using `-d:xxxConan` + ## or `-d:xxxJBB` which download pre-built binaries. + ## + ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call + ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can + ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. Any dependency + ## libraries downloaded by `Conan` or `JBB` are returned in `const xxxLDeps` as a seq[string]. + ## + ## `libdir` can be used to instruct `getHeader()` to copy shared libraries and their + ## dependencies to that directory. This prevents any runtime failures if `outdir` gets + ## removed or its contents changed. By default, `libdir` is set to the output directory + ## where the program binary will be created. The values of `xxxLPath` and `xxxLDeps` will + ## reflect this new location. `libdir` is ignored for `Std` mode. + ## + ## `-d:xxxStatic` can be specified to statically link with the library instead. This + ## will automatically add a `{.passL.}` call to the static library for convenience. Note + ## that `-d:xxxConan` and `-d:xxxJBB` download all dependency libs as well and the + ## `xxxLPath` will include paths to all of them separated by space in the right order for + ## linking. + ## + ## Note also that Conan currently builds all OSX binaries on 10.14 so older versions of + ## OSX will complain if statically linking to these binaries. Further, all Conan binaries + ## for Windows are built with Visual Studio so static linking the `.lib` files with gcc + ## or clang might lead to incompatibility issues if the library uses Visual Studio + ## specific compiler features. + ## + ## `conFlags`, `cmakeFlags` and `makeFlags` allow sending custom parameters to `configure`, + ## `cmake` and `make` in case additional configuration is required as part of the build + ## process. + ## + ## `altNames` is a list of alternate names for the library - e.g. zlib uses `zlib.h` for + ## the header but the typical lib name is `libz.so` and not `libzlib.so`. However, it is + ## libzlib.dll on Windows if built with cmake. In this case, `altNames = "z,zlib"`. Comma + ## separate for multiple alternate names without spaces. + ## + ## The original header name is not included by default if `altNames` is set since it could + ## cause the wrong lib to be selected. E.g. `SDL2/SDL.h` could pick `libSDL.so` even if + ## `altNames = "SDL2"`. Explicitly include it in `altNames` like the `zlib` example when + ## required. + ## + ## `buildTypes` specifies a list of ordered build strategies to use when building the + ## downloaded source files. Default is [btCmake, btAutoconf] + ## + ## `xxxPreBuild` is a hook that is called after the source code is pulled from Git or + ## downloaded but before the library is built. This might be needed if some initial prep + ## needs to be done before compilation. A few values are provided to the hook to help + ## provide context: + ## + ## `outdir` is the same `outdir` passed in and `header` is the discovered header path + ## in the downloaded source code. + ## + ## Simply define `proc xxxPreBuild(outdir, header: string)` in the wrapper and it will get + ## called prior to the build process. + var + origname = header.extractFilename().split(".")[0] + name = origname.split(seps = AllChars-Letters-Digits).join() + + # Default to origname if not specified + conanuri = if conanuri.len != 0: conanuri else: origname + jbburi = if jbburi.len != 0: jbburi else: origname + + # -d:xxx for this header + stdStr = name & "Std" + gitStr = name & "Git" + dlStr = name & "DL" + conanStr = name & "Conan" + jbbStr = name & "JBB" + + staticStr = name & "Static" + verStr = name & "SetVer" + + # Ident nodes of the -d:xxx to check in when statements + nameStd = newIdentNode(stdStr) + nameGit = newIdentNode(gitStr) + nameDL = newIdentNode(dlStr) + nameConan = newIdentNode(conanStr) + nameJBB = newIdentNode(jbbStr) + + nameStatic = newIdentNode(staticStr) + + # Consts to generate + path = newIdentNode(name & "Path") + lpath = newIdentNode(name & "LPath") + ldeps = newIdentNode(name & "LDeps") + version = newIdentNode(verStr) + lname = newIdentNode(name & "LName") + preBuild = newIdentNode(name & "PreBuild") + + # Regex for library search + lre = "(lib)?$1[_-]?(static)?" + + # If -d:xxx set with setDefines() + stdVal = gDefines.hasKey(stdStr) + gitVal = gDefines.hasKey(gitStr) + dlVal = gDefines.hasKey(dlStr) + conanVal = gDefines.hasKey(conanStr) + jbbVal = gDefines.hasKey(jbbStr) + staticVal = gDefines.hasKey(staticStr) + verVal = + if gDefines.hasKey(verStr): + gDefines[verStr] + else: + "" + mode = getCompilerMode(header) + + libdir = if libdir.len != 0: libdir else: getOutDir() + + # Use alternate library names if specified for regex search + if altNames.len != 0: + lre = lre % ("(" & altNames.replace(",", "|") & ")") + else: + lre = lre % origname + + result = newNimNode(nnkStmtList) + result.add(quote do: + # Need to check -d:xxx or setDefines() + const + `nameStd`* = when defined(`nameStd`): true else: `stdVal` == 1 + `nameGit`* = when defined(`nameGit`): true else: `gitVal` == 1 + `nameDL`* = when defined(`nameDL`): true else: `dlVal` == 1 + `nameConan`* = when defined(`nameConan`): true else: `conanVal` == 1 + `nameJBB`* = when defined(`nameJBB`): true else: `jbbVal` == 1 + `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 + + # Search for header in outdir (after retrieving code) depending on -d:xxx mode + proc getPath(header, giturl, dlurl, conanuri, jbburi, outdir, version: string, shared: bool): string = + when `nameGit`: + getGitPath(header, giturl, outdir, version) + elif `nameDL`: + getDlPath(header, dlurl, outdir, version) + elif `nameConan`: + getConanPath(header, conanuri, outdir, version, shared) + elif `nameJBB`: + getJBBPath(header, jbburi, outdir, version) + else: + getLocalPath(header, outdir) + + const + `version`* {.strdefine.} = `verVal` + `lname` = + when `nameStatic`: + `lre` & "\\.(a|lib)" + else: + `lre` & getDynlibExt() + + # Look in standard path if requested by user + stdPath = + when `nameStd`: getStdPath(`header`, `mode`) else: "" + stdLPath = + when `nameStd`: getStdLibPath(`lname`, `mode`) else: "" + + useStd = stdPath.len != 0 and stdLPath.len != 0 + + # Look elsewhere if requested while prioritizing standard paths + prePath = + when useStd: + stdPath + else: + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) + + # Run preBuild hook before building library if not Std, Conan or JBB + when not (useStd or `nameConan` or `nameJBB`) and declared(`preBuild`): + static: + `preBuild`(`outdir`, prePath) + + let + # Library binary path - build if not standard / conan / jbb + lpath {.compileTime.} = + when useStd: + stdLPath + elif `nameConan` or `nameJBB`: + findFile(`lname`, `outdir`, regex = true) + else: + buildLibrary(`lname`, `outdir`, `conFlags`, `cmakeFlags`, `makeFlags`, `buildTypes`) + + # Library dependecy paths + ldeps {.compileTime.}: seq[string] = + when not useStd: + when `nameConan`: + getConanLDeps(`outdir`) + elif `nameJBB`: + getJBBLDeps(`outdir`, not `nameStatic`) + else: + @[] + else: + @[] + + const + # Header path - search again in case header is generated in build + `path`* = + if prePath.len != 0: + prePath + else: + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) + + static: + doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & + "missing/empty outdir or -d:$1Std -d:$1Git -d:$1DL -d:$1Conan or -d:$1JBB not specified" % `name` + doAssert lpath.len != 0, "\nLibrary " & `lname` & " not found" + + when `nameStatic`: + const + `lpath`* = lpath + `ldeps`* = ldeps + + # Automatically link with static library and dependencies + {.passL: `lpath`.} + if `ldeps`.len != 0: + {.passL: `ldeps`.join(" ").} + + static: + echo "# Including library " & lpath + if `ldeps`.len != 0: + echo "# Including dependencies " & `ldeps`.join(" ") + else: + const + `lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath + `ldeps`* = + when not useStd: + block: + var + ldeps = ldeps + copied: seq[string] + for i in 0 ..< ldeps.len: + let + lname = ldeps[i].extractFilename() + ldeptgt = `libdir` / lname + if not fileExists(ldeptgt) or getFileDate(ldeps[i]) != getFileDate(ldeptgt): + cpFile(ldeps[i], ldeptgt, psymlink = true) + copied.add lname + ldeps[i] = ldeptgt + # Copy downloaded dependencies to `libdir` + if copied.len != 0: + echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` + ldeps + else: + ldeps + + static: + when not useStd: + # Copy downloaded shared libraries to `libdir` + if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): + echo "# Copying " & `lpath`.extractFilename() & " to " & `libdir` + cpFile(lpath, `lpath`) + + echo "# Including library " & `lpath` + ) diff --git a/nimterop/jbb.nim b/nimterop/build/jbb.nim similarity index 99% rename from nimterop/jbb.nim rename to nimterop/build/jbb.nim index c282197..15bc578 100644 --- a/nimterop/jbb.nim +++ b/nimterop/build/jbb.nim @@ -1,4 +1,4 @@ -import json, os, strutils +import json, os, strformat, strutils, tables when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): import marshal diff --git a/nimterop/nimconf.nim b/nimterop/build/nimconf.nim similarity index 100% rename from nimterop/nimconf.nim rename to nimterop/build/nimconf.nim diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim new file mode 100644 index 0000000..4f3c316 --- /dev/null +++ b/nimterop/build/shell.nim @@ -0,0 +1,465 @@ +import os, strformat, strutils + +proc sleep*(milsecs: int) = + ## Sleep at compile time + let + cmd = + when defined(Windows): + "cmd /c timeout " + else: + "sleep " + + discard gorgeEx(cmd & $(milsecs / 1000)) + +proc execAction*(cmd: string, retry = 0, die = true, cache = false, + cacheKey = "", onRetry: proc() = nil): tuple[output: string, ret: int] = + ## Execute an external command - supported at compile time + ## + ## Checks if command exits successfully before returning. If not, an + ## error is raised. Always caches results to be used in nimsuggest or nimcheck + ## mode. + ## + ## `retry` - number of times command should be retried before error + ## `die = false` - return on errors + ## `cache = true` - cache results unless cleared with -f + ## `cacheKey` - key to create unique cache entry + let + ccmd = fixCmd(cmd) + + when nimvm: + # Cache results for speedup if cache = true + # Else cache for preserving functionality in nimsuggest and nimcheck + let + hash = (ccmd & cacheKey).hash().abs() + cachePath = getNimteropCacheDir() / "execCache" / "nimterop_" & $hash + cacheFile = cachePath & ".txt" + retFile = cachePath & "_ret.txt" + + when defined(nimsuggest) or defined(nimcheck): + # Load results from cache file if generated in previous run + if fileExists(cacheFile) and fileExists(retFile): + result.output = cacheFile.readFile() + result.ret = retFile.readFile().parseInt() + elif die: + doAssert false, "Results not cached - run nim c/cpp at least once\n" & ccmd + else: + if cache and fileExists(cacheFile) and fileExists(retFile) and not compileOption("forceBuild"): + # Return from cache when requested + result.output = cacheFile.readFile() + result.ret = retFile.readFile().parseInt() + else: + # Execute command and store results in cache + (result.output, result.ret) = gorgeEx(ccmd) + if result.ret == 0 or die == false: + # mkdir for execCache dir (circular dependency) + let dir = cacheFile.parentDir() + if not dirExists(dir): + let flag = when not defined(Windows): "-p" else: "" + discard execAction(&"mkdir {flag} {dir.sanitizePath}") + cacheFile.writeFile(result.output) + retFile.writeFile($result.ret) + else: + # Used by toast + (result.output, result.ret) = execCmdEx(ccmd) + + # On failure, retry or die as requested + if result.ret != 0: + if retry > 0: + if not onRetry.isNil: + onRetry() + sleep(500) + result = execAction(cmd, retry = retry - 1, die, cache, cacheKey) + elif die: + doAssert false, "Command failed: " & $result.ret & "\ncmd: " & ccmd & + "\nresult:\n" & result.output + +proc findExe*(exe: string): string = + ## Find the specified executable using the `which`/`where` command - supported + ## at compile time + var + cmd = + when defined(Windows): + "where " & exe + else: + "which " & exe + + (output, ret) = execAction(cmd, die = false) + + if ret == 0: + return output.splitLines()[0].strip() + +proc mkDir*(dir: string) = + ## Create a directory at compile time + ## + ## The `os` module is not available at compile time so a few + ## crucial helper functions are included with nimterop. + if not dirExists(dir): + let + flag = when not defined(Windows): "-p" else: "" + discard execAction(&"mkdir {flag} {dir.sanitizePath}", retry = 2) + +proc cpFile*(source, dest: string, psymlink = false, move = false) = + ## Copy a file from `source` to `dest` at compile time + ## + ## `psymlink = true` preserves symlinks instead of dereferencing on posix + let + source = source.replace("/", $DirSep) + dest = dest.replace("/", $DirSep) + cmd = + when defined(Windows): + if move: + "move /y" + else: + "copy /y" + else: + if move: + "mv -f" + else: + if psymlink: + "cp -fa" + else: + "cp -f" + + discard execAction(&"{cmd} {source.sanitizePath} {dest.sanitizePath}", retry = 2) + +proc mvFile*(source, dest: string) = + ## Move a file from `source` to `dest` at compile time + cpFile(source, dest, move=true) + +proc rmFile*(source: string, dir = false) = + ## Remove a file or pattern at compile time + let + source = source.replace("/", $DirSep) + cmd = + when defined(Windows): + if dir: + "rd /s/q" + else: + "del /s/q/f" + else: + "rm -rf" + exists = + if dir: + dirExists(source) + else: + fileExists(source) + + if exists: + discard execAction(&"{cmd} {source.sanitizePath}", retry = 2) + +proc rmDir*(dir: string) = + ## Remove a directory or pattern at compile time + rmFile(dir, dir = true) + +proc cleanDir*(dir: string) = + ## Remove all contents of a directory at compile time + for kind, path in walkDir(dir): + if kind == pcDir: + rmDir(path) + else: + rmFile(path) + +proc cpTree*(source, dest: string, move = false) = + ## Copy contents of source dir to the destination, not the directory itself + for kind, path in walkDir(source, relative = true): + if kind == pcDir: + cpTree(source / path, dest / path, move) + if move: + rmDir(source / path) + else: + if not dirExists(dest): + mkDir(dest) + if move: + mvFile(source / path, dest / path) + else: + cpFile(source / path, dest / path) + +proc mvTree*(source, dest: string) = + ## Move contents of source dir to the destination, not the directory itself + cpTree(source, dest, move = true) + +proc getFileDate*(fullpath: string): string = + ## Get file date for `fullpath` + var + ret = 0 + cmd = + when defined(Windows): + let + (head, tail) = fullpath.splitPath() + &"cmd /c forfiles /P {head.sanitizePath()} /M {tail.sanitizePath} /C \"cmd /c echo @fdate @ftime @fsize\"" + elif defined(Linux): + &"stat -c %y {fullpath.sanitizePath}" + elif defined(OSX) or defined(FreeBSD): + &"stat -f %m {fullpath.sanitizePath}" + + (result, ret) = execAction(cmd) + +proc touchFile*(fullpath: string) = + ## Touch file to update modified date + var + cmd = + when defined(Windows): + &"cmd /c copy /b {fullpath.sanitizePath}+" + else: + &"touch {fullpath.sanitizePath}" + + discard execAction(cmd) + +proc extractZip*(zipfile, outdir: string, quiet = false) = + ## Extract a zip file using `powershell` on Windows and `unzip` on other + ## systems to the specified output directory + var cmd = "unzip -o $#" + if defined(Windows): + cmd = "powershell -nologo -noprofile -command \"& { Add-Type -A " & + "'System.IO.Compression.FileSystem'; " & + "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" + + if not quiet: + echo "# Extracting " & zipfile + discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}") + +proc extractTar*(tarfile, outdir: string, quiet = false) = + ## Extract a tar file using `tar`, `7z` or `7za` to the specified output directory + var + cmd = "" + name = "" + + if findExe("tar").len != 0: + let + ext = tarfile.splitFile().ext.toLowerAscii() + typ = + case ext + of ".gz", ".tgz": "z" + of ".xz": "J" + of ".bz2": "j" + else: "" + + cmd = "tar xvf" & typ & " " & tarfile.sanitizePath + else: + for i in ["7z", "7za"]: + if findExe(i).len != 0: + cmd = i & " x $#" % tarfile.sanitizePath + + name = tarfile.splitFile().name + if ".tar" in name.toLowerAscii(): + cmd &= " && " & i & " x $#" % name.sanitizePath + + break + + doAssert cmd.len != 0, "No extraction tool - tar, 7z, 7za - available for " & tarfile.sanitizePath + + if not quiet: + echo "# Extracting " & tarfile + discard execAction(&"cd {outdir.sanitizePath} && {cmd}") + if name.len != 0: + rmFile(outdir / name) + +proc downloadUrl*(url, outdir: string, quiet = false, retry = 1) = + ## Download a file using `curl` or `wget` (or `powershell` on Windows) to the specified directory + ## + ## If an archive file, it is automatically extracted after download. + let + file = url.extractFilename() + filePath = outdir / file + ext = file.splitFile().ext.toLowerAscii() + archives = @[".zip", ".xz", ".gz", ".bz2", ".tgz", ".tar"] + + if not (ext in archives and fileExists(filePath)): + if not quiet: + echo "# Downloading " & file + mkDir(outdir) + var cmd = findExe("curl") + if cmd.len != 0: + cmd &= " -Lk $# -o $#" + else: + cmd = findExe("wget") + if cmd.len != 0: + cmd &= " $# -O $#" + elif defined(Windows): + cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#" + else: + doAssert false, "No download tool available - curl, wget" + discard execAction(cmd % [url.quoteShell, (filePath).sanitizePath], retry = 3, + onRetry = proc() = rmFile(filePath)) + + if ext == ".zip": + extractZip(file, outdir, quiet) + elif ext in archives: + extractTar(file, outdir, quiet) + +proc gitReset*(outdir: string) = + ## Hard reset the git repository at the specified directory + echo "# Resetting " & outdir + + let cmd = &"cd {outdir.sanitizePath} && git reset --hard" + while execAction(cmd).output.contains("Permission denied"): + sleep(1000) + echo "# Retrying ..." + +proc gitCheckout*(file, outdir: string) = + ## Checkout the specified `file` in the git repository at `outdir` + ## + ## This effectively resets all changes in the file and can be + ## used to undo any changes that were made to source files to enable + ## successful wrapping with `cImport()` or `c2nImport()`. + echo "# Resetting " & file + let file2 = file.relativePath outdir + let cmd = &"cd {outdir.sanitizePath} && git checkout {file2.sanitizePath}" + while execAction(cmd).output.contains("Permission denied"): + sleep(500) + echo "# Retrying ..." + +proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = + ## Pull the specified git repository to the output directory + ## + ## `plist` is the list of specific files and directories or wildcards + ## to sparsely checkout. Multiple values can be specified one entry per + ## line. It is optional and if omitted, the entire repository will be + ## checked out. + ## + ## `checkout` is the git tag, branch or commit hash to checkout once + ## the repository is downloaded. This allows for pinning to a specific + ## version of the code. + if dirExists(outdir/".git"): + gitReset(outdir) + return + + let + outdirQ = outdir.sanitizePath + + mkDir(outdir) + + if not quiet: + echo "# Setting up Git repo: " & url + discard execAction(&"cd {outdirQ} && git init .") + discard execAction(&"cd {outdirQ} && git remote add origin {url}") + + if plist.len != 0: + # If a specific list of files is required, create a sparse checkout + # file for git in its config directory + let sparsefile = outdir / ".git/info/sparse-checkout" + + discard execAction(&"cd {outdirQ} && git config core.sparsecheckout true") + writeFile(sparsefile, plist) + + # In case directory has old files from another run + discard execAction(&"cd {outdirQ} && git clean -fxd") + + if checkout.len != 0: + if not quiet: + echo "# Checking out " & checkout + discard execAction(&"cd {outdirQ} && git fetch", retry = 3) + discard execAction(&"cd {outdirQ} && git checkout {checkout}") + else: + if not quiet: + echo "# Pulling repository" + discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 3) + +proc gitTags*(outdir: string): seq[string] = + ## Get all the git tags in the specified directory + let + cmd = &"cd {outdir.sanitizePath} && git tag" + tags = execAction(cmd).output.splitLines() + for tag in tags: + let + tag = tag.strip() + if tag.len != 0: + result.add tag + +proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[string] = + ## Find all matching files in the specified directory + ## + ## `file` is a regular expression if `regex` is true + ## + ## Turn off recursive search with `recurse` + var + cmd = + when defined(Windows): + "nimgrep --filenames --oneline --nocolor $1 \"$2\" $3" + elif defined(linux): + "find $3 $1 -regextype egrep -regex $2" + elif defined(osx) or defined(FreeBSD): + "find -E $3 $1 -regex $2" + + recursive = "" + + if recurse: + when defined(Windows): + recursive = "--recursive" + else: + when not defined(Windows): + recursive = "-maxdepth 1" + + var + dir = dir + file = file + if not recurse: + let + pdir = file.parentDir() + if pdir.len != 0: + dir = dir / pdir + + file = file.extractFilename + + cmd = cmd % [recursive, (".*[\\\\/]" & file & "$").quoteShell, dir.sanitizePath] + + let + (files, ret) = execAction(cmd, die = false) + if ret == 0: + for line in files.splitLines(): + let f = + when defined(Windows): + if ": " in line: + line.split(": ", maxsplit = 1)[1] + else: + "" + else: + line + if f.len != 0: + result.add f + +proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string = + ## Find the file in the specified directory + ## + ## `file` is a regular expression if `regex` is true + ## + ## Turn off recursive search with `recurse` and stop on first match with + ## `first`. Without it, the shortest match is returned. + let + matches = findFiles(file, dir, recurse, regex) + for match in matches: + if (result.len == 0 or result.len > match.len): + result = match + if first: break + +proc linkLibs*(names: openArray[string], staticLink = true): string = + ## Create linker flags for specified libraries using pkg-config + ## + ## Prepends `lib` to the name so you only need `ssl` for `libssl`. + var + stat = if staticLink: "--static" else: "" + resSet: OrderedSet[string] + resSet.init() + + for name in names: + let + cmd = &"pkg-config --libs --silence-errors {stat} lib{name}" + (libs, _) = execAction(cmd, die = false) + for lib in libs.split(" "): + resSet.incl lib + + if staticLink: + resSet.incl "--static" + + for res in resSet: + result &= " " & res + +proc getNumProcs(): string = + when defined(Windows): + getEnv("NUMBER_OF_PROCESSORS").strip() + elif defined(linux): + execAction("nproc").output.strip() + elif defined(macosx) or defined(FreeBSD): + execAction("sysctl -n hw.ncpu").output.strip() + else: + "1" diff --git a/nimterop/build/tools.nim b/nimterop/build/tools.nim new file mode 100644 index 0000000..772c48b --- /dev/null +++ b/nimterop/build/tools.nim @@ -0,0 +1,260 @@ +import os, strformat, strutils + +type + BuildType* = enum + btAutoconf, btCmake + + BuildStatus = object + built: bool + buildPath: string + error: string + +proc configure*(path, check: string, flags = "") = + ## Run the GNU `configure` command to generate all Makefiles or other + ## build scripts in the specified path + ## + ## If a `configure` script is not present and an `autogen.sh` script + ## is present, it will be run before attempting `configure`. + ## + ## Next, if `configure.ac` or `configure.in` exist, `autoreconf` will + ## be executed. + ## + ## `check` is a file that will be generated by the `configure` command. + ## This is required to prevent configure from running on every build. It + ## is relative to the `path` and should not be an absolute path. + ## + ## `flags` are any flags that should be passed to the `configure` command. + if (path / check).fileExists(): + return + + echo "# Configuring " & path + + if not fileExists(path / "configure"): + for i in @["autogen.sh", "build" / "autogen.sh"]: + if fileExists(path / i): + echo "# Running autogen.sh" + + when defined(unix): + echoDebug execAction( + &"cd {(path / i).parentDir().sanitizePath} && ./autogen.sh").output + else: + echoDebug execAction( + &"cd {(path / i).parentDir().sanitizePath} && bash ./autogen.sh").output + + break + + if not fileExists(path / "configure"): + for i in @["configure.ac", "configure.in"]: + if fileExists(path / i): + echo "# Running autoreconf" + + echoDebug execAction(&"cd {path.sanitizePath} && autoreconf -fi").output + + break + + if fileExists(path / "configure"): + echo "# Running configure " & flags + + when defined(unix): + var + cmd = &"cd {path.sanitizePath} && ./configure" + else: + var + cmd = &"cd {path.sanitizePath} && bash ./configure" + if flags.len != 0: + cmd &= &" {flags}" + + echoDebug execAction(cmd).output + + doAssert (path / check).fileExists(), "Configure failed" + +proc getCmakePropertyStr(name, property, value: string): string = + &"\nset_target_properties({name} PROPERTIES {property} \"{value}\")\n" + +proc getCmakeIncludePath*(paths: openArray[string]): string = + ## Create a `cmake` flag to specify custom include paths + ## + ## Result can be included in the `flag` parameter for `cmake()` or + ## the `cmakeFlags` parameter for `getHeader()`. + for path in paths: + result &= path & ";" + result = " -DCMAKE_INCLUDE_PATH=" & result[0 .. ^2].sanitizePath(sep = "/") + +proc setCmakeProperty*(outdir, name, property, value: string) = + ## Set a `cmake` property in `outdir / CMakeLists.txt` - usable in the `xxxPreBuild` hook + ## for `getHeader()` + ## + ## `set_target_properties(name PROPERTIES property "value")` + let + cm = outdir / "CMakeLists.txt" + if cm.fileExists(): + cm.writeFile( + cm.readFile() & getCmakePropertyStr(name, property, value) + ) + +proc setCmakeLibName*(outdir, name, prefix = "", oname = "", suffix = "") = + ## Set a `cmake` property in `outdir / CMakeLists.txt` to specify a custom library output + ## name - usable in the `xxxPreBuild` hook for `getHeader()` + ## + ## `prefix` is typically `lib` + ## `oname` is the library name + ## `suffix` is typically `.a` + ## + ## Sometimes, `cmake` generates non-standard library names - e.g. zlib compiles to + ## `libzlibstatic.a` on Windows. This proc can help rename it to `libzlib.a` so that `getHeader()` + ## can find it after the library is compiled. + ## + ## ``` + ## set_target_properties(name PROPERTIES PREFIX "prefix") + ## set_target_properties(name PROPERTIES OUTPUT_NAME "oname") + ## set_target_properties(name PROPERTIES SUFFIX "suffix") + ## ``` + let + cm = outdir / "CMakeLists.txt" + if cm.fileExists(): + var + str = "" + if prefix.len != 0: + str &= getCmakePropertyStr(name, "PREFIX", prefix) + if oname.len != 0: + str &= getCmakePropertyStr(name, "OUTPUT_NAME", oname) + if suffix.len != 0: + str &= getCmakePropertyStr(name, "SUFFIX", suffix) + if str.len != 0: + cm.writeFile(cm.readFile() & str) + +proc setCmakePositionIndependentCode*(outdir: string) = + ## Set a `cmake` directive to create libraries with -fPIC enabled + let + cm = outdir / "CMakeLists.txt" + if cm.fileExists(): + let + pic = "set(CMAKE_POSITION_INDEPENDENT_CODE ON)" + cmd = cm.readFile() + if not cmd.contains(pic): + cm.writeFile( + pic & "\n" & cmd + ) + +proc cmake*(path, check, flags: string) = + ## Run the `cmake` command to generate all Makefiles or other + ## build scripts in the specified path + ## + ## `path` will be created since typically `cmake` is run in an + ## empty directory. + ## + ## `check` is a file that will be generated by the `cmake` command. + ## This is required to prevent `cmake` from running on every build. It + ## is relative to the `path` and should not be an absolute path. + ## + ## `flags` are any flags that should be passed to the `cmake` command. + ## Unlike `configure`, it is required since typically it will be the + ## path to the repository, typically `..` when `path` is a subdir. + if (path / check).fileExists(): + return + + echo "# Running cmake " & flags + echo "# Path: " & path + + mkDir(path) + + let + cmd = &"cd {path.sanitizePath} && cmake {flags}" + + echoDebug execAction(cmd).output + + doAssert (path / check).fileExists(), "cmake failed" + +proc make*(path, check: string, flags = "", regex = false) = + ## Run the `make` command to build all binaries in the specified path + ## + ## `check` is a file that will be generated by the `make` command. + ## This is required to prevent `make` from running on every build. It + ## is relative to the `path` and should not be an absolute path. + ## + ## `flags` are any flags that should be passed to the `make` command. + ## + ## `regex` can be set to true if `check` is a regular expression. + ## + ## If `make.exe` is missing and `mingw32-make.exe` is available, it will + ## be copied over to make.exe in the same location. + if findFile(check, path, regex = regex).len != 0: + return + + echo "# Running make " & flags + echo "# Path: " & path + + var + cmd = findExe("make") + + if cmd.len == 0: + cmd = findExe("mingw32-make") + if cmd.len != 0: + cpFile(cmd, cmd.replace("mingw32-make", "make")) + doAssert cmd.len != 0, "Make not found" + + cmd = &"cd {path.sanitizePath} && make" + if flags.len != 0: + cmd &= &" {flags}" + + echoDebug execAction(cmd).output + + doAssert findFile(check, path, regex = regex).len != 0, "make failed" + +proc buildWithCmake(outdir, flags: string): BuildStatus = + if not fileExists(outdir / "Makefile"): + if fileExists(outdir / "CMakeLists.txt"): + if findExe("cmake").len != 0: + var + gen = "" + when defined(Windows): + if findExe("sh").len != 0: + let + uname = execAction("sh -c uname -a").output.toLowerAscii() + if uname.contains("msys"): + gen = "MSYS Makefiles".quoteShell + elif uname.contains("mingw"): + gen = "MinGW Makefiles".quoteShell & " -DCMAKE_SH=\"CMAKE_SH-NOTFOUND\"" + else: + echo "Unsupported system: " & uname + else: + gen = "MinGW Makefiles".quoteShell + else: + gen = "Unix Makefiles".quoteShell + if findExe("ccache").len != 0: + gen &= " -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + result.buildPath = outdir / "buildcache" + cmake(result.buildPath, "Makefile", &".. -G {gen} {flags}") + result.built = true + else: + result.error = "cmake capable but cmake executable missing" + else: + result.buildPath = outdir + +proc buildWithAutoConf(outdir, flags: string): BuildStatus = + if not fileExists(outdir / "Makefile"): + if findExe("bash").len != 0: + for file in @["configure", "configure.ac", "configure.in", "autogen.sh", "build/autogen.sh"]: + if fileExists(outdir / file): + configure(outdir, "Makefile", flags) + result.buildPath = outdir + result.built = true + break + else: + result.error = "configure capable but bash executable missing" + else: + result.buildPath = outdir + +proc flagBuild*(base: string, flags: openArray[string]): string = + ## Simple helper proc to generate flags for `configure`, `cmake`, etc. + ## + ## Every entry in `flags` is replaced into the `base` string and + ## concatenated to the result. + ## + ## E.g. + ## `base = "--disable-$#"` + ## `flags = @["one", "two"]` + ## + ## `flagBuild(base, flags) => " --disable-one --disable-two"` + for i in flags: + result &= " " & base % i diff --git a/nimterop/git.nim b/nimterop/git.nim deleted file mode 100644 index 7a0cf2a..0000000 --- a/nimterop/git.nim +++ /dev/null @@ -1 +0,0 @@ -include build \ No newline at end of file diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 1844a56..495e5d1 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,7 +2,9 @@ import os, osproc, sets, strformat, strutils, tables, times import "."/treesitter/[api, c, cpp] -import "."/[ast, ast2, build, globals, getters, grammar, tshelp] +import "."/[build, globals] + +import "."/toastlib/[ast, ast2, getters, grammar, tshelp] proc process(gState: State, path: string, astTable: AstTable) = doAssert existsFile(path), &"Invalid path {path}" diff --git a/nimterop/ast.nim b/nimterop/toastlib/ast.nim similarity index 99% rename from nimterop/ast.nim rename to nimterop/toastlib/ast.nim index 78501fd..0b62a34 100644 --- a/nimterop/ast.nim +++ b/nimterop/toastlib/ast.nim @@ -2,7 +2,8 @@ import hashes, macros, os, sets, strformat, strutils, tables import regex -import "."/[getters, globals, treesitter/api, tshelp] +import ".."/[globals, treesitter/api] +import "."/[getters, tshelp] proc getHeaderPragma*(gState: State): string = result = diff --git a/nimterop/ast2.nim b/nimterop/toastlib/ast2.nim similarity index 99% rename from nimterop/ast2.nim rename to nimterop/toastlib/ast2.nim index de3b452..ffe961e 100644 --- a/nimterop/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -2,9 +2,9 @@ import macros, os, sequtils, sets, strformat, strutils, tables, times import compiler/[ast, idents, lineinfos, modulegraphs, msgs, options, renderer] -import "."/treesitter/api +import ".."/[globals, treesitter/api] -import "."/[comphelp, exprparser, globals, getters, tshelp] +import "."/[comphelp, exprparser, getters, tshelp] proc getOverrideOrSkip(gState: State, node: TSNode, origname: string, kind: NimSymKind): PNode = # Check if symbol `origname` of `kind` and `origname` has any cOverride defined diff --git a/nimterop/comphelp.nim b/nimterop/toastlib/comphelp.nim similarity index 98% rename from nimterop/comphelp.nim rename to nimterop/toastlib/comphelp.nim index e577717..d404c42 100644 --- a/nimterop/comphelp.nim +++ b/nimterop/toastlib/comphelp.nim @@ -2,7 +2,8 @@ import macros, strutils import compiler/[ast, idents, lineinfos, msgs, options, parser, pathutils, renderer] -import "."/[globals, getters, treesitter/api, tshelp] +import ".."/[globals, treesitter/api] +import "."/[getters, tshelp] proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string) = # Raise exception in parseString() instead of exiting for errors diff --git a/nimterop/exprparser.nim b/nimterop/toastlib/exprparser.nim similarity index 99% rename from nimterop/exprparser.nim rename to nimterop/toastlib/exprparser.nim index 0d2099a..dc9f97e 100644 --- a/nimterop/exprparser.nim +++ b/nimterop/toastlib/exprparser.nim @@ -4,9 +4,9 @@ import regex import compiler/[ast, renderer] -import "."/treesitter/[api, c, cpp] - -import "."/[globals, getters, comphelp, tshelp] +import ".."/treesitter/[api, c, cpp] +import ".."/globals +import "."/[getters, comphelp, tshelp] # This version of exprparser should be able to handle: # diff --git a/nimterop/getters.nim b/nimterop/toastlib/getters.nim similarity index 99% rename from nimterop/getters.nim rename to nimterop/toastlib/getters.nim index b092663..b4070f3 100644 --- a/nimterop/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -2,7 +2,7 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import regex -import "."/[build, globals, plugin] +import ".."/[build, globals, plugin] const gReserved = """ addr and as asm diff --git a/nimterop/grammar.nim b/nimterop/toastlib/grammar.nim similarity index 99% rename from nimterop/grammar.nim rename to nimterop/toastlib/grammar.nim index e061a70..52b19e7 100644 --- a/nimterop/grammar.nim +++ b/nimterop/toastlib/grammar.nim @@ -2,7 +2,8 @@ import macros, strformat, strutils, tables import regex -import "."/[ast, getters, globals, lisp, treesitter/api, tshelp] +import ".."/[globals, treesitter/api] +import "."/[ast, getters, lisp, tshelp] type Grammar = seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode, gState: State) {.nimcall.}]] diff --git a/nimterop/lisp.nim b/nimterop/toastlib/lisp.nim similarity index 97% rename from nimterop/lisp.nim rename to nimterop/toastlib/lisp.nim index 8287cf5..57d71a1 100644 --- a/nimterop/lisp.nim +++ b/nimterop/toastlib/lisp.nim @@ -1,4 +1,5 @@ -import "."/[getters, globals] +import ".."/globals +import "."/getters var gTokens: seq[string] diff --git a/nimterop/tshelp.nim b/nimterop/toastlib/tshelp.nim similarity index 99% rename from nimterop/tshelp.nim rename to nimterop/toastlib/tshelp.nim index ad3e89e..91f77ec 100644 --- a/nimterop/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -1,7 +1,8 @@ import sets, strformat, strutils -import "."/[getters, globals] -import "."/treesitter/[api, c, cpp] +import ".."/treesitter/[api, c, cpp] +import ".."/globals +import "."/getters template withCodeAst*(code: string, mode: string, body: untyped): untyped = ## A simple template to inject the TSNode into a body of code From 69e372beeaa4c1aa306efb13237bf274a33a1f05 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 20 Jun 2020 13:52:01 -0500 Subject: [PATCH 039/106] Improve octal fix --- nimterop/build.nim | 5 ++++- nimterop/build/nimconf.nim | 3 ++- nimterop/toastlib/exprparser.nim | 22 ++++++++++------------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 99adeca..0754615 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1,4 +1,7 @@ -import hashes, macros, osproc, sets, strformat, strutils, tables +import hashes, osproc, sets, strformat, strutils + +when not defined(TOAST): + import macros, tables import os except findExe, sleep diff --git a/nimterop/build/nimconf.nim b/nimterop/build/nimconf.nim index 98fc8fe..3ba5ed0 100644 --- a/nimterop/build/nimconf.nim +++ b/nimterop/build/nimconf.nim @@ -1,4 +1,4 @@ -import json, macros, os, osproc, sets, strformat, strutils +import json, os, osproc, sets, strformat, strutils when nimvm: when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): @@ -55,6 +55,7 @@ proc getProjectDir*(): string = result = querySetting(projectFull).parentDir() else: # Get from `macros` + import macros result = getProjectPath() else: discard diff --git a/nimterop/toastlib/exprparser.nim b/nimterop/toastlib/exprparser.nim index dc9f97e..28b49d3 100644 --- a/nimterop/toastlib/exprparser.nim +++ b/nimterop/toastlib/exprparser.nim @@ -140,15 +140,16 @@ proc getIntNode(number, suffix: string): PNode {.inline.} = var val: BiggestInt flags: TNodeFlags - if number.startsWith("0X") or number.startsWith("0x"): - val = parseHexInt(number) - flags = {nfBase16} - elif number.startsWith("0B") or number.startsWith("0b"): - val = parseBinInt(number) - flags = {nfBase2} - elif number.startsWith("0O") or number.startsWith("0o"): - val = parseOctInt(number) - flags = {nfBase8} + if number.len > 1 and number[0] == '0': + if number[1] in ['x', 'X']: + val = parseHexInt(number) + flags = {nfBase16} + elif number[1] in ['b', 'B']: + val = parseBinInt(number) + flags = {nfBase2} + else: + val = parseOctInt(number) + flags = {nfBase8} else: val = parseInt(number) @@ -201,9 +202,6 @@ proc processNumberLiteral(gState: State, node: TSNode): PNode = if number.startsWith("-"): number = number[1 ..< number.len] prefix = "-" - if number.len > 1 and number[0] == '0' and number[1] notin ['x', 'X']: - # Octal 0123 - number = "0o" & number[1 .. ^1] if tripleEndings.any(proc (s: string): bool = number.endsWith(s)): suffix = number[^3 .. ^1] number = number[0 ..< ^3] From c3228683fa91e01c75a8676479ee13a41c7f5a02 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 20 Jun 2020 14:21:59 -0500 Subject: [PATCH 040/106] Fix macros bug --- nimterop/build/nimconf.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nimterop/build/nimconf.nim b/nimterop/build/nimconf.nim index 3ba5ed0..c87a60f 100644 --- a/nimterop/build/nimconf.nim +++ b/nimterop/build/nimconf.nim @@ -3,6 +3,8 @@ import json, os, osproc, sets, strformat, strutils when nimvm: when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): import std/compilesettings + else: + import macros else: discard @@ -55,7 +57,6 @@ proc getProjectDir*(): string = result = querySetting(projectFull).parentDir() else: # Get from `macros` - import macros result = getProjectPath() else: discard From d07bd2d71e194a3ed1c0bd5eec9537db26b0bf3f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 20 Jun 2020 22:06:02 -0500 Subject: [PATCH 041/106] findExe sanitizePath --- nimterop/build/shell.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 4f3c316..235f7b5 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -86,7 +86,7 @@ proc findExe*(exe: string): string = (output, ret) = execAction(cmd, die = false) if ret == 0: - return output.splitLines()[0].strip() + return output.splitLines()[0].strip().sanitizePath proc mkDir*(dir: string) = ## Create a directory at compile time From 6c3142a27c186ed4b02bf717ad2dafc37fe81440 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 23 Jun 2020 09:50:00 -0500 Subject: [PATCH 042/106] Fix findFile with regex --- nimterop/build/shell.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 235f7b5..5179798 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -393,7 +393,8 @@ proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[s var dir = dir file = file - if not recurse: + # If file = `path/file`, adjust dir = `dir/path` and search for new file + if not (recurse or regex): let pdir = file.parentDir() if pdir.len != 0: From 77380630383f37f08d0ead6438aa4c90966b8d25 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 23 Jun 2020 11:42:50 -0500 Subject: [PATCH 043/106] Fix #231 - const func type, comment in struct --- nimterop/toastlib/ast2.nim | 13 +++--- nimterop/toastlib/tshelp.nim | 2 +- tests/include/tast2.h | 83 ++++++++++++++++++++++++++++++++++-- tests/tast2.nim | 4 ++ 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index ffe961e..701b0b4 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1238,8 +1238,9 @@ proc addType(gState: State, node: TSNode, union = false) = gState.addTypeObject(node[0], union = union) else: let - fdecl = node[1].anyChildInTree("function_declarator") - adecl = node[1].anyChildInTree("array_declarator") + start = node.getStartAtom() + fdecl = node[start+1].anyChildInTree("function_declarator") + adecl = node[start+1].anyChildInTree("array_declarator") if fdlist.isNil: if adecl.isNil and fdecl.isNil: # typedef X Y; @@ -1341,9 +1342,9 @@ proc addType(gState: State, node: TSNode, union = false) = # First add struct as object decho("addType(): case 6") - gState.addTypeObject(node[0], union = union) + gState.addTypeObject(node[start], union = union) - if node.len > 1 and gState.getNodeVal(node[1]) != "": + if node.len > start+1 and gState.getNodeVal(node[start+1]) != "": # Add any additional names gState.addTypeTyped(node) else: @@ -1357,14 +1358,14 @@ proc addType(gState: State, node: TSNode, union = false) = name = block: var name = "" - for i in 1 ..< node.len: + for i in start+1 ..< node.len: if node[i].getName() == "type_identifier": name = gState.getNodeVal(node[i].getAtom()) name # Now add struct as object with specified name - gState.addTypeObject(node[0], fname = name, istype = true, union = union) + gState.addTypeObject(node[start], fname = name, istype = true, union = union) if name.nBl: # Add any additional names diff --git a/nimterop/toastlib/tshelp.nim b/nimterop/toastlib/tshelp.nim index 91f77ec..b055259 100644 --- a/nimterop/toastlib/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -62,7 +62,7 @@ proc getAtom*(node: TSNode): TSNode = if node.getName() in gAtoms: return node elif node.len != 0: - if node[0].getName() == "type_qualifier": + if node[0].getName() in ["type_qualifier", "comment"]: # Skip const, volatile if node.len > 1: return node[1].getAtom() diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 6f569d5..89ad486 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -61,6 +61,7 @@ struct some_struct_s struct parent_struct_s { + /* Random comment */ struct some_struct_s s[SOME_CONST]; }; @@ -108,6 +109,9 @@ typedef struct A20 { char a1; } A20, A21, *A21p; //Expression typedef struct A22 { const int **f1; int *f2[123+132]; } A22; +// #231 +typedef const char *(*A23)(); + //Unions union U1 {int f1; float f2; }; typedef union U2 { const int **f1; int abc[123+132]; } U2; @@ -239,6 +243,7 @@ typedef struct { struct { int f1; } f2; struct NT3 { + /* Random comment */ struct { int f1; union NU1 { @@ -277,6 +282,65 @@ struct TestMyInt { #define C 0x10 #define D "hello" #define E 'c' +#define F 01234 + +#define UEXPR (1234u << 1) +#define ULEXPR (1234ul << 2) +#define ULLEXPR (1234ull << 3) +#define LEXPR (1234l << 4) +#define LLEXPR (1234ll << 5) + +#define SHL1 (1u << 1) +#define SHL2 (1u << 2) +#define SHL3 (1u << 3) +#define COERCE 645635634896ull + 35436 +#define COERCE2 645635634896 + 35436ul +#define BINEXPR ~(-(1u << !-1)) ^ (10 >> 1) +#define POINTEREXPR (int*)0 +#define POINTERPOINTERPOINTEREXPR (int***)0 +#define BOOL true +#define MATHEXPR (1 + 2/3*20 - 100) +#define ANDEXPR (100 & 11000) +#define CASTEXPR (char) 34 +#define AVAL 100 +#define BVAL 200 +#define EQ1 AVAL <= BVAL +#define EQ2 AVAL >= BVAL +#define EQ3 AVAL > BVAL +#define EQ4 AVAL < BVAL +#define EQ5 AVAL != BVAL +#define EQ6 AVAL == BVAL + +// testing integer out of long int range +#define INT_FAST16_MIN (-9223372036854775807L-1) + +#define SIZEOF sizeof(char) +#define REG_STR "regular string" +#define NOTSUPPORTEDSTR "not a " REG_STR + +#define NULLCHAR '\0' +#define OCTCHAR '\012' +#define HEXCHAR '\xFE' +#define TRICKYSTR "\x4E\034\nfoo\0\'\"\r\v\a\b\e\f\t\\\?bar" + +#define ALLSHL (SHL1 | SHL2 | SHL3) + +#ifdef NIMTEROP +#define SOME_CONST 8 +#endif + +struct some_struct_s +{ + int x; +}; + +struct parent_struct_s +{ + /* Random comment */ + struct some_struct_s s[SOME_CONST]; +}; + +typedef struct some_struct_s SOME_ARRAY[SOME_CONST]; struct A0; struct A1 {}; @@ -303,10 +367,10 @@ typedef char *(*A11)[3]; typedef struct A0 *A111[12]; typedef int - **(*A12)(int, int b, int *c, int *, int *count[4], int (*func)(int, int)), + **(*A12)(int, int b, int *c, int *, int /*out*/ *count[4], int (*func)(int, int)), **(*A121)(float, float b, float *c, float *, float *count[4], float (*func)(float, float)), **(*A122)(char, char b, char *c, char *, char *count[4], char (*func)(char, char)); -typedef int A13(int, int, void (*func)(void)); +typedef int (*A13)(int, int, void (*func)(void)); struct A14 { volatile char a1; }; struct A15 { char *a1; const int *a2[1]; }; @@ -320,6 +384,9 @@ typedef struct A20 { char a1; } A20, A21, *A21p; //Expression typedef struct A22 { const int **f1; int *f2[123+132]; } A22; +// #231 +typedef const char *(*A23)(); + //Unions union U1 {int f1; float f2; }; typedef union U2 { const int **f1; int abc[123+132]; } U2; @@ -403,6 +470,16 @@ void int sqlite3_bind_blob(struct A1*, int, const void*, int n, void(*)(void*)); +// Issue #174 - type name[] => UncheckedArray[type] +int ucArrFunc1(int text[]); +int ucArrFunc2(int text[][5], int (*func)(int text[])); + +typedef int ucArrType1[][5]; +struct ucArrType2 { + float f1[5][5]; + int *f2[][5]; +}; + typedef struct fieldfuncfunc { int *(*func1)(int f1, int *(*sfunc1)(int f1, int *(*ssfunc1)(int f1, ...))); }; @@ -441,6 +518,7 @@ typedef struct { struct { int f1; } f2; struct NT3 { + /* Random comment */ struct { int f1; union NU1 { @@ -469,7 +547,6 @@ struct TestMyInt { }; - #endif #ifdef __cplusplus diff --git a/tests/tast2.nim b/tests/tast2.nim index 124c4e4..3f626ae 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -333,6 +333,10 @@ checkPragmas(A22, pHeaderBy, istype = false) var a22: A22 a22.f1 = addr a15.a2[0] +assert A23 is proc(): cstring {.cdecl.} +checkPragmas(A23, pHeaderImp & "cdecl") +var a23: A23 + assert U1 is object assert sizeof(U1) == sizeof(cfloat) checkPragmas(U1, pHeaderBy & @["union"], istype = false) From e44ab30af97b8aceefc2568a83fb59c3e5ac21cd Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 23 Jun 2020 12:48:18 -0500 Subject: [PATCH 044/106] Fix rebase issues --- nimterop/build/conan.nim | 2 -- tests/getheader.nims | 13 ++++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index 7e1bd8e..3430752 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -387,8 +387,6 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = cleanDir(outdir) - echo &"# Downloading {pkg.name} v{pkg.version} from Conan" - pkg.getConanBuilds() doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" diff --git a/tests/getheader.nims b/tests/getheader.nims index 2fe4d9c..916abcf 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -25,8 +25,8 @@ when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): cmd &= " --gc:arc" testCall(cmd & lrcmd, "No build files found", 1) -testCall(cmd & " -d:libssh2Conan" & sshcmd, "Need version for Conan uri", 1) -testCall(cmd & " -d:libssh2JBB" & sshcmd, "Need version for BinaryBuilder.org uri", 1) +testCall(cmd & " -d:libssh2Conan" & sshcmd, "Need version for Conan.io uri", 1) +testCall(cmd & " -d:libssh2JBB -d:libssh2SetVer=1.9.0" & sshcmd, "Version in both uri", 1) when defined(posix): # stdlib @@ -43,17 +43,15 @@ when defined(posix): # conan static testCall(cmd & " -d:libssh2Conan -d:libssh2SetVer=1.9.0 -d:libssh2Static" & sshcmd, zexp, 0) -<<<<<<< HEAD else: # conan static for Windows testCall(cmd & " -d:zlibConan -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) # JBB -testCall(cmd & " -d:libssh2JBB -d:libssh2SetVer=1.9.0" & sshcmd, zexp, 0) +testCall(cmd & " -d:libssh2JBB" & sshcmd, zexp, 0) testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11" & zrcmd, zexp, 0) -testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0) -======= ->>>>>>> c35eb74... Add tests for conan, recurse implies preprocess +testCall(cmd & " -d:zlibJBB -d:zlibSetVer=1.2.11 -d:zlibStatic" & zrcmd, zexp, 0, delete = false) +testCall(cmd & " -d:lzmaJBB -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0) # git testCall(cmd & " -d:envTest" & zrcmd, zexp, 0) @@ -74,3 +72,4 @@ testCall(cmd & " -d:zlibDL -d:zlibStatic -d:zlibSetVer=1.2.11" & zrcmd, zexp & " # conan testCall(cmd & " -d:libssh2Conan -d:libssh2SetVer=1.9.0" & sshcmd, zexp, 0) +testCall(cmd & " -d:lzmaConan -d:lzmaSetVer=5.2.4" & lrcmd, lexp & "5.2.4", 0) \ No newline at end of file From a8ea96055d9d1276fff401ffb1866e1f34d6bc46 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 23 Jun 2020 23:20:15 -0500 Subject: [PATCH 045/106] dynlib allow path, don't delete project, fix proc inline comment --- CHANGES.md | 1 + README.md | 2 +- nimterop/build/getheader.nim | 6 ++++++ nimterop/toast.nim | 2 +- nimterop/toastlib/ast2.nim | 5 ++++- nimterop/toastlib/tshelp.nim | 8 +++++++- 6 files changed, 20 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cedf5a5..bf4b50a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,6 +27,7 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 - `getHeader()` now detects and links against `.lib` files as part of enabling Conan.io. Not all `.lib` files are compatible with MinGW as already stated above but for those that work, this is a required capability. +- The `dynlib` command line parameter to `toast` and `cImport()` can also be the path to a shared library (dll|so|dylib) in place of a Nim const string containing the path. This allows for the traditional use case of passing `"xxxLPath"` to `cImport()` as well as simply passing the path to the library on the command line as is. This allows the creation of standalone cached wrappers as well as the usage of the `--check` and the `--stub` functionality that `toast` provides via `cImport()`. ## Version 0.5.0 diff --git a/README.md b/README.md index ca17b90..af8501a 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ Options: -C=, --convention= string "cdecl" calling convention for wrapped procs -d, --debug bool false enable debug output -D=, --defines= strings {} definitions to pass to preprocessor - -l=, --dynlib= string "" import symbols from library in specified Nim string + -l=, --dynlib= string "" {.dynlib.} pragma to import symbols - Nim const string or file path -f=, --feature= Features ast1 flags to enable experimental features -I=, --includeDirs= strings {} include directory to pass to preprocessor -m=, --mode= string "" language parser: c or cpp diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 3852fbf..69741b0 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -392,6 +392,12 @@ macro getHeader*( else: getLocalPath(header, outdir) + static: + # Don't delete project + when not `nameStd` and (`nameGit` or `nameDL` or `nameConan` or `nameJBB`): + doAssert `outdir`.len != 0, "getHeader():outdir cannot be blank" + doAssert `outdir` != getProjectPath(), "getHeader():outdir cannot be the project path" + const `version`* {.strdefine.} = `verVal` `lname` = diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 495e5d1..e8e14cc 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -240,7 +240,7 @@ when isMainModule: "convention": "calling convention for wrapped procs", "debug": "enable debug output", "defines": "definitions to pass to preprocessor", - "dynlib": "import symbols from library in specified Nim string", + "dynlib": "{.dynlib.} pragma to import symbols - Nim const string or file path", "feature": "flags to enable experimental features", "includeDirs": "include directory to pass to preprocessor", "mode": "language parser: c or cpp", diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 701b0b4..0016e29 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1805,7 +1805,10 @@ proc setupPragmas(gState: State, root: TSNode, fullpath: string) = # {.pragma: impnameDyn, dynlib: libname.} let dynPragma = gState.newPragma(root, "pragma", gState.getIdent(gState.impShort & "Dyn")) - gState.addPragma(root, dynPragma, "dynlib", gState.getIdent(gState.dynlib)) + if '.' in gState.dynlib: + gState.addPragma(root, dynPragma, "dynlib", newStrNode(nkStrLit, gState.dynlib)) + else: + gState.addPragma(root, dynPragma, "dynlib", gState.getIdent(gState.dynlib)) gState.pragmaSection.add dynPragma count += 1 diff --git a/nimterop/toastlib/tshelp.nim b/nimterop/toastlib/tshelp.nim index b055259..e7e8d75 100644 --- a/nimterop/toastlib/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -154,7 +154,13 @@ proc firstChildInTree*(node: TSNode, ntype: string): TSNode = while not cnode.isNil: if cnode.getName() == ntype: return cnode - cnode = cnode[0] + if cnode.len != 0: + for i in 0 ..< cnode.len: + if cnode[i].getName() != "comment": + cnode = cnode[i] + break + else: + cnode = cnode[0] proc anyChildInTree*(node: TSNode, ntype: string): TSNode = # Search for node type anywhere in tree - depth first From d126e9944fee992f46bd51d2f65f3db752249163 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 24 Jun 2020 10:37:38 -0500 Subject: [PATCH 046/106] Remove legacy backend --- CHANGES.md | 2 + README.md | 9 +- nimterop.nimble | 13 +- nimterop/build.nim | 8 +- nimterop/build/getheader.nim | 3 +- nimterop/build/shell.nim | 3 +- nimterop/build/tools.nim | 9 +- nimterop/globals.nim | 27 +- nimterop/toast.nim | 39 +- nimterop/toastlib/ast.nim | 253 ----------- nimterop/toastlib/getters.nim | 125 ------ nimterop/toastlib/grammar.nim | 768 ---------------------------------- nimterop/toastlib/lisp.nim | 60 --- nimterop/toastlib/tshelp.nim | 1 - tests/getheader.nims | 2 +- tests/libssh2.nim | 4 +- tests/rsa.nim | 2 +- 17 files changed, 33 insertions(+), 1295 deletions(-) delete mode 100644 nimterop/toastlib/ast.nim delete mode 100644 nimterop/toastlib/grammar.nim delete mode 100644 nimterop/toastlib/lisp.nim diff --git a/CHANGES.md b/CHANGES.md index bf4b50a..93bb8af 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 ### Breaking changes +- The legacy algorithm has been removed as promised. `ast2` is now the default and wrappers no longer need to explicitly specify `-f:ast2` in order to use it. + - All shared libraries installed by `getHeader()` will now get copied into the `libdir` parameter specified. If left blank, `libdir` will default to the directory where the executable binary gets created (outdir). While this is not really a breaking change, it is a change in behavior compared to older versions of nimterop. Note that `Std` libraries are not copied over. [#154](i154) - `git.nim` has been removed. This module was an artifact from the early days and was renamed to `build.nim` back in v0.2.0. diff --git a/README.md b/README.md index af8501a..b054d97 100644 --- a/README.md +++ b/README.md @@ -177,13 +177,13 @@ Now that this is understood, a user might want any combination of the above in t - By default, generated wrappers will include the `{.header, importc.}` pragmas for types and procs. This can be disabled with the `--noHeader | -H` flag to `toast` or `flags = "-H"` param to `cImport()` which will remove `{.header}` for both and `{.importc.}` for types only. - By default, generated wrappers will assume that the user will link the library implementation themselves. The `--dynlib | -l` flag to `toast` or `dynlib = "headerLPath"` param to `cImport()` will configure the wrapper to generate `{.dynlib.}` pragmas for procs. -This results in four cases: +This results in four supported cases: 1. Default: `{.header, importc.}` for both types and procs 2. With `--noHeader`, types will be pure Nim and procs will be just `{.importc.}` 3. With `--dynlib`, types will still be `{.header, importc.}` but procs will be `{.dynlib, importc.}` 4. With `--dynlib` and `--noHeader`, types will be pure Nim, procs will be `{.dynlib, importc.}` -While `ast2` supports all these modes, the legacy backend does not support the third mixed case and will infer `--noHeader` when `--dynlib` is specified (case 4). Creation of a standalone wrapper (case 4) which does not require the header or library at compile time will require an explicit `--noHeader` and `--dynlib` for `ast2`. +Creation of a standalone wrapper (case 4) which does not require the header or library at compile time will require an explicit `--noHeader` and `--dynlib`. More documentation on on these pragmas can be found in the Nim manual: - [{.importc.}](https://nim-lang.org/docs/manual.html#foreign-function-interface-importc-pragma) @@ -218,7 +218,7 @@ Options: -d, --debug bool false enable debug output -D=, --defines= strings {} definitions to pass to preprocessor -l=, --dynlib= string "" {.dynlib.} pragma to import symbols - Nim const string or file path - -f=, --feature= Features ast1 flags to enable experimental features + -f=, --feature= Features {} flags to enable experimental features -I=, --includeDirs= strings {} include directory to pass to preprocessor -m=, --mode= string "" language parser: c or cpp --nim= string "nim" use a particular Nim executable @@ -226,12 +226,11 @@ Options: -H, --noHeader bool false skip {.header.} pragma in wrapper -o=, --output= string "" file to output content - default: stdout -a, --past bool false print AST output - -g, --pgrammar bool false print grammar --pluginSourcePath= string "" nim file to build and load as a plugin -n, --pnim bool false print Nim output -E=, --prefix= strings {} strip prefix from identifiers -p, --preprocess bool false run preprocessor on header - -r, --recurse bool false process #include files, implies --preprocess + -r, --recurse bool false process #include files - implies --preprocess -G=, --replace= strings {} replace X with Y in identifiers, X1=Y1,X2=Y2, @X for regex -s, --stub bool false stub out undefined type references as objects -F=, --suffix= strings {} strip suffix from identifiers diff --git a/nimterop.nimble b/nimterop.nimble index 1657aed..7be89c6 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.5.9" +version = "0.6.0" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" @@ -55,15 +55,12 @@ task test, "Test": execTest "tests/tast2.nim", "-d:NOHEADER" execTest "tests/tnimterop_c.nim" - execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2\"" - execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-f:ast2 -H\"" + execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-H\"" execCmd "nim cpp --hints:off -f -r tests/tnimterop_cpp.nim" execCmd "./nimterop/toast tests/toast.cfg tests/include/toast.h" - execCmd "./nimterop/toast tests/toast.cfg -f:ast2 tests/include/toast.h" execTest "tests/tpcre.nim" - execTest "tests/tpcre.nim", "-d:FLAGS=\"-f:ast2\"" when defined(Linux): execTest "tests/rsa.nim" @@ -72,12 +69,10 @@ task test, "Test": # Platform specific tests when defined(Windows): execTest "tests/tmath.nim" - execTest "tests/tmath.nim", "-d:FLAGS=\"-f:ast2\"" - execTest "tests/tmath.nim", "-d:FLAGS=\"-f:ast2 -H\"" + execTest "tests/tmath.nim", "-d:FLAGS=\"-H\"" if defined(OSX) or defined(Windows) or not existsEnv("TRAVIS"): execTest "tests/tsoloud.nim" - execTest "tests/tsoloud.nim", "-d:FLAGS=\"-f:ast2\"" - execTest "tests/tsoloud.nim", "-d:FLAGS=\"-f:ast2 -H\"" + execTest "tests/tsoloud.nim", "-d:FLAGS=\"-H\"" # getHeader tests withDir("tests"): diff --git a/nimterop/build.nim b/nimterop/build.nim index 0754615..672cb58 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -14,12 +14,6 @@ var gNimExe* = "" # Misc helpers -proc echoDebug(str: string) = - let str = "\n# " & str.strip().replace("\n", "\n# ") - when nimvm: - if gDebugCT: echo str - else: - if gDebug: echo str proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) @@ -34,7 +28,7 @@ proc getCurrentNimCompiler*(): string = else: result = gNimExe -template fixOutDir() {.dirty.} = +template fixOutDir() {.dirty, used.} = let outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 69741b0..216b599 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -178,7 +178,6 @@ proc getLocalPath(header, outdir: string): string = proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildTypes: openArray[BuildType]): string = var lpath = findFile(lname, outdir, regex = true) - makeFlagsProc = &"-j {getNumProcs()} {makeFlags}" makePath = outdir if lpath.len != 0: @@ -200,7 +199,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT let libraryExists = findFile(lname, buildStatus.buildPath, regex = true).len > 0 if not libraryExists and fileExists(buildStatus.buildPath / "Makefile"): - make(buildStatus.buildPath, lname, makeFlagsProc, regex = true) + make(buildStatus.buildPath, lname, makeFlags, regex = true) buildStatus.built = true let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 5179798..bc07285 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -455,7 +455,8 @@ proc linkLibs*(names: openArray[string], staticLink = true): string = for res in resSet: result &= " " & res -proc getNumProcs(): string = +proc getNumProcs*(): string = + ## Get number of processors when defined(Windows): getEnv("NUMBER_OF_PROCESSORS").strip() elif defined(linux): diff --git a/nimterop/build/tools.nim b/nimterop/build/tools.nim index 772c48b..c6a4283 100644 --- a/nimterop/build/tools.nim +++ b/nimterop/build/tools.nim @@ -9,6 +9,13 @@ type buildPath: string error: string +proc echoDebug(str: string) = + let str = "\n# " & str.strip().replace("\n", "\n# ") + when nimvm: + if gDebugCT: echo str + else: + if gDebug: echo str + proc configure*(path, check: string, flags = "") = ## Run the GNU `configure` command to generate all Makefiles or other ## build scripts in the specified path @@ -193,7 +200,7 @@ proc make*(path, check: string, flags = "", regex = false) = cpFile(cmd, cmd.replace("mingw32-make", "make")) doAssert cmd.len != 0, "Make not found" - cmd = &"cd {path.sanitizePath} && make" + cmd = &"cd {path.sanitizePath} && make -j {getNumProcs()}" if flags.len != 0: cmd &= &" {flags}" diff --git a/nimterop/globals.nim b/nimterop/globals.nim index c79f4a5..fa8f8de 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -3,8 +3,6 @@ import tables when defined(TOAST): import sets, sequtils, strutils - import regex - import "."/plugin import compiler/[ast, idents, modulegraphs, options] @@ -13,7 +11,7 @@ when defined(TOAST): type Feature* = enum - ast1, ast2 + ast2 State* = ref object # Command line arguments to toast - some forwarded from cimport.nim @@ -72,12 +70,6 @@ type # Controls whether or not the current expression # should validate idents against currently defined idents skipIdentValidation*: bool - - # Legacy AST fields, remove when ast2 becomes default - constStr*, enumStr*, procStr*, typeStr*: string - commentStr*, debugStr*, skipStr*: string - data*: seq[tuple[name, val: string]] - nodeBranch*: seq[string] else: # cimport.nim specific compile*: seq[string] # `cCompile()` list of files already processed @@ -114,23 +106,6 @@ when defined(TOAST): ].concat(toSeq(gExpressions.items)) type - Kind* = enum - exactlyOne - oneOrMore # + - zeroOrMore # * - zeroOrOne # ? - orWithNext # ! - - Ast* = object - name*: string - kind*: Kind - recursive*: bool - children*: seq[ref Ast] - tonim*: proc (ast: ref Ast, node: TSNode, gState: State) - regex*: Regex - - AstTable* {.used.} = TableRef[string, seq[ref Ast]] - Status* = enum success, unknown, error diff --git a/nimterop/toast.nim b/nimterop/toast.nim index e8e14cc..b7a993a 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -4,9 +4,9 @@ import "."/treesitter/[api, c, cpp] import "."/[build, globals] -import "."/toastlib/[ast, ast2, getters, grammar, tshelp] +import "."/toastlib/[ast2, getters, tshelp] -proc process(gState: State, path: string, astTable: AstTable) = +proc process(gState: State, path: string) = doAssert existsFile(path), &"Invalid path {path}" if gState.mode.Bl: @@ -21,10 +21,7 @@ proc process(gState: State, path: string, astTable: AstTable) = if gState.past: gecho gState.printLisp(root) elif gState.pnim: - if Feature.ast2 in gState.feature: - ast2.parseNim(gState, path, root) - elif Feature.ast1 in gState.feature: - ast.parseNim(gState, path, root, astTable) + parseNim(gState, path, root) elif gState.preprocess: gecho gState.code @@ -43,7 +40,6 @@ proc main( noHeader = false, output = "", past = false, - pgrammar = false, pluginSourcePath: string = "", pnim = false, prefix: seq[string] = @[], @@ -84,10 +80,6 @@ proc main( build.gDebug = gState.debug build.gNimExe = gState.nim - # Default `ast` mode - if gState.feature.Bl: - gState.feature.add Feature.ast1 - # Split some arguments with , gState.symOverride = gState.symOverride.getSplitComma() gState.prefix = gState.prefix.getSplitComma() @@ -137,31 +129,14 @@ proc main( if gState.debug: echo &"# Writing output to {outputFile}\n" - # Process grammar into AST - let - astTable = - if Feature.ast1 in gState.feature: - parseGrammar() - else: - nil - - if pgrammar: - if Feature.ast1 in gState.feature: - # Print AST of grammar - gState.printGrammar(astTable) - elif source.nBl: + if source.nBl: # Print source after preprocess or Nim output if gState.pnim: gState.initNim() for src in source: - gState.process(src.expandSymlinkAbs(), astTable) + gState.process(src.expandSymlinkAbs()) if gState.pnim: - if Feature.ast2 in gState.feature: - ast2.printNim(gState) - elif Feature.ast1 in gState.feature: - ast.printNim(gState) - gecho """{.hint: "The legacy wrapper generation algorithm is deprecated and will be removed in the next release of Nimterop.".}""" - gecho """{.hint: "Refer to CHANGES.md for details on migrating to the new backend.".}""" + printNim(gState) # Close outputFile if outputFile.len != 0: @@ -249,7 +224,6 @@ when isMainModule: "noHeader": "skip {.header.} pragma in wrapper", "output": "file to output content - default: stdout", "past": "print AST output", - "pgrammar": "print grammar", "pluginSourcePath": "nim file to build and load as a plugin", "pnim": "print Nim output", "prefix": "strip prefix from identifiers", @@ -273,7 +247,6 @@ when isMainModule: "noHeader": 'H', "output": 'o', "past": 'a', - "pgrammar": 'g', "pnim": 'n', "prefix": 'E', "preprocess": 'p', diff --git a/nimterop/toastlib/ast.nim b/nimterop/toastlib/ast.nim deleted file mode 100644 index 0b62a34..0000000 --- a/nimterop/toastlib/ast.nim +++ /dev/null @@ -1,253 +0,0 @@ -import hashes, macros, os, sets, strformat, strutils, tables - -import regex - -import ".."/[globals, treesitter/api] -import "."/[getters, tshelp] - -proc getHeaderPragma*(gState: State): string = - result = - if not gState.noHeader and gState.dynlib.Bl: - &", header: {gState.currentHeader}" - else: - "" - -proc getDynlib*(gState: State): string = - result = - if gState.dynlib.nBl: - &", dynlib: {gState.dynlib}" - else: - "" - -proc getImportC*(gState: State, origName, nimName: string): string = - if nimName != origName: - result = &"importc: \"{origName}\"{gState.getHeaderPragma()}" - else: - result = gState.impShort - -proc getPragma*(gState: State, pragmas: varargs[string]): string = - result = "" - for pragma in pragmas.items(): - if pragma.nBl: - result &= pragma & ", " - if result.nBl: - result = " {." & result[0 .. ^3] & ".}" - - result = result.replace(gState.impShort & ", cdecl", gState.impShort & "C") - - let - dy = gState.getDynlib() - - if ", cdecl" in result and dy.nBl: - result = result.replace(".}", dy & ".}") - -proc saveNodeData(node: TSNode, gState: State): bool = - let name = $node.tsNodeType() - - # Atoms are nodes whose values are to be saved - if name in gAtoms: - let - pname = node.getPxName(1) - ppname = node.getPxName(2) - pppname = node.getPxName(3) - ppppname = node.getPxName(4) - - var - val = gState.getNodeVal(node) - - # Skip since value already obtained from parent atom - if name == "primitive_type" and pname == "sized_type_specifier": - return true - - # Skip since value already obtained from parent expression - if name in ["number_literal", "identifier"] and pname in gExpressions: - return true - - # Add reference point in saved data for bitfield_clause - if name in ["number_literal"] and pname == "bitfield_clause": - gState.data.add(("bitfield_clause", val)) - return true - - # Process value as a type - if name in ["primitive_type", "sized_type_specifier"]: - val = val.getType() - - if node.tsNodePrevNamedSibling().tsNodeIsNull(): - if pname == "pointer_declarator": - if ppname notin ["function_declarator", "array_declarator"]: - gState.data.add(("pointer_declarator", "")) - elif ppname == "array_declarator": - gState.data.add(("array_pointer_declarator", "")) - - # Double pointer - if ppname == "pointer_declarator": - gState.data.add(("pointer_declarator", "")) - elif pname in ["function_declarator", "array_declarator"]: - if ppname == "pointer_declarator": - gState.data.add(("pointer_declarator", "")) - if pppname == "pointer_declarator": - gState.data.add(("pointer_declarator", "")) - - gState.data.add((name, val)) - - if pname == "pointer_declarator" and - ppname == "function_declarator": - if name == "field_identifier": - if pppname == "pointer_declarator": - gState.data.insert(("pointer_declarator", ""), gState.data.len-1) - if ppppname == "pointer_declarator": - gState.data.insert(("pointer_declarator", ""), gState.data.len-1) - gState.data.add(("function_declarator", "")) - elif name == "identifier": - gState.data.add(("pointer_declarator", "")) - - # Save node value for a top-level expression - elif name in gExpressions and name != "escape_sequence": - if $node.tsNodeParent.tsNodeType() notin gExpressions: - gState.data.add((name, gState.getNodeVal(node))) - - elif name in ["abstract_pointer_declarator", "enumerator", "field_declaration", "function_declarator"]: - gState.data.add((name.replace("abstract_", ""), "")) - - return true - -proc searchAstForNode(ast: ref Ast, node: TSNode, gState: State): bool = - let - childNames = node.getTSNodeNamedChildNames().join() - - if ast.isNil: - return - - if gState.debug: - gState.nodeBranch.add $node.tsNodeType() - gecho "#" & spaces(gState.nodeBranch.len * 2) & gState.nodeBranch[^1] - - if ast.children.nBl: - if childNames.contains(ast.regex) or - (childNames.Bl and ast.recursive): - if node.getTSNodeNamedChildCountSansComments() != 0: - var flag = true - - for i in 0 .. node.tsNodeNamedChildCount()-1: - if $node.tsNodeNamedChild(i).tsNodeType() != "comment": - let - nodeChild = node.tsNodeNamedChild(i) - astChild = - if not ast.recursive: - ast.getAstChildByName($nodeChild.tsNodeType()) - else: - ast - - if not searchAstForNode(astChild, nodeChild, gState): - flag = false - break - - if flag: - result = node.saveNodeData(gState) - else: - result = node.saveNodeData(gState) - else: - if gState.debug: - gecho "#" & spaces(gState.nodeBranch.len * 2) & &" {ast.getRegexForAstChildren()} !=~ {childNames}" - elif node.getTSNodeNamedChildCountSansComments() == 0: - result = node.saveNodeData(gState) - - if gState.debug: - discard gState.nodeBranch.pop() - if gState.nodeBranch.Bl: - gecho "" - -proc searchAst(root: TSNode, astTable: AstTable, gState: State) = - var - node = root - nextnode: TSNode - depth = 0 - - while true: - if not node.tsNodeIsNull() and depth > -1: - let - name = $node.tsNodeType() - if name in astTable: - for ast in astTable[name]: - if gState.debug: - gecho "\n# " & gState.getNodeVal(node).replace("\n", "\n# ") & "\n" - if searchAstForNode(ast, node, gState): - ast.tonim(ast, node, gState) - if gState.debug: - gState.debugStr &= "\n# " & gState.data.join("\n# ") & "\n" - break - gState.data = @[] - else: - break - - if $node.tsNodeType() notin astTable and node.tsNodeNamedChildCount() != 0: - nextnode = node.tsNodeNamedChild(0) - depth += 1 - else: - nextnode = node.tsNodeNextNamedSibling() - - if nextnode.tsNodeIsNull(): - while true: - node = node.tsNodeParent() - depth -= 1 - if depth == -1: - break - if node == root: - break - if not node.tsNodeNextNamedSibling().tsNodeIsNull(): - node = node.tsNodeNextNamedSibling() - break - else: - node = nextnode - - if node == root: - break - -proc parseNim*(gState: State, fullpath: string, root: TSNode, astTable: AstTable) = - # Generate Nim from tree-sitter AST root node - var - fp = fullpath.replace("\\", "/") - - gState.currentHeader = getCurrentHeader(fullpath) - gState.impShort = gState.currentHeader.replace("header", "imp") - gState.sourceFile = fullpath - - if not gState.noHeader and gState.dynlib.Bl: - gState.constStr &= &"\n {gState.currentHeader} {{.used.}} = \"{fp}\"" - - root.searchAst(astTable, gState) - -proc printNim*(gState: State) = - # Print Nim generated by parseNim() - if gState.enumStr.nBl: - gecho &"{gState.enumStr}\n" - - gState.constStr = gState.getOverrideFinal(nskConst) & gState.constStr - if gState.constStr.nBl: - gecho &"const{gState.constStr}\n" - - gecho &""" -{{.pragma: {gState.impShort}, importc{gState.getHeaderPragma()}.}} -{{.pragma: {gState.impShort}C, {gState.impShort}, cdecl{gState.getDynlib()}.}} -""" - - gState.typeStr = gState.getOverrideFinal(nskType) & gState.typeStr - if gState.typeStr.nBl: - gecho &"type{gState.typeStr}\n" - - gState.procStr = gState.getOverrideFinal(nskProc) & gState.procStr - if gState.procStr.nBl: - gecho &"{gState.procStr}\n" - - gecho "{.pop.}" - - if gState.debug: - if gState.debugStr.nBl: - gecho gState.debugStr - - if gState.skipStr.nBl: - let - hash = gState.skipStr.hash().abs() - sname = getTempDir() / &"nimterop_{$hash}.h" - gecho &"# Writing skipped definitions to {sname}\n" - writeFile(sname, gState.skipStr) diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index b4070f3..0c1b784 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -304,131 +304,6 @@ proc getPreprocessor*(gState: State, fullpath: string) = rdata.add line gState.code = rdata.join("\n") -converter toString*(kind: Kind): string = - return case kind: - of exactlyOne: - "" - of oneOrMore: - "+" - of zeroOrMore: - "*" - of zeroOrOne: - "?" - of orWithNext: - "!" - -converter toKind*(kind: string): Kind = - return case kind: - of "+": - oneOrMore - of "*": - zeroOrMore - of "?": - zeroOrOne - of "!": - orWithNext - else: - exactlyOne - -proc getNameKind*(name: string): tuple[name: string, kind: Kind, recursive: bool] = - if name[0] == '^': - result.recursive = true - result.name = name[1 .. ^1] - else: - result.name = name - result.kind = $name[^1] - - if result.kind != exactlyOne: - result.name = result.name[0 .. ^2] - -proc getRegexForAstChildren*(ast: ref Ast): string = - result = "^" - for i in 0 .. ast.children.len-1: - let - kind: string = ast.children[i].kind - begin = if result[^1] == '|': "" else: "(?:" - case kind: - of "!": - result &= &"{begin}{ast.children[i].name}|" - else: - result &= &"{begin}{ast.children[i].name}){kind}" - result &= "$" - -proc getAstChildByName*(ast: ref Ast, name: string): ref Ast = - for i in 0 .. ast.children.len-1: - if name in ast.children[i].name.split("|"): - return ast.children[i] - - if ast.children.len == 1 and ast.children[0].name == ".": - return ast.children[0] - -proc getNimExpression*(gState: State, expr: string, name = ""): string = - # Convert C/C++ expression into Nim - cast identifiers to `name` if specified - var - clean = expr.multiReplace([("\n", " "), ("\r", "")]) - ident = "" - gen = "" - hex = false - - for i in 0 .. clean.len: - if i != clean.len: - if clean[i] in IdentChars: - if clean[i] in Digits and ident.Bl: - # Identifiers cannot start with digits - gen = $clean[i] - elif clean[i] in HexDigits and hex == true: - # Part of a hex number - gen = $clean[i] - elif i > 0 and i < clean.len-1 and clean[i] in ['x', 'X'] and - clean[i-1] == '0' and clean[i+1] in HexDigits: - # Found a hex number - gen = $clean[i] - hex = true - else: - # Part of an identifier - ident &= clean[i] - hex = false - else: - gen = (block: - if (i == 0 or clean[i-1] != '\'') or - (i == clean.len - 1 or clean[i+1] != '\''): - # If unquoted, convert logical ops to Nim - case clean[i] - of '^': " xor " - of '&': " and " - of '|': " or " - of '~': " not " - else: $clean[i] - else: - $clean[i] - ) - hex = false - - if i == clean.len or gen.nBl: - # Process identifier - if ident.nBl: - # Issue #178 - if ident != "_": - ident = gState.getIdentifier(ident, nskConst, name) - if name.nBl and ident in gState.constIdentifiers: - ident = ident & "." & name - result &= ident - ident = "" - result &= gen - gen = "" - - # Convert shift ops to Nim - result = result.multiReplace([ - ("<<", " shl "), (">>", " shr ") - ]) - -proc getComments*(gState: State, strip = false): string = - if not gState.noComments and gState.commentStr.nBl: - result = "\n" & gState.commentStr - if strip: - result = result.replace("\n ", "\n") - gState.commentStr = "" - # Plugin related proc dll*(path: string): string = diff --git a/nimterop/toastlib/grammar.nim b/nimterop/toastlib/grammar.nim deleted file mode 100644 index 52b19e7..0000000 --- a/nimterop/toastlib/grammar.nim +++ /dev/null @@ -1,768 +0,0 @@ -import macros, strformat, strutils, tables - -import regex - -import ".."/[globals, treesitter/api] -import "."/[ast, getters, lisp, tshelp] - -type - Grammar = seq[tuple[grammar: string, call: proc(ast: ref Ast, node: TSNode, gState: State) {.nimcall.}]] - -proc getPtrType(str: string): string = - result = case str: - of "ptr cchar": - "cstring" - of "ptr ptr cchar": - "ptr cstring" - of "ptr object": - "pointer" - of "ptr ptr object": - "ptr pointer" - of "ptr FILE": - "File" - else: - str - -proc getLit(str: string): string = - # Used to convert #define literals into const - let - str = str.replace(re"/[/*].*?(?:\*/)?$", "").strip() - - if str.contains(re"^[\-]?[\d]*[.]?[\d]+$") or # decimal - str.contains(re"^0x[\da-fA-F]+$") or # hexadecimal - str.contains(re"^'[[:ascii:]]'$") or # char - str.contains(re"""^"[[:ascii:]]+"$"""): # char * - return str - -proc initGrammar(): Grammar = - # #define X Y - result.add((""" - (preproc_def - (identifier) - (preproc_arg) - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# define X Y" - - let - name = gState.data[0].val - nname = gState.getIdentifier(name, nskConst) - val = gState.data[1].val.getLit() - - if not nname.nBl: - let - override = gState.getOverride(name, nskConst) - if override.nBl: - gState.constStr &= &"{gState.getComments()}\n{override}" - else: - gState.constStr &= &"{gState.getComments()}\n # Const '{name}' skipped" - if gState.debug: - gState.skipStr &= &"\n{gState.getNodeVal(node)}" - elif val.nBl and gState.addNewIdentifer(nname): - gState.constStr &= &"{gState.getComments()}\n {nname}* = {val}" - )) - - let - typeGrammar = """ - (type_qualifier?) - (primitive_type|type_identifier?) - (type_qualifier?) - (sized_type_specifier? - (primitive_type?) - ) - (struct_specifier|union_specifier|enum_specifier? - (type_identifier) - ) - """ - - arrGrammar = &""" - (array_declarator! - (pointer_declarator! - (pointer_declarator! - (type_identifier) - ) - (type_identifier) - ) - (type_identifier|identifier) - (identifier|number_literal) - ) - """ - - paramListGrammar = &""" - (parameter_list - (parameter_declaration* - {typeGrammar} - (identifier|type_identifier?) - (pointer_declarator? - (type_qualifier?) - (pointer_declarator! - (type_qualifier?) - {arrGrammar} - (identifier|type_identifier) - ) - {arrGrammar} - (identifier|type_identifier) - ) - {arrGrammar} - (abstract_pointer_declarator? - (abstract_pointer_declarator?) - ) - ) - ) - """ - - funcGrammar = &""" - (function_declarator* - (identifier|type_identifier!) - (pointer_declarator - (pointer_declarator! - (type_identifier) - ) - (type_identifier|identifier) - ) - {paramListGrammar} - (noexcept|throw_specifier?) - ) - """ - - template funcParamCommon(fname, pname, ptyp, pptr, pout, count, i, flen: untyped): untyped = - ptyp = gState.getIdentifier(gState.data[i].val, nskType, fname).getType() - - pptr = "" - while i+1 < gState.data.len and gState.data[i+1].name == "pointer_declarator": - pptr &= "ptr " - i += 1 - - if i+1 < gState.data.len and gState.data[i+1].name == "identifier": - pname = gState.getIdentifier(gState.data[i+1].val, nskParam, fname) - i += 2 - else: - pname = "a" & $count - count += 1 - i += 1 - - if i < gState.data.len and gState.data[i].name in ["identifier", "number_literal"]: - flen = gState.data[i].val - if gState.data[i].name == "identifier": - flen = gState.getIdentifier(flen, nskConst, fname) - - pout &= &"{pname}: array[{flen}, {getPtrType(pptr&ptyp)}], " - i += 1 - elif pptr.nBl or ptyp != "object": - pout &= &"{pname}: {getPtrType(pptr&ptyp)}, " - - # typedef int X - # typedef X Y - # typedef struct X Y - # typedef ?* Y - result.add((&""" - (type_definition - {typeGrammar} - (type_identifier!) - {arrGrammar} - (pointer_declarator! - (pointer_declarator! - (type_identifier!) - {arrGrammar} - {funcGrammar} - ) - (type_identifier!) - {arrGrammar} - {funcGrammar} - ) - {funcGrammar} - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# typedef X Y" - - var - i = 0 - typ = gState.getIdentifier(gState.data[i].val, nskType, "IgnoreSkipSymbol").getType() - name = "" - nname = "" - tptr = "" - aptr = "" - pragmas: seq[string] = @[] - - i += 1 - while i < gState.data.len and "pointer" in gState.data[i].name: - case gState.data[i].name: - of "pointer_declarator": - tptr &= "ptr " - i += 1 - of "array_pointer_declarator": - aptr &= "ptr " - i += 1 - - if i < gState.data.len: - name = gState.data[i].val - nname = gState.getIdentifier(name, nskType) - i += 1 - - if not gState.noHeader and gState.dynlib.Bl: - pragmas.add gState.getImportC(name, nname) - - let - pragma = gState.getPragma(pragmas) - - if not nname.nBl: - let - override = gState.getOverride(name, nskType) - if override.nBl: - gState.typeStr &= &"{gState.getComments()}\n{override}" - elif nname notin gTypeMap and typ.nBl and gState.addNewIdentifer(nname): - if i < gState.data.len and gState.data[^1].name == "function_declarator": - var - fname = nname - pout, pname, ptyp, pptr = "" - count = 1 - flen = "" - - while i < gState.data.len: - if gState.data[i].name == "function_declarator": - break - - funcParamCommon(fname, pname, ptyp, pptr, pout, count, i, flen) - - if pout.nBl and pout[^2 .. ^1] == ", ": - pout = pout[0 .. ^3] - - if tptr.nBl or typ != "object": - gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = proc({pout}): {getPtrType(tptr&typ)} {{.cdecl.}}" - else: - gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = proc({pout}) {{.cdecl.}}" - else: - if i < gState.data.len and gState.data[i].name in ["identifier", "number_literal"]: - var - flen = gState.data[i].val - if gState.data[i].name == "identifier": - flen = gState.getIdentifier(flen, nskConst, nname) - - gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = {aptr}array[{flen}, {getPtrType(tptr&typ)}]" - else: - if nname == typ: - pragmas.add "incompleteStruct" - let - pragma = gState.getPragma(pragmas) - gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = object" - else: - gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = {getPtrType(tptr&typ)}" - )) - - proc pDupTypeCommon(nname: string, fend: int, gState: State, isEnum=false) = - if gState.debug: - gState.debugStr &= "\n# pDupTypeCommon()" - - var - dname = gState.data[^1].val - ndname = gState.getIdentifier(dname, nskType) - dptr = - if fend == 2: - "ptr " - else: - "" - - if ndname.nBl and ndname != nname: - if isEnum: - if gState.addNewIdentifer(ndname): - gState.enumStr &= &"{gState.getComments(true)}\ntype {ndname}* = {dptr}{nname}" - else: - if gState.addNewIdentifer(ndname): - let - pragma = gState.getPragma(gState.getImportc(dname, ndname), "bycopy") - gState.typeStr &= - &"{gState.getComments()}\n {ndname}*{pragma} = {dptr}{nname}" - - proc pStructCommon(ast: ref Ast, node: TSNode, name: string, fstart, fend: int, gState: State) = - if gState.debug: - gState.debugStr &= "\n# pStructCommon" - - var - nname = gState.getIdentifier(name, nskType) - prefix = "" - union = "" - - case $node.tsNodeType(): - of "struct_specifier": - prefix = "struct " - of "union_specifier": - prefix = "union " - union = ", union" - of "type_definition": - if node.getTSNodeNamedChildCountSansComments() != 0: - for i in 0 .. node.tsNodeNamedChildCount()-1: - let - nchild = $node.tsNodeNamedChild(i).tsNodeType() - if nchild != "comment": - case nchild: - of "struct_specifier": - if fstart == 1: - prefix = "struct " - of "union_specifier": - if fstart == 1: - prefix = "union " - union = ", union" - break - - if not nname.nBl: - let - override = gState.getOverride(name, nskType) - if override.nBl: - gState.typeStr &= &"{gState.getComments()}\n{override}" - elif gState.addNewIdentifer(nname): - if gState.data.len == 1: - gState.typeStr &= &"{gState.getComments()}\n {nname}* {{.bycopy{union}.}} = object" - else: - var - pragmas: seq[string] = @[] - if not gState.noHeader and gState.dynlib.Bl: - pragmas.add gState.getImportC(prefix & name, nname) - pragmas.add "bycopy" - if union.nBl: - pragmas.add "union" - - let - pragma = gState.getPragma(pragmas) - - gState.typeStr &= &"{gState.getComments()}\n {nname}*{pragma} = object" - - var - i = fstart - ftyp, fname: string - fptr = "" - aptr = "" - flen = "" - while i < gState.data.len-fend: - fptr = "" - aptr = "" - if gState.data[i].name == "field_declaration": - i += 1 - continue - - if gState.data[i].name notin ["field_identifier", "pointer_declarator", "array_pointer_declarator"]: - ftyp = gState.getIdentifier(gState.data[i].val, nskType, nname).getType() - i += 1 - - while i < gState.data.len-fend and "pointer" in gState.data[i].name: - case gState.data[i].name: - of "pointer_declarator": - fptr &= "ptr " - i += 1 - of "array_pointer_declarator": - aptr &= "ptr " - i += 1 - - fname = gState.getIdentifier(gState.data[i].val, nskField, nname) - - if i+1 < gState.data.len-fend and gState.data[i+1].name in gEnumVals: - # Struct field is an array where size is an expression - var - flen = gState.getNimExpression(gState.data[i+1].val) - if "/" in flen: - flen = &"({flen}).int" - gState.typeStr &= &"{gState.getComments()}\n {fname}*: {aptr}array[{flen}, {getPtrType(fptr&ftyp)}]" - i += 2 - elif i+1 < gState.data.len-fend and gState.data[i+1].name == "bitfield_clause": - let - size = gState.data[i+1].val - gState.typeStr &= &"{gState.getComments()}\n {fname}* {{.bitsize: {size}.}} : {getPtrType(fptr&ftyp)} " - i += 2 - elif i+1 < gState.data.len-fend and gState.data[i+1].name == "function_declarator": - var - pout, pname, ptyp, pptr = "" - count = 1 - - i += 2 - while i < gState.data.len-fend: - if gState.data[i].name == "function_declarator": - i += 1 - continue - - if gState.data[i].name == "field_declaration": - break - - funcParamCommon(fname, pname, ptyp, pptr, pout, count, i, flen) - - if pout.nBl and pout[^2 .. ^1] == ", ": - pout = pout[0 .. ^3] - if fptr.nBl or ftyp != "object": - gState.typeStr &= &"{gState.getComments()}\n {fname}*: proc({pout}): {getPtrType(fptr&ftyp)} {{.cdecl.}}" - else: - gState.typeStr &= &"{gState.getComments()}\n {fname}*: proc({pout}) {{.cdecl.}}" - i += 1 - else: - if ftyp == "object": - gState.typeStr &= &"{gState.getComments()}\n {fname}*: pointer" - else: - gState.typeStr &= &"{gState.getComments()}\n {fname}*: {getPtrType(fptr&ftyp)}" - i += 1 - - if node.tsNodeType() == "type_definition" and - gState.data[^1].name == "type_identifier" and gState.data[^1].val.nBl: - pDupTypeCommon(nname, fend, gState, false) - - let - fieldGrammar = &""" - (field_identifier!) - (bitfield_clause! - (number_literal) - ) - (array_declarator! - (field_identifier!) - (pointer_declarator - (pointer_declarator! - (field_identifier) - ) - (field_identifier) - ) - (^$1+) - ) - (function_declarator+ - (pointer_declarator - (pointer_declarator! - (field_identifier) - ) - (field_identifier) - ) - {paramListGrammar} - ) - """ % gEnumVals.join("|") - - fieldListGrammar = &""" - (field_declaration_list? - (field_declaration+ - {typeGrammar} - (pointer_declarator! - (pointer_declarator! - {fieldGrammar} - ) - {fieldGrammar} - ) - {fieldGrammar} - ) - ) - """ - - # struct X {} - result.add((&""" - (struct_specifier|union_specifier - (type_identifier) - {fieldListGrammar} - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# struct X {}" - - pStructCommon(ast, node, gState.data[0].val, 1, 1, gState) - )) - - # typedef struct X {} - result.add((&""" - (type_definition - (struct_specifier|union_specifier - (type_identifier?) - {fieldListGrammar} - ) - (type_identifier!) - (pointer_declarator - (pointer_declarator! - (type_identifier) - ) - (type_identifier) - ) - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# typedef struct X {}" - - var - fstart = 0 - fend = 1 - - if gState.data[^2].name == "pointer_declarator": - fend = 2 - - if gState.data.len > 1 and - gState.data[0].name == "type_identifier" and - gState.data[1].name notin ["field_identifier", "pointer_declarator"]: - - fstart = 1 - pStructCommon(ast, node, gState.data[0].val, fstart, fend, gState) - else: - pStructCommon(ast, node, gState.data[^1].val, fstart, fend, gState) - )) - - proc pEnumCommon(ast: ref Ast, node: TSNode, name: string, fstart, fend: int, gState: State) = - if gState.debug: - gState.debugStr &= "\n# pEnumCommon()" - - let nname = - if name.Bl: - getUniqueIdentifier(gState, "Enum") - else: - gState.getIdentifier(name, nskType) - - if nname.nBl and gState.addNewIdentifer(nname): - gState.enumStr &= &"{gState.getComments(true)}\ndefineEnum({nname})" - - var - i = fstart - count = 0 - while i < gState.data.len-fend: - if gState.data[i].name == "enumerator": - i += 1 - continue - - let - fname = gState.getIdentifier(gState.data[i].val, nskEnumField) - - if i+1 < gState.data.len-fend and - gState.data[i+1].name in gEnumVals: - if fname.nBl and gState.addNewIdentifer(fname): - gState.constStr &= &"{gState.getComments()}\n {fname}* = ({gState.getNimExpression(gState.data[i+1].val)}).{nname}" - try: - count = gState.data[i+1].val.parseInt() + 1 - except: - count += 1 - i += 2 - else: - if fname.nBl and gState.addNewIdentifer(fname): - gState.constStr &= &"{gState.getComments()}\n {fname}* = {count}.{nname}" - i += 1 - count += 1 - - if node.tsNodeType() == "type_definition" and - gState.data[^1].name == "type_identifier" and gState.data[^1].val.nBl: - pDupTypeCommon(nname, fend, gState, true) - - # enum X {} - result.add((""" - (enum_specifier - (type_identifier?) - (enumerator_list - (enumerator+ - (identifier?) - (^$1+) - ) - ) - ) - """ % gEnumVals.join("|"), - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# enum X {}" - - var - name = "" - offset = 0 - - if gState.data[0].name == "type_identifier": - name = gState.data[0].val - offset = 1 - - pEnumCommon(ast, node, name, offset, 0, gState) - )) - - # typedef enum {} X - result.add((&""" - (type_definition - {result[^1].grammar} - (type_identifier!) - (pointer_declarator - (pointer_declarator! - (type_identifier) - ) - (type_identifier) - ) - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# typedef enum {}" - - var - fstart = 0 - fend = 1 - - if gState.data[^2].name == "pointer_declarator": - fend = 2 - - if gState.data[0].name == "type_identifier": - fstart = 1 - - pEnumCommon(ast, node, gState.data[0].val, fstart, fend, gState) - else: - pEnumCommon(ast, node, gState.data[^1].val, fstart, fend, gState) - )) - - # typ function(typ param1, ...) - result.add((&""" - (declaration - (storage_class_specifier?) - {typeGrammar} - (pointer_declarator! - (pointer_declarator! - {funcGrammar} - ) - {funcGrammar} - ) - {funcGrammar} - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - if gState.debug: - gState.debugStr &= "\n# typ function" - - var - fptr = "" - i = 1 - - while i < gState.data.len: - if gState.data[i].name == "function_declarator": - i += 1 - continue - - fptr = "" - while i < gState.data.len and gState.data[i].name == "pointer_declarator": - fptr &= "ptr " - i += 1 - - var - fname = gState.data[i].val - fnname = gState.getIdentifier(fname, nskProc) - pout, pname, ptyp, pptr = "" - count = 1 - flen = "" - fVar = false - - i += 1 - if i < gState.data.len and gState.data[i].name == "pointer_declarator": - fVar = true - i += 1 - - while i < gState.data.len: - if gState.data[i].name == "function_declarator": - break - - funcParamCommon(fnname, pname, ptyp, pptr, pout, count, i, flen) - - if pout.nBl and pout[^2 .. ^1] == ", ": - pout = pout[0 .. ^3] - - if not fnname.nBl: - let - override = gState.getOverride(fname, nskProc) - if override.nBl: - gState.typeStr &= &"{gState.getComments()}\n{override}" - elif gState.addNewIdentifer(fnname): - let - ftyp = gState.getIdentifier(gState.data[0].val, nskType, fnname).getType() - pragma = gState.getPragma(gState.getImportC(fname, fnname), "cdecl") - - if fptr.nBl or ftyp != "object": - if fVar: - gState.procStr &= &"{gState.getComments(true)}\nvar {fnname}*: proc ({pout}): {getPtrType(fptr&ftyp)}{{.cdecl.}}" - else: - gState.procStr &= &"{gState.getComments(true)}\nproc {fnname}*({pout}): {getPtrType(fptr&ftyp)}{pragma}" - else: - if fVar: - gState.procStr &= &"{gState.getComments(true)}\nvar {fnname}*: proc ({pout}){{.cdecl.}}" - else: - gState.procStr &= &"{gState.getComments(true)}\nproc {fnname}*({pout}){pragma}" - )) - - # // comment - result.add((&""" - (comment - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - let - cmt = $gState.getNodeVal(node) - - for line in cmt.splitLines(): - let - line = line.multiReplace([("//", ""), ("/*", ""), ("*/", "")]) - - gState.commentStr &= &"\n # {line.strip(leading=false)}" - )) - - # // unknown - result.add((&""" - (type_definition|struct_specifier|union_specifier|enum_specifier|declaration - (^.*) - ) - """, - proc (ast: ref Ast, node: TSNode, gState: State) = - var - done = false - for i in gState.data: - case $node.tsNodeType() - of "declaration": - if i.name == "identifier": - let - override = gState.getOverride(i.val, nskProc) - - if override.nBl: - gState.procStr &= &"{gState.getComments(true)}\n{override}" - done = true - break - else: - gState.procStr &= &"{gState.getComments(true)}\n# Declaration '{i.val}' skipped" - - else: - if i.name == "type_identifier": - let - override = gState.getOverride(i.val, nskType) - - if override.nBl: - gState.typeStr &= &"{gState.getComments()}\n{override}" - done = true - break - else: - gState.typeStr &= &"{gState.getComments()}\n # Type '{i.val}' skipped" - - if gState.debug and not done: - gState.skipStr &= &"\n{gState.getNodeVal(node)}" - )) - -proc initRegex(ast: ref Ast) = - if ast.children.nBl: - if not ast.recursive: - for child in ast.children: - child.initRegex() - - var - reg: string - try: - reg = ast.getRegexForAstChildren() - ast.regex = reg.re() - except: - echo reg - raise newException(Exception, getCurrentExceptionMsg()) - -proc parseGrammar*(): AstTable = - const grammars = initGrammar() - - result = newTable[string, seq[ref Ast]]() - for i in 0 .. grammars.len-1: - var - ast = grammars[i].grammar.parseLisp() - - ast.tonim = grammars[i].call - ast.initRegex() - for n in ast.name.split("|"): - if n notin result: - result[n] = @[ast] - else: - result[n].add(ast) - -proc printGrammar*(gState: State, astTable: AstTable) = - for name in astTable.keys(): - for ast in astTable[name]: - gecho ast.printAst() diff --git a/nimterop/toastlib/lisp.nim b/nimterop/toastlib/lisp.nim deleted file mode 100644 index 57d71a1..0000000 --- a/nimterop/toastlib/lisp.nim +++ /dev/null @@ -1,60 +0,0 @@ -import ".."/globals -import "."/getters - -var - gTokens: seq[string] - idx = 0 - -proc tokenize(tree: string) = - var collect = "" - - gTokens = @[] - idx = 0 - for i in tree: - case i: - of ' ', '\n', '\r', '(', ')': - if collect.nBl: - gTokens.add(collect) - collect = "" - if i in ['(', ')']: - gTokens.add($i) - else: - collect &= $i - -proc readFromTokens(): ref Ast = - if idx == gTokens.len: - doAssert false, "Bad AST " & $(idx: idx) - - if gTokens[idx] == "(": - if gTokens.len - idx < 2: - doAssert false, "Corrupt AST " & $(gTokensLen: gTokens.len, idx: idx) - result = new(Ast) - (result.name, result.kind, result.recursive) = gTokens[idx+1].getNameKind() - result.children = @[] - if result.recursive: - result.children.add(result) - idx += 2 - while gTokens[idx] != ")": - var res = readFromTokens() - if not res.isNil: - result.children.add(res) - elif gTokens[idx] == ")": - doAssert false, "Poor AST " & $(idx: idx) - - idx += 1 - -proc printAst*(node: ref Ast, offset=""): string = - result = offset & "(" & (if node.recursive: "^" else: "") & node.name & node.kind.toString() - - if node.children.nBl and not node.recursive: - result &= "\n" - for child in node.children: - result &= printAst(child, offset & " ") - result &= offset & ")\n" - else: - result &= ")\n" - -proc parseLisp*(tree: string): ref Ast = - tokenize(tree) - - return readFromTokens() diff --git a/nimterop/toastlib/tshelp.nim b/nimterop/toastlib/tshelp.nim index e7e8d75..9310cf7 100644 --- a/nimterop/toastlib/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -2,7 +2,6 @@ import sets, strformat, strutils import ".."/treesitter/[api, c, cpp] import ".."/globals -import "."/getters template withCodeAst*(code: string, mode: string, body: untyped): untyped = ## A simple template to inject the TSNode into a body of code diff --git a/tests/getheader.nims b/tests/getheader.nims index 916abcf..e5d964c 100644 --- a/tests/getheader.nims +++ b/tests/getheader.nims @@ -14,7 +14,7 @@ proc testCall(cmd, output: string, exitCode: int, delete = true) = doAssert outp.contains(output), outp var - cmd = "nim c -f --hints:off -d:FLAGS=\"-f:ast2\" -d:checkAbi" + cmd = "nim c -f --hints:off -d:checkAbi" lrcmd = " -r lzma.nim" zrcmd = " -r zlib.nim" sshcmd = " -r libssh2.nim" diff --git a/tests/libssh2.nim b/tests/libssh2.nim index 172f6ae..09343bc 100644 --- a/tests/libssh2.nim +++ b/tests/libssh2.nim @@ -17,12 +17,12 @@ cOverride: SOCKET = object when not libssh2Static: - cImport(libssh2Path, recurse = true, dynlib = "libssh2LPath", flags = "-f:ast2 -c -E_ -F_") + cImport(libssh2Path, recurse = true, dynlib = "libssh2LPath", flags = "-c -E_ -F_") when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc, dynlib: libssh2LPath.} else: - cImport(libssh2Path, recurse = true, flags = "-f:ast2 -c -E_ -F_") + cImport(libssh2Path, recurse = true, flags = "-c -E_ -F_") when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc.} diff --git a/tests/rsa.nim b/tests/rsa.nim index a4aa64b..f5c343c 100644 --- a/tests/rsa.nim +++ b/tests/rsa.nim @@ -39,7 +39,7 @@ cOverride: cImport(@[ basePath / "rsa.h", basePath / "err.h", -], recurse = true, flags = "-f:ast2 -s -c " & FLAGS) +], recurse = true, flags = "-s -c " & FLAGS) {.passL: cryptoLPath.} From 63a75bce4ee36ad2e09ff4957408c7f49e319c2a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 24 Jun 2020 12:24:15 -0500 Subject: [PATCH 047/106] No more includes in build --- nimterop/build.nim | 129 +++++----------------------------- nimterop/build/ccompiler.nim | 2 + nimterop/build/conan.nim | 6 ++ nimterop/build/getheader.nim | 7 +- nimterop/build/jbb.nim | 6 ++ nimterop/build/nimconf.nim | 6 ++ nimterop/build/shell.nim | 85 ++++++++++++++++------ nimterop/build/tools.nim | 22 +++--- nimterop/cimport.nim | 1 - nimterop/globals.nim | 20 ++++-- nimterop/paths.nim | 2 +- nimterop/setup.nim | 3 +- nimterop/toast.nim | 10 ++- nimterop/toastlib/getters.nim | 3 +- nimterop/treesitter/cpp.nim | 3 +- 15 files changed, 144 insertions(+), 161 deletions(-) diff --git a/nimterop/build.nim b/nimterop/build.nim index 672cb58..3a73f56 100644 --- a/nimterop/build.nim +++ b/nimterop/build.nim @@ -1,130 +1,39 @@ -import hashes, osproc, sets, strformat, strutils - when not defined(TOAST): - import macros, tables - -import os except findExe, sleep + import os except findExe, sleep +else: + import os export extractFilename, `/` -# build specific debug since we cannot import globals (yet) -var - gDebug* = false - gDebugCT* {.compileTime.} = false - gNimExe* = "" - # Misc helpers - -proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = - result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) - if not noQuote: - result = result.quoteShell - -proc getCurrentNimCompiler*(): string = - when nimvm: - result = getCurrentCompilerExe() - when defined(nimsuggest): - result = result.replace("nimsuggest", "nim") - else: - result = gNimExe - -template fixOutDir() {.dirty, used.} = - let - outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir - -proc compareVersions*(ver1, ver2: string): int = - ## Compare two version strings x.y.z and return -1, 0, 1 - ## - ## ver1 < ver2 = -1 - ## ver1 = ver2 = 0 - ## ver1 > ver2 = 1 - let - ver1seq = ver1.replace("-", "").split('.') - ver2seq = ver2.replace("-", "").split('.') - for i in 0 ..< ver1seq.len: - let - p1 = ver1seq[i] - p2 = if i < ver2seq.len: ver2seq[i] else: "0" - - try: - let - h1 = p1.parseHexInt() - h2 = p2.parseHexInt() - - if h1 < h2: return -1 - elif h1 > h2: return 1 - except ValueError: - if p1 < p2: return -1 - elif p1 > p2: return 1 - -proc fixCmd(cmd: string): string = - when defined(Windows): - # Replace 'cd d:\abc' with 'd: && cd d:\abc` - var filteredCmd = cmd - if cmd.toLower().startsWith("cd"): - var - colonIndex = cmd.find(":") - driveLetter = cmd.substr(colonIndex-1, colonIndex) - if (driveLetter[0].isAlphaAscii() and - driveLetter[1] == ':' and - colonIndex == 4): - filteredCmd = &"{driveLetter} && {cmd}" - result = "cmd /c " & filteredCmd - elif defined(posix): - result = cmd - else: - doAssert false +import "."/build/misc +export misc # Nim cfg file related functionality -include "."/build/nimconf - -proc getNimteropCacheDir(): string = - # Get location to cache all nimterop artifacts - result = getNimcacheDir() / "nimterop" +import "."/build/nimconf +export nimconf # Functionality shelled out to external executables -include "."/build/shell - -proc getProjectCacheDir*(name: string, forceClean = true): string = - ## Get a cache directory where all nimterop artifacts can be stored - ## - ## Projects can use this location to download source code and build binaries - ## that can be then accessed by multiple apps. This is created under the - ## per-user Nim cache directory. - ## - ## Use `name` to specify the subdirectory name for a project. - ## - ## `forceClean` is enabled by default and effectively deletes the folder - ## if Nim is compiled with the `-f` or `--forceBuild` flag. This allows - ## any project to start out with a clean cache dir on a forced build. - ## - ## NOTE: avoid calling `getProjectCacheDir()` multiple times on the same - ## `name` when `forceClean = true` else checked out source might get deleted - ## at the wrong time during build. - ## - ## E.g. - ## `nimgit2` downloads `libgit2` source so `name = "libgit2"` - ## - ## `nimarchive` downloads `libarchive`, `bzlib`, `liblzma` and `zlib` so - ## `name = "nimarchive" / "libarchive"` for `libarchive`, etc. - result = getNimteropCacheDir() / name - - if forceClean and compileOption("forceBuild"): - echo "# Removing " & result - rmDir(result) +import "."/build/shell +export shell # C compiler support -include "."/build/ccompiler +import "."/build/ccompiler +export ccompiler when not defined(TOAST): # configure, cmake, make support - include "."/build/tools + import "."/build/tools + export tools # Conan.io support - include "."/build/conan + import "."/build/conan + export conan # Julia BinaryBuilder.org support - include "."/build/jbb + import "."/build/jbb + export jbb # getHeader support - include "."/build/getheader + import "."/build/getheader + export getheader diff --git a/nimterop/build/ccompiler.nim b/nimterop/build/ccompiler.nim index c33c94a..550e1b2 100644 --- a/nimterop/build/ccompiler.nim +++ b/nimterop/build/ccompiler.nim @@ -1,5 +1,7 @@ import os, strformat, strutils +import "."/shell + proc getCompilerMode*(path: string): string = ## Determines a target language mode from an input filename, if one is not already specified. let file = path.splitFile() diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index 3430752..31e7f26 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -1,5 +1,7 @@ import os, strformat, strutils, tables +import "."/[ccompiler, misc, nimconf, shell] + when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): import marshal else: @@ -77,6 +79,10 @@ proc jsonGet(url: string): JsonNode = discard rmFile(file) +template fixOutDir() {.dirty.} = + let + outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir + proc `==`*(pkg1, pkg2: ConanPackage): bool = ## Check if two ConanPackage objects are equal (not pkg1.isNil and not pkg2.isNil and diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 216b599..8bb13fd 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -1,4 +1,9 @@ -import macros, os, strutils, tables +import macros, strformat, strutils, tables + +import os except findExe + +import ".."/globals +import "."/[ccompiler, conan, jbb, nimconf, shell, tools] var gDefines {.compileTime.} = initTable[string, string]() diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index 15bc578..cb48a27 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -1,5 +1,7 @@ import json, os, strformat, strutils, tables +import "."/[ccompiler, nimconf, shell] + when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): import marshal @@ -27,6 +29,10 @@ var # Reuse dependencies already downloaded gJBBRequires {.compileTime.}: Table[string, JBBPackage] +template fixOutDir() {.dirty.} = + let + outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir + proc `==`*(pkg1, pkg2: JBBPackage): bool = ## Check if two JBBPackage objects are equal (not pkg1.isNil and not pkg2.isNil and diff --git a/nimterop/build/nimconf.nim b/nimterop/build/nimconf.nim index c87a60f..70e47a3 100644 --- a/nimterop/build/nimconf.nim +++ b/nimterop/build/nimconf.nim @@ -1,5 +1,7 @@ import json, os, osproc, sets, strformat, strutils +import "."/misc + when nimvm: when (NimMajor, NimMinor, NimPatch) >= (1, 2, 0): import std/compilesettings @@ -227,3 +229,7 @@ proc getOutDir*(projectDir = ""): string = let cfg = getNimConfig(projectDir) result = cfg.outDir + +proc getNimteropCacheDir*(): string = + ## Get location to cache all nimterop artifacts + result = getNimcacheDir() / "nimterop" diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index bc07285..7f84db3 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -1,15 +1,25 @@ -import os, strformat, strutils +import hashes, osproc, sets, strformat, strutils -proc sleep*(milsecs: int) = - ## Sleep at compile time - let - cmd = - when defined(Windows): - "cmd /c timeout " - else: - "sleep " +when not defined(TOAST): + import os except findExe, sleep +else: + import os - discard gorgeEx(cmd & $(milsecs / 1000)) +import "."/[misc, nimconf] + +when not defined(TOAST): + proc sleep*(milsecs: int) = + ## Sleep at compile time + let + cmd = + when defined(Windows): + "cmd /c timeout " + else: + "sleep " + + discard gorgeEx(cmd & $(milsecs / 1000)) +else: + export sleep proc execAction*(cmd: string, retry = 0, die = true, cache = false, cacheKey = "", onRetry: proc() = nil): tuple[output: string, ret: int] = @@ -73,20 +83,23 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, doAssert false, "Command failed: " & $result.ret & "\ncmd: " & ccmd & "\nresult:\n" & result.output -proc findExe*(exe: string): string = - ## Find the specified executable using the `which`/`where` command - supported - ## at compile time - var - cmd = - when defined(Windows): - "where " & exe - else: - "which " & exe +when not defined(TOAST): + proc findExe*(exe: string): string = + ## Find the specified executable using the `which`/`where` command - supported + ## at compile time + var + cmd = + when defined(Windows): + "where " & exe + else: + "which " & exe - (output, ret) = execAction(cmd, die = false) + (output, ret) = execAction(cmd, die = false) - if ret == 0: - return output.splitLines()[0].strip().sanitizePath + if ret == 0: + return output.splitLines()[0].strip().sanitizePath +else: + export findExe proc mkDir*(dir: string) = ## Create a directory at compile time @@ -465,3 +478,31 @@ proc getNumProcs*(): string = execAction("sysctl -n hw.ncpu").output.strip() else: "1" + +proc getProjectCacheDir*(name: string, forceClean = true): string = + ## Get a cache directory where all nimterop artifacts can be stored + ## + ## Projects can use this location to download source code and build binaries + ## that can be then accessed by multiple apps. This is created under the + ## per-user Nim cache directory. + ## + ## Use `name` to specify the subdirectory name for a project. + ## + ## `forceClean` is enabled by default and effectively deletes the folder + ## if Nim is compiled with the `-f` or `--forceBuild` flag. This allows + ## any project to start out with a clean cache dir on a forced build. + ## + ## NOTE: avoid calling `getProjectCacheDir()` multiple times on the same + ## `name` when `forceClean = true` else checked out source might get deleted + ## at the wrong time during build. + ## + ## E.g. + ## `nimgit2` downloads `libgit2` source so `name = "libgit2"` + ## + ## `nimarchive` downloads `libarchive`, `bzlib`, `liblzma` and `zlib` so + ## `name = "nimarchive" / "libarchive"` for `libarchive`, etc. + result = getNimteropCacheDir() / name + + if forceClean and compileOption("forceBuild"): + echo "# Removing " & result + rmDir(result) diff --git a/nimterop/build/tools.nim b/nimterop/build/tools.nim index c6a4283..5be1f4b 100644 --- a/nimterop/build/tools.nim +++ b/nimterop/build/tools.nim @@ -1,20 +1,16 @@ -import os, strformat, strutils +import strformat, strutils -type - BuildType* = enum - btAutoconf, btCmake +import os except findExe - BuildStatus = object - built: bool - buildPath: string - error: string +import ".."/globals +import "."/[misc, shell] proc echoDebug(str: string) = let str = "\n# " & str.strip().replace("\n", "\n# ") - when nimvm: - if gDebugCT: echo str + when defined(TOAST): + if gState.debug: echo str else: - if gDebug: echo str + if gStateCT.debug: echo str proc configure*(path, check: string, flags = "") = ## Run the GNU `configure` command to generate all Makefiles or other @@ -208,7 +204,7 @@ proc make*(path, check: string, flags = "", regex = false) = doAssert findFile(check, path, regex = regex).len != 0, "make failed" -proc buildWithCmake(outdir, flags: string): BuildStatus = +proc buildWithCmake*(outdir, flags: string): BuildStatus = if not fileExists(outdir / "Makefile"): if fileExists(outdir / "CMakeLists.txt"): if findExe("cmake").len != 0: @@ -238,7 +234,7 @@ proc buildWithCmake(outdir, flags: string): BuildStatus = else: result.buildPath = outdir -proc buildWithAutoConf(outdir, flags: string): BuildStatus = +proc buildWithAutoConf*(outdir, flags: string): BuildStatus = if not fileExists(outdir / "Makefile"): if findExe("bash").len != 0: for file in @["configure", "configure.ac", "configure.in", "autogen.sh", "build/autogen.sh"]: diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index c81c35b..38a75e7 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -376,7 +376,6 @@ proc cSearchPath*(path: string): string {.compileTime.}= proc cDebug*() {.compileTime.} = ## Enable debug messages and display the generated Nim code gStateCT.debug = true - build.gDebugCT = true proc cDisableCaching*() {.compileTime.} = ## Disable caching of generated Nim code - useful during wrapper development diff --git a/nimterop/globals.nim b/nimterop/globals.nim index fa8f8de..e8117c8 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -7,7 +7,7 @@ when defined(TOAST): import compiler/[ast, idents, modulegraphs, options] - import "."/treesitter/api + # import "."/treesitter/api type Feature* = enum @@ -78,6 +78,21 @@ type pluginSource*: string # `cPlugin()` generated code to write to plugin file from searchDirs*: seq[string] # `cSearchPath()` added directories for header search + BuildType* = enum + btAutoconf, btCmake + + BuildStatus* = object + built*: bool + buildPath*: string + error*: string + +when nimvm: + var + gStateCT* {.compileTime, used.} = new(State) +else: + var + gState*: State + when defined(TOAST): const gAtoms* {.used.} = @[ @@ -119,9 +134,6 @@ when defined(TOAST): template decho*(args: varargs[string, `$`]): untyped = if gState.debug: gecho join(args, "").getCommented() -else: - var - gStateCT* {.compileTime, used.} = new(State) template nBl*(s: typed): untyped {.used.} = (s.len != 0) diff --git a/nimterop/paths.nim b/nimterop/paths.nim index fa245b3..b336d46 100644 --- a/nimterop/paths.nim +++ b/nimterop/paths.nim @@ -1,6 +1,6 @@ import os -import "."/build +import "."/build/shell const cacheDir* = getProjectCacheDir("nimterop", forceClean = false) diff --git a/nimterop/setup.nim b/nimterop/setup.nim index 3f16dd7..998d9e9 100644 --- a/nimterop/setup.nim +++ b/nimterop/setup.nim @@ -1,6 +1,7 @@ import os, strutils -import "."/[build, paths] +import "."/[paths] +import "."/build/[shell] proc treesitterSetup*() = gitPull("https://github.com/tree-sitter/tree-sitter", cacheDir / "treesitter", """ diff --git a/nimterop/toast.nim b/nimterop/toast.nim index b7a993a..e888e30 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -2,10 +2,12 @@ import os, osproc, sets, strformat, strutils, tables, times import "."/treesitter/[api, c, cpp] -import "."/[build, globals] +import "."/[globals] import "."/toastlib/[ast2, getters, tshelp] +import "."/build/[ccompiler, misc] + proc process(gState: State, path: string) = doAssert existsFile(path), &"Invalid path {path}" @@ -54,7 +56,7 @@ proc main( ) = # Setup global state with arguments - var gState = State( + gState = State( convention: convention, debug: debug, defines: defines, @@ -76,10 +78,6 @@ proc main( symOverride: symOverride ) - # Set gDebug in build.nim - build.gDebug = gState.debug - build.gNimExe = gState.nim - # Split some arguments with , gState.symOverride = gState.symOverride.getSplitComma() gState.prefix = gState.prefix.getSplitComma() diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 0c1b784..c3f48ae 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -2,7 +2,8 @@ import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times import regex -import ".."/[build, globals, plugin] +import ".."/[globals, plugin] +import ".."/build/[ccompiler, misc, nimconf, shell] const gReserved = """ addr and as asm diff --git a/nimterop/treesitter/cpp.nim b/nimterop/treesitter/cpp.nim index 2fe3128..4f5be17 100644 --- a/nimterop/treesitter/cpp.nim +++ b/nimterop/treesitter/cpp.nim @@ -1,6 +1,7 @@ import strutils, os -import ".."/[build, setup, paths] +import ".."/[setup, paths] +import ".."/build/shell static: treesitterCppSetup() From a0413f1595be040c57bc3d75cea1679e5d671d71 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 24 Jun 2020 13:17:25 -0500 Subject: [PATCH 048/106] Use g/decho in build --- nimterop/build/conan.nim | 10 +++--- nimterop/build/getheader.nim | 10 +++--- nimterop/build/jbb.nim | 3 +- nimterop/build/misc.nim | 62 ++++++++++++++++++++++++++++++++++++ nimterop/build/nimconf.nim | 3 +- nimterop/build/shell.nim | 23 ++++++------- nimterop/build/tools.nim | 37 +++++++++------------ nimterop/cimport.nim | 18 +++++------ nimterop/globals.nim | 29 +++++++++++++---- nimterop/toast.nim | 3 +- nimterop/toastlib/tshelp.nim | 3 -- 11 files changed, 135 insertions(+), 66 deletions(-) create mode 100644 nimterop/build/misc.nim diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index 31e7f26..792de01 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -1,11 +1,11 @@ -import os, strformat, strutils, tables +import json, os, strformat, strutils, tables + +import ".."/globals import "."/[ccompiler, misc, nimconf, shell] when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): import marshal -else: - import json type ConanPackage* = ref object @@ -158,7 +158,7 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac if channel.len != 0: query &= "/" & channel - echo &"# Searching Conan.io for latest version of {name}" + gecho &"# Searching Conan.io for latest version of {name}" let j1 = jsonGet(conanSearchUrl % ["query", query]) @@ -397,7 +397,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" - echo &"# Downloading {pkg.name} v{pkg.version} from Conan.io" + gecho &"# Downloading {pkg.name} v{pkg.version} from Conan.io" for recipe, builds in pkg.recipes: for build in builds: if pkg.bhash.len == 0 or pkg.bhash == build.bhash: diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 8bb13fd..6972ccc 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -476,9 +476,9 @@ macro getHeader*( {.passL: `ldeps`.join(" ").} static: - echo "# Including library " & lpath + gecho "# Including library " & lpath if `ldeps`.len != 0: - echo "# Including dependencies " & `ldeps`.join(" ") + gecho "# Including dependencies " & `ldeps`.join(" ") else: const `lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath @@ -498,7 +498,7 @@ macro getHeader*( ldeps[i] = ldeptgt # Copy downloaded dependencies to `libdir` if copied.len != 0: - echo "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` + gecho "# Copying dependencies: " & copied.join(" ") & "\n# to " & `libdir` ldeps else: ldeps @@ -507,8 +507,8 @@ macro getHeader*( when not useStd: # Copy downloaded shared libraries to `libdir` if not fileExists(`lpath`) or getFileDate(lpath) != getFileDate(`lpath`): - echo "# Copying " & `lpath`.extractFilename() & " to " & `libdir` + gecho "# Copying " & `lpath`.extractFilename() & " to " & `libdir` cpFile(lpath, `lpath`) - echo "# Including library " & `lpath` + gecho "# Including library " & `lpath` ) diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index cb48a27..0fb163f 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -1,5 +1,6 @@ import json, os, strformat, strutils, tables +import ".."/globals import "."/[ccompiler, nimconf, shell] when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): @@ -198,7 +199,7 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = else: "" path = outdir / pkg.name - echo &"# Downloading {pkg.name}{vstr} from BinaryBuilder.org" + gecho &"# Downloading {pkg.name}{vstr} from BinaryBuilder.org" downloadUrl(pkg.url, path, quiet = true) pkg.findJBBLibs(path) diff --git a/nimterop/build/misc.nim b/nimterop/build/misc.nim new file mode 100644 index 0000000..c6c0e48 --- /dev/null +++ b/nimterop/build/misc.nim @@ -0,0 +1,62 @@ +import os, strutils + +when defined(Windows): + import strformat + +import ".."/globals + +proc sanitizePath*(path: string, noQuote = false, sep = $DirSep): string = + result = path.multiReplace([("\\\\", sep), ("\\", sep), ("/", sep)]) + if not noQuote: + result = result.quoteShell + +proc getCurrentNimCompiler*(): string = + when nimvm: + result = getCurrentCompilerExe() + when defined(nimsuggest): + result = result.replace("nimsuggest", "nim") + else: + result = gState.nim + +proc compareVersions*(ver1, ver2: string): int = + ## Compare two version strings x.y.z and return -1, 0, 1 + ## + ## ver1 < ver2 = -1 + ## ver1 = ver2 = 0 + ## ver1 > ver2 = 1 + let + ver1seq = ver1.replace("-", "").split('.') + ver2seq = ver2.replace("-", "").split('.') + for i in 0 ..< ver1seq.len: + let + p1 = ver1seq[i] + p2 = if i < ver2seq.len: ver2seq[i] else: "0" + + try: + let + h1 = p1.parseHexInt() + h2 = p2.parseHexInt() + + if h1 < h2: return -1 + elif h1 > h2: return 1 + except ValueError: + if p1 < p2: return -1 + elif p1 > p2: return 1 + +proc fixCmd*(cmd: string): string = + when defined(Windows): + # Replace 'cd d:\abc' with 'd: && cd d:\abc` + var filteredCmd = cmd + if cmd.toLower().startsWith("cd"): + var + colonIndex = cmd.find(":") + driveLetter = cmd.substr(colonIndex-1, colonIndex) + if (driveLetter[0].isAlphaAscii() and + driveLetter[1] == ':' and + colonIndex == 4): + filteredCmd = &"{driveLetter} && {cmd}" + result = "cmd /c " & filteredCmd + elif defined(posix): + result = cmd + else: + doAssert false diff --git a/nimterop/build/nimconf.nim b/nimterop/build/nimconf.nim index 70e47a3..3bbe521 100644 --- a/nimterop/build/nimconf.nim +++ b/nimterop/build/nimconf.nim @@ -1,5 +1,6 @@ import json, os, osproc, sets, strformat, strutils +import ".."/globals import "."/misc when nimvm: @@ -42,7 +43,7 @@ proc getJson(projectDir: string): JsonNode = try: result = parseJson(dump) except JsonParsingError as e: - echo "# Failed to parse `nim dump` output: " & e.msg + gecho "# Failed to parse `nim dump` output: " & e.msg proc getOsCacheDir(): string = # OS default cache directory diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 7f84db3..a5d49de 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -5,6 +5,7 @@ when not defined(TOAST): else: import os +import ".."/globals import "."/[misc, nimconf] when not defined(TOAST): @@ -228,7 +229,7 @@ proc extractZip*(zipfile, outdir: string, quiet = false) = "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\"" if not quiet: - echo "# Extracting " & zipfile + gecho "# Extracting " & zipfile discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}") proc extractTar*(tarfile, outdir: string, quiet = false) = @@ -262,7 +263,7 @@ proc extractTar*(tarfile, outdir: string, quiet = false) = doAssert cmd.len != 0, "No extraction tool - tar, 7z, 7za - available for " & tarfile.sanitizePath if not quiet: - echo "# Extracting " & tarfile + gecho "# Extracting " & tarfile discard execAction(&"cd {outdir.sanitizePath} && {cmd}") if name.len != 0: rmFile(outdir / name) @@ -279,7 +280,7 @@ proc downloadUrl*(url, outdir: string, quiet = false, retry = 1) = if not (ext in archives and fileExists(filePath)): if not quiet: - echo "# Downloading " & file + gecho "# Downloading " & file mkDir(outdir) var cmd = findExe("curl") if cmd.len != 0: @@ -302,12 +303,12 @@ proc downloadUrl*(url, outdir: string, quiet = false, retry = 1) = proc gitReset*(outdir: string) = ## Hard reset the git repository at the specified directory - echo "# Resetting " & outdir + gecho "# Resetting " & outdir let cmd = &"cd {outdir.sanitizePath} && git reset --hard" while execAction(cmd).output.contains("Permission denied"): sleep(1000) - echo "# Retrying ..." + gecho "# Retrying ..." proc gitCheckout*(file, outdir: string) = ## Checkout the specified `file` in the git repository at `outdir` @@ -315,12 +316,12 @@ proc gitCheckout*(file, outdir: string) = ## This effectively resets all changes in the file and can be ## used to undo any changes that were made to source files to enable ## successful wrapping with `cImport()` or `c2nImport()`. - echo "# Resetting " & file + gecho "# Resetting " & file let file2 = file.relativePath outdir let cmd = &"cd {outdir.sanitizePath} && git checkout {file2.sanitizePath}" while execAction(cmd).output.contains("Permission denied"): sleep(500) - echo "# Retrying ..." + gecho "# Retrying ..." proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = ## Pull the specified git repository to the output directory @@ -343,7 +344,7 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false mkDir(outdir) if not quiet: - echo "# Setting up Git repo: " & url + gecho "# Setting up Git repo: " & url discard execAction(&"cd {outdirQ} && git init .") discard execAction(&"cd {outdirQ} && git remote add origin {url}") @@ -360,12 +361,12 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false if checkout.len != 0: if not quiet: - echo "# Checking out " & checkout + gecho "# Checking out " & checkout discard execAction(&"cd {outdirQ} && git fetch", retry = 3) discard execAction(&"cd {outdirQ} && git checkout {checkout}") else: if not quiet: - echo "# Pulling repository" + gecho "# Pulling repository" discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 3) proc gitTags*(outdir: string): seq[string] = @@ -504,5 +505,5 @@ proc getProjectCacheDir*(name: string, forceClean = true): string = result = getNimteropCacheDir() / name if forceClean and compileOption("forceBuild"): - echo "# Removing " & result + gecho "# Removing " & result rmDir(result) diff --git a/nimterop/build/tools.nim b/nimterop/build/tools.nim index 5be1f4b..c164ffd 100644 --- a/nimterop/build/tools.nim +++ b/nimterop/build/tools.nim @@ -5,13 +5,6 @@ import os except findExe import ".."/globals import "."/[misc, shell] -proc echoDebug(str: string) = - let str = "\n# " & str.strip().replace("\n", "\n# ") - when defined(TOAST): - if gState.debug: echo str - else: - if gStateCT.debug: echo str - proc configure*(path, check: string, flags = "") = ## Run the GNU `configure` command to generate all Makefiles or other ## build scripts in the specified path @@ -30,18 +23,18 @@ proc configure*(path, check: string, flags = "") = if (path / check).fileExists(): return - echo "# Configuring " & path + gecho "# Configuring " & path if not fileExists(path / "configure"): for i in @["autogen.sh", "build" / "autogen.sh"]: if fileExists(path / i): - echo "# Running autogen.sh" + gecho "# Running autogen.sh" when defined(unix): - echoDebug execAction( + decho execAction( &"cd {(path / i).parentDir().sanitizePath} && ./autogen.sh").output else: - echoDebug execAction( + decho execAction( &"cd {(path / i).parentDir().sanitizePath} && bash ./autogen.sh").output break @@ -49,14 +42,14 @@ proc configure*(path, check: string, flags = "") = if not fileExists(path / "configure"): for i in @["configure.ac", "configure.in"]: if fileExists(path / i): - echo "# Running autoreconf" + gecho "# Running autoreconf" - echoDebug execAction(&"cd {path.sanitizePath} && autoreconf -fi").output + decho execAction(&"cd {path.sanitizePath} && autoreconf -fi").output break if fileExists(path / "configure"): - echo "# Running configure " & flags + gecho "# Running configure " & flags when defined(unix): var @@ -67,7 +60,7 @@ proc configure*(path, check: string, flags = "") = if flags.len != 0: cmd &= &" {flags}" - echoDebug execAction(cmd).output + decho execAction(cmd).output doAssert (path / check).fileExists(), "Configure failed" @@ -156,15 +149,15 @@ proc cmake*(path, check, flags: string) = if (path / check).fileExists(): return - echo "# Running cmake " & flags - echo "# Path: " & path + gecho "# Running cmake " & flags + gecho "# Path: " & path mkDir(path) let cmd = &"cd {path.sanitizePath} && cmake {flags}" - echoDebug execAction(cmd).output + decho execAction(cmd).output doAssert (path / check).fileExists(), "cmake failed" @@ -184,8 +177,8 @@ proc make*(path, check: string, flags = "", regex = false) = if findFile(check, path, regex = regex).len != 0: return - echo "# Running make " & flags - echo "# Path: " & path + gecho "# Running make " & flags + gecho "# Path: " & path var cmd = findExe("make") @@ -200,7 +193,7 @@ proc make*(path, check: string, flags = "", regex = false) = if flags.len != 0: cmd &= &" {flags}" - echoDebug execAction(cmd).output + decho execAction(cmd).output doAssert findFile(check, path, regex = regex).len != 0, "make failed" @@ -219,7 +212,7 @@ proc buildWithCmake*(outdir, flags: string): BuildStatus = elif uname.contains("mingw"): gen = "MinGW Makefiles".quoteShell & " -DCMAKE_SH=\"CMAKE_SH-NOTFOUND\"" else: - echo "Unsupported system: " & uname + gecho "Unsupported system: " & uname else: gen = "MinGW Makefiles".quoteShell else: diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 38a75e7..edfdeb9 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -246,8 +246,8 @@ proc onSymbolOverride*(sym: var Symbol) {.exportc, dynlib.} = gStateCT.symOverride.add name - if gStateCT.debug and names.nBl: - echo "# Overriding " & names.join(" ") + if names.nBl: + decho "Overriding " & names.join(" ") proc cSkipSymbol*(skips: seq[string]) {.compileTime.} = ## Similar to `cOverride() `_, this macro allows @@ -412,7 +412,7 @@ macro cDefine*(name: static string, val: static string = ""): untyped = {.passC: `str`.} if gStateCT.debug: - echo result.repr & "\n" + gecho result.repr & "\n" proc cAddSearchDir*(dir: string) {.compileTime.} = ## Add directory `dir` to the search path used in calls to @@ -442,7 +442,7 @@ macro cIncludeDir*(dir: static string): untyped = result.add quote do: {.passC: `str`.} if gStateCT.debug: - echo result.repr + gecho result.repr proc cAddStdDir*(mode = "c") {.compileTime.} = ## Add the standard `c` [default] or `cpp` include paths to search @@ -545,7 +545,7 @@ macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = result.add stmt.parseStmt() if gStateCT.debug: - echo result.repr + gecho result.repr macro cImport*(filenames: static seq[string], recurse: static bool = false, dynlib: static string = "", mode: static string = "c", flags: static string = ""): untyped = @@ -565,7 +565,7 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl if gStateCT.pluginSourcePath.Bl: cPluginHelper(gStateCT.pluginSource) - echo "# Importing " & fullpaths.join(", ").sanitizePath + gecho "# Importing " & fullpaths.join(", ").sanitizePath let output = getToast(fullpaths, recurse, dynlib, mode, flags) @@ -576,7 +576,7 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl gStateCT.overrides = "" if gStateCT.debug: - echo output + gecho output try: let body = parseStmt(output) @@ -661,7 +661,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: let fullpath = findPath(filename) - echo "# Importing " & fullpath & " with c2nim" + gecho "# Importing " & fullpath & " with c2nim" let output = getToast(@[fullpath], recurse, dynlib, mode, noNimout = true) @@ -700,7 +700,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: nimout = &"const {header} = \"{fullpath}\"\n\n" & readFile(npath) if gStateCT.debug: - echo nimout + gecho nimout try: let body = parseStmt(nimout) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index e8117c8..a89bac4 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -1,4 +1,4 @@ -import tables +import strutils, tables when defined(TOAST): import sets, sequtils, strutils @@ -124,16 +124,31 @@ when defined(TOAST): Status* = enum success, unknown, error - # Redirect output to file when required - template gecho*(args: string) = - if gState.outputHandle.isNil: +proc getCommented*(str: string): string = + "\n# " & str.strip().replace("\n", "\n# ") + +# Redirect output to file when required +template gecho*(args: string) = + when defined(TOAST): + when nimvm: echo args else: - gState.outputHandle.writeLine(args) + if gState.outputHandle.isNil: + echo args + else: + gState.outputHandle.writeLine(args) + else: + echo args - template decho*(args: varargs[string, `$`]): untyped = +template decho*(args: varargs[string, `$`]): untyped = + let + str = join(args, "").getCommented() + when defined(TOAST): if gState.debug: - gecho join(args, "").getCommented() + gecho str + else: + if gStateCT.debug: + echo str template nBl*(s: typed): untyped {.used.} = (s.len != 0) diff --git a/nimterop/toast.nim b/nimterop/toast.nim index e888e30..6fe5180 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -124,8 +124,7 @@ proc main( doAssert gState.outputHandle.open(outputFile, fmWrite), &"Failed to write to {outputFile}" - if gState.debug: - echo &"# Writing output to {outputFile}\n" + decho &"# Writing output to {outputFile}\n" if source.nBl: # Print source after preprocess or Nim output diff --git a/nimterop/toastlib/tshelp.nim b/nimterop/toastlib/tshelp.nim index 9310cf7..3b7c2cc 100644 --- a/nimterop/toastlib/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -30,9 +30,6 @@ template withCodeAst*(code: string, mode: string, body: untyped): untyped = defer: tree.tsTreeDelete() -proc getCommented*(str: string): string = - "\n# " & str.strip().replace("\n", "\n# ") - proc isNil*(node: TSNode): bool = node.tsNodeIsNull() From 85a4365e3cef38e6490d3ff591e33fee9d2167a9 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 25 Jun 2020 00:10:13 -0500 Subject: [PATCH 049/106] Adjust git checkout if changed, tree-sitter version bump --- nimterop/build/shell.nim | 18 ++++++++++++++---- nimterop/globals.nim | 4 +++- nimterop/setup.nim | 6 +++--- nimterop/toastlib/ast2.nim | 5 ++++- nimterop/toastlib/exprparser.nim | 2 +- 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index a5d49de..53c0b5f 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -323,6 +323,12 @@ proc gitCheckout*(file, outdir: string) = sleep(500) gecho "# Retrying ..." +proc gitAtCheckout*(outdir, checkout: string): bool = + ## Check if specified git repository is checked out to the specified + ## commit hash, tag or branch + result = checkout in execAction( + &"cd {outdir.sanitizePath} && git log --decorate --no-color -n 1 --format=oneline").output + proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = ## Pull the specified git repository to the output directory ## @@ -334,13 +340,17 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false ## `checkout` is the git tag, branch or commit hash to checkout once ## the repository is downloaded. This allows for pinning to a specific ## version of the code. - if dirExists(outdir/".git"): - gitReset(outdir) - return - let outdirQ = outdir.sanitizePath + if dirExists(outdir/".git"): + gitReset(outdir) + if checkout.nBl and not gitAtCheckout(outdir, checkout): + gecho &"# Updating repository to checkout {checkout}" + discard execAction( + &"cd {outdirQ} && git clean -fxd && git fetch && git checkout {checkout}", retry = 3) + return + mkDir(outdir) if not quiet: diff --git a/nimterop/globals.nim b/nimterop/globals.nim index a89bac4..c831589 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -111,7 +111,9 @@ when defined(TOAST): "bitwise_expression", "shift_expression", "math_expression", - "escape_sequence" + "escape_sequence", + "binary_expression", + "unary_expression" ].toHashSet() gEnumVals* {.used.} = @[ diff --git a/nimterop/setup.nim b/nimterop/setup.nim index 998d9e9..a0940d2 100644 --- a/nimterop/setup.nim +++ b/nimterop/setup.nim @@ -7,7 +7,7 @@ proc treesitterSetup*() = gitPull("https://github.com/tree-sitter/tree-sitter", cacheDir / "treesitter", """ lib/include/* lib/src/* -""", "0.15.5") +""", "0.15.14") gitPull("https://github.com/JuliaStrings/utf8proc", cacheDir / "utf8proc", """ *.c @@ -37,7 +37,7 @@ src/*.h src/*.c src/*.cc src/tree_sitter/parser.h -""", "v0.15.0") +""", "v0.15.3") writeFile(cacheDir / "treesitter_c" / "src" / "api.h", """ const TSLanguage *tree_sitter_c(); @@ -49,7 +49,7 @@ src/*.h src/*.c src/*.cc src/tree_sitter/parser.h -""", "v0.15.0") +""", "v0.15.1") writeFile(cacheDir / "treesitter_cpp" / "src" / "api.h", """ const TSLanguage *tree_sitter_cpp(); diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 0016e29..783b6e4 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1099,7 +1099,10 @@ proc getTypeProc(gState: State, name: string, node, rnode: TSNode): PNode = ncount = if not afdecl.isNil: # Pointer to function pointer - afdecl[0].getXCount("abstract_pointer_declarator") + if afdecl[0].getName() == "abstract_parenthesized_declarator": + afdecl[0][0].getXCount("abstract_pointer_declarator") + else: + afdecl[0].getXCount("abstract_pointer_declarator") else: node.getAtom().tsNodeParent().getPtrCount(reverse = true) diff --git a/nimterop/toastlib/exprparser.nim b/nimterop/toastlib/exprparser.nim index 28b49d3..13b1a98 100644 --- a/nimterop/toastlib/exprparser.nim +++ b/nimterop/toastlib/exprparser.nim @@ -535,7 +535,7 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = # once we upgrade of "math_expression", "logical_expression", "relational_expression", "bitwise_expression", "equality_expression", "binary_expression", - "shift_expression": + "shift_expression", "unary_expression": # Input -> a == b, a != b, !a, ~a, a < b, a > b, a <= b, a >= b, a >> b, a << b # Output -> # typeof(a)(a == typeof(a)(b)) From f42ba492e3b4649e9fb0e18c095464669aed463e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 25 Jun 2020 00:59:42 -0500 Subject: [PATCH 050/106] Bump to latest tree-sitter --- nimterop/setup.nim | 11 +- nimterop/treesitter/api.nim | 306 ++++++++++++++++++++++------------ nimterop/treesitter/tsgen.nim | 9 +- 3 files changed, 199 insertions(+), 127 deletions(-) diff --git a/nimterop/setup.nim b/nimterop/setup.nim index a0940d2..6da0bbe 100644 --- a/nimterop/setup.nim +++ b/nimterop/setup.nim @@ -7,12 +7,7 @@ proc treesitterSetup*() = gitPull("https://github.com/tree-sitter/tree-sitter", cacheDir / "treesitter", """ lib/include/* lib/src/* -""", "0.15.14") - - gitPull("https://github.com/JuliaStrings/utf8proc", cacheDir / "utf8proc", """ -*.c -*.h -""") +""", "0.16.8") let tbase = cacheDir / "treesitter" / "lib" @@ -37,7 +32,7 @@ src/*.h src/*.c src/*.cc src/tree_sitter/parser.h -""", "v0.15.3") +""", "0.16.1") writeFile(cacheDir / "treesitter_c" / "src" / "api.h", """ const TSLanguage *tree_sitter_c(); @@ -49,7 +44,7 @@ src/*.h src/*.c src/*.cc src/tree_sitter/parser.h -""", "v0.15.1") +""", "v0.16.0") writeFile(cacheDir / "treesitter_cpp" / "src" / "api.h", """ const TSLanguage *tree_sitter_cpp(); diff --git a/nimterop/treesitter/api.nim b/nimterop/treesitter/api.nim index 09e328d..116bc10 100644 --- a/nimterop/treesitter/api.nim +++ b/nimterop/treesitter/api.nim @@ -12,56 +12,70 @@ const sourcePath = cacheDir / "treesitter" / "lib" when defined(Linux): {.passC: "-std=c11".} -{.passC: "-DUTF8PROC_STATIC".} {.passC: "-I$1" % (sourcePath / "include").} {.passC: "-I$1" % (sourcePath / "src").} -{.passC: "-I$1" % (sourcePath / ".." / ".." / "utf8proc").} {.compile: sourcePath / "src" / "lib.c".} ### Generated below -{.hint[ConvFromXtoItselfNotNeeded]: off.} +{.push hint[ConvFromXtoItselfNotNeeded]: off.} +{.pragma: impapiHdr, header: sourcePath / "include" / "tree_sitter" / "api.h".} defineEnum(TSInputEncoding) defineEnum(TSSymbolType) defineEnum(TSLogType) +defineEnum(TSQueryPredicateStepType) +defineEnum(TSQueryError) + const - headerapi {.used.} = sourcePath / "include" / "tree_sitter" / "api.h" - TREE_SITTER_LANGUAGE_VERSION* = 9 - TSInputEncodingUTF8* = 0.TSInputEncoding - TSInputEncodingUTF16* = 1.TSInputEncoding - TSSymbolTypeRegular* = 0.TSSymbolType - TSSymbolTypeAnonymous* = 1.TSSymbolType - TSSymbolTypeAuxiliary* = 2.TSSymbolType - TSLogTypeParse* = 0.TSLogType - TSLogTypeLex* = 1.TSLogType + TREE_SITTER_LANGUAGE_VERSION* = 11 + TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION* = 9 + TSInputEncodingUTF8* = (0).TSInputEncoding + TSInputEncodingUTF16* = (TSInputEncodingUTF8 + 1).TSInputEncoding + TSSymbolTypeRegular* = (0).TSSymbolType + TSSymbolTypeAnonymous* = (TSSymbolTypeRegular + 1).TSSymbolType + TSSymbolTypeAuxiliary* = (TSSymbolTypeAnonymous + 1).TSSymbolType + TSLogTypeParse* = (0).TSLogType + TSLogTypeLex* = (TSLogTypeParse + 1).TSLogType + TSQueryPredicateStepTypeDone* = (0).TSQueryPredicateStepType + TSQueryPredicateStepTypeCapture* = (TSQueryPredicateStepTypeDone + 1).TSQueryPredicateStepType + TSQueryPredicateStepTypeString* = (TSQueryPredicateStepTypeCapture + 1).TSQueryPredicateStepType + TSQueryErrorNone* = (0).TSQueryError + TSQueryErrorSyntax* = (TSQueryErrorNone + 1).TSQueryError + TSQueryErrorNodeType* = (TSQueryErrorSyntax + 1).TSQueryError + TSQueryErrorField* = (TSQueryErrorNodeType + 1).TSQueryError + TSQueryErrorCapture* = (TSQueryErrorField + 1).TSQueryError + type - TSSymbol* = uint16 - TSLanguage* = object - TSParser* = object - TSTree* = object - TSPoint* {.importc, header: headerapi, bycopy.} = object + TSSymbol* {.importc, impapiHdr.} = uint16 + TSFieldId* {.importc, impapiHdr.} = uint16 + TSLanguage* {.importc, impapiHdr, incompleteStruct.} = object + TSParser* {.importc, impapiHdr, incompleteStruct.} = object + TSTree* {.importc, impapiHdr, incompleteStruct.} = object + TSQuery* {.importc, impapiHdr, incompleteStruct.} = object + TSQueryCursor* {.importc, impapiHdr, incompleteStruct.} = object + TSPoint* {.bycopy, importc, impapiHdr.} = object row*: uint32 column*: uint32 - TSRange* {.importc, header: headerapi, bycopy.} = object + TSRange* {.bycopy, importc, impapiHdr.} = object start_point*: TSPoint end_point*: TSPoint start_byte*: uint32 end_byte*: uint32 - TSInput* {.importc, header: headerapi, bycopy.} = object + TSInput* {.bycopy, importc, impapiHdr.} = object payload*: pointer read*: proc (payload: pointer; byte_index: uint32; position: TSPoint; - bytes_read: ptr uint32): cstring {.nimcall.} + bytes_read: ptr uint32): cstring {.cdecl.} encoding*: TSInputEncoding - TSLogger* {.importc, header: headerapi, bycopy.} = object + TSLogger* {.bycopy, importc, impapiHdr.} = object payload*: pointer - log*: proc (payload: pointer; a1: TSLogType; a2: cstring) {.nimcall.} + log*: proc (payload: pointer; a2: TSLogType; a3: cstring) {.cdecl.} - TSInputEdit* {.importc, header: headerapi, bycopy.} = object + TSInputEdit* {.bycopy, importc, impapiHdr.} = object start_byte*: uint32 old_end_byte*: uint32 new_end_byte*: uint32 @@ -69,106 +83,176 @@ type old_end_point*: TSPoint new_end_point*: TSPoint - TSNode* {.importc, header: headerapi, bycopy.} = object + TSNode* {.bycopy, importc, impapiHdr.} = object context*: array[4, uint32] id*: pointer tree*: ptr TSTree - TSTreeCursor* {.importc, header: headerapi, bycopy.} = object + TSTreeCursor* {.bycopy, importc, impapiHdr.} = object tree*: pointer id*: pointer context*: array[2, uint32] -proc ts_parser_new*(): ptr TSParser {.importc, header: headerapi.} -proc ts_parser_delete*(a1: ptr TSParser) {.importc, header: headerapi.} -proc ts_parser_language*(a1: ptr TSParser): ptr TSLanguage {.importc, header: headerapi.} -proc ts_parser_set_language*(a1: ptr TSParser; a2: ptr TSLanguage): bool {.importc, - header: headerapi.} -proc ts_parser_logger*(a1: ptr TSParser): TSLogger {.importc, header: headerapi.} -proc ts_parser_set_logger*(a1: ptr TSParser; a2: TSLogger) {.importc, header: headerapi.} -proc ts_parser_print_dot_graphs*(a1: ptr TSParser; a2: cint) {.importc, - header: headerapi.} -proc ts_parser_halt_on_error*(a1: ptr TSParser; a2: bool) {.importc, header: headerapi.} -proc ts_parser_parse*(a1: ptr TSParser; a2: ptr TSTree; a3: TSInput): ptr TSTree {.importc, - header: headerapi.} -proc ts_parser_parse_string*(a1: ptr TSParser; a2: ptr TSTree; a3: cstring; a4: uint32): ptr TSTree {. - importc, header: headerapi.} -proc ts_parser_parse_string_encoding*(a1: ptr TSParser; a2: ptr TSTree; a3: cstring; - a4: uint32; a5: TSInputEncoding): ptr TSTree {. - importc, header: headerapi.} -proc ts_parser_enabled*(a1: ptr TSParser): bool {.importc, header: headerapi.} -proc ts_parser_set_enabled*(a1: ptr TSParser; a2: bool) {.importc, header: headerapi.} -proc ts_parser_operation_limit*(a1: ptr TSParser): cuint {.importc, header: headerapi.} -proc ts_parser_set_operation_limit*(a1: ptr TSParser; a2: cuint) {.importc, - header: headerapi.} -proc ts_parser_reset*(a1: ptr TSParser) {.importc, header: headerapi.} -proc ts_parser_set_included_ranges*(a1: ptr TSParser; a2: ptr TSRange; a3: uint32) {. - importc, header: headerapi.} -proc ts_parser_included_ranges*(a1: ptr TSParser; a2: ptr uint32): ptr TSRange {.importc, - header: headerapi.} -proc ts_tree_copy*(a1: ptr TSTree): ptr TSTree {.importc, header: headerapi.} -proc ts_tree_delete*(a1: ptr TSTree) {.importc, header: headerapi.} -proc ts_tree_root_node*(a1: ptr TSTree): TSNode {.importc, header: headerapi.} -proc ts_tree_edit*(a1: ptr TSTree; a2: ptr TSInputEdit) {.importc, header: headerapi.} -proc ts_tree_get_changed_ranges*(a1: ptr TSTree; a2: ptr TSTree; a3: ptr uint32): ptr TSRange {. - importc, header: headerapi.} -proc ts_tree_print_dot_graph*(a1: ptr TSTree; a2: ptr FILE) {.importc, header: headerapi.} -proc ts_tree_language*(a1: ptr TSTree): ptr TSLanguage {.importc, header: headerapi.} -proc ts_node_start_byte*(a1: TSNode): uint32 {.importc, header: headerapi.} -proc ts_node_start_point*(a1: TSNode): TSPoint {.importc, header: headerapi.} -proc ts_node_end_byte*(a1: TSNode): uint32 {.importc, header: headerapi.} -proc ts_node_end_point*(a1: TSNode): TSPoint {.importc, header: headerapi.} -proc ts_node_symbol*(a1: TSNode): TSSymbol {.importc, header: headerapi.} -proc ts_node_type*(a1: TSNode): cstring {.importc, header: headerapi.} -proc ts_node_string*(a1: TSNode): cstring {.importc, header: headerapi.} -proc ts_node_eq*(a1: TSNode; a2: TSNode): bool {.importc, header: headerapi.} -proc ts_node_is_null*(a1: TSNode): bool {.importc, header: headerapi.} -proc ts_node_is_named*(a1: TSNode): bool {.importc, header: headerapi.} -proc ts_node_is_missing*(a1: TSNode): bool {.importc, header: headerapi.} -proc ts_node_has_changes*(a1: TSNode): bool {.importc, header: headerapi.} -proc ts_node_has_error*(a1: TSNode): bool {.importc, header: headerapi.} -proc ts_node_parent*(a1: TSNode): TSNode {.importc, header: headerapi.} -proc ts_node_child*(a1: TSNode; a2: uint32): TSNode {.importc, header: headerapi.} -proc ts_node_named_child*(a1: TSNode; a2: uint32): TSNode {.importc, header: headerapi.} -proc ts_node_child_count*(a1: TSNode): uint32 {.importc, header: headerapi.} -proc ts_node_named_child_count*(a1: TSNode): uint32 {.importc, header: headerapi.} -proc ts_node_next_sibling*(a1: TSNode): TSNode {.importc, header: headerapi.} -proc ts_node_next_named_sibling*(a1: TSNode): TSNode {.importc, header: headerapi.} -proc ts_node_prev_sibling*(a1: TSNode): TSNode {.importc, header: headerapi.} -proc ts_node_prev_named_sibling*(a1: TSNode): TSNode {.importc, header: headerapi.} -proc ts_node_first_child_for_byte*(a1: TSNode; a2: uint32): TSNode {.importc, - header: headerapi.} + TSQueryCapture* {.bycopy, importc, impapiHdr.} = object + node*: TSNode + index*: uint32 + + TSQueryMatch* {.bycopy, importc, impapiHdr.} = object + id*: uint32 + pattern_index*: uint16 + capture_count*: uint16 + captures*: ptr TSQueryCapture + + TSQueryPredicateStep* {.bycopy, importc, impapiHdr.} = object + `type`*: TSQueryPredicateStepType + value_id*: uint32 + +proc ts_parser_new*(): ptr TSParser {.importc, cdecl, impapiHdr.} +proc ts_parser_delete*(parser: ptr TSParser) {.importc, cdecl, impapiHdr.} +proc ts_parser_set_language*(self: ptr TSParser; language: ptr TSLanguage): bool {. + importc, cdecl, impapiHdr.} +proc ts_parser_language*(self: ptr TSParser): ptr TSLanguage {.importc, cdecl, impapiHdr.} +proc ts_parser_set_included_ranges*(self: ptr TSParser; ranges: ptr TSRange; + length: uint32) {.importc, cdecl, impapiHdr.} +proc ts_parser_included_ranges*(self: ptr TSParser; length: ptr uint32): ptr TSRange {. + importc, cdecl, impapiHdr.} +proc ts_parser_parse*(self: ptr TSParser; old_tree: ptr TSTree; input: TSInput): ptr TSTree {. + importc, cdecl, impapiHdr.} +proc ts_parser_parse_string*(self: ptr TSParser; old_tree: ptr TSTree; string: cstring; + length: uint32): ptr TSTree {.importc, cdecl, impapiHdr.} +proc ts_parser_parse_string_encoding*(self: ptr TSParser; old_tree: ptr TSTree; + string: cstring; length: uint32; + encoding: TSInputEncoding): ptr TSTree {. + importc, cdecl, impapiHdr.} +proc ts_parser_reset*(self: ptr TSParser) {.importc, cdecl, impapiHdr.} +proc ts_parser_set_timeout_micros*(self: ptr TSParser; timeout: uint64) {.importc, + cdecl, impapiHdr.} +proc ts_parser_timeout_micros*(self: ptr TSParser): uint64 {.importc, cdecl, impapiHdr.} +proc ts_parser_set_cancellation_flag*(self: ptr TSParser; flag: ptr uint) {.importc, + cdecl, impapiHdr.} +proc ts_parser_cancellation_flag*(self: ptr TSParser): ptr uint {.importc, cdecl, + impapiHdr.} +proc ts_parser_set_logger*(self: ptr TSParser; logger: TSLogger) {.importc, cdecl, + impapiHdr.} +proc ts_parser_logger*(self: ptr TSParser): TSLogger {.importc, cdecl, impapiHdr.} +proc ts_parser_print_dot_graphs*(self: ptr TSParser; file: cint) {.importc, cdecl, + impapiHdr.} +proc ts_parser_halt_on_error*(self: ptr TSParser; halt: bool) {.importc, cdecl, + impapiHdr.} +proc ts_tree_copy*(self: ptr TSTree): ptr TSTree {.importc, cdecl, impapiHdr.} +proc ts_tree_delete*(self: ptr TSTree) {.importc, cdecl, impapiHdr.} +proc ts_tree_root_node*(self: ptr TSTree): TSNode {.importc, cdecl, impapiHdr.} +proc ts_tree_language*(a1: ptr TSTree): ptr TSLanguage {.importc, cdecl, impapiHdr.} +proc ts_tree_edit*(self: ptr TSTree; edit: ptr TSInputEdit) {.importc, cdecl, impapiHdr.} +proc ts_tree_get_changed_ranges*(old_tree: ptr TSTree; new_tree: ptr TSTree; + length: ptr uint32): ptr TSRange {.importc, cdecl, + impapiHdr.} +proc ts_tree_print_dot_graph*(a1: ptr TSTree; a2: File) {.importc, cdecl, impapiHdr.} +proc ts_node_type*(a1: TSNode): cstring {.importc, cdecl, impapiHdr.} +proc ts_node_symbol*(a1: TSNode): TSSymbol {.importc, cdecl, impapiHdr.} +proc ts_node_start_byte*(a1: TSNode): uint32 {.importc, cdecl, impapiHdr.} +proc ts_node_start_point*(a1: TSNode): TSPoint {.importc, cdecl, impapiHdr.} +proc ts_node_end_byte*(a1: TSNode): uint32 {.importc, cdecl, impapiHdr.} +proc ts_node_end_point*(a1: TSNode): TSPoint {.importc, cdecl, impapiHdr.} +proc ts_node_string*(a1: TSNode): cstring {.importc, cdecl, impapiHdr.} +proc ts_node_is_null*(a1: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_node_is_named*(a1: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_node_is_missing*(a1: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_node_is_extra*(a1: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_node_has_changes*(a1: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_node_has_error*(a1: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_node_parent*(a1: TSNode): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_child*(a1: TSNode; a2: uint32): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_child_count*(a1: TSNode): uint32 {.importc, cdecl, impapiHdr.} +proc ts_node_named_child*(a1: TSNode; a2: uint32): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_named_child_count*(a1: TSNode): uint32 {.importc, cdecl, impapiHdr.} +proc ts_node_child_by_field_name*(self: TSNode; field_name: cstring; + field_name_length: uint32): TSNode {.importc, + cdecl, impapiHdr.} +proc ts_node_child_by_field_id*(a1: TSNode; a2: TSFieldId): TSNode {.importc, cdecl, + impapiHdr.} +proc ts_node_next_sibling*(a1: TSNode): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_prev_sibling*(a1: TSNode): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_next_named_sibling*(a1: TSNode): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_prev_named_sibling*(a1: TSNode): TSNode {.importc, cdecl, impapiHdr.} +proc ts_node_first_child_for_byte*(a1: TSNode; a2: uint32): TSNode {.importc, cdecl, + impapiHdr.} proc ts_node_first_named_child_for_byte*(a1: TSNode; a2: uint32): TSNode {.importc, - header: headerapi.} + cdecl, impapiHdr.} proc ts_node_descendant_for_byte_range*(a1: TSNode; a2: uint32; a3: uint32): TSNode {. - importc, header: headerapi.} -proc ts_node_named_descendant_for_byte_range*(a1: TSNode; a2: uint32; a3: uint32): TSNode {. - importc, header: headerapi.} + importc, cdecl, impapiHdr.} proc ts_node_descendant_for_point_range*(a1: TSNode; a2: TSPoint; a3: TSPoint): TSNode {. - importc, header: headerapi.} + importc, cdecl, impapiHdr.} +proc ts_node_named_descendant_for_byte_range*(a1: TSNode; a2: uint32; a3: uint32): TSNode {. + importc, cdecl, impapiHdr.} proc ts_node_named_descendant_for_point_range*(a1: TSNode; a2: TSPoint; a3: TSPoint): TSNode {. - importc, header: headerapi.} -proc ts_node_edit*(a1: ptr TSNode; a2: ptr TSInputEdit) {.importc, header: headerapi.} -proc ts_tree_cursor_new*(a1: TSNode): TSTreeCursor {.importc, header: headerapi.} -proc ts_tree_cursor_delete*(a1: ptr TSTreeCursor) {.importc, header: headerapi.} -proc ts_tree_cursor_reset*(a1: ptr TSTreeCursor; a2: TSNode) {.importc, - header: headerapi.} -proc ts_tree_cursor_current_node*(a1: ptr TSTreeCursor): TSNode {.importc, - header: headerapi.} -proc ts_tree_cursor_goto_parent*(a1: ptr TSTreeCursor): bool {.importc, - header: headerapi.} -proc ts_tree_cursor_goto_next_sibling*(a1: ptr TSTreeCursor): bool {.importc, - header: headerapi.} -proc ts_tree_cursor_goto_first_child*(a1: ptr TSTreeCursor): bool {.importc, - header: headerapi.} + importc, cdecl, impapiHdr.} +proc ts_node_edit*(a1: ptr TSNode; a2: ptr TSInputEdit) {.importc, cdecl, impapiHdr.} +proc ts_node_eq*(a1: TSNode; a2: TSNode): bool {.importc, cdecl, impapiHdr.} +proc ts_tree_cursor_new*(a1: TSNode): TSTreeCursor {.importc, cdecl, impapiHdr.} +proc ts_tree_cursor_delete*(a1: ptr TSTreeCursor) {.importc, cdecl, impapiHdr.} +proc ts_tree_cursor_reset*(a1: ptr TSTreeCursor; a2: TSNode) {.importc, cdecl, impapiHdr.} +proc ts_tree_cursor_current_node*(a1: ptr TSTreeCursor): TSNode {.importc, cdecl, + impapiHdr.} +proc ts_tree_cursor_current_field_name*(a1: ptr TSTreeCursor): cstring {.importc, + cdecl, impapiHdr.} +proc ts_tree_cursor_current_field_id*(a1: ptr TSTreeCursor): TSFieldId {.importc, + cdecl, impapiHdr.} +proc ts_tree_cursor_goto_parent*(a1: ptr TSTreeCursor): bool {.importc, cdecl, + impapiHdr.} +proc ts_tree_cursor_goto_next_sibling*(a1: ptr TSTreeCursor): bool {.importc, cdecl, + impapiHdr.} +proc ts_tree_cursor_goto_first_child*(a1: ptr TSTreeCursor): bool {.importc, cdecl, + impapiHdr.} proc ts_tree_cursor_goto_first_child_for_byte*(a1: ptr TSTreeCursor; a2: uint32): int64 {. - importc, header: headerapi.} -proc ts_language_symbol_count*(a1: ptr TSLanguage): uint32 {.importc, - header: headerapi.} + importc, cdecl, impapiHdr.} +proc ts_tree_cursor_copy*(a1: ptr TSTreeCursor): TSTreeCursor {.importc, cdecl, + impapiHdr.} +proc ts_query_new*(language: ptr TSLanguage; source: cstring; source_len: uint32; + error_offset: ptr uint32; error_type: ptr TSQueryError): ptr TSQuery {. + importc, cdecl, impapiHdr.} +proc ts_query_delete*(a1: ptr TSQuery) {.importc, cdecl, impapiHdr.} +proc ts_query_pattern_count*(a1: ptr TSQuery): uint32 {.importc, cdecl, impapiHdr.} +proc ts_query_capture_count*(a1: ptr TSQuery): uint32 {.importc, cdecl, impapiHdr.} +proc ts_query_string_count*(a1: ptr TSQuery): uint32 {.importc, cdecl, impapiHdr.} +proc ts_query_start_byte_for_pattern*(a1: ptr TSQuery; a2: uint32): uint32 {.importc, + cdecl, impapiHdr.} +proc ts_query_predicates_for_pattern*(self: ptr TSQuery; pattern_index: uint32; + length: ptr uint32): ptr TSQueryPredicateStep {. + importc, cdecl, impapiHdr.} +proc ts_query_capture_name_for_id*(a1: ptr TSQuery; id: uint32; length: ptr uint32): cstring {. + importc, cdecl, impapiHdr.} +proc ts_query_string_value_for_id*(a1: ptr TSQuery; id: uint32; length: ptr uint32): cstring {. + importc, cdecl, impapiHdr.} +proc ts_query_disable_capture*(a1: ptr TSQuery; a2: cstring; a3: uint32) {.importc, + cdecl, impapiHdr.} +proc ts_query_cursor_new*(): ptr TSQueryCursor {.importc, cdecl, impapiHdr.} +proc ts_query_cursor_delete*(a1: ptr TSQueryCursor) {.importc, cdecl, impapiHdr.} +proc ts_query_cursor_exec*(a1: ptr TSQueryCursor; a2: ptr TSQuery; a3: TSNode) {.importc, + cdecl, impapiHdr.} +proc ts_query_cursor_set_byte_range*(a1: ptr TSQueryCursor; a2: uint32; a3: uint32) {. + importc, cdecl, impapiHdr.} +proc ts_query_cursor_set_point_range*(a1: ptr TSQueryCursor; a2: TSPoint; a3: TSPoint) {. + importc, cdecl, impapiHdr.} +proc ts_query_cursor_next_match*(a1: ptr TSQueryCursor; match: ptr TSQueryMatch): bool {. + importc, cdecl, impapiHdr.} +proc ts_query_cursor_remove_match*(a1: ptr TSQueryCursor; id: uint32) {.importc, cdecl, + impapiHdr.} +proc ts_query_cursor_next_capture*(a1: ptr TSQueryCursor; match: ptr TSQueryMatch; + capture_index: ptr uint32): bool {.importc, cdecl, + impapiHdr.} +proc ts_language_symbol_count*(a1: ptr TSLanguage): uint32 {.importc, cdecl, impapiHdr.} proc ts_language_symbol_name*(a1: ptr TSLanguage; a2: TSSymbol): cstring {.importc, - header: headerapi.} -proc ts_language_symbol_for_name*(a1: ptr TSLanguage; a2: cstring): TSSymbol {.importc, - header: headerapi.} + cdecl, impapiHdr.} +proc ts_language_symbol_for_name*(self: ptr TSLanguage; string: cstring; + length: uint32; is_named: bool): TSSymbol {.importc, + cdecl, impapiHdr.} +proc ts_language_field_count*(a1: ptr TSLanguage): uint32 {.importc, cdecl, impapiHdr.} +proc ts_language_field_name_for_id*(a1: ptr TSLanguage; a2: TSFieldId): cstring {. + importc, cdecl, impapiHdr.} +proc ts_language_field_id_for_name*(a1: ptr TSLanguage; a2: cstring; a3: uint32): TSFieldId {. + importc, cdecl, impapiHdr.} proc ts_language_symbol_type*(a1: ptr TSLanguage; a2: TSSymbol): TSSymbolType {. - importc, header: headerapi.} -proc ts_language_version*(a1: ptr TSLanguage): uint32 {.importc, header: headerapi.} + importc, cdecl, impapiHdr.} +proc ts_language_version*(a1: ptr TSLanguage): uint32 {.importc, cdecl, impapiHdr.} +{.pop.} diff --git a/nimterop/treesitter/tsgen.nim b/nimterop/treesitter/tsgen.nim index 484483c..ee2c8e0 100644 --- a/nimterop/treesitter/tsgen.nim +++ b/nimterop/treesitter/tsgen.nim @@ -5,14 +5,7 @@ import os import nimterop/[cimport, paths] -cPlugin: - import strutils - - proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = - if "_CRT" in sym.name: - sym.name = sym.name.strip(chars={'_'}) - static: cDebug() -cImport(cacheDir / "treesitter" /"lib" / "include" / "tree_sitter" / "api.h") +cImport(cacheDir / "treesitter" / "lib" / "include" / "tree_sitter" / "api.h", flags = "-E_ -c") From e3e1d80a55777b695db393fc440226093044472b Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 25 Jun 2020 10:00:12 -0500 Subject: [PATCH 051/106] Add minitest for nim CI --- CHANGES.md | 2 ++ nimterop.nimble | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 93bb8af..6748521 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -31,6 +31,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 - The `dynlib` command line parameter to `toast` and `cImport()` can also be the path to a shared library (dll|so|dylib) in place of a Nim const string containing the path. This allows for the traditional use case of passing `"xxxLPath"` to `cImport()` as well as simply passing the path to the library on the command line as is. This allows the creation of standalone cached wrappers as well as the usage of the `--check` and the `--stub` functionality that `toast` provides via `cImport()`. +- `gitPull()` now checks if an existing repository is at the `checkout` value specified. If not, it will pull the latest changes and checkout the specified commit, tag or branch. + ## Version 0.5.0 diff --git a/nimterop.nimble b/nimterop.nimble index 7be89c6..101805e 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -45,6 +45,11 @@ task btd, "build toast": task docs, "Generate docs": buildDocs(@["nimterop/all.nim"], "build/htmldocs") +task minitest, "Test for Nim CI": + exec "nim c -f -d:danger nimterop/toast" + exec "nim c -f -d:checkAbi -r tests/tast2.nim" + exec "nim c -f -d:checkAbi -d:zlibStd -d:zlibDL -d:zlibSetVer=1.2.11 -r tests/zlib.nim" + task test, "Test": rmFile("tests/timeit.txt") From 4ece07969d9aea6e9d8966b64fa445477115483a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 27 Jun 2020 12:28:22 -0500 Subject: [PATCH 052/106] Bump dep versions --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 101805e..bf9f520 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -9,7 +9,7 @@ bin = @["nimterop/toast"] installDirs = @["nimterop"] # Dependencies -requires "nim >= 0.20.2", "regex >= 0.14.1", "cligen >= 0.9.45" +requires "nim >= 0.20.2", "regex >= 0.15.0", "cligen >= 1.0.0" import nimterop/docs import os From f543124d7e2bec7f7ca8a47b70d88438a96443ed Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 29 Jun 2020 21:47:00 -0500 Subject: [PATCH 053/106] Fix #125 - no more types import --- nimterop/all.nim | 2 +- nimterop/cimport.nim | 5 +-- nimterop/enumtype.nim | 42 ++++++++++++++++++ nimterop/globals.nim | 5 ++- nimterop/toastlib/ast2.nim | 11 +++-- nimterop/toastlib/comphelp.nim | 2 +- nimterop/toastlib/exprparser.nim | 2 +- nimterop/toastlib/getters.nim | 41 +++++++++++++++++- nimterop/treesitter/api.nim | 3 +- nimterop/types.nim | 74 -------------------------------- 10 files changed, 99 insertions(+), 88 deletions(-) create mode 100644 nimterop/enumtype.nim delete mode 100644 nimterop/types.nim diff --git a/nimterop/all.nim b/nimterop/all.nim index 6be6d11..20ab4e8 100644 --- a/nimterop/all.nim +++ b/nimterop/all.nim @@ -2,4 +2,4 @@ The following modules are available to users of Nimterop. ]## -import "."/[docs, cimport, build, types, plugin] +import "."/[build, cimport, docs, plugin] diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index edfdeb9..b5c62a2 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -17,8 +17,7 @@ All `{.compileTime.}` procs must be used in a compile time context, e.g. using: import hashes, macros, os, strformat, strutils -import "."/[build, globals, paths, types] -export types +import "."/[build, globals, paths] proc interpPath(dir: string): string= # TODO: more robust: needs a DirSep after "$projpath" @@ -153,7 +152,7 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" # see https://github.com/nimterop/nimterop/issues/69 (result, ret) = execAction(cmd, die = false, cache = (not gStateCT.nocache), - cacheKey = getCacheValue(fullpaths)) + cacheKey = getCacheValue(toastExe) & getCacheValue(fullpaths)) doAssert ret == 0, getToastError(result) macro cOverride*(body): untyped = diff --git a/nimterop/enumtype.nim b/nimterop/enumtype.nim new file mode 100644 index 0000000..3c20b79 --- /dev/null +++ b/nimterop/enumtype.nim @@ -0,0 +1,42 @@ +import macros + +macro defineEnum(typ: untyped): untyped = + result = newNimNode(nnkStmtList) + + # Enum mapped to distinct cint + result.add quote do: + type `typ`* = distinct cint + + for i in ["+", "-", "*", "div", "mod", "shl", "shr", "or", "and", "xor", "<", "<=", "==", ">", ">="]: + let + ni = newIdentNode(i) + typout = if i[0] in "<=>": newIdentNode("bool") else: typ # comparisons return bool + if i[0] == '>': # cannot borrow `>` and `>=` from templates + let + nopp = if i.len == 2: newIdentNode("<=") else: newIdentNode("<") + result.add quote do: + proc `ni`*(x: `typ`, y: cint): `typout` = `nopp`(y, x) + proc `ni`*(x: cint, y: `typ`): `typout` = `nopp`(y, x) + proc `ni`*(x, y: `typ`): `typout` = `nopp`(y, x) + else: + result.add quote do: + proc `ni`*(x: `typ`, y: cint): `typout` {.borrow.} + proc `ni`*(x: cint, y: `typ`): `typout` {.borrow.} + proc `ni`*(x, y: `typ`): `typout` {.borrow.} + result.add quote do: + proc `ni`*(x: `typ`, y: int): `typout` = `ni`(x, y.cint) + proc `ni`*(x: int, y: `typ`): `typout` = `ni`(x.cint, y) + + let + divop = newIdentNode("/") # `/`() + dlrop = newIdentNode("$") # `$`() + notop = newIdentNode("not") # `not`() + result.add quote do: + proc `divop`*(x, y: `typ`): `typ` = `typ`((x.float / y.float).cint) + proc `divop`*(x: `typ`, y: cint): `typ` = `divop`(x, `typ`(y)) + proc `divop`*(x: cint, y: `typ`): `typ` = `divop`(`typ`(x), y) + proc `divop`*(x: `typ`, y: int): `typ` = `divop`(x, y.cint) + proc `divop`*(x: int, y: `typ`): `typ` = `divop`(x.cint, y) + + proc `dlrop`*(x: `typ`): string {.borrow.} + proc `notop`*(x: `typ`): `typ` {.borrow.} diff --git a/nimterop/globals.nim b/nimterop/globals.nim index c831589..d1e22a6 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -54,7 +54,7 @@ type constIdentifiers*: HashSet[string] # Const names for enum casting identifiers*: TableRef[string, string] # Symbols that have been declared so far indexed by nimName skippedSyms*: HashSet[string] # Symbols that have been skipped due to being unwrappable or - # the user provided override is blank + # the user provided override is blank # Nim compiler objects constSection*, enumSection*, pragmaSection*, procSection*, typeSection*, varSection*: PNode @@ -70,6 +70,9 @@ type # Controls whether or not the current expression # should validate idents against currently defined idents skipIdentValidation*: bool + + # Top level header for wrapper output - include imported types, pragmas and other info + wrapperHeader*: string else: # cimport.nim specific compile*: seq[string] # `cCompile()` list of files already processed diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 783b6e4..0147463 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1481,6 +1481,11 @@ proc addEnum(gState: State, node: TSNode) = if node.getName() == "type_definition" and node.len > 1: gState.addTypeTyped(node, ftname = name, offset = offset) + if gEnumMacro.nBl: + # Add enum generation macro once + gState.wrapperHeader &= gEnumMacro + gEnumMacro = "" + proc addProc(gState: State, node, rnode: TSNode, commentNodes: seq[TSNode]) = # Add a proc # @@ -1822,10 +1827,7 @@ proc setupPragmas(gState: State, root: TSNode, fullpath: string) = proc initNim*(gState: State) = # Initialize for parseNim() one time - gecho """import nimterop/types - -{.push hint[ConvFromXtoItselfNotNeeded]: off.} -""" + gState.wrapperHeader = "{.push hint[ConvFromXtoItselfNotNeeded]: off.}\n" # Track identifiers already rendered and corresponding PNodes gState.identifiers = newTable[string, string]() @@ -1876,6 +1878,7 @@ proc printNim*(gState: State) = tree.add gState.varSection tree.add gState.procSection + gecho gState.wrapperHeader gecho tree.renderTree() gecho "{.pop.}" \ No newline at end of file diff --git a/nimterop/toastlib/comphelp.nim b/nimterop/toastlib/comphelp.nim index d404c42..642fbbd 100644 --- a/nimterop/toastlib/comphelp.nim +++ b/nimterop/toastlib/comphelp.nim @@ -95,7 +95,7 @@ proc getNameInfo*(gState: State, node: TSNode, kind: NimSymKind, parent = ""): result.name = gState.getIdentifier(result.origname, kind, parent) if result.name.nBl: if kind == nskType: - result.name = result.name.getType() + result.name = gState.getType(result.name, parent) result.info = gState.getLineInfo(node) proc getPtrType*(str: string): string = diff --git a/nimterop/toastlib/exprparser.nim b/nimterop/toastlib/exprparser.nim index 13b1a98..a77dbf9 100644 --- a/nimterop/toastlib/exprparser.nim +++ b/nimterop/toastlib/exprparser.nim @@ -580,7 +580,7 @@ proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode = of "sized_type_specifier", "primitive_type", "type_identifier": # Input -> int, unsigned int, long int, etc # Output -> cint, cuint, clong, etc - let ty = getType(node.val) + let ty = gState.getType(node.val, parent = node.getName()) if ty.len > 0: # If ty is not empty, one of C's builtin types has been found result = gState.getExprIdent(ty, nskType, parent=node.getName()) diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index c3f48ae..43a4f6f 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -29,7 +29,13 @@ yield""".split(Whitespace).toHashSet() # Types related +const + # Enum macro read from file - written into wrapper when required + gEnumMacroConst = staticRead(currentSourcePath.parentDir().parentDir() / "enumtype.nim") + var + gEnumMacro* = gEnumMacroConst + gTypeMap* = { # char "char": "cchar", @@ -107,13 +113,40 @@ var "long double": "clongdouble", # Misc Nim types - "Bool": "bool" + "Bool": "bool", + "ptrdiff_t": "ByteAddress" }.toTable() # Nim type names that shouldn't need to be wrapped again gTypeMapValues* = toSeq(gTypeMap.values).toHashSet() -proc getType*(str: string): string = + # Types to import from C/Nim if used in wrapper + gTypeImport* = { + "time_t": """ +import std/time_t as std_time_t +type time_t* = Time +""", + + "time64_t": """ +import std/time_t as std_time64_t +type time64_t* = Time +""", + + "wchar_t": """ +when defined(cpp): + # http://www.cplusplus.com/reference/cwchar/wchar_t/ + # In C++, wchar_t is a distinct fundamental type (and thus it is + # not defined in nor any other header). + type wchar_t* {.importc.} = object +else: + type wchar_t* {.importc, header:"".} = object +""", + + "va_list": """ +type va_list* {.importc, header:"".} = object +"""}.toTable() + +proc getType*(gState: State, str, parent: string): string = if str == "void": return "object" @@ -121,6 +154,10 @@ proc getType*(str: string): string = if gTypeMap.hasKey(result): result = gTypeMap[result] + elif parent.nBl and gTypeImport.hasKey(result) and not gState.identifierNodes.hasKey(result): + # Include C/Nim type imports once if a field/param and not already declared + gState.wrapperHeader &= "\n" & gTypeImport[result] + gTypeImport.del result # Identifier related diff --git a/nimterop/treesitter/api.nim b/nimterop/treesitter/api.nim index 116bc10..753178c 100644 --- a/nimterop/treesitter/api.nim +++ b/nimterop/treesitter/api.nim @@ -2,7 +2,8 @@ import strutils, os -import ".."/[setup, paths, types] +include ".."/enumtype +import ".."/[paths, setup] static: treesitterSetup() diff --git a/nimterop/types.nim b/nimterop/types.nim deleted file mode 100644 index 38f95d1..0000000 --- a/nimterop/types.nim +++ /dev/null @@ -1,74 +0,0 @@ -# see https://github.com/nimterop/nimterop/issues/79 - -import std/time_t as time_t_temp -type - time_t* = time_t_temp.Time - time64_t* = time_t_temp.Time - -when defined(cpp): - # http://www.cplusplus.com/reference/cwchar/wchar_t/ - # In C++, wchar_t is a distinct fundamental type (and thus it is - # not defined in nor any other header). - type - wchar_t* {.importc.} = object -else: - type - wchar_t* {.importc, header:"".} = object - -type - ptrdiff_t* = ByteAddress - -type - va_list* {.importc, header:"".} = object - -template enumOp*(op, typ, typout) = - proc op*(x: typ, y: cint): typout {.borrow.} - proc op*(x: cint, y: typ): typout {.borrow.} - proc op*(x, y: typ): typout {.borrow.} - - proc op*(x: typ, y: int): typout = op(x, y.cint) - proc op*(x: int, y: typ): typout = op(x.cint, y) - -template defineEnum*(typ) = - # Create a `distinct cint` type for C enums since Nim enums - # need to be in order and cannot have duplicates. - type - typ* = distinct cint - - # Enum operations allowed - enumOp(`+`, typ, typ) - enumOp(`-`, typ, typ) - enumOp(`*`, typ, typ) - enumOp(`<`, typ, bool) - enumOp(`<=`, typ, bool) - enumOp(`==`, typ, bool) - enumOp(`div`, typ, typ) - enumOp(`mod`, typ, typ) - - # These don't work with `enumOp()` for some reason - proc `shl`*(x: typ, y: cint): typ {.borrow.} - proc `shl`*(x: cint, y: typ): typ {.borrow.} - proc `shl`*(x, y: typ): typ {.borrow.} - - proc `shr`*(x: typ, y: cint): typ {.borrow.} - proc `shr`*(x: cint, y: typ): typ {.borrow.} - proc `shr`*(x, y: typ): typ {.borrow.} - - proc `or`*(x: typ, y: cint): typ {.borrow.} - proc `or`*(x: cint, y: typ): typ {.borrow.} - proc `or`*(x, y: typ): typ {.borrow.} - - proc `and`*(x: typ, y: cint): typ {.borrow.} - proc `and`*(x: cint, y: typ): typ {.borrow.} - proc `and`*(x, y: typ): typ {.borrow.} - - proc `xor`*(x: typ, y: cint): typ {.borrow.} - proc `xor`*(x: cint, y: typ): typ {.borrow.} - proc `xor`*(x, y: typ): typ {.borrow.} - - proc `/`*(x, y: typ): typ = - return (x.float / y.float).cint.typ - proc `/`*(x: typ, y: cint): typ = `/`(x, y.typ) - proc `/`*(x: cint, y: typ): typ = `/`(x.typ, y) - - proc `$`*(x: typ): string {.borrow.} From a28c3e7cb4f6da157667b921ab725b92f6774512 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Jun 2020 14:08:09 -0500 Subject: [PATCH 054/106] Improve errors - don't point to macros.nim --- CHANGES.md | 6 ++ nimterop.nimble | 2 +- nimterop/build/shell.nim | 14 +++-- nimterop/cimport.nim | 124 +++++++++++++++++++-------------------- nimterop/toast.nim | 9 ++- 5 files changed, 84 insertions(+), 71 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6748521..3488252 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -33,6 +33,12 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 - `gitPull()` now checks if an existing repository is at the `checkout` value specified. If not, it will pull the latest changes and checkout the specified commit, tag or branch. +### Other improvements + +- Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125](i125) (since v0.6.1) + +- `cImport()` now includes wrapper output from a file rather than inline. Errors in generated wrappers will no longer point to a line in `macros.nim` making debugging easier. + ## Version 0.5.0 diff --git a/nimterop.nimble b/nimterop.nimble index bf9f520..ebc5f0a 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.0" +version = "0.6.1" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 53c0b5f..2b29734 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -23,7 +23,8 @@ else: export sleep proc execAction*(cmd: string, retry = 0, die = true, cache = false, - cacheKey = "", onRetry: proc() = nil): tuple[output: string, ret: int] = + cacheKey = "", onRetry: proc() = nil, + onError: proc(output: string, err: int) = nil): tuple[output: string, ret: int] = ## Execute an external command - supported at compile time ## ## Checks if command exits successfully before returning. If not, an @@ -34,6 +35,8 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, ## `die = false` - return on errors ## `cache = true` - cache results unless cleared with -f ## `cacheKey` - key to create unique cache entry + ## `onRetry()` - proc to call before retrying + ## `onError(output, err)` - proc to call on error let ccmd = fixCmd(cmd) @@ -80,9 +83,12 @@ proc execAction*(cmd: string, retry = 0, die = true, cache = false, onRetry() sleep(500) result = execAction(cmd, retry = retry - 1, die, cache, cacheKey) - elif die: - doAssert false, "Command failed: " & $result.ret & "\ncmd: " & ccmd & - "\nresult:\n" & result.output + else: + if not onError.isNil: + onError(result.output, result.ret) + + doAssert not die, "Command failed: " & $result.ret & "\ncmd: " & ccmd & + "\nresult:\n" & result.output when not defined(TOAST): proc findExe*(exe: string): string = diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index b5c62a2..6e0a80f 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -17,7 +17,8 @@ All `{.compileTime.}` procs must be used in a compile time context, e.g. using: import hashes, macros, os, strformat, strutils -import "."/[build, globals, paths] +import "."/[globals, paths] +import "."/build/[ccompiler, misc, nimconf, shell] proc interpPath(dir: string): string= # TODO: more robust: needs a DirSep after "$projpath" @@ -90,35 +91,30 @@ proc getToastError(output: string): string = if result.Bl: result = "\n\n" & output -proc getNimCheckError(output: string): tuple[tmpFile, errors: string] = - let - hash = output.hash().abs() - - result.tmpFile = getProjectCacheDir("failed", forceClean = false) / "nimterop_" & $hash & ".nim" - - if not fileExists(result.tmpFile) or gStateCT.nocache or compileOption("forceBuild"): - mkDir(result.tmpFile.parentDir()) - writeFile(result.tmpFile, output) - - doAssert fileExists(result.tmpFile), "Failed to write to cache dir: " & result.tmpFile - +proc getNimCheckError(nimFile: string) = let (check, _) = execAction( - &"{getCurrentNimCompiler()} check {result.tmpFile.sanitizePath}", + &"{getCurrentNimCompiler()} check {nimFile.sanitizePath}", die = false ) - result.errors = "\n\n" & check + doAssert false, &"\n\n{check}\n\n" & + "Codegen limitation or error - review 'nim check' output above generated for " & nimFile proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "", mode = "c", flags = "", noNimout = false): string = var - ret = 0 cmd = when defined(Windows): "cmd /c " else: "" + ext = "h" + + let + toastExe = toastExePath() + # see https://github.com/nimterop/nimterop/issues/69 + cacheKey = getCacheValue(toastExe) & getCacheValue(fullpaths) - let toastExe = toastExePath() doAssert fileExists(toastExe), "toast not compiled: " & toastExe.sanitizePath & " make sure 'nimble build' or 'nimble install' built it" + cmd &= &"{toastExe} --preprocess -m:{mode}" if recurse: @@ -147,13 +143,29 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" if gStateCT.pluginSourcePath.nBl: cmd.add &" --pluginSourcePath={gStateCT.pluginSourcePath.sanitizePath}" + ext = "nim" + for fullpath in fullpaths: cmd.add &" {fullpath.sanitizePath}" - # see https://github.com/nimterop/nimterop/issues/69 - (result, ret) = execAction(cmd, die = false, cache = (not gStateCT.nocache), - cacheKey = getCacheValue(toastExe) & getCacheValue(fullpaths)) - doAssert ret == 0, getToastError(result) + # Generate filename for toast output + let + hash = (cmd & cacheKey).hash().abs() + cachePath = getNimteropCacheDir() / "toastCache" / "nimterop_" & $hash + + result = cachePath.addFileExt(ext) + + if not fileExists(result) or compileOption("forceBuild"): + let + dir = cachePath.parentDir() + if not dirExists(dir): + mkDir(dir) + + cmd.add &" -o {result.sanitizePath}" + + var + (_, ret) = execAction(cmd, die = false) + doAssert ret == 0, getToastError(result.readFile()) macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not @@ -567,7 +579,7 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl gecho "# Importing " & fullpaths.join(", ").sanitizePath let - output = getToast(fullpaths, recurse, dynlib, mode, flags) + nimFile = getToast(fullpaths, recurse, dynlib, mode, flags) # Reset plugin and overrides for next cImport if gStateCT.overrides.nBl: @@ -575,16 +587,12 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl gStateCT.overrides = "" if gStateCT.debug: - gecho output + gecho nimFile.readFile() try: - let body = parseStmt(output) - - result.add body + result.add parseStmt("include " & nimFile.changeFileExt("")) except: - let - (tmpFile, errors) = getNimCheckError(output) - doAssert false, errors & "\n\nNimterop codegen limitation or error - review 'nim check' output above generated for " & tmpFile + getNimCheckError(nimFile) macro cImport*(filename: static string, recurse: static bool = false, dynlib: static string = "", mode: static string = "c", flags: static string = ""): untyped = @@ -663,49 +671,37 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: gecho "# Importing " & fullpath & " with c2nim" let - output = getToast(@[fullpath], recurse, dynlib, mode, noNimout = true) - hash = output.hash().abs() - hpath = getProjectCacheDir("c2nimCache", forceClean = false) / "nimterop_" & $hash & ".h" - npath = hpath[0 .. hpath.rfind('.')] & "nim" + hFile = getToast(@[fullpath], recurse, dynlib, mode, noNimout = true) + nimFile = hFile.changeFileExt("nim") header = "header" & fullpath.splitFile().name.split(seps = {'-', '.'}).join() - if not fileExists(hpath) or gStateCT.nocache or compileOption("forceBuild"): - mkDir(hpath.parentDir()) - writeFile(hpath, output) + if not fileExists(nimFile) or compileOption("forceBuild"): + var + cmd = when defined(Windows): "cmd /c " else: "" + cmd &= &"c2nim {hFile} --header:{header}" - doAssert fileExists(hpath), "Unable to write temporary header file: " & hpath + if dynlib.nBl: + cmd.add &" --dynlib:{dynlib}" + if mode.contains("cpp"): + cmd.add " --cpp" + if flags.nBl: + cmd.add &" {flags}" - var - cmd = when defined(Windows): "cmd /c " else: "" - cmd &= &"c2nim {hpath} --header:{header}" + for i in gStateCT.defines: + cmd.add &" --assumedef:{i.quoteShell}" - if dynlib.nBl: - cmd.add &" --dynlib:{dynlib}" - if mode.contains("cpp"): - cmd.add " --cpp" - if flags.nBl: - cmd.add &" {flags}" + let + (c2nimout, ret) = execAction(cmd) + if ret != 0: + rmFile(nimFile) + doAssert false, "\n\nc2nim codegen limitation or error - " & c2nimout - for i in gStateCT.defines: - cmd.add &" --assumedef:{i.quoteShell}" - - let - (c2nimout, ret) = execAction(cmd, cache = not gStateCT.nocache, - cacheKey = getCacheValue(hpath)) - - doAssert ret == 0, "\n\nc2nim codegen limitation or error - " & c2nimout - - var - nimout = &"const {header} = \"{fullpath}\"\n\n" & readFile(npath) + nimFile.writeFile(&"const {header} = \"{fullpath}\"\n\n" & readFile(nimFile)) if gStateCT.debug: - gecho nimout + gecho nimFile.readFile() try: - let body = parseStmt(nimout) - - result.add body + result.add parseStmt("include " & nimFile.changeFileExt("")) except: - let - (tmpFile, errors) = getNimCheckError(nimout) - doAssert false, errors & "\n\nc2nim codegen limitation or error - review 'nim check' output above generated for " & tmpFile + getNimCheckError(nimFile) diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 6fe5180..fbaa960 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -8,6 +8,10 @@ import "."/toastlib/[ast2, getters, tshelp] import "."/build/[ccompiler, misc] +var + # Output generated before main() is called + preMainOut = "" + proc process(gState: State, path: string) = doAssert existsFile(path), &"Invalid path {path}" @@ -129,6 +133,7 @@ proc main( if source.nBl: # Print source after preprocess or Nim output if gState.pnim: + gecho preMainOut gState.initNim() for src in source: gState.process(src.expandSymlinkAbs()) @@ -187,7 +192,7 @@ proc mergeParams(cmdNames: seq[string], cmdLine = commandLineParams()): seq[stri # https://github.com/c-blake/cligen/issues/149 for param in cmdLine: if param.fileExists() and param.splitFile().ext == ".cfg": - echo &"# Loading flags from '{param}'" + preMainOut &= &"# Loading flags from '{param}'\n" for line in param.readFile().splitLines(): let line = line.strip() @@ -197,7 +202,7 @@ proc mergeParams(cmdNames: seq[string], cmdLine = commandLineParams()): seq[stri result.add param if result.len != 0 and "-h" notin result and "--help" notin result: - echo &"""# Generated @ {$now()} + preMainOut &= &"""# Generated @ {$now()} # Command line: # {getAppFilename()} {result.join(" ")} """ From b22b23620da088f8728802199bb1a8c0f5999ec6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Jun 2020 15:43:09 -0500 Subject: [PATCH 055/106] Fix Windows include issue --- nimterop/cimport.nim | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 6e0a80f..109b172 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -154,6 +154,8 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" cachePath = getNimteropCacheDir() / "toastCache" / "nimterop_" & $hash result = cachePath.addFileExt(ext) + when defined(Windows): + result = result.replace(DirSep, '/') if not fileExists(result) or compileOption("forceBuild"): let @@ -590,7 +592,10 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl gecho nimFile.readFile() try: - result.add parseStmt("include " & nimFile.changeFileExt("")) + let + nimFileNode = newStrLitNode(nimFile.changeFileExt("")) + result.add quote do: + include `nimFileNode` except: getNimCheckError(nimFile) @@ -702,6 +707,9 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: gecho nimFile.readFile() try: - result.add parseStmt("include " & nimFile.changeFileExt("")) + let + nimFileNode = newStrLitNode(nimFile.changeFileExt("")) + result.add quote do: + include `nimFileNode` except: getNimCheckError(nimFile) From 2aef7c99b36bbc9db9bcbcd62c06c6a5e2d4d0c0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 30 Jun 2020 22:33:10 -0500 Subject: [PATCH 056/106] Fix #127 - write wrapper to file --- CHANGES.md | 14 +++-- nimterop.nimble | 30 +++++++--- nimterop/cimport.nim | 135 ++++++++++++++++++++----------------------- tests/tast2.nim | 6 +- 4 files changed, 100 insertions(+), 85 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3488252..43f1d34 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,13 +15,13 @@ Refer to the documentation for `getHeader()` for details on how to use this new See the full list of changes here: -https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 +https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.1 ### Breaking changes - The legacy algorithm has been removed as promised. `ast2` is now the default and wrappers no longer need to explicitly specify `-f:ast2` in order to use it. -- All shared libraries installed by `getHeader()` will now get copied into the `libdir` parameter specified. If left blank, `libdir` will default to the directory where the executable binary gets created (outdir). While this is not really a breaking change, it is a change in behavior compared to older versions of nimterop. Note that `Std` libraries are not copied over. [#154](i154) +- All shared libraries installed by `getHeader()` will now get copied into the `libdir` parameter specified. If left blank, `libdir` will default to the directory where the executable binary gets created (outdir). While this is not really a breaking change, it is a change in behavior compared to older versions of nimterop. Note that `Std` libraries are not copied over. [#154][i154] - `git.nim` has been removed. This module was an artifact from the early days and was renamed to `build.nim` back in v0.2.0. @@ -33,11 +33,13 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.0 - `gitPull()` now checks if an existing repository is at the `checkout` value specified. If not, it will pull the latest changes and checkout the specified commit, tag or branch. +- `cImport()` can now write the generated wrapper output to a user-defined file with the `nimFile` param. [#127][i127] (since v0.6.1) + ### Other improvements -- Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125](i125) (since v0.6.1) +- Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) -- `cImport()` now includes wrapper output from a file rather than inline. Errors in generated wrappers will no longer point to a line in `macros.nim` making debugging easier. +- `cImport()` now includes wrapper output from a file rather than inline. Errors in generated wrappers will no longer point to a line in `macros.nim` making debugging easier. (since v0.6.1) ## Version 0.5.0 @@ -50,7 +52,7 @@ Version 0.6.0 of Nimterop will make `ast2` the default backend and the legacy al See the full list of changes here: -https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 +https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.4 ### Breaking changes @@ -108,6 +110,8 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.0 [i54]: https://github.com/nimterop/nimterop/issues/54 [i74]: https://github.com/nimterop/nimterop/issues/74 [i76]: https://github.com/nimterop/nimterop/issues/76 +[i125]: https://github.com/nimterop/nimterop/issues/125 +[i127]: https://github.com/nimterop/nimterop/issues/127 [i137]: https://github.com/nimterop/nimterop/issues/137 [i147]: https://github.com/nimterop/nimterop/issues/147 [i148]: https://github.com/nimterop/nimterop/issues/148 diff --git a/nimterop.nimble b/nimterop.nimble index ebc5f0a..628ef40 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -50,14 +50,10 @@ task minitest, "Test for Nim CI": exec "nim c -f -d:checkAbi -r tests/tast2.nim" exec "nim c -f -d:checkAbi -d:zlibStd -d:zlibDL -d:zlibSetVer=1.2.11 -r tests/zlib.nim" -task test, "Test": - rmFile("tests/timeit.txt") - - buildTimeitTask() - buildToastTask() - +task basic, "Basic tests": execTest "tests/tast2.nim" execTest "tests/tast2.nim", "-d:NOHEADER" + execTest "tests/tast2.nim", "-d:NOHEADER -d:WRAPPED" execTest "tests/tnimterop_c.nim" execTest "tests/tnimterop_c.nim", "-d:FLAGS=\"-H\"" @@ -65,6 +61,7 @@ task test, "Test": execCmd "nim cpp --hints:off -f -r tests/tnimterop_cpp.nim" execCmd "./nimterop/toast tests/toast.cfg tests/include/toast.h" +task wrapper, "Wrapper tests": execTest "tests/tpcre.nim" when defined(Linux): @@ -79,12 +76,29 @@ task test, "Test": execTest "tests/tsoloud.nim" execTest "tests/tsoloud.nim", "-d:FLAGS=\"-H\"" - # getHeader tests +task getheader, "getHeader tests": withDir("tests"): exec "nim e getheader.nims" - if not existsEnv("APPVEYOR"): + +task package, "Wrapper package tests": + if not existsEnv("APPVEYOR"): + withDir("tests"): exec "nim e wrappers.nims" +task test, "Test": + rmFile("tests/timeit.txt") + + buildTimeitTask() + buildToastTask() + + basicTask() + + wrapperTask() + + getheaderTask() + + packageTask() + docsTask() echo readFile("tests/timeit.txt") diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 109b172..d992c04 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -70,6 +70,9 @@ proc walkDirImpl(indir, inext: string, file=true): seq[string] = if ret == 0: result = output.splitLines() +proc fixRelFile(file: string): string = + if file.isAbsolute(): file else: getProjectDir() / file + proc getCacheValue(fullpath: string): string = if not gStateCT.nocache: result = fullpath.getFileDate() @@ -102,7 +105,7 @@ proc getNimCheckError(nimFile: string) = "Codegen limitation or error - review 'nim check' output above generated for " & nimFile proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "", - mode = "c", flags = "", noNimout = false): string = + mode = "c", flags = "", outFile = "", noNimout = false): string = var cmd = when defined(Windows): "cmd /c " else: "" ext = "h" @@ -148,18 +151,17 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" for fullpath in fullpaths: cmd.add &" {fullpath.sanitizePath}" - # Generate filename for toast output - let - hash = (cmd & cacheKey).hash().abs() - cachePath = getNimteropCacheDir() / "toastCache" / "nimterop_" & $hash + result = if outFile.nBl: fixRelFile(outFile) else: + # Generate filename for toast output if not specified + getNimteropCacheDir() / "toastCache" / "nimterop_" & + ($(cmd & cacheKey).hash().abs()).addFileExt(ext) - result = cachePath.addFileExt(ext) when defined(Windows): result = result.replace(DirSep, '/') if not fileExists(result) or compileOption("forceBuild"): let - dir = cachePath.parentDir() + dir = result.parentDir() if not dirExists(dir): mkDir(dir) @@ -171,9 +173,8 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not - ## accurate, it may be required to hand wrap them. Define them in a - ## `cOverride() `_ macro block so that Nimterop uses - ## these definitions instead. + ## accurate, it may be required to hand wrap them. Define them in a `cOverride()` + ## macro block so that Nimterop uses these definitions instead. ## ## For example: ## @@ -194,9 +195,9 @@ macro cOverride*(body): untyped = ## cOverride: ## proc svGetCallerInfo(fileName: var cstring; lineNumber: var cint) ## - ## Using the `cOverride() `_ block, nimterop - ## can be instructed to use this definition of `svGetCallerInfo()` instead. - ## This works for procs, consts and types. + ## Using the `cOverride()` block, nimterop can be instructed to use this + ## definition of `svGetCallerInfo()` instead. This works for procs, consts + ## and types. ## ## `cOverride()` only affects the next `cImport()` call. This is because any ## recognized symbols get overridden in place and any remaining symbols get @@ -263,11 +264,10 @@ proc onSymbolOverride*(sym: var Symbol) {.exportc, dynlib.} = decho "Overriding " & names.join(" ") proc cSkipSymbol*(skips: seq[string]) {.compileTime.} = - ## Similar to `cOverride() `_, this macro allows - ## filtering out symbols not of interest from the generated output. + ## Similar to `cOverride()`, this macro allows filtering out symbols not of + ## interest from the generated output. ## - ## `cSkipSymbol() `_ only affects calls to - ## `cImport() `_ that follow it. + ## `cSkipSymbol()` only affects calls to `cImport()` that follow it. runnableExamples: static: cSkipSymbol @["proc1", "Type2"] gStateCT.symOverride.add skips @@ -291,11 +291,9 @@ proc cPluginHelper(body: string, imports = "import macros, nimterop/plugin\n\n") gStateCT.pluginSourcePath = path macro cPlugin*(body): untyped = - ## When `cOverride() `_ and - ## `cSkipSymbol() `_ - ## are not adequate, the `cPlugin() `_ macro can be used - ## to customize the generated Nim output. The following callbacks are available at - ## this time. + ## When `cOverride()` and `cSkipSymbol()` are not adequate, the `cPlugin()` + ## macro can be used to customize the generated Nim output. The following + ## callbacks are available at this time. ## ## .. code-block:: nim ## @@ -325,9 +323,7 @@ macro cPlugin*(body): untyped = ## `macros` and `nimterop/plugins` are implicitly imported to provide access to standard ## plugin facilities. ## - ## `cPlugin() `_ only affects calls to - ## `cImport() `_ that - ## follow it. + ## `cPlugin()` only affects calls to `cImport()` that follow it. runnableExamples: cPlugin: import strutils @@ -367,14 +363,10 @@ macro cPluginPath*(path: static[string]): untyped = proc cSearchPath*(path: string): string {.compileTime.}= ## Get full path to file or directory `path` in search path configured - ## using `cAddSearchDir() `_ and - ## `cAddStdDir() `_. + ## using `cAddSearchDir()` and `cAddStdDir()`. ## ## This can be used to locate files or directories that can be passed onto - ## `cCompile() `_, - ## `cIncludeDir() `_ and - ## `cImport() `_. - + ## `cCompile()`, `cIncludeDir()` and `cImport()`. result = findPath(path, fail = false) if result.Bl: var found = false @@ -394,21 +386,17 @@ proc cDisableCaching*() {.compileTime.} = ## Disable caching of generated Nim code - useful during wrapper development ## ## If files included by header being processed by - ## `cImport() `_ - ## change and affect the generated content, they will be ignored and the cached - ## value will continue to be used . Use `cDisableCaching() `_ - ## to avoid this scenario during development. + ## `cImport()` change and affect the generated content, they will be ignored + ## and the cached value will continue to be used . Use `cDisableCaching()` to + ## avoid this scenario during development. ## - ## `nim -f` was broken prior to 0.19.4 but can also be used to flush the cached content. - + ## `nim -f` can also be used to flush the cached content. gStateCT.nocache = true macro cDefine*(name: static string, val: static string = ""): untyped = ## `#define` an identifer that is forwarded to the C/C++ preprocessor if - ## called within `cImport() `_ - ## or `c2nImport() `_ - ## as well as to the C/C++ compiler during Nim compilation using `{.passC: "-DXXX".}` - + ## called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-DXXX".}` result = newNimNode(nnkStmtList) var str = name @@ -429,7 +417,7 @@ macro cDefine*(name: static string, val: static string = ""): untyped = proc cAddSearchDir*(dir: string) {.compileTime.} = ## Add directory `dir` to the search path used in calls to - ## `cSearchPath() `_. + ## `cSearchPath()`. runnableExamples: import nimterop/paths, os static: @@ -441,10 +429,8 @@ proc cAddSearchDir*(dir: string) {.compileTime.} = macro cIncludeDir*(dir: static string): untyped = ## Add an include directory that is forwarded to the C/C++ preprocessor if - ## called within `cImport() `_ - ## or `c2nImport() `_ - ## as well as to the C/C++ compiler during Nim compilation using `{.passC: "-IXXX".}`. - + ## called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-IXXX".}`. var dir = interpPath(dir) result = newNimNode(nnkStmtList) @@ -459,7 +445,7 @@ macro cIncludeDir*(dir: static string): untyped = proc cAddStdDir*(mode = "c") {.compileTime.} = ## Add the standard `c` [default] or `cpp` include paths to search - ## path used in calls to `cSearchPath() `_ + ## path used in calls to `cSearchPath()`. runnableExamples: static: cAddStdDir() import os @@ -561,7 +547,7 @@ macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = gecho result.repr macro cImport*(filenames: static seq[string], recurse: static bool = false, dynlib: static string = "", - mode: static string = "c", flags: static string = ""): untyped = + mode: static string = "c", flags: static string = "", nimFile: static string = ""): untyped = ## Import multiple headers in one shot ## ## This macro is preferable over multiple individual `cImport()` calls, especially @@ -581,7 +567,7 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl gecho "# Importing " & fullpaths.join(", ").sanitizePath let - nimFile = getToast(fullpaths, recurse, dynlib, mode, flags) + nimFile = getToast(fullpaths, recurse, dynlib, mode, flags, nimFile) # Reset plugin and overrides for next cImport if gStateCT.overrides.nBl: @@ -600,16 +586,15 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl getNimCheckError(nimFile) macro cImport*(filename: static string, recurse: static bool = false, dynlib: static string = "", - mode: static string = "c", flags: static string = ""): untyped = + mode: static string = "c", flags: static string = "", nimFile: static string = ""): untyped = ## Import all supported definitions from specified header file. Generated ## content is cached in `nimcache` until `filename` changes unless - ## `cDisableCaching() `_ is set. `nim -f` - ## can also be used after Nim v0.19.4 to flush the cache. + ## `cDisableCaching()` is set. `nim -f` can also be used to flush the cache. ## ## `recurse` can be used to generate Nim wrappers from `#include` files ## referenced in `filename`. This is only done for files in the same ## directory as `filename` or in a directory added using - ## `cIncludeDir() `_ + ## `cIncludeDir()`. ## ## `dynlib` can be used to specify the Nim string to use to specify the dynamic ## library to load the imported symbols from. For example: @@ -630,9 +615,9 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st ## ## cImport("pcre.h", dynlib="dynpcre") ## - ## If `dynlib` is not specified, the C/C++ implementation files can be compiled in - ## with `cCompile() `_, or the - ## `{.passL.}` pragma can be used to specify the static lib to link. + ## If `dynlib` is not specified, the C/C++ implementation files can be compiled + ## in with `cCompile()`, or the `{.passL.}` pragma can be used to specify the + ## static lib to link. ## ## `mode` selects the preprocessor and tree-sitter parser to be used to process ## the header. @@ -641,33 +626,41 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st ## good example would be `--prefix` and `--suffix` which strip leading and ## trailing strings from identifiers, `_` being quite common. ## + ## `nimFile` is the location where the generated wrapper should get written. + ## By default, the generated wrapper is written to `nimcache` and included from + ## there. `nimFile` makes it possible to write the wrapper to a predetermined + ## location which can then be directly imported into the main application and + ## checked into source control if preferred. Importing the nimterop wrapper with + ## `nimFile` specified still works per usual. If `nimFile` is not an absolute + ## path, it is relative to the project path. + ## ## `cImport()` consumes and resets preceding `cOverride()` calls. `cPlugin()` ## is retained for the next `cImport()` call unless a new `cPlugin()` call is ## defined. return quote do: - cImport(@[`filename`], bool(`recurse`), `dynlib`, `mode`, `flags`) + cImport(@[`filename`], bool(`recurse`), `dynlib`, `mode`, `flags`, `nimFile`) macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: static string = "", - mode: static string = "c", flags: static string = ""): untyped = + mode: static string = "c", flags: static string = "", nimFile: static string = ""): untyped = ## Import all supported definitions from specified header file using `c2nim` ## - ## Similar to `cImport() `_ - ## but uses `c2nim` to generate the Nim wrapper instead of `toast`. Note that neither - ## `cOverride() `_, `cSkipSymbol() `_ - ## nor `cPlugin() `_ have any impact on `c2nim`. + ## Similar to `cImport()` but uses `c2nim` to generate the Nim wrapper instead + ## of `toast`. Note that neither `cOverride()`, `cSkipSymbol()` nor `cPlugin()` + ## have any impact on `c2nim`. ## - ## `toast` is only used to preprocess the header file and recurse - ## if specified. + ## `toast` is only used to preprocess the header file and `recurse` if specified. ## ## `mode` should be set to `cpp` for c2nim to wrap C++ headers. ## ## `flags` can be used to pass other command line arguments to `c2nim`. ## - ## `nimterop` does not depend on `c2nim` as a `nimble` dependency so it - ## does not get installed automatically. Any wrapper or library that requires this proc - ## needs to install `c2nim` with `nimble install c2nim` or add it as a dependency in - ## its own `.nimble` file. - + ## `nimFile` is the location where the generated wrapper should get written, + ## similar to `cImport()`. + ## + ## `nimterop` does not depend on `c2nim` as a `nimble` dependency so it does not + ## get installed automatically. Any wrapper or library that requires this proc + ## needs to install `c2nim` with `nimble install c2nim` or add it as a dependency + ## in its own `.nimble` file. result = newNimNode(nnkStmtList) let @@ -677,13 +670,13 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: let hFile = getToast(@[fullpath], recurse, dynlib, mode, noNimout = true) - nimFile = hFile.changeFileExt("nim") + nimFile = if nimFile.nBl: fixRelFile(nimFile) else: hFile.changeFileExt("nim") header = "header" & fullpath.splitFile().name.split(seps = {'-', '.'}).join() if not fileExists(nimFile) or compileOption("forceBuild"): var cmd = when defined(Windows): "cmd /c " else: "" - cmd &= &"c2nim {hFile} --header:{header}" + cmd &= &"c2nim {hFile} --header:{header} --out:{nimFile.sanitizePath}" if dynlib.nBl: cmd.add &" --dynlib:{dynlib}" diff --git a/tests/tast2.nim b/tests/tast2.nim index 3f626ae..669ec98 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -38,7 +38,11 @@ cOverride: A1* = A0 cDefine("SOME_CONST=100") -cImport(path, flags="-f:ast2 -ENK_,SDL_ -GVICE=SLICE -TMyInt=cint" & flags) + +when not defined(WRAPPED): + cImport(path, flags="-f:ast2 -ENK_,SDL_ -GVICE=SLICE -TMyInt=cint" & flags, nimFile = "tast2wrapped.nim") +else: + import tast2wrapped proc getPragmas(n: NimNode): HashSet[string] = # Find all pragmas in AST, return as "name" or "name:value" in set From a63f67f6850dab10ec0c8784d370f8178edc7bf9 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Jul 2020 00:10:16 -0500 Subject: [PATCH 057/106] Minor fixes, improvements --- nimterop/globals.nim | 8 ++-- nimterop/setup.nim | 8 ---- nimterop/toastlib/getters.nim | 83 ++++++++++++++++++----------------- nimterop/treesitter/c.nim | 2 +- nimterop/treesitter/cpp.nim | 2 +- 5 files changed, 48 insertions(+), 55 deletions(-) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index d1e22a6..e0b9b8f 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -129,7 +129,7 @@ when defined(TOAST): Status* = enum success, unknown, error -proc getCommented*(str: string): string = +template getCommented*(str: string): string = "\n# " & str.strip().replace("\n", "\n# ") # Redirect output to file when required @@ -147,13 +147,13 @@ template gecho*(args: string) = template decho*(args: varargs[string, `$`]): untyped = let - str = join(args, "").getCommented() + str = join(args, "") when defined(TOAST): if gState.debug: - gecho str + gecho str.getCommented() else: if gStateCT.debug: - echo str + echo str.getCommented() template nBl*(s: typed): untyped {.used.} = (s.len != 0) diff --git a/nimterop/setup.nim b/nimterop/setup.nim index 6da0bbe..6da1358 100644 --- a/nimterop/setup.nim +++ b/nimterop/setup.nim @@ -34,10 +34,6 @@ src/*.cc src/tree_sitter/parser.h """, "0.16.1") - writeFile(cacheDir / "treesitter_c" / "src" / "api.h", """ -const TSLanguage *tree_sitter_c(); -""") - proc treesitterCppSetup*() = gitPull("https://github.com/tree-sitter/tree-sitter-cpp", cacheDir / "treesitter_cpp", """ src/*.h @@ -45,7 +41,3 @@ src/*.c src/*.cc src/tree_sitter/parser.h """, "v0.16.0") - - writeFile(cacheDir / "treesitter_cpp" / "src" / "api.h", """ -const TSLanguage *tree_sitter_cpp(); -""") diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 43a4f6f..534b6fa 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -1,4 +1,4 @@ -import dynlib, macros, os, sequtils, sets, strformat, strutils, tables, times +import dynlib, macros, os, osproc, sequtils, sets, streams, strformat, strutils, tables, times import regex @@ -124,12 +124,12 @@ var gTypeImport* = { "time_t": """ import std/time_t as std_time_t -type time_t* = Time +type time_t* = std_time_t.Time """, "time64_t": """ import std/time_t as std_time64_t -type time64_t* = Time +type time64_t* = std_time_t.Time """, "wchar_t": """ @@ -286,10 +286,7 @@ proc getCurrentHeader*(fullpath: string): string = proc getPreprocessor*(gState: State, fullpath: string) = var - cmts = if gState.noComments: "" else: "-CC" - cmd = &"""{getCompiler()} -E {cmts} -dD {getGccModeArg(gState.mode)} -w """ - - rdata: seq[string] + args: seq[string] start = false sfile = fullpath.sanitizePath(noQuote = true) @@ -297,50 +294,54 @@ proc getPreprocessor*(gState: State, fullpath: string) = pDir = sfile.expandFilename().parentDir() includeDirs: seq[string] + args.add @["-E", "-dD", getGccModeArg(gState.mode), "-w"] + if not gState.noComments: + args.add "-CC" + for inc in gState.includeDirs: - cmd &= &"-I{inc.sanitizePath} " + args.add &"-I{inc.sanitizePath}" includeDirs.add inc.absolutePath().sanitizePath(noQuote = true) for def in gState.defines: - cmd &= &"-D{def} " + args.add &"-D{def}" # Remove gcc special calls - if defined(posix): - cmd &= "-D__attribute__\\(x\\)= " - else: - cmd &= "-D__attribute__(x)= " - - cmd &= "-D__restrict= -D__extension__= -D__inline__=inline -D__inline=inline " - # https://github.com/tree-sitter/tree-sitter-c/issues/43 - cmd &= "-D_Noreturn= " - - cmd &= &"{fullpath.sanitizePath}" + args.add @["-D__attribute__(x)=", "-D__restrict=", "-D__extension__=", "-D__inline__=inline", + "-D__inline=inline", "-D_Noreturn=", &"{fullpath.sanitizePath}"] # Include content only from file - for line in execAction(cmd).output.splitLines(): - # We want to keep blank lines here for comment processing - if line.len > 1 and line[0 .. 1] == "# ": - start = false - let - saniLine = line.sanitizePath(noQuote = true) - if sfile in saniLine or - (DirSep notin saniLine and sfileName in saniLine): - start = true - elif gState.recurse: - if pDir.Bl or pDir in saniLine: + var + p = startProcess(getCompiler(), args = args, options = {poStdErrToStdOut, poUsePath}) + outp = p.outputStream() + line = "" + + # Include content only from file + gState.code = "" + while true: + if outp.readLine(line): + # We want to keep blank lines here for comment processing + if line.len > 1 and line[0] == '#' and line[1] == ' ': + start = false + line = line.sanitizePath(noQuote = true) + if sfile in line or + (DirSep notin line and sfileName in line): start = true - else: - for inc in includeDirs: - if inc in saniLine: - start = true - break - else: - if start: - if "#undef" in line: - continue - rdata.add line - gState.code = rdata.join("\n") + elif gState.recurse: + if pDir.Bl or pDir in line: + start = true + else: + for inc in includeDirs: + if inc in line: + start = true + break + else: + if start: + if "#undef" in line: + continue + gState.code.add line & "\n" + elif not p.running(): break + p.close() # Plugin related diff --git a/nimterop/treesitter/c.nim b/nimterop/treesitter/c.nim index d779bfd..4280b91 100644 --- a/nimterop/treesitter/c.nim +++ b/nimterop/treesitter/c.nim @@ -13,4 +13,4 @@ import "."/api {.compile: srcDir / "parser.c".} -proc treeSitterC*(): ptr TSLanguage {.importc: "tree_sitter_c", header: srcDir / "api.h".} +proc treeSitterC*(): ptr TSLanguage {.importc: "tree_sitter_c".} diff --git a/nimterop/treesitter/cpp.nim b/nimterop/treesitter/cpp.nim index 4f5be17..fd0437e 100644 --- a/nimterop/treesitter/cpp.nim +++ b/nimterop/treesitter/cpp.nim @@ -18,4 +18,4 @@ static: {.compile: srcDir / "parser_cpp.c".} {.compile: srcDir / "scanner.cc".} -proc treeSitterCpp*(): ptr TSLanguage {.importc: "tree_sitter_cpp", header: srcDir / "api.h".} +proc treeSitterCpp*(): ptr TSLanguage {.importc: "tree_sitter_cpp".} From 94acdeb5f1e0576decdb116570d5b2f05af82bf7 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Jul 2020 02:29:28 -0500 Subject: [PATCH 058/106] Fix #238 crash --- nimterop/toastlib/ast2.nim | 2 +- nimterop/toastlib/getters.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 0147463..5769893 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1668,7 +1668,7 @@ proc addDecl(gState: State, node: TSNode) = commentNodes: seq[TSNode] for i in start+1 ..< node.len: - if node[i].getName() == "comment": + if node[i].getName() in ["comment", "type_qualifier"]: continue if firstDecl: diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 534b6fa..3a71c78 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -129,7 +129,7 @@ type time_t* = std_time_t.Time "time64_t": """ import std/time_t as std_time64_t -type time64_t* = std_time_t.Time +type time64_t* = std_time64_t.Time """, "wchar_t": """ From 1e88a3b42ec6d27c1d924ee9f6840852b1a68797 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Jul 2020 12:41:56 -0500 Subject: [PATCH 059/106] Fix static inline and restrict --- nimterop/toastlib/ast2.nim | 3 ++- nimterop/toastlib/getters.nim | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 5769893..dc4d220 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1703,8 +1703,9 @@ proc addDef(gState: State, node: TSNode) = let start = getStartAtom(node) commentNodes = gState.getCommentNodes(node) + fdecl = node[start+1].firstChildInTree("function_declarator") - if node[start+1].getName() == "function_declarator": + if not fdecl.isNil: if not gState.noHeader: gState.addProc(node[start+1], node[start], commentNodes) else: diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 3a71c78..08ba604 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -307,7 +307,7 @@ proc getPreprocessor*(gState: State, fullpath: string) = # Remove gcc special calls # https://github.com/tree-sitter/tree-sitter-c/issues/43 - args.add @["-D__attribute__(x)=", "-D__restrict=", "-D__extension__=", "-D__inline__=inline", + args.add @["-D__attribute__(x)=", "-D__restrict=", "-D__restrict__=", "-D__extension__=", "-D__inline__=inline", "-D__inline=inline", "-D_Noreturn=", &"{fullpath.sanitizePath}"] # Include content only from file From c3f4f3e487ce8bf8bd9e6e9a370a808e00638091 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Jul 2020 20:11:11 -0500 Subject: [PATCH 060/106] Handle tserror in static inline, wrap headers only once --- nimterop/globals.nim | 1 + nimterop/toast.nim | 6 +++++- nimterop/toastlib/ast2.nim | 23 ++++++++++++----------- nimterop/toastlib/getters.nim | 15 ++++++++++----- nimterop/toastlib/tshelp.nim | 9 +++++++++ 5 files changed, 37 insertions(+), 17 deletions(-) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index e0b9b8f..17b9518 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -55,6 +55,7 @@ type identifiers*: TableRef[string, string] # Symbols that have been declared so far indexed by nimName skippedSyms*: HashSet[string] # Symbols that have been skipped due to being unwrappable or # the user provided override is blank + headersProcessed*: HashSet[string] # Headers already processed directly or recursively # Nim compiler objects constSection*, enumSection*, pragmaSection*, procSection*, typeSection*, varSection*: PNode diff --git a/nimterop/toast.nim b/nimterop/toast.nim index fbaa960..1cda239 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -136,7 +136,11 @@ proc main( gecho preMainOut gState.initNim() for src in source: - gState.process(src.expandSymlinkAbs()) + let + src = src.expandSymlinkAbs() + if src notin gState.headersProcessed: + gState.process(src) + gState.headersProcessed.incl src if gState.pnim: printNim(gState) diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index dc4d220..43ec8d3 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -137,7 +137,7 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode = # In case symbol was skipped earlier gState.skippedSyms.excl origname else: - gecho &"# const '{origname}' is duplicate, skipped" + decho &"const '{origname}' is duplicate, skipped" else: gecho &"# const '{origname}' has unsupported value '{val.strip()}'" gState.skippedSyms.incl origname @@ -391,7 +391,7 @@ proc newXIdent(gState: State, node: TSNode, kind = nskType, fname = "", pragmas: gState.identifierNodes[name] = result else: - gecho &"# {getKeyword(kind)} '{origname}' is duplicate, skipped" + decho &"{getKeyword(kind)} '{origname}' is duplicate, skipped" proc newArrayTree(gState: State, node: TSNode, typ, size: PNode = nil): PNode = # Create nkBracketExpr tree depending on input @@ -1706,6 +1706,9 @@ proc addDef(gState: State, node: TSNode) = fdecl = node[start+1].firstChildInTree("function_declarator") if not fdecl.isNil: + if gState.getNodeError(fdecl): + return + if not gState.noHeader: gState.addProc(node[start+1], node[start], commentNodes) else: @@ -1716,19 +1719,14 @@ proc processNode(gState: State, node: TSNode): Status = const known = ["preproc_def", "type_definition", "struct_specifier", "union_specifier", "enum_specifier", - "declaration", "function_definition"].toHashSet() + "declaration"].toHashSet() result = success let name = node.getName() if name in known: # Recognized top-level nodes - let - err = node.anyChildInTree("ERROR") - if not err.isNil: - # Bail on errors - gState.printDebug(node) - gecho &"# tree-sitter parse error: '{gState.getNodeVal(node).splitLines()[0]}', skipped" + if gState.getNodeError(node): result = Status.error else: # Process nodes @@ -1750,8 +1748,11 @@ proc processNode(gState: State, node: TSNode): Status = gState.addEnum(node) of "declaration": gState.addDecl(node) - of "function_definition": - gState.addDef(node) + elif name == "function_definition": + # Separate since we only need to check function_declarator for errors and + # not the compound_statement which could have errors but does not impact + # wrapper generation + gState.addDef(node) else: # Unknown, will check child nodes result = unknown diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 08ba604..bfbd4be 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -315,6 +315,7 @@ proc getPreprocessor*(gState: State, fullpath: string) = p = startProcess(getCompiler(), args = args, options = {poStdErrToStdOut, poUsePath}) outp = p.outputStream() line = "" + newHeaders: HashSet[string] # Include content only from file gState.code = "" @@ -323,17 +324,19 @@ proc getPreprocessor*(gState: State, fullpath: string) = # We want to keep blank lines here for comment processing if line.len > 1 and line[0] == '#' and line[1] == ' ': start = false - line = line.sanitizePath(noQuote = true) - if sfile in line or - (DirSep notin line and sfileName in line): + line = line.split('"')[1].sanitizePath(noQuote = true) + if sfile == line or + (DirSep notin line and sfileName == line): start = true elif gState.recurse: - if pDir.Bl or pDir in line: + if (pDir.Bl or pDir in line) and line notin gState.headersProcessed: start = true + newHeaders.incl line else: for inc in includeDirs: - if inc in line: + if line.startsWith(inc) and line notin gState.headersProcessed: start = true + newHeaders.incl line break else: if start: @@ -342,6 +345,7 @@ proc getPreprocessor*(gState: State, fullpath: string) = gState.code.add line & "\n" elif not p.running(): break p.close() + gState.headersProcessed.incl newHeaders # Plugin related @@ -393,3 +397,4 @@ proc expandSymlinkAbs*(path: string): string = result = path.expandFilename().normalizedPath() except: result = path + result = result.sanitizePath(noQuote = true) \ No newline at end of file diff --git a/nimterop/toastlib/tshelp.nim b/nimterop/toastlib/tshelp.nim index 3b7c2cc..bbe029f 100644 --- a/nimterop/toastlib/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -399,3 +399,12 @@ proc getTSNodeNamedChildNames*(node: TSNode): seq[string] = if name != "comment": result.add(name) + +proc getNodeError*(gState: State, node: TSNode): bool = + let + err = node.anyChildInTree("ERROR") + if not err.isNil: + # Bail on errors + gState.printDebug(node) + gecho &"# tree-sitter parse error: '{gState.getNodeVal(node).splitLines()[0]}', skipped" + result = true \ No newline at end of file From f191ea7244df545c3dc7b4dc2451a600694fe298 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 2 Jul 2020 23:55:26 -0500 Subject: [PATCH 061/106] Fix #237 - anonymous nested struct/unions, fix #236 - unnamed enums --- CHANGES.md | 8 +++- README.md | 5 ++- nimterop/toastlib/ast2.nim | 78 +++++++++++++++++++++++++++++++++----- tests/include/tast2.h | 41 ++++++++++++++++++++ tests/tast2.nim | 20 +++++++++- tests/tnimterop_c.nim | 2 +- 6 files changed, 139 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 43f1d34..e85d261 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -25,6 +25,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.1 - `git.nim` has been removed. This module was an artifact from the early days and was renamed to `build.nim` back in v0.2.0. +- Nameless enum values are no longer typed to the made-up enum type name, they are instead typed as `cint` to match the underlying type. This allows using such enums without having to depend on the made-up name which could change if enum ordering changes upstream. [#236][i236] (since v0.6.1) + ### New functionality - `getHeader()` now detects and links against `.lib` files as part of enabling Conan.io. Not all `.lib` files are compatible with MinGW as already stated above but for those that work, this is a required capability. @@ -35,6 +37,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.1 - `cImport()` can now write the generated wrapper output to a user-defined file with the `nimFile` param. [#127][i127] (since v0.6.1) +- Nimterop now supports anonymous nested structs/unions but it only works correctly for unions when `noHeader` is turned off (the default). This is because Nim does not support nested structs/unions and is unaware of the underlying memory structure. [#237][i237] (since v0.6.1) + ### Other improvements - Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) @@ -128,4 +132,6 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.4 [i181]: https://github.com/nimterop/nimterop/issues/181 [i196]: https://github.com/nimterop/nimterop/issues/196 [i197]: https://github.com/nimterop/nimterop/issues/197 -[i200]: https://github.com/nimterop/nimterop/issues/200 \ No newline at end of file +[i200]: https://github.com/nimterop/nimterop/issues/200 +[i236]: https://github.com/nimterop/nimterop/issues/236 +[i237]: https://github.com/nimterop/nimterop/issues/237 \ No newline at end of file diff --git a/README.md b/README.md index b054d97..f9e9307 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,9 @@ This will download and install nimterop in the standard Nimble package location, ## Usage -Nimterop can be used in two ways: +Nimterop can be used in three ways: - Creating a wrapper file - a `.nim` file that contains calls to the high-level API that can download and build the C library as well as generate the required Nim code to interface with the library. This wrapper file can then be imported into Nim code like any other module and it will be processed at compile time. +- Same as the first option except using the `nimFile` param to `cImport()` to write the generated wrapper to a file during build time just once and then importing that generated wrapper into the application like any other Nim module. - Using the command line `toast` tool to generate the Nim code which can then be stored into a file and imported separately. Any combination of the above is possible - only download, build or wrapping and nimterop avoids imposing any particular workflow. @@ -169,7 +170,7 @@ For types, `{.header: "header.h".}` informs Nim that `header.h` has the symbol a For functions, `{.header.}` works the same as types and can be omitted if preferred. The `{.importc.}` pragma is still required, unlike types since functions need to be linked to the implementation in the library. The user will need to provide this information at link time with `{.passL.}` and linking to a library with `-lheader` or `path/to/libheader.a`. It is also possible to just use `cCompile()` or `{.compile.}` to compile some C source files which contain the implementation. -While `{.header.}` can be omitted for convenience, it does prevent wrapping of `static inline` functions as well as type checking of the wrapper ABI with `-d:checkAbi` at compile time. The user will need to choose based on the library in question. +While `{.header.}` can be omitted for convenience, it does prevent wrapping of `static inline` functions as well as type checking of the wrapper ABI with `-d:checkAbi` at compile time. Further, anonymous nested structs/unions within unions will be rendered incorrectly by Nim since it is unaware of the true memory structure of the type. The user will need to choose based on the library in question. Going further, the `{.dynlib: "path/to/libheader.so".}` pragma can be used to inform Nim to load the library at runtime and link the function instead of linking at compile time. This enables creation of a wrapper that does not need the library present at compile time. diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 43ec8d3..fb2b55f 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -704,6 +704,17 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = edecl = node[i].anyChildInTree("enumerator_list") commentNodes = gState.getCommentNodes(node[i]) + # Check if struct/union field is anonymous + isNamedField = block: + var found = false + if not fdecl.isNil: + var sibling = fdecl.tsNodeParent().tsNodeNextNamedSibling() + while not sibling.isNil and sibling.getName() != "field_identifier": + sibling = sibling.tsNodeNextNamedSibling() + if not sibling.isNil: + found = true + found + # `tname` is name of nested struct / union / enum just # added, passed on as type name for field in `newIdentDefs()` (processed, tname) = @@ -725,11 +736,48 @@ proc newRecListTree(gState: State, name: string, node: TSNode): PNode = if processed != success: return nil - # Add nkIdentDefs for each field - for field in gState.newIdentDefs(name, node[i], i, ftname = tname, exported = true): - if not field.isNil: - field.comment = gState.getCommentsStr(commentNodes) - result.add field + if not fdecl.isNil and not isNamedField: + # Since anonymous, add fields directly to this struct/union + + # nkTypeDef( <= last + # nkPragmaExpr( + # .. + # ), + # nkEmpty(), + # nkObjectTy( <= last[2] + # nkEmpty(), + # nkEmpty(), + # nkRecList( <= last[2][2] + # nkIdentDefs( <= field1 + # .. + # ), + # nkIdentDefs( <= field2 + # .. + # ) + # ) + # ) + # ) + let + last = gState.typeSection[^1] + obj = + if last.len > 2 and last[2].kind == nkObjectTy: + last[2] + else: nil + recList = + if not obj.isNil and obj.len > 2 and obj[2].kind == nkRecList: + obj[2] + else: nil + + if not recList.isNil: + for identdef in recList: + result.add identdef + gState.typeSection.sons.del(gState.typeSection.len-1) + else: + # Add nkIdentDefs for each field + for field in gState.newIdentDefs(name, node[i], i, ftname = tname, exported = true): + if not field.isNil: + field.comment = gState.getCommentsStr(commentNodes) + result.add field proc addTypeObject(gState: State, node: TSNode, typeDef: PNode = nil, fname = "", istype = false, union = false) = # Add a type of object @@ -1444,14 +1492,16 @@ proc addEnum(gState: State, node: TSNode) = fval = "" if prev.Bl: # Starting default value - fval = &"(0).{name}" + fval = &"(0)" else: # One greater than previous - fval = &"({prev} + 1).{name}" + fval = &"({prev} + 1)" if en.len > 1 and en[1].getName() in gEnumVals: + # Enum value specified, evaluate later, don't use calculated value fieldDeclarations.add((fname, forigname, "", gState.getNodeVal(en[1]), commentNodes)) else: + # Set calculated value fieldDeclarations.add((fname, forigname, fval, "", commentNodes)) fnames.incl fname @@ -1464,10 +1514,18 @@ proc addEnum(gState: State, node: TSNode) = # parseCExpression requires all const identifiers to be present for the enum for (fname, forigname, fval, cexpr, commentNodes) in fieldDeclarations: let - fval = + fval = block: + var fval = fval if fval.Bl: - "(" & $gState.parseCExpression(cexpr, name) & ")." & name - else: fval + # Evaluate enum value from expression + fval = &"({$gState.parseCExpression(cexpr, name)})" + if origname.nBl: + # Named enum so cast to type - #236 + fval &= &".{name}" + else: + # Cast to cint to match underlying type + fval &= ".cint" + fval # Cannot use newConstDef() since parseString(fval) adds backticks to and/or constNode = gState.parseString(&"const {fname}* = {fval}")[0][0] diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 89ad486..06a0078 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -271,6 +271,27 @@ struct TestMyInt { MyInt f1; }; +// Issue #237 +typedef union sx_ivec3 { + struct { + int x; + int y; + struct z { + int z; + }; + }; + + int n[3]; +} sx_ivec3; + +// Issue #236 +enum { + SG_INVALID_ID = 0, + SG_NUM_SHADER_STAGES = 2, + SG_MAX_MIPMAPS = 16, + SG_MAX_TEXTUREARRAY_LAYERS = 128 +}; + // DUPLICATES @@ -546,6 +567,26 @@ struct TestMyInt { MyInt f1; }; +// Issue #237 +typedef union sx_ivec3 { + struct { + int x; + int y; + struct z { + int z; + }; + }; + + int n[3]; +} sx_ivec3; + +// Issue #236 +enum { + SG_INVALID_ID = 0, + SG_NUM_SHADER_STAGES = 2, + SG_MAX_MIPMAPS = 16, + SG_MAX_TEXTUREARRAY_LAYERS = 128 +}; #endif diff --git a/tests/tast2.nim b/tests/tast2.nim index 669ec98..ade5dff 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -353,6 +353,7 @@ checkPragmas(U2, pHeaderBy & @["union"], istype = false) var u2: U2 u2.f1 = addr a15.a2[0] +assert PANEL_WINDOW is nk_panel_type assert PANEL_WINDOW == 1 assert PANEL_GROUP == 2 assert PANEL_POPUP == 4 @@ -497,4 +498,21 @@ when not defined(NOHEADER): when declared(MyInt): assert false, "MyInt is defined!" testFields(TestMyInt, "f1!cint") -checkPragmas(TestMyInt, pHeaderBy, isType = false) \ No newline at end of file +checkPragmas(TestMyInt, pHeaderBy, isType = false) + +# #237 +assert sx_ivec3 is object +testFields(sx_ivec3, "x|y|z|n!cint|cint|cint|array[3, cint]") +checkPragmas(sx_ivec3, pHeaderBy & @["union"], istype = false) +var sx: sx_ivec3 +sx.x = 5 +assert sx.n[0] == 5 +when not defined(NOHEADER): + # Nim doesn't know of the anonymous nested struct so when the header + # isn't present, the test below breaks + sx.n[1] = 4 + assert sx.y == 4 + +# #236 +assert SG_MAX_MIPMAPS is cint +assert SG_MAX_MIPMAPS == 16 \ No newline at end of file diff --git a/tests/tnimterop_c.nim b/tests/tnimterop_c.nim index ef814ea..ac7193c 100644 --- a/tests/tnimterop_c.nim +++ b/tests/tnimterop_c.nim @@ -70,7 +70,7 @@ var e: ENUM e2: ENUM2 = enum5 - e3: Enum_testh1 = enum7 + e3 = enum7 e4: ENUM4 = enum11 vptr: VOIDPTR From 1ad32e4574a70d48cc0f50e71f354207bcdd2bd0 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 3 Jul 2020 01:46:27 -0500 Subject: [PATCH 062/106] Fix missing preprocessor errors, toast errors, cDisableCache not working, print generated wrapper location --- nimterop/cimport.nim | 24 ++++++++---------------- nimterop/toastlib/getters.nim | 7 +++++++ tests/include/tast2.h | 2 +- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index d992c04..114ab80 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -82,18 +82,6 @@ proc getCacheValue(fullpaths: seq[string]): string = for fullpath in fullpaths: result &= getCacheValue(fullpath) -proc getToastError(output: string): string = - # Filter out preprocessor errors - for line in output.splitLines(): - if "fatal error:" in line.toLowerAscii: - if result.len == 0: - result = "\n\nFailed in preprocessing, check if `cIncludeDir()` is needed or compiler `mode` is correct (c/cpp)" - result &= "\n\nERROR:$1\n" % line.split("fatal error:")[1] - - # Toast error - if result.Bl: - result = "\n\n" & output - proc getNimCheckError(nimFile: string) = let (check, _) = execAction( @@ -159,7 +147,7 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" when defined(Windows): result = result.replace(DirSep, '/') - if not fileExists(result) or compileOption("forceBuild"): + if not fileExists(result) or gStateCT.nocache or compileOption("forceBuild"): let dir = result.parentDir() if not dirExists(dir): @@ -168,8 +156,8 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" cmd.add &" -o {result.sanitizePath}" var - (_, ret) = execAction(cmd, die = false) - doAssert ret == 0, getToastError(result.readFile()) + (output, ret) = execAction(cmd, die = false) + doAssert ret == 0, result.readFile() & output macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not @@ -577,6 +565,8 @@ macro cImport*(filenames: static seq[string], recurse: static bool = false, dynl if gStateCT.debug: gecho nimFile.readFile() + gecho "# Saved to " & nimFile + try: let nimFileNode = newStrLitNode(nimFile.changeFileExt("")) @@ -673,7 +663,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: nimFile = if nimFile.nBl: fixRelFile(nimFile) else: hFile.changeFileExt("nim") header = "header" & fullpath.splitFile().name.split(seps = {'-', '.'}).join() - if not fileExists(nimFile) or compileOption("forceBuild"): + if not fileExists(nimFile) or gStateCT.nocache or compileOption("forceBuild"): var cmd = when defined(Windows): "cmd /c " else: "" cmd &= &"c2nim {hFile} --header:{header} --out:{nimFile.sanitizePath}" @@ -699,6 +689,8 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: if gStateCT.debug: gecho nimFile.readFile() + gecho "# Saved to " & nimFile + try: let nimFileNode = newStrLitNode(nimFile.changeFileExt("")) diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index bfbd4be..df930bb 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -338,6 +338,10 @@ proc getPreprocessor*(gState: State, fullpath: string) = start = true newHeaders.incl line break + elif ": fatal error:" in line: + doAssert false, + "\n\nFailed in preprocessing, check if `cIncludeDir()` is needed or compiler `mode` is correct (c/cpp)" & + "\n\nERROR:$1\n" % line.split(": fatal error:")[1] else: if start: if "#undef" in line: @@ -345,6 +349,9 @@ proc getPreprocessor*(gState: State, fullpath: string) = gState.code.add line & "\n" elif not p.running(): break p.close() + assert p.peekExitCode() == 0, + gState.code & "\n\nFailed in preprocessing:\n " & + getCompiler() & " " & args.join(" ") gState.headersProcessed.incl newHeaders # Plugin related diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 06a0078..f9e0017 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -276,7 +276,7 @@ typedef union sx_ivec3 { struct { int x; int y; - struct z { + struct { int z; }; }; From 4d344e5807c17d8644cf23753d8445443cdb72bb Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 3 Jul 2020 16:17:34 -0500 Subject: [PATCH 063/106] Handle (*field), ident starting w/ digit error, fix detection of plugin and toast errors --- nimterop/cimport.nim | 2 +- nimterop/toastlib/ast2.nim | 3 ++- nimterop/toastlib/getters.nim | 13 ++++++++----- nimterop/toastlib/tshelp.nim | 2 +- tests/include/tast2.h | 7 +++++++ tests/tast2.nim | 6 +++++- 6 files changed, 24 insertions(+), 9 deletions(-) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 114ab80..cedd903 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -157,7 +157,7 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" var (output, ret) = execAction(cmd, die = false) - doAssert ret == 0, result.readFile() & output + doAssert ret == 0, "\n\n" & (if result.fileExists(): result.readFile() else: "") & output macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index fb2b55f..083d4b2 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -523,7 +523,8 @@ proc newIdentDef(gState: State, name: string, node: TSNode, tname: string, tinfo result.add pident let - count = node[offset].getPtrCount() + # Could be parenthesized + count = node[offset].getAtom().tsNodeParent().getPtrCount(reverse = true) if count > 0: result.add gState.newPtrTree(count, tident) else: diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index df930bb..1e70d59 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -167,13 +167,15 @@ proc checkIdentifier(name, kind, parent, origName: string) = if name.nBl: let - origStr = if name != origName: &", originally '{origName}' before 'cPlugin:onSymbol()', still" else: "" - errmsg = &"Identifier '{parentStr}{name}' ({kind}){origStr} contains $1 " & + origStr = if name != origName: &", originally '{origName}' before 'cPlugin:onSymbol()'," else: "" + errmsg = &"Identifier '{parentStr}{name}' ({kind}){origStr} $1 " & "which Nim does not allow. Use toast flag '$2' or 'cPlugin()' to modify." - doAssert name[0] != '_' and name[^1] != '_', errmsg % ["leading/trailing underscores '_'", "--prefix or --suffix"] + doAssert name[0] != '_' and name[^1] != '_', errmsg % ["has leading/trailing underscores '_'", "--prefix or --suffix"] - doAssert (not name.contains("__")): errmsg % ["consecutive underscores '_'", "--replace"] + doAssert (not name.contains("__")): errmsg % ["has consecutive underscores '_'", "--replace"] + + doAssert not name[0].isDigit(), errmsg % [&"starts with a digit '{name[0]}'", "--prefix"] # Cannot blank out symbols which are fields or params # @@ -381,7 +383,8 @@ proc loadPlugin*(gState: State, sourcePath: string) = # Compile plugin as library with `markAndSweep` GC cmd = &"{gState.nim} c --app:lib --gc:markAndSweep {flags} {outflags} {sourcePath.sanitizePath}" - discard execAction(cmd) + (output, ret) = execAction(cmd, die = false) + doAssert ret == 0, output & "\nFailed to compile cPlugin()\n\ncmd: " & cmd doAssert fileExists(pdll), "No plugin binary generated for " & sourcePath let lib = loadLib(pdll) diff --git a/nimterop/toastlib/tshelp.nim b/nimterop/toastlib/tshelp.nim index bbe029f..81c8d0d 100644 --- a/nimterop/toastlib/tshelp.nim +++ b/nimterop/toastlib/tshelp.nim @@ -116,7 +116,7 @@ proc getXCount*(node: TSNode, ntype: string, reverse = false): int = break proc getPtrCount*(node: TSNode, reverse = false): int = - node.getXCount("pointer_declarator") + node.getXCount("pointer_declarator", reverse) proc getArrayCount*(node: TSNode, reverse = false): int = node.getXCount("array_declarator") diff --git a/tests/include/tast2.h b/tests/include/tast2.h index f9e0017..38c5f18 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -292,6 +292,9 @@ enum { SG_MAX_TEXTUREARRAY_LAYERS = 128 }; +struct parenpoin { + void (*gtk_reserved1); +}; // DUPLICATES @@ -588,6 +591,10 @@ enum { SG_MAX_TEXTUREARRAY_LAYERS = 128 }; +struct parenpoin { + void (*__gtk_reserved1); +}; + #endif #ifdef __cplusplus diff --git a/tests/tast2.nim b/tests/tast2.nim index ade5dff..17edfb0 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -515,4 +515,8 @@ when not defined(NOHEADER): # #236 assert SG_MAX_MIPMAPS is cint -assert SG_MAX_MIPMAPS == 16 \ No newline at end of file +assert SG_MAX_MIPMAPS == 16 + +assert parenpoin is object +var pp: parenpoin +assert pp.gtk_reserved1 is pointer \ No newline at end of file From 03d1b54d6c10317a7bd005f35ecb7a9b426051fb Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 3 Jul 2020 23:51:52 -0500 Subject: [PATCH 064/106] gitPull non-master default branch, rmFile fix --- nimterop/build/shell.nim | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 2b29734..f07f70c 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -155,7 +155,7 @@ proc rmFile*(source: string, dir = false) = if dir: "rd /s/q" else: - "del /s/q/f" + "del /q/f" else: "rm -rf" exists = @@ -335,6 +335,18 @@ proc gitAtCheckout*(outdir, checkout: string): bool = result = checkout in execAction( &"cd {outdir.sanitizePath} && git log --decorate --no-color -n 1 --format=oneline").output +proc gitDefaultBranch*(outdir: string): string = + ## Get the default branch for a git repository before it is pulled + result = "master" + let + output = execAction( + &"cd {outdir.sanitizePath} && git remote show origin" + ).output + + for line in output.splitLines(): + if "HEAD branch: " in line: + result = line.split("branch: ")[1].strip() + proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false) = ## Pull the specified git repository to the output directory ## @@ -375,15 +387,14 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "", quiet = false # In case directory has old files from another run discard execAction(&"cd {outdirQ} && git clean -fxd") - if checkout.len != 0: - if not quiet: - gecho "# Checking out " & checkout - discard execAction(&"cd {outdirQ} && git fetch", retry = 3) - discard execAction(&"cd {outdirQ} && git checkout {checkout}") - else: - if not quiet: - gecho "# Pulling repository" - discard execAction(&"cd {outdirQ} && git pull --depth=1 origin master", retry = 3) + # Checkout specified branch/tag/commit or default branch - typically master + let + checkout = if checkout.Bl: gitDefaultBranch(outdir) else: checkout + + if not quiet: + gecho "# Checking out " & checkout + discard execAction(&"cd {outdirQ} && git fetch", retry = 3) + discard execAction(&"cd {outdirQ} && git checkout {checkout}") proc gitTags*(outdir: string): seq[string] = ## Get all the git tags in the specified directory From 90c5f54dc3b21c32e6666065c7a192c89a37fb4f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 4 Jul 2020 00:49:46 -0500 Subject: [PATCH 065/106] v0.6.2 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 628ef40..adbd171 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.1" +version = "0.6.2" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 37cdc5b94ec42c268fc2f7dc63cea967b3d521cd Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sat, 11 Jul 2020 17:29:21 -0500 Subject: [PATCH 066/106] Add jbbFlags to customize xxxJBB --- CHANGES.md | 4 ++- nimterop/build/getheader.nim | 42 ++++++++++++++++++++++--------- nimterop/build/jbb.nim | 48 ++++++++++++++++++++++++------------ 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e85d261..70e1f76 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,7 @@ Refer to the documentation for `getHeader()` for details on how to use this new See the full list of changes here: -https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.1 +https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 ### Breaking changes @@ -39,6 +39,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.1 - Nimterop now supports anonymous nested structs/unions but it only works correctly for unions when `noHeader` is turned off (the default). This is because Nim does not support nested structs/unions and is unaware of the underlying memory structure. [#237][i237] (since v0.6.1) +- `xxxJBB` now allows for customizing the base location to search packages with the `jbbFlags` param to `getHeader()`. Specifying `giturl=xxx` where `xxx` could be a full Git URL or just the username for Github.com allows changing the default Git repo. In addition, `url=xxx` is also supported to download project info and binaries compiled with BinaryBuilder.org but hosted at another non-Git location. (since v0.6.3) + ### Other improvements - Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 6972ccc..d5b398a 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -141,18 +141,14 @@ proc getConanLDeps(outdir: string): seq[string] = result = pkg.getConanLDeps(outdir) -proc getJBBPath(header, uri, outdir, version: string): string = +proc getJBBPath(header, uri, flags, outdir, version: string): string = let spl = uri.split('/', 1) name = spl[0] hasVersion = version.len != 0 var - ver = - if spl.len == 2: - spl[1] - else: - "" + ver = if spl.len == 2: spl[1] else: "" if ver.len != 0: if "$#" in ver or "$1" in ver: @@ -166,6 +162,20 @@ proc getJBBPath(header, uri, outdir, version: string): string = let pkg = newJBBPackage(name, ver) + + # Handle `jbbFlags` + if flags.nBl: + if flags.startsWith("giturl="): + let + val = flags["giturl=".len .. ^1] + if val.contains("://"): + pkg.baseUrl = val + else: + pkg.baseUrl = "https://github.com/" & val + elif flags.startsWith("url="): + pkg.baseUrl = flags["url=".len .. ^1] + pkg.isGit = false + downloadJBB(pkg, outdir) result = findFile(header, outdir) @@ -217,7 +227,8 @@ macro getHeader*( conanuri: static[string] = "", jbburi: static[string] = "", outdir: static[string] = "", libdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", - altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = + jbbFlags: static[string] = "", altNames: static[string] = "", + buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = ## Get the path to a header file for wrapping with ## `cImport() `_ or ## `c2nImport() `_. @@ -289,6 +300,13 @@ macro getHeader*( ## `cmake` and `make` in case additional configuration is required as part of the build ## process. ## + ## `jbbFlags` allows changing the BinaryBuilder.org defaults: + ## - `giturl=customUrl` changes the default `https://github.com/JuliaBinaryWrappers` to + ## another Git URL. If no hostname is specified, `https://github.com` is assumed. + ## - `url=customUrl` uses regular HTTP instead of Git and looks for `Artifacts.toml` and + ## `Project.toml` files at that location. `$1` or `$#` are replaced with the version + ## if specified. + ## ## `altNames` is a list of alternate names for the library - e.g. zlib uses `zlib.h` for ## the header but the typical lib name is `libz.so` and not `libzlib.so`. However, it is ## libzlib.dll on Windows if built with cmake. In this case, `altNames = "z,zlib"`. Comma @@ -384,7 +402,7 @@ macro getHeader*( `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, conanuri, jbburi, outdir, version: string, shared: bool): string = + proc getPath(header, giturl, dlurl, conanuri, jbburi, jbbFlags, outdir, version: string, shared: bool): string = when `nameGit`: getGitPath(header, giturl, outdir, version) elif `nameDL`: @@ -392,7 +410,7 @@ macro getHeader*( elif `nameConan`: getConanPath(header, conanuri, outdir, version, shared) elif `nameJBB`: - getJBBPath(header, jbburi, outdir, version) + getJBBPath(header, jbburi, jbbFlags, outdir, version) else: getLocalPath(header, outdir) @@ -423,7 +441,8 @@ macro getHeader*( when useStd: stdPath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `jbbFlags`, + `outdir`, `version`, not `nameStatic`) # Run preBuild hook before building library if not Std, Conan or JBB when not (useStd or `nameConan` or `nameJBB`) and declared(`preBuild`): @@ -458,7 +477,8 @@ macro getHeader*( if prePath.len != 0: prePath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `outdir`, `version`, not `nameStatic`) + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `jbbFlags`, + `outdir`, `version`, not `nameStatic`) static: doAssert `path`.len != 0, "\nHeader " & `header` & " not found - " & diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index 0fb163f..b9272c7 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -12,7 +12,10 @@ type name*: string version*: string - url*: string + baseUrl*: string # Location to find package + isGit*: bool # Git or HTTP + + url*: string # Download URL sharedLibs*: seq[string] staticLibs*: seq[string] @@ -20,7 +23,7 @@ type const # JBB URLs - jbbBaseUrl = "https://github.com/JuliaBinaryWrappers/$1_jll.jl" + jbbBaseUrl = "https://github.com/JuliaBinaryWrappers" jbbInfo = "jbbinfo.json" jbbProject = "Project.toml" @@ -45,6 +48,8 @@ proc newJBBPackage*(name, version: string): JBBPackage = result = new(JBBPackage) result.name = name result.version = version + result.baseUrl = jbbBaseUrl + result.isGit = true proc parseJBBProject(pkg: JBBPackage, outdir: string) = # Get all dependencies from Project.toml @@ -127,21 +132,32 @@ proc getJBBRepo*(pkg: JBBPackage, outdir: string) = let path = outdir / "repos" / pkg.name - gitPull( - jbbBaseUrl % pkg.name, - outdir = path, - plist = "*.toml", - "master", - quiet = true - ) + if pkg.isGit: + # Get package info using Git + gitPull( + pkg.baseUrl & ("/$1_jll.jl" % pkg.name), + outdir = path, + plist = "*.toml", + "master", + quiet = true + ) - if pkg.version.len != 0: - # Checkout correct tag - let - tags = gitTags(path) - for i in tags.len - 1 .. 0: - if pkg.version in tags[i] and i != tags.len - 1: - gitCheckout(path, tags[i-1]) + if pkg.version.len != 0: + # Checkout correct tag + let + tags = gitTags(path) + for i in tags.len - 1 .. 0: + if pkg.version in tags[i] and i != tags.len - 1: + gitCheckout(path, tags[i-1]) + else: + # Download package info from HTTP + var + url = pkg.baseUrl + if "$#" in url or "$1" in url: + doAssert pkg.version.len != 0, "Need version for custom BinaryBuilder.org url: " & url + url = url % pkg.version + downloadUrl(url & "Artifacts.toml", path, quiet = true) + downloadUrl(url & "Project.toml", path, quiet = true) pkg.parseJBBProject(path) pkg.parseJBBArtifacts(path) From fb2acc752a283223b5c4647bfe85d75cd23a5b4f Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 12 Jul 2020 00:12:41 -0500 Subject: [PATCH 067/106] v0.6.3 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index adbd171..16a6c6a 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.2" +version = "0.6.3" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From a9887cc6b2569bddd1daf56ae66960fb500be3e9 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 12 Jul 2020 17:35:29 -0500 Subject: [PATCH 068/106] Add ability to exclude files or directories from wrapped output --- CHANGES.md | 4 +++ README.md | 2 ++ nimterop/cimport.nim | 56 ++++++++++++++++++++++++++++------- nimterop/globals.nim | 1 + nimterop/toast.nim | 6 +++- nimterop/toastlib/getters.nim | 16 ++++++++-- 6 files changed, 70 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 70e1f76..23aa2bb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -41,12 +41,16 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 - `xxxJBB` now allows for customizing the base location to search packages with the `jbbFlags` param to `getHeader()`. Specifying `giturl=xxx` where `xxx` could be a full Git URL or just the username for Github.com allows changing the default Git repo. In addition, `url=xxx` is also supported to download project info and binaries compiled with BinaryBuilder.org but hosted at another non-Git location. (since v0.6.3) +- It is now possible to exclude the contents of specific files or entire directories from the wrapped output using `--exclude | -X` with `toast` or `cExclude()` from a wrapper. This might be required when a header uses `#include` to pull in external dependencies. E.g. `sciter` has a `#include ` which pulls in the entire GTK ecosystem which is needed for successful preprocessing but we do not want to include those headers in the wrapped output when using `--recurse | -r`. + ### Other improvements - Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) - `cImport()` now includes wrapper output from a file rather than inline. Errors in generated wrappers will no longer point to a line in `macros.nim` making debugging easier. (since v0.6.1) +- `cIncludeDir()` can now accept a `seq[string]` of directories and an optional `exclude` param which sets those include directories to not be included in the wrapped output. + ## Version 0.5.0 diff --git a/README.md b/README.md index f9e9307..0e79a6f 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,7 @@ cDefine("HAS_ABC") # Set #defines for preprocessor and compiler cDefine("HAS_ABC", "DEF") cIncludeDir("clib/include") # Setup any include directories +cExclude("clib/file.h") # Exclude file from wrapped output cImport("clib.h") # Generate wrappers for header specified @@ -219,6 +220,7 @@ Options: -d, --debug bool false enable debug output -D=, --defines= strings {} definitions to pass to preprocessor -l=, --dynlib= string "" {.dynlib.} pragma to import symbols - Nim const string or file path + -X=, --exclude= strings {} files or directories to exclude from the wrapped output -f=, --feature= Features {} flags to enable experimental features -I=, --includeDirs= strings {} include directory to pass to preprocessor -m=, --mode= string "" language parser: c or cpp diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index cedd903..95004ad 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -120,6 +120,9 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" for i in gStateCT.includeDirs: cmd.add &" --includeDirs+={i.sanitizePath}" + for i in gStateCT.exclude: + cmd.add &" --exclude+={i.sanitizePath}" + if not noNimout: cmd.add &" --pnim" @@ -415,21 +418,52 @@ proc cAddSearchDir*(dir: string) {.compileTime.} = if dir notin gStateCT.searchDirs: gStateCT.searchDirs.add(dir) -macro cIncludeDir*(dir: static string): untyped = +macro cIncludeDir*(dirs: static seq[string], exclude: static[bool] = false): untyped = + ## Add include directories that are forwarded to the C/C++ preprocessor if + ## called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-IXXX".}`. + ## + ## Set `exclude = true` if the contents of these include directories should + ## not be included in the wrapped output. + for dir in dirs: + var dir = interpPath(dir) + result = newNimNode(nnkStmtList) + + let fullpath = findPath(dir) + if fullpath notin gStateCT.includeDirs: + gStateCT.includeDirs.add(fullpath) + if exclude: + gStateCT.exclude.add(fullpath) + let str = &"-I{fullpath.quoteShell}" + result.add quote do: + {.passC: `str`.} + if gStateCT.debug: + gecho result.repr + +macro cIncludeDir*(dir: static[string], exclude: static[bool] = false): untyped = ## Add an include directory that is forwarded to the C/C++ preprocessor if ## called within `cImport()` or `c2nImport()` as well as to the C/C++ ## compiler during Nim compilation using `{.passC: "-IXXX".}`. - var dir = interpPath(dir) - result = newNimNode(nnkStmtList) + ## + ## Set `exclude = true` if the contents of this include directory should + ## not be included in the wrapped output. + return quote do: + cIncludeDir(@[`dir`], `exclude` == 1) - let fullpath = findPath(dir) - if fullpath notin gStateCT.includeDirs: - gStateCT.includeDirs.add(fullpath) - let str = &"-I{fullpath.quoteShell}" - result.add quote do: - {.passC: `str`.} - if gStateCT.debug: - gecho result.repr +macro cExclude*(paths: static seq[string]): untyped = + ## Exclude specified paths - files or directories from the wrapped output + ## + ## Full path to file or directory is required. + result = newNimNode(nnkStmtList) + for path in paths: + gStateCT.exclude.add path + +macro cExclude*(path: static string): untyped = + ## Exclude specified path - file or directory from the wrapped output. + ## + ## Full path to file or directory is required. + return quote do: + cExclude(@[`path`]) proc cAddStdDir*(mode = "c") {.compileTime.} = ## Add the standard `c` [default] or `cpp` include paths to search diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 17b9518..89e403f 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -19,6 +19,7 @@ type debug*: bool # `cDebug()` or `--debug | -d` to enable debug mode defines*: seq[string] # Symbols added by `cDefine()` and `--define | -D` for C/C++ preprocessor/compiler dynlib*: string # `cImport(dynlib)` or `--dynlib | -l` to specify variable containing library name + exclude*: seq[string] # files or directories to exclude from the wrapped output feature*: seq[Feature] # `--feature | -f` feature flags enabled includeDirs*: seq[string] # Paths added by `cIncludeDir()` and `--includeDirs | -I` for C/C++ preprocessor/compiler mode*: string # `cImport(mode)` or `--mode | -m` to override detected compiler mode - c or cpp diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 1cda239..20dc79e 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -13,7 +13,7 @@ var preMainOut = "" proc process(gState: State, path: string) = - doAssert existsFile(path), &"Invalid path {path}" + doAssert fileExists(path), &"Invalid path {path}" if gState.mode.Bl: gState.mode = getCompilerMode(path) @@ -38,6 +38,7 @@ proc main( debug = false, defines: seq[string] = @[], dynlib: string = "", + exclude: seq[string] = @[], feature: seq[Feature] = @[], includeDirs: seq[string] = @[], mode = "", @@ -65,6 +66,7 @@ proc main( debug: debug, defines: defines, dynlib: dynlib, + exclude: exclude, feature: feature, includeDirs: includeDirs, mode: mode, @@ -222,6 +224,7 @@ when isMainModule: "debug": "enable debug output", "defines": "definitions to pass to preprocessor", "dynlib": "{.dynlib.} pragma to import symbols - Nim const string or file path", + "exclude": "files or directories to exclude from the wrapped output", "feature": "flags to enable experimental features", "includeDirs": "include directory to pass to preprocessor", "mode": "language parser: c or cpp", @@ -247,6 +250,7 @@ when isMainModule: "debug": 'd', "defines": 'D', "dynlib": 'l', + "exclude": 'X', "feature": 'f', "includeDirs": 'I', "noComments": 'c', diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 1e70d59..8c6abe1 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -286,7 +286,16 @@ proc getKeyword*(kind: NimSymKind): string = proc getCurrentHeader*(fullpath: string): string = ("header" & fullpath.splitFile().name.multiReplace([(".", ""), ("-", "")])) +proc isIncluded(gState: State, file: string): bool {.inline.} = + # Check if the specified file should be excluded from wrapped output + if gState.exclude.nBl: + for excl in gState.exclude: + if file.startsWith(excl): + return + result = true + proc getPreprocessor*(gState: State, fullpath: string) = + # Get preprocessed output from the C/C++ compiler var args: seq[string] start = false @@ -332,14 +341,15 @@ proc getPreprocessor*(gState: State, fullpath: string) = start = true elif gState.recurse: if (pDir.Bl or pDir in line) and line notin gState.headersProcessed: - start = true newHeaders.incl line + start = gState.isIncluded(line) else: for inc in includeDirs: if line.startsWith(inc) and line notin gState.headersProcessed: - start = true newHeaders.incl line - break + start = gState.isIncluded(line) + if start: + break elif ": fatal error:" in line: doAssert false, "\n\nFailed in preprocessing, check if `cIncludeDir()` is needed or compiler `mode` is correct (c/cpp)" & From d7c94bb70bb754ffbe85669e550589089d6597d1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 12 Jul 2020 22:21:35 -0500 Subject: [PATCH 069/106] Bump to v0.6.4 --- .travis.yml | 3 ++- CHANGES.md | 4 ++-- nimterop.nimble | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 173da68..b58b29d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ language: c env: - BRANCH=0.20.2 - BRANCH=1.0.6 - - BRANCH=1.2.2 + - BRANCH=1.2.4 - BRANCH=devel cache: @@ -26,6 +26,7 @@ install: - source travis.sh script: + - set -e - nimble develop -y - nimble test - nimble --verbose --nimbleDir:`pwd`/build/fakenimble install nimterop@#head -y diff --git a/CHANGES.md b/CHANGES.md index 23aa2bb..91cecac 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -41,7 +41,7 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 - `xxxJBB` now allows for customizing the base location to search packages with the `jbbFlags` param to `getHeader()`. Specifying `giturl=xxx` where `xxx` could be a full Git URL or just the username for Github.com allows changing the default Git repo. In addition, `url=xxx` is also supported to download project info and binaries compiled with BinaryBuilder.org but hosted at another non-Git location. (since v0.6.3) -- It is now possible to exclude the contents of specific files or entire directories from the wrapped output using `--exclude | -X` with `toast` or `cExclude()` from a wrapper. This might be required when a header uses `#include` to pull in external dependencies. E.g. `sciter` has a `#include ` which pulls in the entire GTK ecosystem which is needed for successful preprocessing but we do not want to include those headers in the wrapped output when using `--recurse | -r`. +- It is now possible to exclude the contents of specific files or entire directories from the wrapped output using `--exclude | -X` with `toast` or `cExclude()` from a wrapper. This might be required when a header uses `#include` to pull in external dependencies. E.g. `sciter` has a `#include ` which pulls in the entire GTK ecosystem which is needed for successful preprocessing but we do not want to include those headers in the wrapped output when using `--recurse | -r`. (since v0.6.4) ### Other improvements @@ -49,7 +49,7 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 - `cImport()` now includes wrapper output from a file rather than inline. Errors in generated wrappers will no longer point to a line in `macros.nim` making debugging easier. (since v0.6.1) -- `cIncludeDir()` can now accept a `seq[string]` of directories and an optional `exclude` param which sets those include directories to not be included in the wrapped output. +- `cIncludeDir()` can now accept a `seq[string]` of directories and an optional `exclude` param which sets those include directories to not be included in the wrapped output. (since v0.6.4) ## Version 0.5.0 diff --git a/nimterop.nimble b/nimterop.nimble index 16a6c6a..1eb35ef 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.3" +version = "0.6.4" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 27fcecaa26a186dbc1e45c475baefe7182abb425 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 13 Jul 2020 17:47:17 -0500 Subject: [PATCH 070/106] Caching fixes, wchar_t --- nimterop/cimport.nim | 42 ++++++++++++++++++++++++++++++----- nimterop/toastlib/getters.nim | 4 ++-- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 95004ad..aa7895a 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -142,15 +142,35 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" for fullpath in fullpaths: cmd.add &" {fullpath.sanitizePath}" - result = if outFile.nBl: fixRelFile(outFile) else: - # Generate filename for toast output if not specified - getNimteropCacheDir() / "toastCache" / "nimterop_" & + let + cacheFile = getNimteropCacheDir() / "toastCache" / "nimterop_" & ($(cmd & cacheKey).hash().abs()).addFileExt(ext) + if outFile.nBl: + result = fixRelFile(outFile) + else: + result = cacheFile + when defined(Windows): result = result.replace(DirSep, '/') - if not fileExists(result) or gStateCT.nocache or compileOption("forceBuild"): + let + # When to regenerate the wrapper + regen = + if gStateCT.nocache or compileOption("forceBuild"): + # No caching or forced + true + elif not fileExists(result): + # Cache or outfile doesn't exist + true + elif outFile.nBl and (not fileExists(cacheFile) or + result.getFileDate() > cacheFile.getFileDate()): + # Outfile exists but cache doesn't or outdated + true + else: + false + + if regen: let dir = result.parentDir() if not dirExists(dir): @@ -160,7 +180,19 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" var (output, ret) = execAction(cmd, die = false) - doAssert ret == 0, "\n\n" & (if result.fileExists(): result.readFile() else: "") & output + if ret != 0: + # If toast fails, print failure to output and delete any generated files + let errout = if result.fileExists(): result.readFile() & output else: output + rmFile(result) + doAssert false, "\n\n" & errout & "\n" + + # Write empty cache file to track changes when outFile specified + if outFile.nBl: + let dir = cacheFile.parentDir() + if not dirExists(dir): + mkdir(dir) + + writeFile(cacheFile, "") macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 8c6abe1..8a4586d 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -139,7 +139,7 @@ when defined(cpp): # not defined in nor any other header). type wchar_t* {.importc.} = object else: - type wchar_t* {.importc, header:"".} = object + type wchar_t* {.importc, header:"stddef.h".} = object """, "va_list": """ @@ -417,4 +417,4 @@ proc expandSymlinkAbs*(path: string): string = result = path.expandFilename().normalizedPath() except: result = path - result = result.sanitizePath(noQuote = true) \ No newline at end of file + result = result.sanitizePath(noQuote = true) From 6ffacf92fc19d2cd0754e2409868a2dca4501e93 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 14 Jul 2020 18:08:59 -0500 Subject: [PATCH 071/106] Forward pragmas to toast --- README.md | 2 + nimterop/build/conan.nim | 20 ++-- nimterop/build/jbb.nim | 17 ++- nimterop/build/nimconf.nim | 4 + nimterop/cimport.nim | 212 ++++++++++++++++--------------------- nimterop/globals.nim | 5 +- nimterop/template.nim | 111 ------------------- nimterop/templite.nim | 36 ------- nimterop/toast.nim | 9 ++ nimterop/toastlib/ast2.nim | 24 ++++- tests/libssh2.nim | 4 +- tests/rsa.nim | 4 +- tests/tast2.nim | 2 +- tests/tsoloud.nim | 8 +- tests/zlib.nim | 10 +- 15 files changed, 162 insertions(+), 306 deletions(-) delete mode 100644 nimterop/template.nim delete mode 100644 nimterop/templite.nim diff --git a/README.md b/README.md index 0e79a6f..cda3b5e 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,8 @@ cImport("clib.h") # Generate wrappers for header specified cCompile("clib/src/*.c") # Compile in any implementation source files ``` +All `{.compileTime.}` procs must be used in a compile time context, like `cDebug()` and `cDisableCaching()` above. + Module documentation for the wrapper API can be found [here](https://nimterop.github.io/nimterop/cimport.html). #### Preprocessing diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index 792de01..dd43dc9 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -79,10 +79,6 @@ proc jsonGet(url: string): JsonNode = discard rmFile(file) -template fixOutDir() {.dirty.} = - let - outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir - proc `==`*(pkg1, pkg2: ConanPackage): bool = ## Check if two ConanPackage objects are equal (not pkg1.isNil and not pkg2.isNil and @@ -282,9 +278,8 @@ proc getConanRevisions*(pkg: ConanPackage, bld: ConanBuild) = proc loadConanInfo*(outdir: string): ConanPackage = ## Load cached package info from `outdir/conaninfo.json` - fixOutDir() let - file = outdir / conanInfo + file = fixRelPath(outdir) / conanInfo if fileExists(file): when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): @@ -297,9 +292,8 @@ proc loadConanInfo*(outdir: string): ConanPackage = proc saveConanInfo*(pkg: ConanPackage, outdir: string) = ## Save downloaded package info to `outdir/conaninfo.json` - fixOutDir() let - file = outdir / conanInfo + file = fixRelPath(outdir) / conanInfo when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): writeFile(file, $$pkg) @@ -329,11 +323,11 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision ## Download specific `revision` of `bld` to `outdir` ## ## If omitted, the latest revision (first) is downloaded - fixOutDir() - doAssert bld.revisions.len != 0, "No build revisions found for Conan.io package " & pkg.getUriFromConanPackage() let + outdir = fixRelPath(outdir) + revision = if revision.len != 0: revision @@ -376,8 +370,9 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = ## ## High-level API that handles the end to end Conan process flow to find ## latest package binary and downloads and extracts it to `outdir`. - fixOutDir() let + outdir = fixRelPath(outdir) + pkg = if pkg.version.len == 0: searchConan(pkg) @@ -415,7 +410,8 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = ## ## This is not required for shared libs since conan builds them ## with all dependencies statically linked in - fixOutDir() + let + outdir = fixRelPath(outdir) if bld.options["shared"] == "False": for req in bld.requires: let diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index b9272c7..a430a55 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -33,10 +33,6 @@ var # Reuse dependencies already downloaded gJBBRequires {.compileTime.}: Table[string, JBBPackage] -template fixOutDir() {.dirty.} = - let - outdir = if outdir.isAbsolute(): outdir else: getProjectDir() / outdir - proc `==`*(pkg1, pkg2: JBBPackage): bool = ## Check if two JBBPackage objects are equal (not pkg1.isNil and not pkg2.isNil and @@ -164,9 +160,8 @@ proc getJBBRepo*(pkg: JBBPackage, outdir: string) = proc loadJBBInfo*(outdir: string): JBBPackage = ## Load cached package info from `outdir/jbbinfo.json` - fixOutDir() let - file = outdir / jbbInfo + file = fixRelPath(outdir) / jbbInfo if fileExists(file): when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): @@ -179,9 +174,8 @@ proc loadJBBInfo*(outdir: string): JBBPackage = proc saveJBBInfo*(pkg: JBBPackage, outdir: string) = ## Save downloaded package info to `outdir/jbbinfo.json` - fixOutDir() let - file = outdir / jbbInfo + file = fixRelPath(outdir) / jbbInfo when (NimMajor, NimMinor, NimPatch) < (1, 2, 0): writeFile(file, $$pkg) @@ -194,7 +188,9 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = ## ## High-level API that handles the end to end JBB process flow to find ## latest package binary and downloads and extracts it to `outdir`. - fixOutDir() + let + outdir = fixRelPath(outdir) + if main: let cpkg = loadJBBInfo(outdir) @@ -226,7 +222,8 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = proc dlJBBRequires*(pkg: JBBPackage, outdir: string) = ## Download all required dependencies of this `pkg` - fixOutDir() + let + outdir = fixRelPath(outdir) for i in 0 ..< pkg.requires.len: let rpkg = pkg.requires[i] diff --git a/nimterop/build/nimconf.nim b/nimterop/build/nimconf.nim index 3bbe521..7624bf2 100644 --- a/nimterop/build/nimconf.nim +++ b/nimterop/build/nimconf.nim @@ -234,3 +234,7 @@ proc getOutDir*(projectDir = ""): string = proc getNimteropCacheDir*(): string = ## Get location to cache all nimterop artifacts result = getNimcacheDir() / "nimterop" + +proc fixRelPath*(path: string): string = + ## If `path` is relative, consider relative to `projectPath` + if path.isAbsolute: path else: getProjectDir() / path \ No newline at end of file diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index aa7895a..6120cb5 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -1,78 +1,17 @@ -##[ -This is the main nimterop import file to help with wrapping C/C++ source code. - -Check out `template.nim `_ -as a starting point for wrapping a new library. The template can be copied and -trimmed down and modified as required. `templite.nim `_ is a shorter -version for more experienced users. - -All `{.compileTime.}` procs must be used in a compile time context, e.g. using: - -.. code-block:: c - - static: - cAddStdDir() - -]## - import hashes, macros, os, strformat, strutils import "."/[globals, paths] import "."/build/[ccompiler, misc, nimconf, shell] -proc interpPath(dir: string): string= - # TODO: more robust: needs a DirSep after "$projpath" - # disabling this interpolation as this is error prone, but other less - # interpolations can be added, eg see https://github.com/nim-lang/Nim/pull/10530 - # result = dir.replace("$projpath", getProjectPath()) - result = dir - -proc joinPathIfRel(path1: string, path2: string): string = - if path2.isAbsolute: - result = path2 - else: - result = joinPath(path1, path2) - proc findPath(path: string, fail = true): string = # Relative to project path - result = joinPathIfRel(getProjectPath(), path).replace("\\", "/") + let + path = fixRelPath(path) + result = path.replace("\\", "/") if not fileExists(result) and not dirExists(result): doAssert (not fail), "File or directory not found: " & path result = "" -proc walkDirImpl(indir, inext: string, file=true): seq[string] = - let - dir = joinPathIfRel(getProjectPath(), indir) - ext = - if inext.nBl: - when not defined(Windows): - "-name " & inext - else: - "\\" & inext - else: - "" - - let - cmd = - when defined(Windows): - if file: - "cmd /c dir /s/b/a-d " & dir.replace("/", "\\") & ext - else: - "cmd /c dir /s/b/ad " & dir.replace("/", "\\") - else: - if file: - "find $1 -type f $2" % [dir, ext] - else: - "find $1 -type d" % dir - - (output, ret) = execAction(cmd, die = false) - - if ret == 0: - result = output.splitLines() - -proc fixRelFile(file: string): string = - if file.isAbsolute(): file else: getProjectDir() / file - proc getCacheValue(fullpath: string): string = if not gStateCT.nocache: result = fullpath.getFileDate() @@ -123,6 +62,15 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" for i in gStateCT.exclude: cmd.add &" --exclude+={i.sanitizePath}" + for i in gStateCT.passC: + cmd.add &" --passC+={i.quoteShell}" + + for i in gStateCT.passL: + cmd.add &" --passL+={i.quoteShell}" + + for i in gStateCT.compile: + cmd.add &" --compile+={i.sanitizePath}" + if not noNimout: cmd.add &" --pnim" @@ -147,7 +95,7 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" ($(cmd & cacheKey).hash().abs()).addFileExt(ext) if outFile.nBl: - result = fixRelFile(outFile) + result = fixRelPath(outFile) else: result = cacheFile @@ -384,7 +332,7 @@ macro cPluginPath*(path: static[string]): untyped = doAssert fileExists(path), "Plugin file not found: " & path cPluginHelper(readFile(path), imports = "") -proc cSearchPath*(path: string): string {.compileTime.}= +proc cSearchPath*(path: string): string {.compileTime.} = ## Get full path to file or directory `path` in search path configured ## using `cAddSearchDir()` and `cAddStdDir()`. ## @@ -416,27 +364,38 @@ proc cDisableCaching*() {.compileTime.} = ## `nim -f` can also be used to flush the cached content. gStateCT.nocache = true -macro cDefine*(name: static string, val: static string = ""): untyped = +macro cDefine*(name: static[string], val: static[string] = ""): untyped = ## `#define` an identifer that is forwarded to the C/C++ preprocessor if ## called within `cImport()` or `c2nImport()` as well as to the C/C++ ## compiler during Nim compilation using `{.passC: "-DXXX".}` - result = newNimNode(nnkStmtList) - var str = name - # todo: see https://github.com/nimterop/nimterop/issues/100 for - # edge case of empty strings if val.nBl: str &= &"={val.quoteShell}" if str notin gStateCT.defines: gStateCT.defines.add(str) - str = "-D" & str - result.add quote do: - {.passC: `str`.} +macro cDefine*(values: static seq[string]): untyped = + ## `#define` multiple identifers that are forwarded to the C/C++ preprocessor + ## if called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-DXXX".}` + for value in values: + let + spl = value.split("=", maxsplit = 1) + name = spl[0] + val = if spl.len == 2: spl[1] else: "" + discard quote do: + cDefine(`name`, `val`) - if gStateCT.debug: - gecho result.repr & "\n" +macro cPassC*(value: static string): untyped = + ## Create a `{.passC.}` entry that gets forwarded to the C/C++ compiler + ## during Nim compilation. + gStateCT.passC.add value + +macro cPassL*(value: static string): untyped = + ## Create a `{.passL.}` entry that gets forwarded to the C/C++ compiler + ## during Nim compilation. + gStateCT.passL.add value proc cAddSearchDir*(dir: string) {.compileTime.} = ## Add directory `dir` to the search path used in calls to @@ -445,8 +404,8 @@ proc cAddSearchDir*(dir: string) {.compileTime.} = import nimterop/paths, os static: cAddSearchDir testsIncludeDir() - doAssert cSearchPath("test.h").existsFile - var dir = interpPath(dir) + doAssert cSearchPath("test.h").fileExists + if dir notin gStateCT.searchDirs: gStateCT.searchDirs.add(dir) @@ -458,19 +417,11 @@ macro cIncludeDir*(dirs: static seq[string], exclude: static[bool] = false): unt ## Set `exclude = true` if the contents of these include directories should ## not be included in the wrapped output. for dir in dirs: - var dir = interpPath(dir) - result = newNimNode(nnkStmtList) - let fullpath = findPath(dir) if fullpath notin gStateCT.includeDirs: gStateCT.includeDirs.add(fullpath) if exclude: gStateCT.exclude.add(fullpath) - let str = &"-I{fullpath.quoteShell}" - result.add quote do: - {.passC: `str`.} - if gStateCT.debug: - gecho result.repr macro cIncludeDir*(dir: static[string], exclude: static[bool] = false): untyped = ## Add an include directory that is forwarded to the C/C++ preprocessor if @@ -501,16 +452,17 @@ proc cAddStdDir*(mode = "c") {.compileTime.} = ## Add the standard `c` [default] or `cpp` include paths to search ## path used in calls to `cSearchPath()`. runnableExamples: - static: cAddStdDir() import os - doAssert cSearchPath("math.h").existsFile + static: + cAddStdDir() + doAssert cSearchPath("math.h").fileExists for inc in getGccPaths(mode): cAddSearchDir inc -macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = +macro cCompile*(path: static string, mode: static[string] = "c", exclude: static[string] = ""): untyped = ## Compile and link C/C++ implementation into resulting binary using `{.compile.}` ## - ## `path` can be a specific file or contain wildcards: + ## `path` can be a specific file or contain `*` wildcard for filename: ## ## .. code-block:: nim ## @@ -532,26 +484,21 @@ macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = ## ## cCompile("path/to/dir", exclude="test2.c") - result = newNimNode(nnkStmtList) - - var - stmt = "" - - proc fcompile(file: string): string = + proc fcompile(file: string) = let (_, fn, ext) = file.splitFile() var ufn = fn uniq = 1 - while ufn in gStateCT.compile: + while ufn in gStateCT.compcache: ufn = fn & $uniq uniq += 1 # - https://github.com/nim-lang/Nim/issues/10299 # - https://github.com/nim-lang/Nim/issues/10486 - gStateCT.compile.add(ufn) + gStateCT.compcache.add(ufn) if fn == ufn: - return "{.compile: \"$#\".}\n" % file.replace("\\", "/") + gStateCT.compile.add file.replace("\\", "/") else: # - https://github.com/nim-lang/Nim/issues/9370 let @@ -559,7 +506,7 @@ macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = tmpFile = file.parentDir() / &"_nimterop_{$hash}_{ufn}{ext}" if not tmpFile.fileExists() or file.getFileDate() > tmpFile.getFileDate(): cpFile(file, tmpFile) - return "{.compile: \"$#\".}\n" % tmpFile.replace("\\", "/") + gStateCT.compile.add tmpFile.replace("\\", "/") # Due to https://github.com/nim-lang/Nim/issues/9863 # cannot use seq[string] for excludes @@ -572,33 +519,37 @@ macro cCompile*(path: static string, mode = "c", exclude = ""): untyped = if excl in file: result = false - proc dcompile(dir, exclude: string, ext=""): string = + proc dcompile(dir, exclude: string, ext="") = let - files = walkDirImpl(dir, ext) + (dir, pat) = + if "*" in dir: + dir.splitPath() + else: + (dir, "") - for f in files: - if f.nBl and f.notExcluded(exclude): - result &= fcompile(f) + for file in walkDirRec(dir): + if ext.nBl or pat.nBl: + let + fext = file.splitFile().ext + if (ext.nBl and fext != ext) or (pat.nBl and fext != pat[1 .. ^1]): + continue + if file.notExcluded(exclude): + fcompile(file) - if path.contains("*") or path.contains("?"): - stmt &= dcompile(path, exclude.strVal()) + if "*" in path: + dcompile(path, exclude) else: let fpath = findPath(path) - if fileExists(fpath) and fpath.notExcluded(exclude.strVal()): - stmt &= fcompile(fpath) + if fileExists(fpath) and fpath.notExcluded(exclude): + fcompile(fpath) elif dirExists(fpath): - if mode.strVal().contains("cpp"): - for i in @["*.cpp", "*.c++", "*.cc", "*.cxx"]: - stmt &= dcompile(fpath, exclude.strVal(), i) + if mode.contains("cpp"): + for i in @[".cpp", ".c++", ".cc", ".cxx"]: + dcompile(fpath, exclude, i) when not defined(Windows): - stmt &= dcompile(fpath, exclude.strVal(), "*.C") + dcompile(fpath, exclude, ".C") else: - stmt &= dcompile(fpath, exclude.strVal(), "*.c") - - result.add stmt.parseStmt() - - if gStateCT.debug: - gecho result.repr + dcompile(fpath, exclude, ".c") macro cImport*(filenames: static seq[string], recurse: static bool = false, dynlib: static string = "", mode: static string = "c", flags: static string = "", nimFile: static string = ""): untyped = @@ -726,7 +677,7 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: let hFile = getToast(@[fullpath], recurse, dynlib, mode, noNimout = true) - nimFile = if nimFile.nBl: fixRelFile(nimFile) else: hFile.changeFileExt("nim") + nimFile = if nimFile.nBl: fixRelPath(nimFile) else: hFile.changeFileExt("nim") header = "header" & fullpath.splitFile().name.split(seps = {'-', '.'}).join() if not fileExists(nimFile) or gStateCT.nocache or compileOption("forceBuild"): @@ -741,8 +692,29 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: if flags.nBl: cmd.add &" {flags}" + # Have to create pragmas for c2nim since toast handles this at runtime for i in gStateCT.defines: cmd.add &" --assumedef:{i.quoteShell}" + let str = "-D" & i + result.add quote do: + {.passC: `str`.} + + for i in gStateCT.includeDirs: + let str = &"-I{i.quoteShell}" + result.add quote do: + {.passC: `str`.} + + for i in gStateCT.passC: + result.add quote do: + {.passC: `i`.} + + for i in gStateCT.passL: + result.add quote do: + {.passL: `i`.} + + for i in gStateCT.compile: + result.add quote do: + {.compile: `i`.} let (c2nimout, ret) = execAction(cmd) diff --git a/nimterop/globals.nim b/nimterop/globals.nim index 89e403f..2900272 100644 --- a/nimterop/globals.nim +++ b/nimterop/globals.nim @@ -15,6 +15,7 @@ type State* = ref object # Command line arguments to toast - some forwarded from cimport.nim + compile*: seq[string] # `--compile` to create `{.compile.}` entries in generated wrapper convention*: string # `--convention | -C` to change calling convention from cdecl default debug*: bool # `cDebug()` or `--debug | -d` to enable debug mode defines*: seq[string] # Symbols added by `cDefine()` and `--define | -D` for C/C++ preprocessor/compiler @@ -26,6 +27,8 @@ type nim*: string # `--nim` to specify full path to Nim compiler noComments*: bool # `--noComments | -c` to disable rendering comments in wrappers noHeader*: bool # `--noHeader | -H` to skip {.header.} pragma in wrapper + passC*: seq[string] # `--passC` to create `{.passC.}` entries in the generated wrapper + passL*: seq[string] # `--passL` to create `{.passL.}` entries in the generated wrapper past*: bool # `--past | -a` to print tree-sitter AST of code pluginSourcePath*: string # `--pluginSourcePath` specified path to plugin file to compile and load pnim*: bool # `--pnim | -n` to render Nim wrapper for header @@ -77,7 +80,7 @@ type wrapperHeader*: string else: # cimport.nim specific - compile*: seq[string] # `cCompile()` list of files already processed + compcache*: seq[string] # `cCompile()` list of files already processed nocache*: bool # `cDisableCaching()` to disable caching of artifacts overrides*: string # `cOverride()` code which gets added to `cPlugin()` output pluginSource*: string # `cPlugin()` generated code to write to plugin file from diff --git a/nimterop/template.nim b/nimterop/template.nim deleted file mode 100644 index c6e846f..0000000 --- a/nimterop/template.nim +++ /dev/null @@ -1,111 +0,0 @@ -import os, strutils - -import nimterop/[cimport, build, paths] - -# Documentation: -# https://github.com/nimterop/nimterop -# https://nimterop.github.io/nimterop/cimport.html - -const - # Location where any sources should get downloaded. Adjust depending on - # actual location of wrapper file relative to project. - baseDir = currentSourcePath.parentDir()/"build" - - # All files and dirs should be inside to baseDir - srcDir = baseDir/"project" - -static: - # Print generated Nim to output - cDebug() - - # Disable caching so that wrapper is generated every time. Useful during - # development. Remove once wrapper is working as expected. - cDisableCaching() - - # Download C/C++ source code from a git repository - gitPull("https://github.com/user/project", outdir = srcDir, plist = """ -include/*.h -src/*.c -""", checkout = "tag/branch/hash") - - # Download source from the web - zip files are auto extracted - downloadUrl("https://hostname.com/file.h", outdir = srcDir) - - # Run GNU configure on the source - when defined(posix): - configure(srcDir, fileThatShouldGetGenerated, flagsToConfigure) - - # Run cmake on the source - cmake(srcDir/"build", fileThatShouldGetGenerated, flagsToCmake) - - # Run standard file/directory operations with mkDir(), cpFile(), mvFile() - - # Edit file contents if required with readFile(), writeFile() and standard - # string operations - - # Run any other external commands with execAction() - - # Skip any symbols from being wrapped - cSkipSymbol(@["type1", "proc2"]) - -# Manually wrap any symbols since nimterop cannot or incorrectly wraps them -cOverride: - # Standard Nim code to wrap types, consts, procs, etc. - type - symbol = object - -# Specify include directories for gcc and Nim -cIncludeDir(srcDir/"include") - -# Define global symbols -cDefine("SYMBOL", "value") - -# Any global compiler options -{.passC: "flags".} - -# Any global linker options -{.passL: "flags".} - -# Compile in any common source code -cCompile(srcDir/"file.c") - -# Perform OS specific tasks -when defined(Windows): - # Windows specific symbols, options and files - - # Dynamic library to link against - const dynlibFile = - when defined(cpu64): - "xyz64.dll" - else: - "xyz32.dll" -elif defined(posix): - # Common posix symbols, options and files - - when defined(linux): - # Linux specific - const dynlibFile = "libxyz.so(.2|.1|)" - elif defined(osx): - # MacOSX specific - const dynlibFile = "libxyz(.2|.1|).dylib" - else: - static: doAssert false -else: - static: doAssert false - -# Use cPlugin() to make any symbol changes -cPlugin: - import strutils - - # Symbol renaming examples - proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = - # Get rid of leading and trailing underscores - sym.name = sym.name.strip(chars = {'_'}) - - # Remove prefixes or suffixes from procs - if sym.kind == nskProc and sym.name.contains("SDL_"): - sym.name = sym.name.replace("SDL_", "") - -# Finally import wrapped header file. Recurse if #include files should also -# be wrapped. Set dynlib if binding to dynamic library. -cImport(srcDir/"include/file.h", recurse = true, dynlib="dynlibFile") diff --git a/nimterop/templite.nim b/nimterop/templite.nim deleted file mode 100644 index 7d8eb5b..0000000 --- a/nimterop/templite.nim +++ /dev/null @@ -1,36 +0,0 @@ -import os, strutils - -import nimterop/[cimport, build, paths] - -const - baseDir = currentSourcePath.parentDir()/"build" - - srcDir = baseDir/"project" - -static: - cDebug() - cDisableCaching() - - gitPull("https://github.com/user/project", outdir = srcDir, plist = """ -include/*.h -src/*.c -""", checkout = "tag/branch/hash") - - downloadUrl("https://hostname.com/file.h", outdir = srcDir) - -cIncludeDir(srcDir/"include") - -cDefine("SYMBOL", "value") - -{.passC: "flags".} -{.passL: "flags".} - -cCompile(srcDir/"file.c") - -cPlugin: - import strutils - - proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = - sym.name = sym.name.strip(chars = {'_'}) - -cImport(srcDir/"include/file.h", recurse = true) diff --git a/nimterop/toast.nim b/nimterop/toast.nim index 20dc79e..8695216 100644 --- a/nimterop/toast.nim +++ b/nimterop/toast.nim @@ -34,6 +34,7 @@ proc process(gState: State, path: string) = # CLI processing with default values proc main( check = false, + compile: seq[string] = @[], convention = "cdecl", debug = false, defines: seq[string] = @[], @@ -46,6 +47,8 @@ proc main( noComments = false, noHeader = false, output = "", + passC: seq[string] = @[], + passL: seq[string] = @[], past = false, pluginSourcePath: string = "", pnim = false, @@ -62,6 +65,7 @@ proc main( # Setup global state with arguments gState = State( + compile: compile, convention: convention, debug: debug, defines: defines, @@ -73,6 +77,8 @@ proc main( nim: nim.sanitizePath, noComments: noComments, noHeader: noHeader, + passC: passC, + passL: passL, past: past, pluginSourcePath: pluginSourcePath, pnim: pnim, @@ -220,6 +226,7 @@ when isMainModule: import cligen dispatch(main, help = { "check": "check generated wrapper with compiler", + "compile": "create {.compile.} entries in generated wrapper", "convention": "calling convention for wrapped procs", "debug": "enable debug output", "defines": "definitions to pass to preprocessor", @@ -232,6 +239,8 @@ when isMainModule: "noComments": "exclude top-level comments from output", "noHeader": "skip {.header.} pragma in wrapper", "output": "file to output content - default: stdout", + "passC": "create {.passC.} entries in generated wrapper", + "passL": "create {.passL.} entries in generated wrapper", "past": "print AST output", "pluginSourcePath": "nim file to build and load as a plugin", "pnim": "print Nim output", diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 083d4b2..7f2b4ad 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -1881,11 +1881,31 @@ proc setupPragmas(gState: State, root: TSNode, fullpath: string) = gState.pragmaSection.add dynPragma count += 1 - # Add `{.experimental: "codeReordering".} for #206 + # Only if not already done if gState.pragmaSection.len == count: - # Only if not already done + # Add `{.experimental: "codeReordering".} for #206 gState.pragmaSection.add gState.newPragma(root, "experimental", newStrNode(nkStrLit, "codeReordering")) + # Create `{.passC.}` from defines + for define in gState.defines: + gState.pragmaSection.add gState.newPragma(root, "passC", newStrNode(nkStrLit, "-D" & define)) + + # Create `{.passC.}` from include directories + for inc in gState.includeDirs: + gState.pragmaSection.add gState.newPragma(root, "passC", newStrNode(nkStrLit, "-I" & inc.quoteShell)) + + # Create `{.passC.}` from passC + for passC in gState.passC: + gState.pragmaSection.add gState.newPragma(root, "passC", newStrNode(nkStrLit, passC)) + + # Create `{.passL.}` from passL + for passL in gState.passL: + gState.pragmaSection.add gState.newPragma(root, "passL", newStrNode(nkStrLit, passL)) + + # Create `{.compile.}` for specified files + for file in gState.compile: + gState.pragmaSection.add gState.newPragma(root, "compile", newStrNode(nkStrLit, file)) + proc initNim*(gState: State) = # Initialize for parseNim() one time gState.wrapperHeader = "{.push hint[ConvFromXtoItselfNotNeeded]: off.}\n" diff --git a/tests/libssh2.nim b/tests/libssh2.nim index 09343bc..940b2df 100644 --- a/tests/libssh2.nim +++ b/tests/libssh2.nim @@ -22,13 +22,13 @@ when not libssh2Static: when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc, dynlib: libssh2LPath.} else: + cPassL("-lpthread") + cImport(libssh2Path, recurse = true, flags = "-c -E_ -F_") when not defined(Windows) and not isDefined(libssh2JBB): proc zlibVersion(): cstring {.importc.} - {.passL: "-lpthread".} - assert libssh2_init(0) == 0 let diff --git a/tests/rsa.nim b/tests/rsa.nim index f5c343c..ee36286 100644 --- a/tests/rsa.nim +++ b/tests/rsa.nim @@ -35,13 +35,13 @@ cPlugin: cOverride: proc OPENSSL_die*(assertion: cstring; file: cstring; line: cint) {.importc.} +cPassL(cryptoLPath) + # Skip comments for https://github.com/tree-sitter/tree-sitter-c/issues/44 cImport(@[ basePath / "rsa.h", basePath / "err.h", ], recurse = true, flags = "-s -c " & FLAGS) -{.passL: cryptoLPath.} - OpensslInit() echo $OPENSSL_VERSION_TEXT diff --git a/tests/tast2.nim b/tests/tast2.nim index 17edfb0..a410de4 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -2,7 +2,7 @@ import macros, os, sets, strutils import nimterop/[cimport] -{.passC: "-DNIMTEROP".} +cPassC("-DNIMTEROP") static: # Skip casting on lower nim compilers because diff --git a/tests/tsoloud.nim b/tests/tsoloud.nim index 4d9a28b..d594516 100644 --- a/tests/tsoloud.nim +++ b/tests/tsoloud.nim @@ -24,15 +24,15 @@ cIncludeDir(incl) when defined(osx): cDefine("WITH_COREAUDIO") - {.passL: "-framework CoreAudio -framework AudioToolbox".} + cPassL("-framework CoreAudio -framework AudioToolbox") cCompile(src/"backend/coreaudio/*.cpp") elif defined(Linux): - {.passL: "-lpthread".} + cPassL("-lpthread") cDefine("WITH_OSS") cCompile(src/"backend/oss/*.cpp") elif defined(Windows): - {.passC: "-msse".} - {.passL: "-lwinmm".} + cPassC("-msse") + cPassL("-lwinmm") cDefine("WITH_WINMM") cCompile(src/"backend/winmm/*.cpp") else: diff --git a/tests/zlib.nim b/tests/zlib.nim index 53384c3..d4744eb 100644 --- a/tests/zlib.nim +++ b/tests/zlib.nim @@ -65,12 +65,12 @@ when zlibGit or zlibDL: when dirExists(baseDir / "buildcache"): cIncludeDir(baseDir / "buildcache") -when not zlibStatic: +when not isDefined(zlibStatic): cImport(zlibPath, recurse = true, dynlib = "zlibLPath", flags = FLAGS) else: + when isDefined(zlibJBB): + cPassL("-no-pie") + cImport(zlibPath, recurse = true, flags = FLAGS) -echo "zlib version = " & $zlibVersion() - -when isDefined(zlibJBB) and isDefined(zlibStatic): - {.passL: "-no-pie".} +echo "zlib version = " & $zlibVersion() \ No newline at end of file From aa813bdf3212f22642ec81d79f712911d9df4226 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 14 Jul 2020 20:20:42 -0500 Subject: [PATCH 072/106] Forward getHeader passL --- nimterop/build/getheader.nim | 54 +++++++++++++++++------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index d5b398a..dbd965e 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -33,7 +33,7 @@ macro setDefines*(defs: static openArray[string]): untyped = for def in defs: let nv = def.strip().split("=", maxsplit = 1) - if nv.len != 0: + if nv.nBl: let n = nv[0] v = @@ -70,33 +70,33 @@ proc getDynlibExt(): string = proc getStdPath(header, mode: string): string = for inc in getGccPaths(mode): result = findFile(header, inc, recurse = false, first = true) - if result.len != 0: + if result.nBl: break proc getStdLibPath(lname, mode: string): string = for lib in getGccLibPaths(mode): result = findFile(lname, lib, recurse = false, first = true, regex = true) - if result.len != 0: + if result.nBl: break proc getGitPath(header, url, outdir, version: string): string = - doAssert url.len != 0, "No git url setup for " & header - doAssert findExe("git").len != 0, "git executable missing" + doAssert url.nBl, "No git url setup for " & header + doAssert findExe("git").nBl, "git executable missing" gitPull(url, outdir, checkout = version) result = findFile(header, outdir) proc getDlPath(header, url, outdir, version: string): string = - doAssert url.len != 0, "No download url setup for " & header + doAssert url.nBl, "No download url setup for " & header var dlurl = url if "$#" in url or "$1" in url: - doAssert version.len != 0, "Need version for download url" + doAssert version.nBl, "Need version for download url" dlurl = url % version else: - doAssert version.len == 0, "Download url does not contain version" + doAssert version.Bl, "Download url does not contain version" downloadUrl(dlurl, outdir) @@ -107,13 +107,13 @@ proc getDlPath(header, url, outdir, version: string): string = dirname = "" break elif kind == pcDir: - if dirname.len == 0: + if dirname.Bl: dirname = path else: dirname = "" break - if dirname.len != 0: + if dirname.nBl: for kind, path in walkDir(outdir / dirname, relative = true): mvFile(outdir / dirname / path, outdir / path) @@ -124,9 +124,9 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string = uri = uri if "$#" in uri or "$1" in uri: - doAssert version.len != 0, "Need version for Conan.io uri: " & uri + doAssert version.nBl, "Need version for Conan.io uri: " & uri uri = uri % version - elif version.len != 0: + elif version.nBl: uri = uri & "/" & version let @@ -145,12 +145,12 @@ proc getJBBPath(header, uri, flags, outdir, version: string): string = let spl = uri.split('/', 1) name = spl[0] - hasVersion = version.len != 0 + hasVersion = version.nBl var ver = if spl.len == 2: spl[1] else: "" - if ver.len != 0: + if ver.nBl: if "$#" in ver or "$1" in ver: doAssert hasVersion, "Need version for BinaryBuilder.org uri: " & uri ver = ver % version @@ -187,7 +187,7 @@ proc getJBBLDeps(outdir: string, shared: bool): seq[string] = result = pkg.getJBBLDeps(outdir, shared) proc getLocalPath(header, outdir: string): string = - if outdir.len != 0: + if outdir.nBl: result = findFile(header, outdir) proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildTypes: openArray[BuildType]): string = @@ -195,7 +195,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT lpath = findFile(lname, outdir, regex = true) makePath = outdir - if lpath.len != 0: + if lpath.nBl: return lpath var buildStatus: BuildStatus @@ -275,7 +275,7 @@ macro getHeader*( ## ## The header path is stored in `const xxxPath` and can be used in a `cImport()` call ## in the calling wrapper. The dynamic library path is stored in `const xxxLPath` and can - ## be used for the `dynlib` parameter (within quotes) or with `{.passL.}`. Any dependency + ## be used for the `dynlib` parameter (within quotes) or with `cPassL()`. Any dependency ## libraries downloaded by `Conan` or `JBB` are returned in `const xxxLDeps` as a seq[string]. ## ## `libdir` can be used to instruct `getHeader()` to copy shared libraries and their @@ -285,7 +285,7 @@ macro getHeader*( ## reflect this new location. `libdir` is ignored for `Std` mode. ## ## `-d:xxxStatic` can be specified to statically link with the library instead. This - ## will automatically add a `{.passL.}` call to the static library for convenience. Note + ## will automatically add a `cPassL()` call to the static library for convenience. Note ## that `-d:xxxConan` and `-d:xxxJBB` download all dependency libs as well and the ## `xxxLPath` will include paths to all of them separated by space in the right order for ## linking. @@ -335,8 +335,8 @@ macro getHeader*( name = origname.split(seps = AllChars-Letters-Digits).join() # Default to origname if not specified - conanuri = if conanuri.len != 0: conanuri else: origname - jbburi = if jbburi.len != 0: jbburi else: origname + conanuri = if conanuri.nBl: conanuri else: origname + jbburi = if jbburi.nBl: jbburi else: origname # -d:xxx for this header stdStr = name & "Std" @@ -382,10 +382,10 @@ macro getHeader*( "" mode = getCompilerMode(header) - libdir = if libdir.len != 0: libdir else: getOutDir() + libdir = if libdir.nBl: libdir else: getOutDir() # Use alternate library names if specified for regex search - if altNames.len != 0: + if altNames.nBl: lre = lre % ("(" & altNames.replace(",", "|") & ")") else: lre = lre % origname @@ -491,14 +491,12 @@ macro getHeader*( `ldeps`* = ldeps # Automatically link with static library and dependencies - {.passL: `lpath`.} - if `ldeps`.len != 0: - {.passL: `ldeps`.join(" ").} - static: gecho "# Including library " & lpath - if `ldeps`.len != 0: - gecho "# Including dependencies " & `ldeps`.join(" ") + gStateCT.passL.add lpath + if ldeps.len != 0: + gecho "# Including dependencies " & ldeps.join(" ") + gStateCT.passL.add ldeps.join(" ") else: const `lpath`* = when not useStd: `libdir` / lpath.extractFilename() else: lpath From 44f335c87ffdf153e8e1cad4d7f59f638f19c76b Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Jul 2020 09:08:10 -0500 Subject: [PATCH 073/106] Doc updates --- CHANGES.md | 14 ++++++++++++-- README.md | 6 +++++- nimterop/cimport.nim | 12 ++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 91cecac..8609474 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,7 @@ Refer to the documentation for `getHeader()` for details on how to use this new See the full list of changes here: -https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 +https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.5 ### Breaking changes @@ -27,6 +27,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 - Nameless enum values are no longer typed to the made-up enum type name, they are instead typed as `cint` to match the underlying type. This allows using such enums without having to depend on the made-up name which could change if enum ordering changes upstream. [#236][i236] (since v0.6.1) +- Static libraries installed and linked with `getHeader()` now have their `{.passL.}` pragmas forwarded to the generated wrapper. This might lead to link errors in existing wrappers if other dependencies are specified with `{.passL.}` calls and the order of linking is wrong. This can be fixed by changing such explicit `{.passL.}` calls with `cPassL()` which will forward the link call to the generated wrapper as well. (since v0.6.5) + ### New functionality - `getHeader()` now detects and links against `.lib` files as part of enabling Conan.io. Not all `.lib` files are compatible with MinGW as already stated above but for those that work, this is a required capability. @@ -43,6 +45,12 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 - It is now possible to exclude the contents of specific files or entire directories from the wrapped output using `--exclude | -X` with `toast` or `cExclude()` from a wrapper. This might be required when a header uses `#include` to pull in external dependencies. E.g. `sciter` has a `#include ` which pulls in the entire GTK ecosystem which is needed for successful preprocessing but we do not want to include those headers in the wrapped output when using `--recurse | -r`. (since v0.6.4) +- All `cDefine()`, `cIncludeDir()` and `cCompile()` calls now forward relevant pragmas into the generated wrapper further enabling standalone wrappers. [#239][i239] + +- Added `cPassC()` and `cPassL()` to forward C/C++ compilation pragmas into the generated wrapper. (since v0.6.5) + +- Added `--compile`, `--passC` and `--passL` flags to `toast` to enable the previous two improvements. (since v0.6.5) + ### Other improvements - Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) @@ -51,6 +59,7 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.3 - `cIncludeDir()` can now accept a `seq[string]` of directories and an optional `exclude` param which sets those include directories to not be included in the wrapped output. (since v0.6.4) +- `cDefine()` can now accept a `seq[string]` of values. (since v0.6.5) ## Version 0.5.0 @@ -140,4 +149,5 @@ https://github.com/nimterop/nimterop/compare/v0.4.4...v0.5.4 [i197]: https://github.com/nimterop/nimterop/issues/197 [i200]: https://github.com/nimterop/nimterop/issues/200 [i236]: https://github.com/nimterop/nimterop/issues/236 -[i237]: https://github.com/nimterop/nimterop/issues/237 \ No newline at end of file +[i237]: https://github.com/nimterop/nimterop/issues/237 +[i239]: https://github.com/nimterop/nimterop/issues/239 \ No newline at end of file diff --git a/README.md b/README.md index cda3b5e..4770563 100644 --- a/README.md +++ b/README.md @@ -218,10 +218,12 @@ Options: -h, --help print this cligen-erated help --help-syntax advanced: prepend,plurals,.. -k, --check bool false check generated wrapper with compiler + --compile= strings {} create {.compile.} entries in generated wrapper -C=, --convention= string "cdecl" calling convention for wrapped procs -d, --debug bool false enable debug output -D=, --defines= strings {} definitions to pass to preprocessor - -l=, --dynlib= string "" {.dynlib.} pragma to import symbols - Nim const string or file path + -l=, --dynlib= string "" {.dynlib.} pragma to import symbols - Nim const string or + file path -X=, --exclude= strings {} files or directories to exclude from the wrapped output -f=, --feature= Features {} flags to enable experimental features -I=, --includeDirs= strings {} include directory to pass to preprocessor @@ -230,6 +232,8 @@ Options: -c, --noComments bool false exclude top-level comments from output -H, --noHeader bool false skip {.header.} pragma in wrapper -o=, --output= string "" file to output content - default: stdout + --passC= strings {} create {.passC.} entries in generated wrapper + --passL= strings {} create {.passL.} entries in generated wrapper -a, --past bool false print AST output --pluginSourcePath= string "" nim file to build and load as a plugin -n, --pnim bool false print Nim output diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index 6120cb5..c82f884 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -368,6 +368,8 @@ macro cDefine*(name: static[string], val: static[string] = ""): untyped = ## `#define` an identifer that is forwarded to the C/C++ preprocessor if ## called within `cImport()` or `c2nImport()` as well as to the C/C++ ## compiler during Nim compilation using `{.passC: "-DXXX".}` + ## + ## This needs to be called before `cImport()` to take effect. var str = name if val.nBl: str &= &"={val.quoteShell}" @@ -379,6 +381,8 @@ macro cDefine*(values: static seq[string]): untyped = ## `#define` multiple identifers that are forwarded to the C/C++ preprocessor ## if called within `cImport()` or `c2nImport()` as well as to the C/C++ ## compiler during Nim compilation using `{.passC: "-DXXX".}` + ## + ## This needs to be called before `cImport()` to take effect. for value in values: let spl = value.split("=", maxsplit = 1) @@ -390,11 +394,15 @@ macro cDefine*(values: static seq[string]): untyped = macro cPassC*(value: static string): untyped = ## Create a `{.passC.}` entry that gets forwarded to the C/C++ compiler ## during Nim compilation. + ## + ## This needs to be called before `cImport()` to take effect. gStateCT.passC.add value macro cPassL*(value: static string): untyped = ## Create a `{.passL.}` entry that gets forwarded to the C/C++ compiler ## during Nim compilation. + ## + ## This needs to be called before `cImport()` to take effect. gStateCT.passL.add value proc cAddSearchDir*(dir: string) {.compileTime.} = @@ -416,6 +424,8 @@ macro cIncludeDir*(dirs: static seq[string], exclude: static[bool] = false): unt ## ## Set `exclude = true` if the contents of these include directories should ## not be included in the wrapped output. + ## + ## This needs to be called before `cImport()` to take effect. for dir in dirs: let fullpath = findPath(dir) if fullpath notin gStateCT.includeDirs: @@ -430,6 +440,8 @@ macro cIncludeDir*(dir: static[string], exclude: static[bool] = false): untyped ## ## Set `exclude = true` if the contents of this include directory should ## not be included in the wrapped output. + ## + ## This needs to be called before `cImport()` to take effect. return quote do: cIncludeDir(@[`dir`], `exclude` == 1) From e61dc54a898a395902205b5794e09ece0e9b35ba Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Jul 2020 11:33:26 -0500 Subject: [PATCH 074/106] renderPragma, cleanup --- CHANGES.md | 5 +- README.md | 4 +- nimterop/cimport.nim | 548 +++++++++++++++++++++++-------------------- 3 files changed, 294 insertions(+), 263 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8609474..e7e9213 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -47,10 +47,12 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.5 - All `cDefine()`, `cIncludeDir()` and `cCompile()` calls now forward relevant pragmas into the generated wrapper further enabling standalone wrappers. [#239][i239] -- Added `cPassC()` and `cPassL()` to forward C/C++ compilation pragmas into the generated wrapper. (since v0.6.5) +- Added `cPassC()` and `cPassL()` to forward C/C++ compilation pragmas into the generated wrapper. These should be used in place of `{.passC.}` and `{.passL.}` and need to be called before `cImport()` to take effect. (since v0.6.5) - Added `--compile`, `--passC` and `--passL` flags to `toast` to enable the previous two improvements. (since v0.6.5) +- Added `renderPragma()` to create pragmas inline in case `cImport()` is not being used. (since v0.6.5) + ### Other improvements - Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) @@ -61,6 +63,7 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.5 - `cDefine()` can now accept a `seq[string]` of values. (since v0.6.5) + ## Version 0.5.0 This release introduces a new backend for wrapper generation dubbed `ast2` that leverages the Nim compiler AST and renderer. The new design simplifies feature development and already includes all the functionality of the legacy algorithm plus fixes for several open issues. diff --git a/README.md b/README.md index 4770563..d652d0d 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Flags can be specified to these tools via `getHeader()` or directly via the unde If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`. -All dependency libraries (supported by Conan and JBB) will be returned in `headerLDeps`. Static libraries and dependencies are automatically linked using `{.passL.}`. Conan shared libs include all dependencies whereas JBB shared libs expect the required dependencies to be in the same location or in `LD_LIBRARY_PATH`. +All dependency libraries (supported by Conan and JBB) will be returned in `headerLDeps`. Static libraries and dependencies are automatically linked using `cPassL()`. Conan shared libs include all dependencies whereas JBB shared libs expect the required dependencies to be in the same location or in `LD_LIBRARY_PATH`. `getHeader()` searches for libraries based on the header name by default: - `libheader.so` or `libheader.a` on Linux @@ -171,7 +171,7 @@ Nim provides some flexibility when it comes to using C/C++ libraries. In order t For types, `{.header: "header.h".}` informs Nim that `header.h` has the symbol and to `#include "header.h"` in the generated code. However, types can be mostly recreated in pure Nim so it is also possible to omit both `{.importc.}` and `{.header}` and it will work just fine except with a different name in the generated C code. This allows the user to compile the wrapper without requiring `header.h` to be present. -For functions, `{.header.}` works the same as types and can be omitted if preferred. The `{.importc.}` pragma is still required, unlike types since functions need to be linked to the implementation in the library. The user will need to provide this information at link time with `{.passL.}` and linking to a library with `-lheader` or `path/to/libheader.a`. It is also possible to just use `cCompile()` or `{.compile.}` to compile some C source files which contain the implementation. +For functions, `{.header.}` works the same as types and can be omitted if preferred. The `{.importc.}` pragma is still required, unlike types since functions need to be linked to the implementation in the library. The user will need to provide this information at link time with `cPassL()` and linking to a library with `-lheader` or `path/to/libheader.a`. It is also possible to just use `cCompile()` or `{.compile.}` to compile some C source files which contain the implementation. While `{.header.}` can be omitted for convenience, it does prevent wrapping of `static inline` functions as well as type checking of the wrapper ABI with `-d:checkAbi` at compile time. Further, anonymous nested structs/unions within unions will be rendered incorrectly by Nim since it is unaware of the true memory structure of the type. The user will need to choose based on the library in question. diff --git a/nimterop/cimport.nim b/nimterop/cimport.nim index c82f884..88aed1a 100644 --- a/nimterop/cimport.nim +++ b/nimterop/cimport.nim @@ -64,12 +64,15 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" for i in gStateCT.passC: cmd.add &" --passC+={i.quoteShell}" + gStateCT.passC = @[] for i in gStateCT.passL: cmd.add &" --passL+={i.quoteShell}" + gStateCT.passL = @[] for i in gStateCT.compile: cmd.add &" --compile+={i.sanitizePath}" + gStateCT.compile = @[] if not noNimout: cmd.add &" --pnim" @@ -142,6 +145,288 @@ proc getToast(fullpaths: seq[string], recurse: bool = false, dynlib: string = "" writeFile(cacheFile, "") +proc cDebug*() {.compileTime.} = + ## Enable debug messages and display the generated Nim code + gStateCT.debug = true + +proc cDisableCaching*() {.compileTime.} = + ## Disable caching of generated Nim code - useful during wrapper development + ## + ## If files included by header being processed by + ## `cImport()` change and affect the generated content, they will be ignored + ## and the cached value will continue to be used . Use `cDisableCaching()` to + ## avoid this scenario during development. + ## + ## `nim -f` can also be used to flush the cached content. + gStateCT.nocache = true + +proc cSearchPath*(path: string): string {.compileTime.} = + ## Get full path to file or directory `path` in search path configured + ## using `cAddSearchDir()` and `cAddStdDir()`. + ## + ## This can be used to locate files or directories that can be passed onto + ## `cCompile()`, `cIncludeDir()` and `cImport()`. + result = findPath(path, fail = false) + if result.Bl: + var found = false + for inc in gStateCT.searchDirs: + result = findPath(inc / path, fail = false) + if result.nBl: + found = true + break + doAssert found, "File or directory not found: " & path & + " gStateCT.searchDirs: " & $gStateCT.searchDirs + +proc cAddSearchDir*(dir: string) {.compileTime.} = + ## Add directory `dir` to the search path used in calls to + ## `cSearchPath()`. + runnableExamples: + import nimterop/paths, os + static: + cAddSearchDir testsIncludeDir() + doAssert cSearchPath("test.h").fileExists + + if dir notin gStateCT.searchDirs: + gStateCT.searchDirs.add(dir) + +proc cAddStdDir*(mode = "c") {.compileTime.} = + ## Add the standard `c` [default] or `cpp` include paths to search + ## path used in calls to `cSearchPath()`. + runnableExamples: + import os + static: + cAddStdDir() + doAssert cSearchPath("math.h").fileExists + for inc in getGccPaths(mode): + cAddSearchDir inc + +macro cDefine*(name: static[string], val: static[string] = ""): untyped = + ## `#define` an identifer that is forwarded to the C/C++ preprocessor if + ## called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-DXXX".}` + ## + ## This needs to be called before `cImport()` to take effect. + var str = name + if val.nBl: + str &= &"={val.quoteShell}" + + if str notin gStateCT.defines: + gStateCT.defines.add(str) + +macro cDefine*(values: static seq[string]): untyped = + ## `#define` multiple identifers that are forwarded to the C/C++ preprocessor + ## if called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-DXXX".}` + ## + ## This needs to be called before `cImport()` to take effect. + for value in values: + let + spl = value.split("=", maxsplit = 1) + name = spl[0] + val = if spl.len == 2: spl[1] else: "" + discard quote do: + cDefine(`name`, `val`) + +macro cIncludeDir*(dirs: static seq[string], exclude: static[bool] = false): untyped = + ## Add include directories that are forwarded to the C/C++ preprocessor if + ## called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-IXXX".}`. + ## + ## Set `exclude = true` if the contents of these include directories should + ## not be included in the wrapped output. + ## + ## This needs to be called before `cImport()` to take effect. + for dir in dirs: + let fullpath = findPath(dir) + if fullpath notin gStateCT.includeDirs: + gStateCT.includeDirs.add(fullpath) + if exclude: + gStateCT.exclude.add(fullpath) + +macro cIncludeDir*(dir: static[string], exclude: static[bool] = false): untyped = + ## Add an include directory that is forwarded to the C/C++ preprocessor if + ## called within `cImport()` or `c2nImport()` as well as to the C/C++ + ## compiler during Nim compilation using `{.passC: "-IXXX".}`. + ## + ## Set `exclude = true` if the contents of this include directory should + ## not be included in the wrapped output. + ## + ## This needs to be called before `cImport()` to take effect. + return quote do: + cIncludeDir(@[`dir`], `exclude` == 1) + +macro cExclude*(paths: static seq[string]): untyped = + ## Exclude specified paths - files or directories from the wrapped output + ## + ## Full path to file or directory is required. + result = newNimNode(nnkStmtList) + for path in paths: + gStateCT.exclude.add path + +macro cExclude*(path: static string): untyped = + ## Exclude specified path - file or directory from the wrapped output. + ## + ## Full path to file or directory is required. + return quote do: + cExclude(@[`path`]) + +macro cPassC*(value: static string): untyped = + ## Create a `{.passC.}` entry that gets forwarded to the C/C++ compiler + ## during Nim compilation. + ## + ## `cPassC()` needs to be called before `cImport()` to take effect and gets + ## consumed and reset so as not to impact subsequent `cImport()` calls. + gStateCT.passC.add value + +macro cPassL*(value: static string): untyped = + ## Create a `{.passL.}` entry that gets forwarded to the C/C++ compiler + ## during Nim compilation. + ## + ## `cPassL()` needs to be called before `cImport()` to take effect and gets + ## consumed and reset so as not to impact subsequent `cImport()` calls. + gStateCT.passL.add value + +macro cCompile*(path: static string, mode: static[string] = "c", exclude: static[string] = ""): untyped = + ## Compile and link C/C++ implementation into resulting binary using `{.compile.}` + ## + ## `path` can be a specific file or contain `*` wildcard for filename: + ## + ## .. code-block:: nim + ## + ## cCompile("file.c") + ## cCompile("path/to/*.c") + ## + ## `mode` recursively searches for code files in `path`. + ## + ## `c` searches for `*.c` whereas `cpp` searches for `*.C *.cpp *.c++ *.cc *.cxx` + ## + ## .. code-block:: nim + ## + ## cCompile("path/to/dir", "cpp") + ## + ## `exclude` can be used to exclude files by partial string match. Comma separated to + ## specify multiple exclude strings + ## + ## .. code-block:: nim + ## + ## cCompile("path/to/dir", exclude="test2.c") + ## + ## `cCompile()` needs to be called before `cImport()` to take effect and gets + ## consumed and reset so as not to impact subsequent `cImport()` calls. + + proc fcompile(file: string) = + let + (_, fn, ext) = file.splitFile() + var + ufn = fn + uniq = 1 + while ufn in gStateCT.compcache: + ufn = fn & $uniq + uniq += 1 + + # - https://github.com/nim-lang/Nim/issues/10299 + # - https://github.com/nim-lang/Nim/issues/10486 + gStateCT.compcache.add(ufn) + if fn == ufn: + gStateCT.compile.add file.replace("\\", "/") + else: + # - https://github.com/nim-lang/Nim/issues/9370 + let + hash = file.hash().abs() + tmpFile = file.parentDir() / &"_nimterop_{$hash}_{ufn}{ext}" + if not tmpFile.fileExists() or file.getFileDate() > tmpFile.getFileDate(): + cpFile(file, tmpFile) + gStateCT.compile.add tmpFile.replace("\\", "/") + + # Due to https://github.com/nim-lang/Nim/issues/9863 + # cannot use seq[string] for excludes + proc notExcluded(file, exclude: string): bool = + result = true + if "_nimterop_" in file: + result = false + elif exclude.nBl: + for excl in exclude.split(","): + if excl in file: + result = false + + proc dcompile(dir, exclude: string, ext="") = + let + (dir, pat) = + if "*" in dir: + dir.splitPath() + else: + (dir, "") + + for file in walkDirRec(dir): + if ext.nBl or pat.nBl: + let + fext = file.splitFile().ext + if (ext.nBl and fext != ext) or (pat.nBl and fext != pat[1 .. ^1]): + continue + if file.notExcluded(exclude): + fcompile(file) + + if "*" in path: + dcompile(path, exclude) + else: + let fpath = findPath(path) + if fileExists(fpath) and fpath.notExcluded(exclude): + fcompile(fpath) + elif dirExists(fpath): + if mode.contains("cpp"): + for i in @[".cpp", ".c++", ".cc", ".cxx"]: + dcompile(fpath, exclude, i) + when not defined(Windows): + dcompile(fpath, exclude, ".C") + else: + dcompile(fpath, exclude, ".c") + +macro renderPragma*(): untyped = + ## All `cDefine()`, `cIncludeDir()`, `cCompile()`, `cPassC()` and `cPassL()` + ## content typically gets forwarded via `cImport()` to the generated wrapper to be + ## rendered as part of the output so as to enable standalone wrappers. If `cImport()` + ## is not being used for some reason, `renderPragma()` can create these pragmas + ## in the nimterop wrapper itself. A good example is using `getHeader()` without + ## calling `cImport()`. + ## + ## `c2nImport()` already uses this macro so there's no need to use it when typically + ## wrapping headers. + result = newNimNode(nnkStmtList) + + for i in gStateCT.defines: + let str = "-D" & i + result.add quote do: + {.passC: `str`.} + + for i in gStateCT.includeDirs: + let str = &"-I{i.quoteShell}" + result.add quote do: + {.passC: `str`.} + + for i in gStateCT.passC: + result.add quote do: + {.passC: `i`.} + gStateCT.passC = @[] + + for i in gStateCT.passL: + result.add quote do: + {.passL: `i`.} + gStateCT.passL = @[] + + for i in gStateCT.compile: + result.add quote do: + {.compile: `i`.} + gStateCT.compile = @[] + +proc cSkipSymbol*(skips: seq[string]) {.compileTime.} = + ## Similar to `cOverride()`, this macro allows filtering out symbols not of + ## interest from the generated output. + ## + ## `cSkipSymbol()` only affects calls to `cImport()` that follow it. + runnableExamples: + static: cSkipSymbol @["proc1", "Type2"] + gStateCT.symOverride.add skips + macro cOverride*(body): untyped = ## When the wrapper code generated by nimterop is missing certain symbols or not ## accurate, it may be required to hand wrap them. Define them in a `cOverride()` @@ -234,15 +519,6 @@ proc onSymbolOverride*(sym: var Symbol) {.exportc, dynlib.} = if names.nBl: decho "Overriding " & names.join(" ") -proc cSkipSymbol*(skips: seq[string]) {.compileTime.} = - ## Similar to `cOverride()`, this macro allows filtering out symbols not of - ## interest from the generated output. - ## - ## `cSkipSymbol()` only affects calls to `cImport()` that follow it. - runnableExamples: - static: cSkipSymbol @["proc1", "Type2"] - gStateCT.symOverride.add skips - proc cPluginHelper(body: string, imports = "import macros, nimterop/plugin\n\n") = gStateCT.pluginSource = body @@ -332,237 +608,6 @@ macro cPluginPath*(path: static[string]): untyped = doAssert fileExists(path), "Plugin file not found: " & path cPluginHelper(readFile(path), imports = "") -proc cSearchPath*(path: string): string {.compileTime.} = - ## Get full path to file or directory `path` in search path configured - ## using `cAddSearchDir()` and `cAddStdDir()`. - ## - ## This can be used to locate files or directories that can be passed onto - ## `cCompile()`, `cIncludeDir()` and `cImport()`. - result = findPath(path, fail = false) - if result.Bl: - var found = false - for inc in gStateCT.searchDirs: - result = findPath(inc / path, fail = false) - if result.nBl: - found = true - break - doAssert found, "File or directory not found: " & path & - " gStateCT.searchDirs: " & $gStateCT.searchDirs - -proc cDebug*() {.compileTime.} = - ## Enable debug messages and display the generated Nim code - gStateCT.debug = true - -proc cDisableCaching*() {.compileTime.} = - ## Disable caching of generated Nim code - useful during wrapper development - ## - ## If files included by header being processed by - ## `cImport()` change and affect the generated content, they will be ignored - ## and the cached value will continue to be used . Use `cDisableCaching()` to - ## avoid this scenario during development. - ## - ## `nim -f` can also be used to flush the cached content. - gStateCT.nocache = true - -macro cDefine*(name: static[string], val: static[string] = ""): untyped = - ## `#define` an identifer that is forwarded to the C/C++ preprocessor if - ## called within `cImport()` or `c2nImport()` as well as to the C/C++ - ## compiler during Nim compilation using `{.passC: "-DXXX".}` - ## - ## This needs to be called before `cImport()` to take effect. - var str = name - if val.nBl: - str &= &"={val.quoteShell}" - - if str notin gStateCT.defines: - gStateCT.defines.add(str) - -macro cDefine*(values: static seq[string]): untyped = - ## `#define` multiple identifers that are forwarded to the C/C++ preprocessor - ## if called within `cImport()` or `c2nImport()` as well as to the C/C++ - ## compiler during Nim compilation using `{.passC: "-DXXX".}` - ## - ## This needs to be called before `cImport()` to take effect. - for value in values: - let - spl = value.split("=", maxsplit = 1) - name = spl[0] - val = if spl.len == 2: spl[1] else: "" - discard quote do: - cDefine(`name`, `val`) - -macro cPassC*(value: static string): untyped = - ## Create a `{.passC.}` entry that gets forwarded to the C/C++ compiler - ## during Nim compilation. - ## - ## This needs to be called before `cImport()` to take effect. - gStateCT.passC.add value - -macro cPassL*(value: static string): untyped = - ## Create a `{.passL.}` entry that gets forwarded to the C/C++ compiler - ## during Nim compilation. - ## - ## This needs to be called before `cImport()` to take effect. - gStateCT.passL.add value - -proc cAddSearchDir*(dir: string) {.compileTime.} = - ## Add directory `dir` to the search path used in calls to - ## `cSearchPath()`. - runnableExamples: - import nimterop/paths, os - static: - cAddSearchDir testsIncludeDir() - doAssert cSearchPath("test.h").fileExists - - if dir notin gStateCT.searchDirs: - gStateCT.searchDirs.add(dir) - -macro cIncludeDir*(dirs: static seq[string], exclude: static[bool] = false): untyped = - ## Add include directories that are forwarded to the C/C++ preprocessor if - ## called within `cImport()` or `c2nImport()` as well as to the C/C++ - ## compiler during Nim compilation using `{.passC: "-IXXX".}`. - ## - ## Set `exclude = true` if the contents of these include directories should - ## not be included in the wrapped output. - ## - ## This needs to be called before `cImport()` to take effect. - for dir in dirs: - let fullpath = findPath(dir) - if fullpath notin gStateCT.includeDirs: - gStateCT.includeDirs.add(fullpath) - if exclude: - gStateCT.exclude.add(fullpath) - -macro cIncludeDir*(dir: static[string], exclude: static[bool] = false): untyped = - ## Add an include directory that is forwarded to the C/C++ preprocessor if - ## called within `cImport()` or `c2nImport()` as well as to the C/C++ - ## compiler during Nim compilation using `{.passC: "-IXXX".}`. - ## - ## Set `exclude = true` if the contents of this include directory should - ## not be included in the wrapped output. - ## - ## This needs to be called before `cImport()` to take effect. - return quote do: - cIncludeDir(@[`dir`], `exclude` == 1) - -macro cExclude*(paths: static seq[string]): untyped = - ## Exclude specified paths - files or directories from the wrapped output - ## - ## Full path to file or directory is required. - result = newNimNode(nnkStmtList) - for path in paths: - gStateCT.exclude.add path - -macro cExclude*(path: static string): untyped = - ## Exclude specified path - file or directory from the wrapped output. - ## - ## Full path to file or directory is required. - return quote do: - cExclude(@[`path`]) - -proc cAddStdDir*(mode = "c") {.compileTime.} = - ## Add the standard `c` [default] or `cpp` include paths to search - ## path used in calls to `cSearchPath()`. - runnableExamples: - import os - static: - cAddStdDir() - doAssert cSearchPath("math.h").fileExists - for inc in getGccPaths(mode): - cAddSearchDir inc - -macro cCompile*(path: static string, mode: static[string] = "c", exclude: static[string] = ""): untyped = - ## Compile and link C/C++ implementation into resulting binary using `{.compile.}` - ## - ## `path` can be a specific file or contain `*` wildcard for filename: - ## - ## .. code-block:: nim - ## - ## cCompile("file.c") - ## cCompile("path/to/*.c") - ## - ## `mode` recursively searches for code files in `path`. - ## - ## `c` searches for `*.c` whereas `cpp` searches for `*.C *.cpp *.c++ *.cc *.cxx` - ## - ## .. code-block:: nim - ## - ## cCompile("path/to/dir", "cpp") - ## - ## `exclude` can be used to exclude files by partial string match. Comma separated to - ## specify multiple exclude strings - ## - ## .. code-block:: nim - ## - ## cCompile("path/to/dir", exclude="test2.c") - - proc fcompile(file: string) = - let - (_, fn, ext) = file.splitFile() - var - ufn = fn - uniq = 1 - while ufn in gStateCT.compcache: - ufn = fn & $uniq - uniq += 1 - - # - https://github.com/nim-lang/Nim/issues/10299 - # - https://github.com/nim-lang/Nim/issues/10486 - gStateCT.compcache.add(ufn) - if fn == ufn: - gStateCT.compile.add file.replace("\\", "/") - else: - # - https://github.com/nim-lang/Nim/issues/9370 - let - hash = file.hash().abs() - tmpFile = file.parentDir() / &"_nimterop_{$hash}_{ufn}{ext}" - if not tmpFile.fileExists() or file.getFileDate() > tmpFile.getFileDate(): - cpFile(file, tmpFile) - gStateCT.compile.add tmpFile.replace("\\", "/") - - # Due to https://github.com/nim-lang/Nim/issues/9863 - # cannot use seq[string] for excludes - proc notExcluded(file, exclude: string): bool = - result = true - if "_nimterop_" in file: - result = false - elif exclude.nBl: - for excl in exclude.split(","): - if excl in file: - result = false - - proc dcompile(dir, exclude: string, ext="") = - let - (dir, pat) = - if "*" in dir: - dir.splitPath() - else: - (dir, "") - - for file in walkDirRec(dir): - if ext.nBl or pat.nBl: - let - fext = file.splitFile().ext - if (ext.nBl and fext != ext) or (pat.nBl and fext != pat[1 .. ^1]): - continue - if file.notExcluded(exclude): - fcompile(file) - - if "*" in path: - dcompile(path, exclude) - else: - let fpath = findPath(path) - if fileExists(fpath) and fpath.notExcluded(exclude): - fcompile(fpath) - elif dirExists(fpath): - if mode.contains("cpp"): - for i in @[".cpp", ".c++", ".cc", ".cxx"]: - dcompile(fpath, exclude, i) - when not defined(Windows): - dcompile(fpath, exclude, ".C") - else: - dcompile(fpath, exclude, ".c") - macro cImport*(filenames: static seq[string], recurse: static bool = false, dynlib: static string = "", mode: static string = "c", flags: static string = "", nimFile: static string = ""): untyped = ## Import multiple headers in one shot @@ -704,29 +749,12 @@ macro c2nImport*(filename: static string, recurse: static bool = false, dynlib: if flags.nBl: cmd.add &" {flags}" - # Have to create pragmas for c2nim since toast handles this at runtime for i in gStateCT.defines: cmd.add &" --assumedef:{i.quoteShell}" - let str = "-D" & i - result.add quote do: - {.passC: `str`.} - for i in gStateCT.includeDirs: - let str = &"-I{i.quoteShell}" - result.add quote do: - {.passC: `str`.} - - for i in gStateCT.passC: - result.add quote do: - {.passC: `i`.} - - for i in gStateCT.passL: - result.add quote do: - {.passL: `i`.} - - for i in gStateCT.compile: - result.add quote do: - {.compile: `i`.} + # Have to create pragmas for c2nim since toast handles this at runtime + result.add quote do: + renderPragma() let (c2nimout, ret) = execAction(cmd) From d95deffb6aa9993eda989a20116cf7fd72eb767a Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Wed, 15 Jul 2020 21:06:59 -0500 Subject: [PATCH 075/106] v0.6.5 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 1eb35ef..a246483 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.4" +version = "0.6.5" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 74c9506529b422333a67e363d0554d91d542f289 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 16 Jul 2020 15:33:41 -0500 Subject: [PATCH 076/106] Skip Conan and JBB deps --- CHANGES.md | 2 ++ README.md | 2 +- nimterop/build/conan.nim | 8 +++++- nimterop/build/getheader.nim | 54 ++++++++++++++++++++++++------------ nimterop/build/jbb.nim | 9 +++++- 5 files changed, 55 insertions(+), 20 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e7e9213..edd6c97 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -53,6 +53,8 @@ https://github.com/nimterop/nimterop/compare/v0.5.9...v0.6.5 - Added `renderPragma()` to create pragmas inline in case `cImport()` is not being used. (since v0.6.5) +- `xxxConan` and `xxxJBB` now allow skipping required dependencies by specifying `skip=pkg1,pkg2` to the `conanFlags` and `jbbFlags` params to `getHeader()`. (since v0.6.6) + ### Other improvements - Generated wrappers no longer depend on nimterop being present - no more `import nimterop/types`. Supporting code is directly included in the wrapper output and only when required. E.g. enum macro is only included if wrapper contains enums. [#125][i125] (since v0.6.1) diff --git a/README.md b/README.md index d652d0d..96f3a46 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Flags can be specified to these tools via `getHeader()` or directly via the unde If `-d:headerStatic` is specified, `getHeader()` will return the static library path in `headerLPath`. The wrapper writer can check for this and call `cImport()` accordingly as in the example above. If `-d:headerStatic` is omitted, the dynamic library is returned in `headerLPath`. -All dependency libraries (supported by Conan and JBB) will be returned in `headerLDeps`. Static libraries and dependencies are automatically linked using `cPassL()`. Conan shared libs include all dependencies whereas JBB shared libs expect the required dependencies to be in the same location or in `LD_LIBRARY_PATH`. +All dependency libraries (supported by Conan and JBB) will be returned in `headerLDeps`. Static libraries and dependencies are automatically linked using `cPassL()`. Conan shared libs typically include dependencies compiled in whereas JBB shared libs expect the required dependencies to be in the same location or in `LD_LIBRARY_PATH`. `conanFlags` and `jbbFlags` can be used to skip required dependencies from being downloaded in case another source is preferred. This can be done with `skip=pkg1,pkg2` to these flags. `getHeader()` searches for libraries based on the header name by default: - `libheader.so` or `libheader.a` on Linux diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index dd43dc9..eb27557 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -22,6 +22,8 @@ type staticLibs*: seq[string] requires*: seq[ConanPackage] + skipRequires*: seq[string] + ConanBuild* = ref object ## Build type that stores build specific info and revisions bhash*: string @@ -245,7 +247,10 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = bld.options = newTable[string, string](8) for key, value in options.getFields(): bld.options[key] = value.getStr() - bld.requires = requires.to(seq[string]) + for req in requires.to(seq[string]): + # Filter skipped dependencies + if req.toLowerAscii() notin pkg.skipRequires: + bld.requires.add req bld.recipe_hash = bdata.getOrDefault("recipe_hash").getStr() if pkg.recipes.hasKey(bld.recipe_hash): @@ -422,6 +427,7 @@ proc dlConanRequires*(pkg: ConanPackage, bld: ConanBuild, outdir: string) = else: let rpkg = newConanPackageFromUri(req, shared = false) + rpkg.skipRequires = pkg.skipRequires downloadConan(rpkg, outdir, main = false) pkg.requires.add rpkg diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index dbd965e..85a0331 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -119,7 +119,7 @@ proc getDlPath(header, url, outdir, version: string): string = result = findFile(header, outdir) -proc getConanPath(header, uri, outdir, version: string, shared: bool): string = +proc getConanPath(header, uri, flags, outdir, version: string, shared: bool): string = var uri = uri @@ -131,6 +131,15 @@ proc getConanPath(header, uri, outdir, version: string, shared: bool): string = let pkg = newConanPackageFromUri(uri, shared) + + # Handle `conanFlags` + if flags.nBl: + for flag in flags.split(" "): + if flag.startsWith("skip="): + for req in flag["skip=".len .. ^1].split(","): + if req.nBl: + pkg.skipRequires.add req.toLowerAscii() + downloadConan(pkg, outdir) result = findFile(header, outdir) @@ -165,16 +174,21 @@ proc getJBBPath(header, uri, flags, outdir, version: string): string = # Handle `jbbFlags` if flags.nBl: - if flags.startsWith("giturl="): - let - val = flags["giturl=".len .. ^1] - if val.contains("://"): - pkg.baseUrl = val - else: - pkg.baseUrl = "https://github.com/" & val - elif flags.startsWith("url="): - pkg.baseUrl = flags["url=".len .. ^1] - pkg.isGit = false + for flag in flags.split(" "): + if flag.startsWith("giturl="): + let + val = flag["giturl=".len .. ^1] + if val.contains("://"): + pkg.baseUrl = val + else: + pkg.baseUrl = "https://github.com/" & val + elif flag.startsWith("url="): + pkg.baseUrl = flag["url=".len .. ^1] + pkg.isGit = false + elif flag.startsWith("skip="): + for req in flag["skip=".len .. ^1].split(","): + if req.nBl: + pkg.skipRequires.add req.toLowerAscii() downloadJBB(pkg, outdir) @@ -227,7 +241,7 @@ macro getHeader*( conanuri: static[string] = "", jbburi: static[string] = "", outdir: static[string] = "", libdir: static[string] = "", conFlags: static[string] = "", cmakeFlags: static[string] = "", makeFlags: static[string] = "", - jbbFlags: static[string] = "", altNames: static[string] = "", + conanFlags: static[string] = "", jbbFlags: static[string] = "", altNames: static[string] = "", buildTypes: static[openArray[BuildType]] = [btCmake, btAutoconf]): untyped = ## Get the path to a header file for wrapping with ## `cImport() `_ or @@ -300,7 +314,12 @@ macro getHeader*( ## `cmake` and `make` in case additional configuration is required as part of the build ## process. ## - ## `jbbFlags` allows changing the BinaryBuilder.org defaults: + ## `conanFlags` and `jbbFlags` allow changing the Conan.io and BinaryBuilder.org defaults: + ## - `skip=pkg1,pkg2` skips the specified packages which are required dependencies of the + ## package in question. This enables downloading those dependencies from other sources + ## if required. + ## + ## `jbbFlags` allows two additional customizations: ## - `giturl=customUrl` changes the default `https://github.com/JuliaBinaryWrappers` to ## another Git URL. If no hostname is specified, `https://github.com` is assumed. ## - `url=customUrl` uses regular HTTP instead of Git and looks for `Artifacts.toml` and @@ -402,13 +421,14 @@ macro getHeader*( `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, conanuri, jbburi, jbbFlags, outdir, version: string, shared: bool): string = + proc getPath(header, giturl, dlurl, conanuri, conanFlags, jbburi, jbbFlags, + outdir, version: string, shared: bool): string = when `nameGit`: getGitPath(header, giturl, outdir, version) elif `nameDL`: getDlPath(header, dlurl, outdir, version) elif `nameConan`: - getConanPath(header, conanuri, outdir, version, shared) + getConanPath(header, conanuri, conanFlags, outdir, version, shared) elif `nameJBB`: getJBBPath(header, jbburi, jbbFlags, outdir, version) else: @@ -441,7 +461,7 @@ macro getHeader*( when useStd: stdPath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `jbbFlags`, + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `conanFlags`, `jbburi`, `jbbFlags`, `outdir`, `version`, not `nameStatic`) # Run preBuild hook before building library if not Std, Conan or JBB @@ -477,7 +497,7 @@ macro getHeader*( if prePath.len != 0: prePath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `jbburi`, `jbbFlags`, + getPath(`header`, `giturl`, `dlurl`, `conanuri`, `conanFlags`, `jbburi`, `jbbFlags`, `outdir`, `version`, not `nameStatic`) static: diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index a430a55..a4ff98f 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -21,6 +21,8 @@ type staticLibs*: seq[string] requires*: seq[JBBPackage] + skipRequires*: seq[string] + const # JBB URLs jbbBaseUrl = "https://github.com/JuliaBinaryWrappers" @@ -73,7 +75,12 @@ proc parseJBBProject(pkg: JBBPackage, outdir: string) = let name = line.split()[0] if name.endsWith("_jll"): - pkg.requires.add newJBBPackage(name[0 .. ^5], "") + # Filter skipped dependencies + let + pname = name[0 .. ^5] + if pname.toLowerAscii() notin pkg.skipRequires: + pkg.requires.add newJBBPackage(pname, "") + pkg.requires[^1].skipRequires = pkg.skipRequires proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = # Get build information from Artifacts.toml From 5acfd0e8197160e8f71274473a4f9f2f75bc8fbc Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 16 Jul 2020 16:05:22 -0500 Subject: [PATCH 077/106] Handle JBB case where no dep for os/arch combo --- nimterop/build/conan.nim | 36 ++++++++++++++++++------------------ nimterop/build/jbb.nim | 17 ++++++++++------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index eb27557..6df25d8 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -134,13 +134,13 @@ proc newConanPackageFromUri*(uri: string, shared = true): ConanPackage = proc getUriFromConanPackage*(pkg: ConanPackage): string = ## Convert a ConanPackage to a conan uri result = pkg.name - if pkg.version.len != 0: + if pkg.version.nBl: result &= "/" & pkg.version - if pkg.user.len != 0: + if pkg.user.nBl: result &= "@" & pkg.user - if pkg.channel.len != 0: + if pkg.channel.nBl: result &= "/" & pkg.channel - if pkg.bhash.len != 0: + if pkg.bhash.nBl: result &= ":" & pkg.bhash proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPackage = @@ -149,11 +149,11 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac ## Search is quite slow so it is preferable to specify a version and use `getConanBuilds()` var query = name - if version.len != 0: + if version.nBl: query &= "/" & version - if user.len != 0: + if user.nBl: query &= "@" & user - if channel.len != 0: + if channel.nBl: query &= "/" & channel gecho &"# Searching Conan.io for latest version of {name}" @@ -172,11 +172,11 @@ proc searchConan*(name: string, version = "", user = "", channel = ""): ConanPac if "@_/_" in str: let ver = str.split('/')[1].split('@')[0] - if latestv.len == 0 or compareVersions(ver, latestv) > 0: + if latestv.Bl or compareVersions(ver, latestv) > 0: latestv = ver latest = str - if latest.len != 0: + if latest.nBl: result = newConanPackageFromUri(latest) proc searchConan*(pkg: ConanPackage): ConanPackage = @@ -200,7 +200,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = vsplit[0] query = - if pkg.bhash.len == 0: + if pkg.bhash.Bl: block: var query = &"?q=arch={arch}&os={os.capitalizeAscii()}" @@ -208,7 +208,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = query &= "&build_type=Release" if "shared=" notin filter: query &= &"&options.shared={($pkg.shared).capitalizeAscii()}" - if filter.len != 0: + if filter.nBl: query &= &"&{filter}" if "compiler=" notin filter and os != "windows": query &= &"&compiler={compiler}&compiler.version=" & vfilter @@ -232,7 +232,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = if not j1.isNil: for bhash, bdata in j1.getFields(): - if pkg.bhash.len == 0 or pkg.bhash == bhash: + if pkg.bhash.Bl or pkg.bhash == bhash: let bld = new(ConanBuild) settings = bdata.getOrDefault("settings") @@ -328,13 +328,13 @@ proc dlConanBuild*(pkg: ConanPackage, bld: ConanBuild, outdir: string, revision ## Download specific `revision` of `bld` to `outdir` ## ## If omitted, the latest revision (first) is downloaded - doAssert bld.revisions.len != 0, "No build revisions found for Conan.io package " & pkg.getUriFromConanPackage() + doAssert bld.revisions.nBl, "No build revisions found for Conan.io package " & pkg.getUriFromConanPackage() let outdir = fixRelPath(outdir) revision = - if revision.len != 0: + if revision.nBl: revision else: bld.revisions[0] @@ -379,7 +379,7 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = outdir = fixRelPath(outdir) pkg = - if pkg.version.len == 0: + if pkg.version.Bl: searchConan(pkg) else: pkg @@ -395,12 +395,12 @@ proc downloadConan*(pkg: ConanPackage, outdir: string, main = true) = pkg.getConanBuilds() - doAssert pkg.recipes.len != 0, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" + doAssert pkg.recipes.nBl, &"Failed to download {pkg.name} v{pkg.version} from Conan - check https://conan.io/center" gecho &"# Downloading {pkg.name} v{pkg.version} from Conan.io" for recipe, builds in pkg.recipes: for build in builds: - if pkg.bhash.len == 0 or pkg.bhash == build.bhash: + if pkg.bhash.Bl or pkg.bhash == build.bhash: pkg.getConanRevisions(build) pkg.dlConanBuild(build, outdir) pkg.dlConanRequires(build, outdir) @@ -442,7 +442,7 @@ proc getConanLDeps*(pkg: ConanPackage, outdir: string, main = true): seq[string] libs = if pkg.shared: pkg.sharedLibs else: pkg.staticLibs str = if pkg.shared: "shared" else: "static" - doAssert libs.len != 0, &"No {str} libs found for {pkg.name} in {outdir}" + doAssert libs.nBl, &"No {str} libs found for {pkg.name} in {outdir}" if not main: for lib in libs: diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index a4ff98f..9dacfe2 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -65,7 +65,7 @@ proc parseJBBProject(pkg: JBBPackage, outdir: string) = for line in data.splitLines(): let line = line.strip() - if line.len != 0: + if line.nBl: if line.startsWith('['): if line == "[deps]": deps = true @@ -100,7 +100,7 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = for line in data.splitLines(): let line = line.strip() - if line.len != 0: + if line.nBl: let spl = line.split(" = ", 1) name = spl[0] @@ -145,7 +145,7 @@ proc getJBBRepo*(pkg: JBBPackage, outdir: string) = quiet = true ) - if pkg.version.len != 0: + if pkg.version.nBl: # Checkout correct tag let tags = gitTags(path) @@ -157,7 +157,7 @@ proc getJBBRepo*(pkg: JBBPackage, outdir: string) = var url = pkg.baseUrl if "$#" in url or "$1" in url: - doAssert pkg.version.len != 0, "Need version for custom BinaryBuilder.org url: " & url + doAssert pkg.version.nBl, "Need version for custom BinaryBuilder.org url: " & url url = url % pkg.version downloadUrl(url & "Artifacts.toml", path, quiet = true) downloadUrl(url & "Project.toml", path, quiet = true) @@ -209,11 +209,14 @@ proc downloadJBB*(pkg: JBBPackage, outdir: string, main = true) = pkg.getJBBRepo(outdir) - doAssert pkg.url.len != 0, &"Failed to download {pkg.name} info from BinaryBuilder.org" + if pkg.url.Bl: + # No url for deps means no package for that os/arch combo - e.g. Attr + doAssert not main, &"Failed to download {pkg.name} info from BinaryBuilder.org" + return let vstr = - if pkg.version.len != 0: + if pkg.version.nBl: &" v{pkg.version}" else: "" @@ -250,7 +253,7 @@ proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): s libs = if shared: pkg.sharedLibs else: pkg.staticLibs str = if shared: "shared" else: "static" - doAssert libs.len != 0, &"No {str} libs found for {pkg.name} in {outdir}" + doAssert libs.nBl, &"No {str} libs found for {pkg.name} in {outdir}" if not main: for lib in libs: From b1ca1c6ffdbf8c2704f62c672ccfb498810970be Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 16 Jul 2020 17:19:41 -0500 Subject: [PATCH 078/106] Skip fix for JBB --- nimterop/build/jbb.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index 9dacfe2..c8d86c3 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -260,4 +260,6 @@ proc getJBBLDeps*(pkg: JBBPackage, outdir: string, shared: bool, main = true): s result.add lib for cpkg in pkg.requires: - result.add cpkg.getJBBLDeps(outdir, shared, main = false) + # No url for deps means no package for that os/arch combo - e.g. Attr + if cpkg.url.nBl: + result.add cpkg.getJBBLDeps(outdir, shared, main = false) From 5ac66286d5d44a50d0a84d1810694f818b4dda62 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 17 Jul 2020 20:56:22 -0500 Subject: [PATCH 079/106] v0.6.6 --- nimterop.nimble | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimterop.nimble b/nimterop.nimble index a246483..3cb733f 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.5" +version = "0.6.6" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" @@ -48,7 +48,7 @@ task docs, "Generate docs": task minitest, "Test for Nim CI": exec "nim c -f -d:danger nimterop/toast" exec "nim c -f -d:checkAbi -r tests/tast2.nim" - exec "nim c -f -d:checkAbi -d:zlibStd -d:zlibDL -d:zlibSetVer=1.2.11 -r tests/zlib.nim" + exec "nim c -f -d:checkAbi -d:zlibJBB -d:zlibSetVer=1.2.11 -r tests/zlib.nim" task basic, "Basic tests": execTest "tests/tast2.nim" From dfc4e09454b5e517185b0f83c9baa642a86c6b61 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 28 Jul 2020 14:09:48 -0500 Subject: [PATCH 080/106] Fix docs for nimble change --- nimterop/docs.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimterop/docs.nim b/nimterop/docs.nim index 5fe87a1..6a5ef0c 100644 --- a/nimterop/docs.nim +++ b/nimterop/docs.nim @@ -1,4 +1,4 @@ -import macros, strformat +import strformat from os import parentDir, getCurrentCompilerExe, DirSep @@ -32,7 +32,7 @@ proc execAction(cmd: string): string = (result, ret) = gorgeEx(ccmd) doAssert ret == 0, "Command failed: " & $ret & "\ncmd: " & ccmd & "\nresult:\n" & result -proc buildDocs*(files: openArray[string], path: string, baseDir = getProjectPath() & $DirSep, +proc buildDocs*(files: openArray[string], path: string, baseDir = getCurrentDir() & $DirSep, defines: openArray[string] = @[], nimArgs = "") = ## Generate docs for all specified nim `files` to the specified `path` ## From 96d4d6a8957eed5d696e164b81353ec6b47c3a87 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 30 Jul 2020 20:33:56 -0500 Subject: [PATCH 081/106] v0.6.7 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 3cb733f..8b07a47 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.6" +version = "0.6.7" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 11814811e32ca5394bdc99c83ccaf2866b6d5c00 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 2 Aug 2020 22:20:04 -0500 Subject: [PATCH 082/106] Multiple getHeader, linkLibs name fix --- nimterop/build/getheader.nim | 8 +++++--- nimterop/build/shell.nim | 15 ++++++++++----- nimterop/toastlib/ast2.nim | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 85a0331..4549894 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -366,6 +366,7 @@ macro getHeader*( staticStr = name & "Static" verStr = name & "SetVer" + getPath = name & "GetPath" # Ident nodes of the -d:xxx to check in when statements nameStd = newIdentNode(stdStr) @@ -375,6 +376,7 @@ macro getHeader*( nameJBB = newIdentNode(jbbStr) nameStatic = newIdentNode(staticStr) + nameGetPath = newIdentNode(getPath) # Consts to generate path = newIdentNode(name & "Path") @@ -421,7 +423,7 @@ macro getHeader*( `nameStatic`* = when defined(`nameStatic`): true else: `staticVal` == 1 # Search for header in outdir (after retrieving code) depending on -d:xxx mode - proc getPath(header, giturl, dlurl, conanuri, conanFlags, jbburi, jbbFlags, + proc `nameGetPath`(header, giturl, dlurl, conanuri, conanFlags, jbburi, jbbFlags, outdir, version: string, shared: bool): string = when `nameGit`: getGitPath(header, giturl, outdir, version) @@ -461,7 +463,7 @@ macro getHeader*( when useStd: stdPath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `conanFlags`, `jbburi`, `jbbFlags`, + `nameGetPath`(`header`, `giturl`, `dlurl`, `conanuri`, `conanFlags`, `jbburi`, `jbbFlags`, `outdir`, `version`, not `nameStatic`) # Run preBuild hook before building library if not Std, Conan or JBB @@ -497,7 +499,7 @@ macro getHeader*( if prePath.len != 0: prePath else: - getPath(`header`, `giturl`, `dlurl`, `conanuri`, `conanFlags`, `jbburi`, `jbbFlags`, + `nameGetPath`(`header`, `giturl`, `dlurl`, `conanuri`, `conanFlags`, `jbburi`, `jbbFlags`, `outdir`, `version`, not `nameStatic`) static: diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index f07f70c..f0af561 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -481,14 +481,19 @@ proc linkLibs*(names: openArray[string], staticLink = true): string = var stat = if staticLink: "--static" else: "" resSet: OrderedSet[string] + cmd = &"pkg-config --libs --silence-errors {stat}" resSet.init() for name in names: - let - cmd = &"pkg-config --libs --silence-errors {stat} lib{name}" - (libs, _) = execAction(cmd, die = false) - for lib in libs.split(" "): - resSet.incl lib + for n in ["lib" & name, name]: + # Try libname and name - e.g. MagickWand doesn't have lib + let + cmd = &"{cmd} {n}" + (libs, _) = execAction(cmd, die = false) + if libs.len != 0: + for lib in libs.split(" "): + resSet.incl lib + break if staticLink: resSet.incl "--static" diff --git a/nimterop/toastlib/ast2.nim b/nimterop/toastlib/ast2.nim index 7f2b4ad..bea6ff6 100644 --- a/nimterop/toastlib/ast2.nim +++ b/nimterop/toastlib/ast2.nim @@ -633,7 +633,7 @@ iterator newIdentDefs(gState: State, name: string, node: TSNode, offset: SomeInt yield result else: for i in start+1 ..< node.len: - if node[i].getName() == "bitfield_clause": + if node[i].getName() in ["bitfield_clause", "comment"]: continue yield gState.newIdentDef(name, node, tname, tinfo, tident, start, i, offset, exported) From 672e634d89f4db7158c9986fa96b0d5f02174259 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Sun, 2 Aug 2020 23:25:54 -0500 Subject: [PATCH 083/106] v0.6.8 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 8b07a47..804f1a5 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.7" +version = "0.6.8" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From dc90a1debb8c5e914173d69f9a8d3fce2670d330 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 27 Aug 2020 20:01:04 -0500 Subject: [PATCH 084/106] Fix -std issue with clang --- nimterop/treesitter/api.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/treesitter/api.nim b/nimterop/treesitter/api.nim index 753178c..b763722 100644 --- a/nimterop/treesitter/api.nim +++ b/nimterop/treesitter/api.nim @@ -10,7 +10,7 @@ static: const sourcePath = cacheDir / "treesitter" / "lib" -when defined(Linux): +when defined(Linux) and defined(gcc): {.passC: "-std=c11".} {.passC: "-I$1" % (sourcePath / "include").} From f07dd4dd56c587021f6477bae3731ff4abb1f1b1 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 27 Aug 2020 21:12:51 -0500 Subject: [PATCH 085/106] v0.6.9 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 804f1a5..1ef3fb6 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.8" +version = "0.6.9" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From c0079db1548882949ded9fb3453a1a433a809aa8 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 27 Aug 2020 22:30:53 -0500 Subject: [PATCH 086/106] Test arm64 and ppc --- .travis.yml | 47 +++++++++++++++++++++++++++++++++++++---------- appveyor.yml | 4 ++-- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index b58b29d..25d8449 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,47 @@ -os: - - windows - - linux - - osx +os: linux +dist: bionic +language: c addons: apt: packages: - autopoint -language: c +matrix: + include: + # Linux - amd64 + - env: BRANCH=0.20.2 + - env: BRANCH=1.0.8 + - env: BRANCH=1.2.6 + - env: BRANCH=devel -env: - - BRANCH=0.20.2 - - BRANCH=1.0.6 - - BRANCH=1.2.4 - - BRANCH=devel + # Linux - arm64 + - arch: arm64 + env: BRANCH=1.2.6 + + # Linux - ppc64 + - arch: ppc64le + env: BRANCH=1.2.6 + + # macOS - amd64 + - os: osx + env: BRANCH=0.20.2 + - os: osx + env: BRANCH=1.0.8 + - os: osx + env: BRANCH=1.2.6 + - os: osx + env: BRANCH=devel + + # windows - amd64 + - os: windows + env: BRANCH=0.20.2 + - os: windows + env: BRANCH=1.0.8 + - os: windows + env: BRANCH=1.2.6 + - os: windows + env: BRANCH=devel cache: directories: diff --git a/appveyor.yml b/appveyor.yml index b572ea1..273eee2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,8 +10,8 @@ matrix: environment: matrix: - NIM_VERSION: 0.20.2 - - NIM_VERSION: 1.0.6 - - NIM_VERSION: 1.2.2 + - NIM_VERSION: 1.0.8 + - NIM_VERSION: 1.2.6 for: - From 650281039c7cb11cf089a86dc82830a347ae8784 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 28 Aug 2020 16:08:11 -0500 Subject: [PATCH 087/106] Fix preprocessing bug --- nimterop/toastlib/getters.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index 8a4586d..f0a4735 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -333,7 +333,8 @@ proc getPreprocessor*(gState: State, fullpath: string) = while true: if outp.readLine(line): # We want to keep blank lines here for comment processing - if line.len > 1 and line[0] == '#' and line[1] == ' ': + if line.len > 10 and line[0] == '#' and line[1] == ' ' and line.contains('"'): + # # 1 "path/to/file.h" 1 start = false line = line.split('"')[1].sanitizePath(noQuote = true) if sfile == line or From 9b848bc726ee37fb1abbce2fb51b30c189aa5a69 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 28 Aug 2020 16:14:36 -0500 Subject: [PATCH 088/106] Adjust CI for failures --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 25d8449..5a88169 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,16 +12,16 @@ matrix: # Linux - amd64 - env: BRANCH=0.20.2 - env: BRANCH=1.0.8 - - env: BRANCH=1.2.6 + - env: BRANCH=1.2.4 - env: BRANCH=devel # Linux - arm64 - - arch: arm64 - env: BRANCH=1.2.6 + # - arch: arm64 + # env: BRANCH=1.2.4 # Linux - ppc64 - - arch: ppc64le - env: BRANCH=1.2.6 + # - arch: ppc64le + # env: BRANCH=1.2.4 # macOS - amd64 - os: osx @@ -29,7 +29,7 @@ matrix: - os: osx env: BRANCH=1.0.8 - os: osx - env: BRANCH=1.2.6 + env: BRANCH=1.2.4 - os: osx env: BRANCH=devel @@ -39,7 +39,7 @@ matrix: - os: windows env: BRANCH=1.0.8 - os: windows - env: BRANCH=1.2.6 + env: BRANCH=1.2.4 - os: windows env: BRANCH=devel From 6e86ebc194d5f56e519802e78f9b5e39d6ea7f51 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 28 Aug 2020 17:19:27 -0500 Subject: [PATCH 089/106] Fix appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 273eee2..d807b19 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,7 +11,7 @@ environment: matrix: - NIM_VERSION: 0.20.2 - NIM_VERSION: 1.0.8 - - NIM_VERSION: 1.2.6 + - NIM_VERSION: 1.2.4 for: - From f2139d53750c673208a3895e6c2b318e83ab7bf6 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 28 Aug 2020 17:19:31 -0500 Subject: [PATCH 090/106] v0.6.10 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 1ef3fb6..1fbd440 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.9" +version = "0.6.10" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From b82d7ce87b2aa3b9ee194230565bfe6d73262b38 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 1 Sep 2020 12:38:21 -0500 Subject: [PATCH 091/106] Fix #248 - static build on Windows --- nimterop/toast.nims | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nimterop/toast.nims b/nimterop/toast.nims index e9a3c99..e67a881 100644 --- a/nimterop/toast.nims +++ b/nimterop/toast.nims @@ -24,3 +24,7 @@ switch("out", currentSourcePath.parentDir() / "toast".addFileExt(ExeExt)) # Define TOAST for globals.nim switch("define", "TOAST") + +# Static for Windows - #248 +when defined(Windows): + switch("passL", "-static") From 674d842415ef68979f6407cb31b08f179238b189 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 1 Sep 2020 14:09:54 -0500 Subject: [PATCH 092/106] v0.6.11 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 1fbd440..49c40d0 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.10" +version = "0.6.11" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 93c83728c71132a63c69ecfaab84c4c91bb05615 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sun, 23 Aug 2020 17:11:18 -0600 Subject: [PATCH 093/106] Fix nimterop imports --- nimterop/enumtypepub.nim | 42 +++++++++++++++++++++++++++++++++++ nimterop/toastlib/getters.nim | 2 +- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 nimterop/enumtypepub.nim diff --git a/nimterop/enumtypepub.nim b/nimterop/enumtypepub.nim new file mode 100644 index 0000000..7700e30 --- /dev/null +++ b/nimterop/enumtypepub.nim @@ -0,0 +1,42 @@ +import macros + +macro defineEnum*(typ: untyped): untyped = + result = newNimNode(nnkStmtList) + + # Enum mapped to distinct cint + result.add quote do: + type `typ`* = distinct cint + + for i in ["+", "-", "*", "div", "mod", "shl", "shr", "or", "and", "xor", "<", "<=", "==", ">", ">="]: + let + ni = newIdentNode(i) + typout = if i[0] in "<=>": newIdentNode("bool") else: typ # comparisons return bool + if i[0] == '>': # cannot borrow `>` and `>=` from templates + let + nopp = if i.len == 2: newIdentNode("<=") else: newIdentNode("<") + result.add quote do: + proc `ni`*(x: `typ`, y: cint): `typout` = `nopp`(y, x) + proc `ni`*(x: cint, y: `typ`): `typout` = `nopp`(y, x) + proc `ni`*(x, y: `typ`): `typout` = `nopp`(y, x) + else: + result.add quote do: + proc `ni`*(x: `typ`, y: cint): `typout` {.borrow.} + proc `ni`*(x: cint, y: `typ`): `typout` {.borrow.} + proc `ni`*(x, y: `typ`): `typout` {.borrow.} + result.add quote do: + proc `ni`*(x: `typ`, y: int): `typout` = `ni`(x, y.cint) + proc `ni`*(x: int, y: `typ`): `typout` = `ni`(x.cint, y) + + let + divop = newIdentNode("/") # `/`() + dlrop = newIdentNode("$") # `$`() + notop = newIdentNode("not") # `not`() + result.add quote do: + proc `divop`*(x, y: `typ`): `typ` = `typ`((x.float / y.float).cint) + proc `divop`*(x: `typ`, y: cint): `typ` = `divop`(x, `typ`(y)) + proc `divop`*(x: cint, y: `typ`): `typ` = `divop`(`typ`(x), y) + proc `divop`*(x: `typ`, y: int): `typ` = `divop`(x, y.cint) + proc `divop`*(x: int, y: `typ`): `typ` = `divop`(x.cint, y) + + proc `dlrop`*(x: `typ`): string {.borrow.} + proc `notop`*(x: `typ`): `typ` {.borrow.} diff --git a/nimterop/toastlib/getters.nim b/nimterop/toastlib/getters.nim index f0a4735..2e1ac4c 100644 --- a/nimterop/toastlib/getters.nim +++ b/nimterop/toastlib/getters.nim @@ -31,7 +31,7 @@ yield""".split(Whitespace).toHashSet() const # Enum macro read from file - written into wrapper when required - gEnumMacroConst = staticRead(currentSourcePath.parentDir().parentDir() / "enumtype.nim") + gEnumMacroConst = "import nimterop / enumtypepub" var gEnumMacro* = gEnumMacroConst From 00941e892588503d18b3d271a65b0596bc6d857b Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 12 Sep 2020 10:59:14 -0600 Subject: [PATCH 094/106] Fix regen issues windows/linux --- nimterop/build/shell.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index f0af561..e5a70a8 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -206,9 +206,9 @@ proc getFileDate*(fullpath: string): string = when defined(Windows): let (head, tail) = fullpath.splitPath() - &"cmd /c forfiles /P {head.sanitizePath()} /M {tail.sanitizePath} /C \"cmd /c echo @fdate @ftime @fsize\"" + &"forfiles /P {head.sanitizePath()} /M {tail.sanitizePath} /C \"cmd /c echo @fdate @ftime\"" elif defined(Linux): - &"stat -c %y {fullpath.sanitizePath}" + &"stat -c %Y {fullpath.sanitizePath}" elif defined(OSX) or defined(FreeBSD): &"stat -f %m {fullpath.sanitizePath}" From 316a56107a4ccfcc0773d5a5b7f59abf0e05f922 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 12 Sep 2020 12:48:45 -0600 Subject: [PATCH 095/106] Add ability to get a define value --- nimterop/build/getheader.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 4549894..bbc810a 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -59,6 +59,11 @@ macro isDefined*(def: untyped): untyped = false ) +macro getDefine*(def: untyped): untyped = + if gDefines.hasKey(def.strVal()): + return newStrLitNode(gDefines[def.strVal()]) + return newStrLitNode("") + proc getDynlibExt(): string = when defined(Windows): result = "[0-9.\\-]*\\.dll" From 1b8aa46b8a5b8254cee5aa0c5d73e29881ca7ed9 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 12 Sep 2020 15:24:29 -0600 Subject: [PATCH 096/106] Improve on getdefine --- nimterop/build/getheader.nim | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index bbc810a..36dd50c 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -60,9 +60,15 @@ macro isDefined*(def: untyped): untyped = ) macro getDefine*(def: untyped): untyped = - if gDefines.hasKey(def.strVal()): - return newStrLitNode(gDefines[def.strVal()]) - return newStrLitNode("") + let version = newIdentNode(def.strVal()) + let verVal = + if gDefines.hasKey(def.strVal()): + gDefines[def.strVal()] + else: + "" + result = quote do: + const `version` {.strdefine.} = `verVal` + `version` proc getDynlibExt(): string = when defined(Windows): From 217bce386a2ef7119e4914b19461d18e0fae821a Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Thu, 17 Sep 2020 19:59:41 -0600 Subject: [PATCH 097/106] Don't die on nimcheck for cached files --- nimterop/build/shell.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index e5a70a8..a8511c2 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -212,7 +212,7 @@ proc getFileDate*(fullpath: string): string = elif defined(OSX) or defined(FreeBSD): &"stat -f %m {fullpath.sanitizePath}" - (result, ret) = execAction(cmd) + (result, ret) = execAction(cmd, die=false) proc touchFile*(fullpath: string) = ## Touch file to update modified date From 60cc0d26efb79e4becb064e2db3bf5973bd25b89 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Fri, 2 Oct 2020 17:46:27 -0500 Subject: [PATCH 098/106] Add musl support --- nimterop/build/ccompiler.nim | 4 +++- nimterop/build/conan.nim | 30 ++++++++++++++++++++++-------- nimterop/build/jbb.nim | 26 +++++++++++++++++++------- nimterop/build/shell.nim | 29 ++++++----------------------- 4 files changed, 50 insertions(+), 39 deletions(-) diff --git a/nimterop/build/ccompiler.nim b/nimterop/build/ccompiler.nim index 550e1b2..f21f45a 100644 --- a/nimterop/build/ccompiler.nim +++ b/nimterop/build/ccompiler.nim @@ -75,7 +75,7 @@ proc getGccLibPaths*(mode: string): seq[string] = when defined(osx): result.add "/usr/lib" -proc getGccInfo*(): tuple[arch, os, compiler, version: string] = +proc getGccInfo*(): tuple[arch, os, compiler, version, libc: string] = let (outp, _) = execAction(&"{getCompiler()} -v") for line in outp.splitLines(): @@ -101,3 +101,5 @@ proc getGccInfo*(): tuple[arch, os, compiler, version: string] = result.compiler = "clang" else: result.compiler = "gcc" + if "musl" in outp: + result.libc = "musl" diff --git a/nimterop/build/conan.nim b/nimterop/build/conan.nim index 6df25d8..ac74089 100644 --- a/nimterop/build/conan.nim +++ b/nimterop/build/conan.nim @@ -16,6 +16,8 @@ type channel*: string recipes*: OrderedTableRef[string, seq[ConanBuild]] + arch*, os*, compiler*, compversion*: string + bhash*: string shared*: bool sharedLibs*: seq[string] @@ -89,6 +91,11 @@ proc `==`*(pkg1, pkg2: ConanPackage): bool = pkg1.user == pkg2.user and pkg1.channel == pkg2.channel and + pkg1.arch == pkg2.arch and + pkg1.os == pkg2.os and + pkg1.compiler == pkg2.compiler and + pkg1.compversion == pkg2.compversion and + pkg1.bhash == pkg2.bhash and pkg1.shared == pkg2.shared) @@ -101,6 +108,15 @@ proc newConanPackage*(name, version, user = "_", channel = "_", bhash = "", shar result.channel = channel result.recipes = newOrderedTable[string, seq[ConanBuild]](2) + let + (arch, os, compiler, compversion, libc) = getGccInfo() + doAssert libc != "musl", "Conan does not provide precompiled binaries using musl" + + result.arch = arch + result.os = os + result.compiler = compiler + result.compversion = compversion + result.bhash = bhash result.shared = shared @@ -189,9 +205,7 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = ## `filter` can be used to tweak search terms ## e.g. build_type=Debug&compiler=clang let - (arch, os, compiler, version) = getGccInfo() - - vsplit = version.split('.') + vsplit = pkg.compversion.split('.') vfilter = when defined(OSX): @@ -203,18 +217,18 @@ proc getConanBuilds*(pkg: ConanPackage, filter = "") = if pkg.bhash.Bl: block: var - query = &"?q=arch={arch}&os={os.capitalizeAscii()}" + query = &"?q=arch={pkg.arch}&os={pkg.os.capitalizeAscii()}" if "build_type" notin filter: query &= "&build_type=Release" if "shared=" notin filter: query &= &"&options.shared={($pkg.shared).capitalizeAscii()}" if filter.nBl: query &= &"&{filter}" - if "compiler=" notin filter and os != "windows": - query &= &"&compiler={compiler}&compiler.version=" & vfilter - if "compiler.runtime=" notin filter and os == "windows": + if "compiler=" notin filter and pkg.os != "windows": + query &= &"&compiler={pkg.compiler}&compiler.version=" & vfilter + if "compiler.runtime=" notin filter and pkg.os == "windows": query &= &"&compiler.runtime=MD" - if "compiler.version=" notin filter and os == "windows": + if "compiler.version=" notin filter and pkg.os == "windows": query &= &"&compiler.version=14" query.replace("&", "%20and%20") diff --git a/nimterop/build/jbb.nim b/nimterop/build/jbb.nim index c8d86c3..392f848 100644 --- a/nimterop/build/jbb.nim +++ b/nimterop/build/jbb.nim @@ -17,6 +17,8 @@ type url*: string # Download URL + arch*, os*, libc*: string # Target + sharedLibs*: seq[string] staticLibs*: seq[string] requires*: seq[JBBPackage] @@ -39,7 +41,11 @@ proc `==`*(pkg1, pkg2: JBBPackage): bool = ## Check if two JBBPackage objects are equal (not pkg1.isNil and not pkg2.isNil and pkg1.name == pkg2.name and - pkg1.version == pkg2.version) + pkg1.version == pkg2.version and + + pkg1.arch == pkg2.arch and + pkg1.os == pkg2.os and + pkg1.libc == pkg2.libc) proc newJBBPackage*(name, version: string): JBBPackage = ## Create a new JBBPackage with specified name and version @@ -49,6 +55,12 @@ proc newJBBPackage*(name, version: string): JBBPackage = result.baseUrl = jbbBaseUrl result.isGit = true + let + (arch, os, _, _, libc) = getGccInfo() + result.arch = arch + result.os = os + result.libc = libc + proc parseJBBProject(pkg: JBBPackage, outdir: string) = # Get all dependencies from Project.toml let @@ -87,8 +99,6 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = let file = outdir / jbbArtifacts - (arch, os, _, _) = getGccInfo() - if fileExists(file): let data = readFile(file) @@ -109,12 +119,14 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = # Match arch, os and glibc on Linux to find download URL case name of "arch": - if val == arch and not found: found = true + if val == pkg.arch and not found: found = true of "os": - if val != os and found: found = false + if val != pkg.os and found: found = false of "libc": when defined(Linux): - if val != "glibc" and found: found = false + if found: + let libc = if pkg.libc.nBl: pkg.libc else: "glibc" + if val != libc: found = false of "url": if found: pkg.url = val @@ -125,7 +137,7 @@ proc parseJBBArtifacts(pkg: JBBPackage, outdir: string) = proc findJBBLibs(pkg: JBBPackage, outdir: string) = pkg.sharedLibs = findFiles("(bin|lib)[\\\\/].*\\.(so|dll|dylib)[0-9.]*", outdir) - for lib in findFiles("lib[\\\\/].*\\.(a|lib)$", outdir): + for lib in findFiles("lib[\\\\/].*\\.(a|lib)", outdir): if not lib.endsWith(".dll.a"): pkg.staticLibs.add lib diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index f0af561..d0a10fb 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -414,22 +414,8 @@ proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[s ## ## Turn off recursive search with `recurse` var - cmd = - when defined(Windows): - "nimgrep --filenames --oneline --nocolor $1 \"$2\" $3" - elif defined(linux): - "find $3 $1 -regextype egrep -regex $2" - elif defined(osx) or defined(FreeBSD): - "find -E $3 $1 -regex $2" - - recursive = "" - - if recurse: - when defined(Windows): - recursive = "--recursive" - else: - when not defined(Windows): - recursive = "-maxdepth 1" + cmd = "nimgrep --follow --filenames --oneline --nocolor $1 \"$2\" $3" + recursive = if recurse: "--recursive" else: "" var dir = dir @@ -443,20 +429,17 @@ proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[s file = file.extractFilename - cmd = cmd % [recursive, (".*[\\\\/]" & file & "$").quoteShell, dir.sanitizePath] + cmd = cmd % [recursive, (".*[\\\\/]" & file & "$"), dir.sanitizePath] let (files, ret) = execAction(cmd, die = false) if ret == 0: for line in files.splitLines(): let f = - when defined(Windows): - if ": " in line: - line.split(": ", maxsplit = 1)[1] - else: - "" + if ": " in line: + line.split(": ", maxsplit = 1)[1] else: - line + "" if f.len != 0: result.add f From 13e5ce7e221f9e675d6747857cd3f9dbef961d40 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 5 Oct 2020 12:17:36 -0500 Subject: [PATCH 099/106] Add loaf, replace find --- nimterop.nimble | 6 +++++- nimterop/build/shell.nim | 16 +++++++--------- nimterop/loaf.nim | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 nimterop/loaf.nim diff --git a/nimterop.nimble b/nimterop.nimble index 49c40d0..8e12ee8 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -5,7 +5,7 @@ author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" -bin = @["nimterop/toast"] +bin = @["nimterop/toast", "nimterop/loaf"] installDirs = @["nimterop"] # Dependencies @@ -33,6 +33,9 @@ proc execTest(test: string, flags = "", runDocs = true) = task buildTimeit, "build timer": exec "nim c --hints:off -d:danger tests/timeit" +task buildLoaf, "build loaf": + execCmd("nim c --hints:off -d:danger nimterop/loaf.nim") + task buildToast, "build toast": execCmd("nim c --hints:off -d:danger nimterop/toast.nim") @@ -89,6 +92,7 @@ task test, "Test": rmFile("tests/timeit.txt") buildTimeitTask() + buildLoafTask() buildToastTask() basicTask() diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index d0a10fb..7c65b71 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -407,6 +407,9 @@ proc gitTags*(outdir: string): seq[string] = if tag.len != 0: result.add tag +proc loafExePath(): string = + currentSourcePath.parentDir.parentDir / ("loaf".addFileExt ExeExt) + proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[string] = ## Find all matching files in the specified directory ## @@ -414,8 +417,8 @@ proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[s ## ## Turn off recursive search with `recurse` var - cmd = "nimgrep --follow --filenames --oneline --nocolor $1 \"$2\" $3" - recursive = if recurse: "--recursive" else: "" + cmd = loafExePath().quoteShell & " find --rexp $1 \"$2\" $3" + recursive = if recurse: "--recurse" else: "" var dir = dir @@ -435,13 +438,8 @@ proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[s (files, ret) = execAction(cmd, die = false) if ret == 0: for line in files.splitLines(): - let f = - if ": " in line: - line.split(": ", maxsplit = 1)[1] - else: - "" - if f.len != 0: - result.add f + if line.len != 0: + result.add line proc findFile*(file: string, dir: string, recurse = true, first = false, regex = false): string = ## Find the file in the specified directory diff --git a/nimterop/loaf.nim b/nimterop/loaf.nim new file mode 100644 index 0000000..04b377e --- /dev/null +++ b/nimterop/loaf.nim @@ -0,0 +1,39 @@ +import system except find + +import os + +import cligen + +import strutils except find +import regex except find + +proc findRec(dir: string, pattern: string | Regex, recurse: bool) = + for kind, path in walkDir(dir): + if kind in [pcDir, pcLinkToDir]: + if recurse: findRec(path, pattern, recurse) + elif pattern in path: + echo path.absolutePath() + +proc find(recurse = false, rexp = false, args: seq[string]) = + var + pat = "" + rpat: Regex + for arg in args: + if not arg.startsWith("-"): + if dirExists(arg): + if rexp: + findRec(arg, rpat, recurse) + else: + findRec(arg, pat, recurse) + else: + pat = arg + if rexp: rpat = re(arg) + +when isMainModule: + dispatchMulti([ + find, help = { + "recurse": "recursive search", + "rexp": "patterns are regular expressions", + "args": "pattern1 dir1 dir2 pattern2 dir3 ..." + } + ]) From 94b3b04e6d72d1e8f02bc116abb6e8d05a90c54e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 5 Oct 2020 13:33:54 -0500 Subject: [PATCH 100/106] Fix build error messaging --- nimterop/build/getheader.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nimterop/build/getheader.nim b/nimterop/build/getheader.nim index 4549894..918e923 100644 --- a/nimterop/build/getheader.nim +++ b/nimterop/build/getheader.nim @@ -208,11 +208,12 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT var lpath = findFile(lname, outdir, regex = true) makePath = outdir + buildStatus: BuildStatus + errors: seq[string] if lpath.nBl: return lpath - var buildStatus: BuildStatus for buildType in buildTypes: case buildType @@ -223,6 +224,8 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT if buildStatus.built: break + elif buildStatus.error.nBl: + errors.add buildStatus.error if buildStatus.buildPath.len > 0: let libraryExists = findFile(lname, buildStatus.buildPath, regex = true).len > 0 @@ -231,7 +234,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string, buildT make(buildStatus.buildPath, lname, makeFlags, regex = true) buildStatus.built = true - let error = if buildStatus.error.len > 0: buildStatus.error else: "No build files found in " & outdir + let error = if errors.len > 0: errors.join("\n") else: "No build files found in " & outdir doAssert buildStatus.built, &"\nBuild configuration failed - {error}\n" result = findFile(lname, outdir, regex = true) From 0c2ca16f7ad9b1798f1c28ca0a3268d98e845a8d Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Mon, 5 Oct 2020 21:25:35 -0500 Subject: [PATCH 101/106] v0.6.12 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 8e12ee8..6d4306c 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.11" +version = "0.6.12" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From 0e003b5dba306143beb586231f17e013a946e4b2 Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 8 Oct 2020 23:17:43 -0500 Subject: [PATCH 102/106] Build loaf for minitest --- nimterop.nimble | 1 + 1 file changed, 1 insertion(+) diff --git a/nimterop.nimble b/nimterop.nimble index 6d4306c..9a9083f 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -49,6 +49,7 @@ task docs, "Generate docs": buildDocs(@["nimterop/all.nim"], "build/htmldocs") task minitest, "Test for Nim CI": + exec "nim c -f -d:danger nimterop/loaf.nim" exec "nim c -f -d:danger nimterop/toast" exec "nim c -f -d:checkAbi -r tests/tast2.nim" exec "nim c -f -d:checkAbi -d:zlibJBB -d:zlibSetVer=1.2.11 -r tests/zlib.nim" From e987b50610c4756a1f5acaaba8fb865c6e0c8693 Mon Sep 17 00:00:00 2001 From: n5m <72841454+n5m@users.noreply.github.com> Date: Thu, 15 Oct 2020 13:48:39 +0000 Subject: [PATCH 103/106] ensure loaf exists before finding (#256) --- nimterop/build/shell.nim | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nimterop/build/shell.nim b/nimterop/build/shell.nim index 7c65b71..24075cf 100644 --- a/nimterop/build/shell.nim +++ b/nimterop/build/shell.nim @@ -416,8 +416,14 @@ proc findFiles*(file: string, dir: string, recurse = true, regex = false): seq[s ## `file` is a regular expression if `regex` is true ## ## Turn off recursive search with `recurse` + let + loafExe = loafExePath() + + doAssert fileExists(loafExe), "loaf not compiled: " & loafExe.sanitizePath & + " make sure 'nimble build' or 'nimble install' built it" + var - cmd = loafExePath().quoteShell & " find --rexp $1 \"$2\" $3" + cmd = loafExe.quoteShell & " find --rexp $1 \"$2\" $3" recursive = if recurse: "--recurse" else: "" var From f7cee5c983650336f93fde5d4fea087863ac0e5e Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Thu, 15 Oct 2020 08:48:57 -0500 Subject: [PATCH 104/106] v0.6.13 --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 9a9083f..15a8ed1 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.12" +version = "0.6.13" author = "genotrance" description = "C/C++ interop for Nim" license = "MIT" From b568863527d8b700922774e64a41db50f9b701cb Mon Sep 17 00:00:00 2001 From: Joey Date: Tue, 20 Oct 2020 11:08:52 -0600 Subject: [PATCH 105/106] Fix #233: make shl/shr use the direct left operand for type casting (#235) --- nimterop/toastlib/exprparser.nim | 16 +++++++++++++--- tests/include/tast2.h | 1 + tests/tast2.nim | 2 ++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/nimterop/toastlib/exprparser.nim b/nimterop/toastlib/exprparser.nim index a77dbf9..a67ed59 100644 --- a/nimterop/toastlib/exprparser.nim +++ b/nimterop/toastlib/exprparser.nim @@ -379,18 +379,28 @@ proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode) result.add gState.getIdent(nimSym) let leftNode = gState.processTSNode(left, typeofNode) + var tyNode = typeofNode if typeofNode.isNil: typeofNode = nkCall.newTree( gState.getIdent("typeof"), leftNode ) + tyNode = typeofNode - let rightNode = gState.processTSNode(right, typeofNode) + # Special case of setting the shift left/right type + # to be the type of the direct left operand + if binarySym in [">>", "<<"]: + tyNode = nkCall.newTree( + gState.getIdent("typeof"), + leftNode + ) + + let rightNode = gState.processTSNode(right, tyNode) result.add leftNode result.add nkCall.newTree( - typeofNode, + tyNode, rightNode ) if binarySym == "/": @@ -399,7 +409,7 @@ proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode) # So we need to emulate C here and cast the whole # expression to the type of the first arg result = nkCall.newTree( - typeofNode, + tyNode, result ) diff --git a/tests/include/tast2.h b/tests/include/tast2.h index 38c5f18..ede71d5 100644 --- a/tests/include/tast2.h +++ b/tests/include/tast2.h @@ -35,6 +35,7 @@ extern "C" { #define EQ4 AVAL < BVAL #define EQ5 AVAL != BVAL #define EQ6 AVAL == BVAL +#define SX_NEAR_ZERO (1.0f / (1 << 28)) // testing integer out of long int range #define INT_FAST16_MIN (-9223372036854775807L-1) diff --git a/tests/tast2.nim b/tests/tast2.nim index a410de4..e301fb5 100644 --- a/tests/tast2.nim +++ b/tests/tast2.nim @@ -134,6 +134,8 @@ assert EQ4 == (AVAL < BVAL) assert EQ5 == (AVAL != BVAL) assert EQ6 == (AVAL == BVAL) +assert SX_NEAR_ZERO == 3.725290298461914e-09 + assert SIZEOF == 1 assert COERCE == 645635670332'u64 From fa9e66b4a371b78d97d074cf531fad8f55135334 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Sat, 15 May 2021 18:39:47 -0600 Subject: [PATCH 106/106] Update cligen --- nimterop.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimterop.nimble b/nimterop.nimble index 15a8ed1..87e1bc4 100644 --- a/nimterop.nimble +++ b/nimterop.nimble @@ -9,7 +9,7 @@ bin = @["nimterop/toast", "nimterop/loaf"] installDirs = @["nimterop"] # Dependencies -requires "nim >= 0.20.2", "regex >= 0.15.0", "cligen >= 1.0.0" +requires "nim >= 0.20.2", "regex >= 0.15.0", "cligen >= 1.5.3" import nimterop/docs import os